@valentinkolb/sync 0.1.0

package/src/jobs.ts

@@ -0,0 +1,568 @@
+import { redis, sleep } from "bun";
+import { randomBytes } from "crypto";
+import type { z } from "zod";
+
+// ==========================
+// Types
+// ==========================
+
+type JobStatus = "waiting" | "delayed" | "active" | "completed" | "failed";
+
+export type Job<T> = {
+  id: string;
+  data: T;
+  status: JobStatus;
+  attempts: number;
+  maxRetries: number;
+  timeout: number;
+  interval?: number;
+  createdAt: number;
+  scheduledAt?: number;
+  startedAt?: number;
+  completedAt?: number;
+  error?: string;
+};
+
+type BaseOptions = {
+  /** Number of retries on failure (default: 0) */
+  retries?: number;
+  /** Max processing time in ms before job is marked as failed (default: 30000) */
+  timeout?: number;
+};
+
+type DelayOptions = BaseOptions & {
+  /** Delay execution by this many milliseconds */
+  delay: number;
+  at?: never;
+  interval?: never;
+  startImmediately?: never;
+};
+
+type AtOptions = BaseOptions & {
+  /** Schedule execution at this timestamp (ms since epoch) */
+  at: number;
+  delay?: never;
+  interval?: never;
+  startImmediately?: never;
+};
+
+type IntervalOptions = BaseOptions & {
+  /** Repeat job every X milliseconds (for periodic tasks) */
+  interval: number;
+  /** Start first run immediately instead of waiting for first interval (default: false) */
+  startImmediately?: boolean;
+  delay?: never;
+  at?: never;
+};
+
+type ImmediateOptions = BaseOptions & {
+  delay?: never;
+  at?: never;
+  interval?: never;
+  startImmediately?: never;
+};
+
+export type SendOptions = DelayOptions | AtOptions | IntervalOptions | ImmediateOptions;
+
+export type JobsConfig<T extends z.ZodType> = {
+  /** Job queue name (used as Redis key prefix) */
+  name: string;
+  /** Zod schema for job data validation */
+  schema: T;
+  /** Prefix for all Redis keys (default: "jobs") */
+  prefix?: string;
+};
+
+export type ProcessOptions = {
+  /** Number of concurrent jobs to process (default: 1) */
+  concurrency?: number;
+  /** Poll interval in ms when queue is empty (default: 1000) */
+  pollInterval?: number;
+  /** Called when a job completes successfully */
+  onSuccess?: (job: Job<unknown>) => void | Promise<void>;
+  /** Called when a job fails (after all retries exhausted, or interval job rescheduled after failure) */
+  onError?: (job: Job<unknown>, error: Error) => void | Promise<void>;
+  /** Called after every job attempt (success or failure) */
+  onFinally?: (job: Job<unknown>) => void | Promise<void>;
+};
+
+export type Jobs<T> = {
+  /** Send a job to the queue */
+  send: (data: T, options?: SendOptions) => Promise<Job<T>>;
+  /** Process jobs from the queue */
+  process: (handler: (job: Job<T>) => Promise<void> | void, options?: ProcessOptions) => () => void;
+};
+
+// ==========================
+// Defaults
+// ==========================
+
+const DEFAULT_TIMEOUT = 30000;
+const DEFAULT_RETRIES = 0;
+const DEFAULT_POLL_INTERVAL = 1000;
+const DEFAULT_CONCURRENCY = 1;
+
+// ==========================
+// Errors
+// ==========================
+
+export class JobsError extends Error {
+  readonly code: string;
+
+  constructor(message: string, code: string) {
+    super(message);
+    this.name = "JobsError";
+    this.code = code;
+  }
+}
+
+export class ValidationError extends JobsError {
+  readonly issues: z.ZodIssue[];
+
+  constructor(issues: z.ZodIssue[]) {
+    super("Job data validation failed", "VALIDATION_ERROR");
+    this.name = "ValidationError";
+    this.issues = issues;
+  }
+}
+
+// ==========================
+// Lua Scripts for Atomic Operations
+// ==========================
+
+// Atomically move job from waiting to active and update job status
+const CLAIM_JOB_SCRIPT = `
+  local jobId = redis.call("RPOPLPUSH", KEYS[1], KEYS[2])
+  if not jobId then
+    return nil
+  end
+
+  local jobData = redis.call("HGET", KEYS[3], jobId)
+  if not jobData then
+    redis.call("LREM", KEYS[2], 1, jobId)
+    return nil
+  end
+
+  local job = cjson.decode(jobData)
+  job.status = "active"
+  job.startedAt = tonumber(ARGV[1])
+  job.attempts = job.attempts + 1
+
+  redis.call("HSET", KEYS[3], jobId, cjson.encode(job))
+
+  -- Store deadline based on job's timeout
+  local deadline = tonumber(ARGV[1]) + job.timeout
+  redis.call("ZADD", KEYS[4], deadline, jobId)
+
+  return cjson.encode(job)
+`;
+
+// Atomically complete a job (and reschedule if interval)
+const COMPLETE_JOB_SCRIPT = `
+  local jobId = ARGV[1]
+  local completedAt = tonumber(ARGV[2])
+
+  local jobData = redis.call("HGET", KEYS[1], jobId)
+  if not jobData then
+    return 0
+  end
+
+  local job = cjson.decode(jobData)
+
+  redis.call("LREM", KEYS[2], 1, jobId)
+  redis.call("ZREM", KEYS[4], jobId)
+
+  -- If job has interval, reschedule it
+  if job.interval and job.interval > 0 then
+    job.status = "delayed"
+    job.attempts = 0
+    job.error = nil
+    job.startedAt = nil
+    job.completedAt = nil
+    job.scheduledAt = completedAt + job.interval
+
+    redis.call("HSET", KEYS[1], jobId, cjson.encode(job))
+    redis.call("ZADD", KEYS[5], job.scheduledAt, jobId)
+    return 2
+  else
+    job.status = "completed"
+    job.completedAt = completedAt
+
+    redis.call("HSET", KEYS[1], jobId, cjson.encode(job))
+    redis.call("SADD", KEYS[3], jobId)
+    return 1
+  end
+`;
+
+// Atomically fail or retry a job
+// Returns: 1 = retried, 2 = failed permanently, 3 = interval rescheduled
+const FAIL_OR_RETRY_SCRIPT = `
+  local jobId = ARGV[1]
+  local errorMsg = ARGV[2]
+  local completedAt = tonumber(ARGV[3])
+
+  local jobData = redis.call("HGET", KEYS[1], jobId)
+  if not jobData then
+    return 0
+  end
+
+  local job = cjson.decode(jobData)
+  job.error = errorMsg
+
+  redis.call("LREM", KEYS[2], 1, jobId)
+  redis.call("ZREM", KEYS[5], jobId)
+
+  -- Check if we can retry (attempts <= maxRetries means 1 initial + N retries)
+  if job.attempts <= job.maxRetries then
+    job.status = "waiting"
+    redis.call("HSET", KEYS[1], jobId, cjson.encode(job))
+    redis.call("LPUSH", KEYS[3], jobId)
+    return 1
+  end
+
+  -- No more retries - check if interval job should be rescheduled
+  if job.interval and job.interval > 0 then
+    job.status = "delayed"
+    job.attempts = 0
+    job.startedAt = nil
+    job.scheduledAt = completedAt + job.interval
+    redis.call("HSET", KEYS[1], jobId, cjson.encode(job))
+    redis.call("ZADD", KEYS[6], job.scheduledAt, jobId)
+    return 3
+  end
+
+  -- Permanent failure
+  job.status = "failed"
+  job.completedAt = completedAt
+  redis.call("HSET", KEYS[1], jobId, cjson.encode(job))
+  redis.call("SADD", KEYS[4], jobId)
+  return 2
+`;
+
+// Fail timed out jobs (hard fail, no recovery)
+const FAIL_TIMED_OUT_SCRIPT = `
+  local now = tonumber(ARGV[1])
+
+  local timedOutJobs = redis.call("ZRANGEBYSCORE", KEYS[1], "0", tostring(now))
+  local failed = 0
+
+  for _, jobId in ipairs(timedOutJobs) do
+    local jobData = redis.call("HGET", KEYS[2], jobId)
+    if jobData then
+      local job = cjson.decode(jobData)
+
+      redis.call("ZREM", KEYS[1], jobId)
+      redis.call("LREM", KEYS[3], 0, jobId)
+
+      job.status = "failed"
+      job.completedAt = now
+      job.error = "Job timed out"
+      redis.call("HSET", KEYS[2], jobId, cjson.encode(job))
+      redis.call("SADD", KEYS[4], jobId)
+      failed = failed + 1
+    else
+      redis.call("ZREM", KEYS[1], jobId)
+    end
+  end
+
+  return failed
+`;
+
+// Promote delayed jobs that are ready
+const PROMOTE_DELAYED_SCRIPT = `
+  local now = tonumber(ARGV[1])
+  local readyJobs = redis.call("ZRANGEBYSCORE", KEYS[1], "0", tostring(now))
+  local promoted = 0
+
+  for _, jobId in ipairs(readyJobs) do
+    local removed = redis.call("ZREM", KEYS[1], jobId)
+    if removed > 0 then
+      local jobData = redis.call("HGET", KEYS[3], jobId)
+      if jobData then
+        local job = cjson.decode(jobData)
+        job.status = "waiting"
+        redis.call("HSET", KEYS[3], jobId, cjson.encode(job))
+        redis.call("LPUSH", KEYS[2], jobId)
+        promoted = promoted + 1
+      end
+    end
+  end
+
+  return promoted
+`;
+
+// ==========================
+// Jobs Factory
+// ==========================
+
+/**
+ * Create a typed job queue with Zod schema validation.
+ * All state is stored in Redis, making it safe to use across multiple processes.
+ *
+ * @example
+ * ```ts
+ * import { z } from "zod";
+ * import { jobs } from "@valentinkolb/sync";
+ *
+ * const emailJobs = jobs.create({
+ *   name: "emails",
+ *   schema: z.object({
+ *     to: z.string().email(),
+ *     subject: z.string(),
+ *     body: z.string(),
+ *   }),
+ * });
+ *
+ * // Send a job immediately
+ * await emailJobs.send({ to: "user@example.com", subject: "Hello", body: "World" });
+ *
+ * // Send with delay and retries
+ * await emailJobs.send(data, { delay: 5000, retries: 3 });
+ *
+ * // Send scheduled with custom timeout
+ * await emailJobs.send(data, { at: Date.now() + 60000, timeout: 60000 });
+ *
+ * // Send periodic job (runs every hour)
+ * await emailJobs.send(data, { interval: 3600000 });
+ *
+ * // Process jobs (in worker process)
+ * const stop = emailJobs.process(async (job) => {
+ *   await sendEmail(job.data);
+ * }, { concurrency: 10 });
+ *
+ * // Stop processing
+ * stop();
+ * ```
+ */
+const create = <T extends z.ZodType>(config: JobsConfig<T>): Jobs<z.infer<T>> => {
+  type Data = z.infer<T>;
+
+  const { name, schema, prefix = "jobs" } = config;
+
+  // Redis keys
+  const keys = {
+    jobs: `${prefix}:${name}:data`,
+    waiting: `${prefix}:${name}:waiting`,
+    delayed: `${prefix}:${name}:delayed`,
+    active: `${prefix}:${name}:active`,
+    deadlines: `${prefix}:${name}:deadlines`,
+    completed: `${prefix}:${name}:completed`,
+    failed: `${prefix}:${name}:failed`,
+    id: `${prefix}:${name}:id`,
+  };
+
+  // ==========================
+  // Internal Helpers
+  // ==========================
+
+  const generateId = async (): Promise<string> => {
+    const counter = await redis.incr(keys.id);
+    const random = randomBytes(4).toString("hex");
+    return `${counter}-${random}`;
+  };
+
+  const deserializeJob = (data: string): Job<Data> => {
+    const job = JSON.parse(data) as Job<Data>;
+    const result = schema.safeParse(job.data);
+    if (!result.success) {
+      throw new ValidationError(result.error.issues);
+    }
+    return job;
+  };
+
+  // ==========================
+  // Public API
+  // ==========================
+
+  const send = async (data: Data, options: SendOptions = {}): Promise<Job<Data>> => {
+    const result = schema.safeParse(data);
+    if (!result.success) {
+      throw new ValidationError(result.error.issues);
+    }
+
+    const id = await generateId();
+    const now = Date.now();
+
+    // Extract interval from options (TypeScript knows it's only present in IntervalOptions)
+    const interval = "interval" in options ? options.interval : undefined;
+    const startImmediately = "startImmediately" in options ? options.startImmediately : undefined;
+
+    const job: Job<Data> = {
+      id,
+      data: result.data,
+      status: "waiting",
+      attempts: 0,
+      maxRetries: options.retries ?? DEFAULT_RETRIES,
+      timeout: options.timeout ?? DEFAULT_TIMEOUT,
+      interval,
+      createdAt: now,
+    };
+
+    // Handle scheduling
+    if ("delay" in options && options.delay !== undefined) {
+      // Delayed job
+      job.status = "delayed";
+      job.scheduledAt = now + options.delay;
+      await redis.hset(keys.jobs, id, JSON.stringify(job));
+      await redis.send("ZADD", [keys.delayed, job.scheduledAt.toString(), id]);
+    } else if ("at" in options && options.at !== undefined) {
+      // Scheduled job
+      job.status = "delayed";
+      job.scheduledAt = options.at;
+      await redis.hset(keys.jobs, id, JSON.stringify(job));
+      await redis.send("ZADD", [keys.delayed, job.scheduledAt.toString(), id]);
+    } else if (interval !== undefined && !startImmediately) {
+      // Interval job, start after first interval
+      job.status = "delayed";
+      job.scheduledAt = now + interval;
+      await redis.hset(keys.jobs, id, JSON.stringify(job));
+      await redis.send("ZADD", [keys.delayed, job.scheduledAt.toString(), id]);
+    } else {
+      // Immediate job (or interval with startImmediately)
+      await redis.hset(keys.jobs, id, JSON.stringify(job));
+      await redis.lpush(keys.waiting, id);
+    }
+
+    return job;
+  };
+
+  const process = (handler: (job: Job<Data>) => Promise<void> | void, options: ProcessOptions = {}): (() => void) => {
+    const {
+      concurrency = DEFAULT_CONCURRENCY,
+      pollInterval = DEFAULT_POLL_INTERVAL,
+      onSuccess,
+      onError,
+      onFinally,
+    } = options;
+
+    let running = true;
+
+    const processLoop = async (): Promise<void> => {
+      while (running) {
+        try {
+          // Promote delayed jobs
+          await redis.send("EVAL", [
+            PROMOTE_DELAYED_SCRIPT,
+            "3",
+            keys.delayed,
+            keys.waiting,
+            keys.jobs,
+            Date.now().toString(),
+          ]);
+
+          // Fail timed out jobs
+          await redis.send("EVAL", [
+            FAIL_TIMED_OUT_SCRIPT,
+            "4",
+            keys.deadlines,
+            keys.jobs,
+            keys.active,
+            keys.failed,
+            Date.now().toString(),
+          ]);
+
+          // Claim a job
+          const now = Date.now();
+
+          const jobJson = (await redis.send("EVAL", [
+            CLAIM_JOB_SCRIPT,
+            "4",
+            keys.waiting,
+            keys.active,
+            keys.jobs,
+            keys.deadlines,
+            now.toString(),
+          ])) as string | null;
+
+          if (!jobJson) {
+            await sleep(pollInterval);
+            continue;
+          }
+
+          const job = deserializeJob(jobJson);
+
+          try {
+            await handler(job);
+
+            await redis.send("EVAL", [
+              COMPLETE_JOB_SCRIPT,
+              "5",
+              keys.jobs,
+              keys.active,
+              keys.completed,
+              keys.deadlines,
+              keys.delayed,
+              job.id,
+              Date.now().toString(),
+            ]);
+
+            // Call onSuccess
+            if (onSuccess) {
+              try {
+                await onSuccess(job);
+              } catch (onSuccessError) {
+                console.error("onSuccess callback failed:", onSuccessError);
+              }
+            }
+          } catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+
+            // Returns: 1 = retried, 2 = failed permanently, 3 = interval rescheduled
+            const result = (await redis.send("EVAL", [
+              FAIL_OR_RETRY_SCRIPT,
+              "6",
+              keys.jobs,
+              keys.active,
+              keys.waiting,
+              keys.failed,
+              keys.deadlines,
+              keys.delayed,
+              job.id,
+              err.message,
+              Date.now().toString(),
+            ])) as number;
+
+            // Call onError if job permanently failed or interval rescheduled after failure
+            if (onError && (result === 2 || result === 3)) {
+              try {
+                await onError(job, err);
+              } catch (onErrorError) {
+                console.error("onError callback failed:", onErrorError);
+              }
+            }
+          } finally {
+            // Call onFinally after every attempt
+            if (onFinally) {
+              try {
+                await onFinally(job);
+              } catch (onFinallyError) {
+                console.error("onFinally callback failed:", onFinallyError);
+              }
+            }
+          }
+        } catch (error) {
+          console.error("Worker error:", error);
+          await sleep(pollInterval);
+        }
+      }
+    };
+
+    // Start workers
+    for (let i = 0; i < concurrency; i++) {
+      processLoop();
+    }
+
+    // Return stop function
+    return () => {
+      running = false;
+    };
+  };
+
+  return { send, process };
+};
+
+// ==========================
+// Export
+// ==========================
+
+export const jobs = { create };