@zerotal/queue 1.0.0

package/src/QueueManager.ts

@@ -0,0 +1,427 @@
+import { randomUUIDv7 } from "bun";
+import { QueueShuttingDownError, QueueBatchingUnsupportedError } from "./errors.ts";
+import type { QueueDriver, JobRecord } from "./drivers/QueueDriver.ts";
+import type { Job } from "./Job.ts";
+import type { PendingBatch } from "./PendingBatch.ts";
+import type { SerializedJob } from "./Batch.ts";
+import { Batch } from "./Batch.ts";
+import { JobRegistry } from "./JobRegistry.ts";
+import { SyncDriver } from "./drivers/SyncDriver.ts";
+import { FrameworkEvents } from "@zerotal/core";
+import { JobRan } from "./events.ts";
+import { frameworkLog } from "@zerotal/core/logger";
+
+export type JobStatus = "dispatched" | "completed" | "failed" | "retried";
+
+interface JobEvent {
+  className: string;
+  queue: string;
+  status: JobStatus;
+  durationMs: number;
+  error?: string | undefined;
+}
+
+function _fireJobEvent(
+  className: string,
+  queue: string,
+  status: JobStatus,
+  durationMs: number,
+  error?: string,
+): void {
+  _recordStat({ className, queue, status, durationMs, error });
+  FrameworkEvents.emit(new JobRan(className, queue, status, durationMs, error));
+}
+
+// ── Throughput stats (in-memory, best-effort — powers the admin Queue panel) ──
+
+interface JobStat {
+  t: number;
+  status: JobStatus;
+}
+const _stats: JobStat[] = [];
+let _processedTotal = 0;
+let _failedTotal = 0;
+
+function _recordStat(e: JobEvent): void {
+  if (e.status === "dispatched") return;
+  const now = Date.now();
+  _stats.push({ t: now, status: e.status });
+  if (e.status === "completed") _processedTotal++;
+  else if (e.status === "failed") _failedTotal++;
+  const cutoff = now - 30 * 60 * 1000; // keep ~30 min of events
+  while (_stats.length && _stats[0]!.t < cutoff) _stats.shift();
+  if (_stats.length > 5000) _stats.splice(0, _stats.length - 5000);
+}
+
+export interface QueueStats {
+  processedTotal: number;
+  failedTotal: number;
+  processedLast5m: number;
+  failedLast5m: number;
+}
+
+/** Best-effort in-process queue throughput counters (since boot). */
+export function queueStats(): QueueStats {
+  const cutoff = Date.now() - 5 * 60 * 1000;
+  let p = 0,
+    f = 0;
+  for (const s of _stats) {
+    if (s.t < cutoff) continue;
+    if (s.status === "completed") p++;
+    else if (s.status === "failed") f++;
+  }
+  return {
+    processedTotal: _processedTotal,
+    failedTotal: _failedTotal,
+    processedLast5m: p,
+    failedLast5m: f,
+  };
+}
+
+// ── QueueManager ──────────────────────────────────────────────────────────────
+
+type JobExecutor = {
+  run(record: JobRecord): Promise<{ success: true } | { success: false; error: string }>;
+};
+
+export class QueueManager {
+  private _driver: QueueDriver;
+  private _isShuttingDown: boolean = false;
+  private _activeJobCount: number = 0;
+  private _drainResolve: (() => void) | undefined = undefined;
+  private _executor: JobExecutor | undefined = undefined;
+
+  constructor(driver: QueueDriver) {
+    this._driver = driver;
+  }
+
+  setWorkerPool(pool: JobExecutor): void {
+    this._executor = pool;
+  }
+
+  // ── Dispatch ───────────────────────────────────────────────────────────
+
+  async dispatch(job: Job): Promise<void> {
+    if (this._isShuttingDown) {
+      throw new QueueShuttingDownError(job.className);
+    }
+
+    if (this._isSyncDriver()) {
+      SyncDriver._pendingJob = job;
+    }
+
+    const payload = job.payload();
+    if (job._chain?.length) {
+      payload["__chain"] = job._chain;
+    }
+
+    await this._driver.push({
+      queue: job.queue,
+      className: job.className,
+      payload: JSON.stringify(payload),
+      attempts: 0,
+      maxAttempts: job.maxAttempts,
+      retryDelay: job.retryDelay,
+      availableAt: Math.floor(Date.now() / 1000),
+      batchId: job.batchId,
+    });
+
+    _fireJobEvent(job.className, job.queue, "dispatched", 0);
+
+    if (this._isSyncDriver()) {
+      const record = await this._driver.pop(job.queue);
+      if (record) await this._processRecord(record, job);
+    }
+  }
+
+  async dispatchBatch(pending: PendingBatch): Promise<Batch> {
+    if (!this._driver.createBatch) {
+      throw new QueueBatchingUnsupportedError();
+    }
+
+    // v7 UUID: time-ordered, so persisted batch IDs sort by creation and give
+    // better index locality than random v4 IDs.
+    const id = randomUUIDv7();
+    const options = pending.options;
+
+    await this._driver.createBatch(
+      id,
+      pending.batchName,
+      pending.jobs.length,
+      JSON.stringify(options),
+    );
+
+    for (const job of pending.jobs) {
+      job.batchId = id;
+      await this.dispatch(job);
+    }
+
+    const record = await this._driver.getBatch!(id);
+    return new Batch(record!);
+  }
+
+  // ── Single job processing ─────────────────────────────────────────────
+
+  async processNext(queue = "default"): Promise<boolean> {
+    const record = await this._driver.pop(queue);
+    if (!record) return false;
+
+    if (this._executor) {
+      this._activeJobCount++;
+      const _t0 = Date.now();
+      try {
+        const result = await this._executor.run(record);
+        if (result.success) {
+          await this._driver.delete(record);
+          _fireJobEvent(record.className, record.queue, "completed", Date.now() - _t0);
+          await this._onJobComplete(record);
+        } else if (record.attempts >= record.maxAttempts) {
+          await this._driver.fail(record, result.error);
+          _fireJobEvent(record.className, record.queue, "failed", Date.now() - _t0, result.error);
+          await this._onJobFailed(record);
+          frameworkLog("queue").error(
+            `Job ${record.className} permanently failed after ${record.attempts} attempts`,
+            { job: record.className, queue: record.queue, attempts: record.attempts },
+            result.error,
+          );
+        } else {
+          await this._driver.retry(record);
+          _fireJobEvent(record.className, record.queue, "retried", Date.now() - _t0, result.error);
+          frameworkLog("queue").warn(
+            `Job ${record.className} failed (attempt ${record.attempts}/${record.maxAttempts}) — retrying`,
+            { job: record.className, queue: record.queue, attempts: record.attempts },
+            result.error,
+          );
+        }
+      } finally {
+        this._activeJobCount--;
+        if (this._isShuttingDown && this._activeJobCount === 0) {
+          this._drainResolve?.();
+        }
+      }
+      return true;
+    }
+
+    await this._processRecord(record);
+    return true;
+  }
+
+  // ── Worker loop control ───────────────────────────────────────────────
+
+  get isShuttingDown(): boolean {
+    return this._isShuttingDown;
+  }
+  get activeJobCount(): number {
+    return this._activeJobCount;
+  }
+
+  async drain(): Promise<void> {
+    this._isShuttingDown = true;
+    if (this._activeJobCount === 0) return;
+    return new Promise((resolve) => {
+      this._drainResolve = resolve;
+      setTimeout(resolve, 25_000);
+    });
+  }
+
+  async size(queue = "default"): Promise<number> {
+    return this._driver.size(queue);
+  }
+
+  // ── Introspection / management (used by tooling & the admin Queue panel) ──
+
+  /** Pending (not-yet-processed) jobs, newest queues first. */
+  async pending(queue?: string, limit = 50, offset = 0): Promise<JobRecord[]> {
+    return this._driver.listPending ? this._driver.listPending(queue, limit, offset) : [];
+  }
+
+  /** Distinct queue names with their pending depth. */
+  async queues(): Promise<{ queue: string; pending: number }[]> {
+    return this._driver.queues ? this._driver.queues() : [];
+  }
+
+  /** Failed jobs, optionally for a single queue. */
+  async failed(queue?: string) {
+    return this._driver.listFailed(queue);
+  }
+
+  /** Re-dispatch a failed job and remove it from the failed list. */
+  async retryFailed(id: number): Promise<boolean> {
+    const rec = (await this._driver.listFailed()).find((r) => r.id === id);
+    if (!rec) return false;
+    await this._driver.push({
+      queue: rec.queue,
+      className: rec.className,
+      payload: rec.payload,
+      attempts: 0,
+      maxAttempts: 3,
+      retryDelay: 0,
+      availableAt: Math.floor(Date.now() / 1000),
+      batchId: undefined,
+    });
+    await this._driver.deleteFailedRecord(id);
+    return true;
+  }
+
+  /** Forget a single failed job. */
+  async forgetFailed(id: number): Promise<void> {
+    return this._driver.deleteFailedRecord(id);
+  }
+
+  /** Clear failed jobs, optionally for a single queue. */
+  async clearFailed(queue?: string): Promise<void> {
+    return this._driver.clearFailed(queue);
+  }
+
+  /** In-process throughput counters. */
+  stats(): QueueStats {
+    return queueStats();
+  }
+
+  async flush(): Promise<void> {
+    return this._driver.flush();
+  }
+
+  // ── Private ───────────────────────────────────────────────────────────
+
+  private async _processRecord(record: JobRecord, jobInstance?: Job): Promise<void> {
+    this._activeJobCount++;
+    const _t0 = Date.now();
+    let instance: Job | undefined = undefined;
+
+    try {
+      instance = jobInstance ?? this._instantiate(record);
+      if (!instance) {
+        await this._driver.fail(record, `Unknown job class: ${record.className}`);
+        return;
+      }
+
+      await instance.handle();
+      await this._driver.delete(record);
+      _fireJobEvent(record.className, record.queue, "completed", Date.now() - _t0);
+      await this._onJobComplete(record);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+
+      if (record.attempts >= record.maxAttempts) {
+        await this._driver.fail(record, message);
+        _fireJobEvent(record.className, record.queue, "failed", Date.now() - _t0, message);
+        await this._onJobFailed(record);
+        frameworkLog("queue").error(
+          `Job ${record.className} permanently failed after ${record.attempts} attempts`,
+          { job: record.className, queue: record.queue, attempts: record.attempts },
+          message,
+        );
+      } else {
+        if (this._isSyncDriver() && instance) {
+          SyncDriver._pendingJob = instance;
+        }
+        await this._driver.retry(record);
+        _fireJobEvent(record.className, record.queue, "retried", Date.now() - _t0, message);
+        frameworkLog("queue").warn(
+          `Job ${record.className} failed (attempt ${record.attempts}/${record.maxAttempts}) — ` +
+            `retrying in ${record.retryDelay}ms`,
+          { job: record.className, queue: record.queue, attempts: record.attempts },
+          message,
+        );
+        if (this._isSyncDriver()) {
+          const nextRecord = await this._driver.pop(record.queue);
+          if (nextRecord) {
+            await this._processRecord(nextRecord, instance);
+          }
+        }
+      }
+    } finally {
+      this._activeJobCount--;
+      if (this._isShuttingDown && this._activeJobCount === 0) {
+        this._drainResolve?.();
+      }
+    }
+  }
+
+  private async _onJobComplete(record: JobRecord): Promise<void> {
+    const payload = JSON.parse(record.payload) as Record<string, unknown>;
+    const chain = payload["__chain"] as SerializedJob[] | undefined;
+    if (chain?.length) {
+      const next = chain[0]!;
+      const rest = chain.slice(1);
+      await this._driver.push({
+        queue: record.queue,
+        className: next.className,
+        payload: JSON.stringify(rest.length ? { ...next.payload, __chain: rest } : next.payload),
+        attempts: 0,
+        maxAttempts: record.maxAttempts,
+        retryDelay: record.retryDelay,
+        availableAt: Math.floor(Date.now() / 1000),
+        batchId: undefined,
+      });
+    }
+
+    if (record.batchId && this._driver.recordBatchJobComplete) {
+      const status = await this._driver.recordBatchJobComplete(record.batchId);
+      if (status !== "pending") {
+        await this._dispatchBatchCallbacks(record.batchId, status);
+      }
+    }
+  }
+
+  private async _onJobFailed(record: JobRecord): Promise<void> {
+    if (record.batchId && this._driver.recordBatchJobFailed) {
+      const status = await this._driver.recordBatchJobFailed(record.batchId, record.id);
+      if (status !== "pending") {
+        await this._dispatchBatchCallbacks(record.batchId, status);
+      }
+    }
+  }
+
+  private async _dispatchBatchCallbacks(
+    batchId: string,
+    status: "complete" | "failed",
+  ): Promise<void> {
+    if (!this._driver.getBatch) return;
+    const batchRecord = await this._driver.getBatch(batchId);
+    if (!batchRecord) return;
+
+    const opts = batchRecord.options;
+    if (status === "complete") {
+      for (const j of opts.thenJobs ?? []) await this._dispatchSerialized(j);
+    } else {
+      for (const j of opts.catchJobs ?? []) await this._dispatchSerialized(j);
+    }
+    for (const j of opts.finallyJobs ?? []) await this._dispatchSerialized(j);
+  }
+
+  private async _dispatchSerialized(serialized: SerializedJob): Promise<void> {
+    await this._driver.push({
+      queue: "default",
+      className: serialized.className,
+      payload: JSON.stringify(serialized.payload),
+      attempts: 0,
+      maxAttempts: 3,
+      retryDelay: 1000,
+      availableAt: Math.floor(Date.now() / 1000),
+      batchId: undefined,
+    });
+  }
+
+  private _instantiate(record: JobRecord): Job | undefined {
+    const JobClass = JobRegistry.resolve(record.className);
+    if (!JobClass) return undefined;
+
+    const payload = JSON.parse(record.payload) as Record<string, unknown>;
+    if (
+      "fromPayload" in JobClass &&
+      typeof (JobClass as unknown as { fromPayload?: (p: Record<string, unknown>) => Job })
+        .fromPayload === "function"
+    ) {
+      return (
+        JobClass as unknown as { fromPayload: (p: Record<string, unknown>) => Job }
+      ).fromPayload(payload);
+    }
+    return new (JobClass as new () => Job)();
+  }
+
+  private _isSyncDriver(): boolean {
+    return this._driver instanceof SyncDriver;
+  }
+}

package/src/WorkerPool.ts

@@ -0,0 +1,242 @@
+import type { JobRecord } from "./drivers/QueueDriver.ts";
+import { frameworkLog } from "@zerotal/core/logger";
+
+export interface WorkerPoolOptions {
+  /** Number of OS threads to spawn. */
+  size: number;
+  /**
+   * Optional absolute path or file URL to a module that imports+registers all job
+   * classes. When omitted, the worker thread discovers jobs by convention (imports
+   * every `app/jobs/*.ts`). Set it only to override that with an explicit bootstrap.
+   */
+  bootstrapPath?: string | undefined;
+  /**
+   * Most jobs allowed to wait in memory for a free thread. Default: `size * 64`.
+   *
+   * A job sitting in this backlog is already *claimed* — the driver has reserved it and is
+   * counting down its visibility timeout. Queue more than the pool can work through and
+   * those jobs are reclaimed and executed a second time while the first copy is still
+   * waiting its turn. Past the cap, `run()` reports failure immediately so the job goes
+   * back to the driver instead of aging out.
+   */
+  maxPending?: number | undefined;
+  /**
+   * How many times a slot may respawn after its worker dies before the pool gives up on it.
+   * Default: 5. A worker whose script throws on load would otherwise respawn forever.
+   */
+  maxRespawns?: number | undefined;
+}
+
+export type WorkerResult = { success: true } | { success: false; error: string };
+
+interface WorkerState {
+  worker: Worker;
+  busy: boolean;
+  resolve?: (r: WorkerResult) => void;
+  /** Consecutive respawns for this slot; reset by a successful job. */
+  respawns: number;
+  /** Set once the slot has exhausted its respawns — it is skipped when handing out work. */
+  dead: boolean;
+}
+
+/**
+ * Manages a pool of Bun Web Worker threads for off-main-thread job execution.
+ *
+ * Each worker is a genuine OS thread (not just a separate V8 context), so
+ * CPU-intensive jobs run without stalling the HTTP event loop.
+ *
+ * Usage:
+ *   const pool = new WorkerPool({ size: 2, bootstrapPath: '...' });
+ *   await pool.start();                    // spawns + bootstraps threads
+ *   const result = await pool.run(record); // executes job on a free thread
+ *   await pool.terminate();                // cleanly shuts threads down
+ */
+export class WorkerPool {
+  private _workers: WorkerState[] = [];
+  private _pending: Array<{ record: JobRecord; resolve: (r: WorkerResult) => void }> = [];
+
+  constructor(private readonly opts: WorkerPoolOptions) {}
+
+  /** True once {@link terminate} has run, so a dying worker is not respawned. */
+  private _terminated = false;
+
+  /** Spawn all worker threads and wait until every thread has bootstrapped. */
+  async start(): Promise<void> {
+    this._terminated = false;
+    const readyPromises: Promise<void>[] = [];
+
+    for (let i = 0; i < this.opts.size; i++) {
+      const state: WorkerState = { worker: null!, busy: false, respawns: 0, dead: false };
+      this._workers.push(state);
+      readyPromises.push(this._spawn(state));
+    }
+
+    await Promise.all(readyPromises);
+  }
+
+  /**
+   * Bring up a worker in `state` and wire its full lifecycle.
+   *
+   * Every path that ends a worker settles the in-flight promise and clears `busy`. Without
+   * `error`/`close` listeners a thread that died mid-job left its caller's promise pending
+   * forever and its slot marked busy — permanently wedging a `size: 1` pool — and a
+   * bootstrap failure that surfaced as an error rather than a message hung `start()`, and
+   * with it `app.boot()`.
+   *
+   * @returns Resolves when the thread reports ready; rejects if it fails to bootstrap.
+   */
+  private _spawn(state: WorkerState): Promise<void> {
+    const scriptUrl = new URL("./queue-worker.ts", import.meta.url);
+    const worker = new Worker(scriptUrl);
+    state.worker = worker;
+    state.busy = false;
+
+    let settleReady: (() => void) | undefined;
+    let failReady: ((err: Error) => void) | undefined;
+    const ready = new Promise<void>((resolve, reject) => {
+      settleReady = resolve;
+      failReady = reject;
+    });
+
+    const onInit = (event: MessageEvent) => {
+      const msg = event.data as { type: string; message?: string };
+      if (msg.type === "ready") {
+        worker.removeEventListener("message", onInit as EventListener);
+        settleReady?.();
+      } else if (msg.type === "bootstrap-error") {
+        worker.removeEventListener("message", onInit as EventListener);
+        failReady?.(new Error(`[Zerotal Queue] Worker bootstrap failed: ${msg.message}`));
+      }
+    };
+    worker.addEventListener("message", onInit as EventListener);
+
+    // Job results.
+    worker.addEventListener("message", (event: MessageEvent) => {
+      const msg = event.data as { type: string; message?: string };
+      if (msg.type !== "done" && msg.type !== "error") return;
+
+      const result: WorkerResult =
+        msg.type === "done"
+          ? { success: true }
+          : { success: false, error: msg.message ?? "Unknown error" };
+
+      // A thread that completed a job is healthy, whatever the job's own outcome.
+      state.respawns = 0;
+      this._settle(state, result);
+    });
+
+    // The thread threw outside a job, or failed to load its script.
+    worker.addEventListener("error", (event: ErrorEvent) => {
+      const detail = event.message || "worker error";
+      failReady?.(new Error(`[Zerotal Queue] Worker failed to start: ${detail}`));
+      this._onWorkerLost(state, `Worker errored: ${detail}`);
+    });
+
+    // The thread exited — crashed, killed, or terminated by us.
+    worker.addEventListener("close", () => {
+      failReady?.(new Error("[Zerotal Queue] Worker exited before bootstrapping"));
+      this._onWorkerLost(state, "Worker exited before the job completed");
+    });
+
+    worker.postMessage({ type: "bootstrap", modulePath: this.opts.bootstrapPath });
+    return ready;
+  }
+
+  /** Release a worker's in-flight job with `result` and hand it the next one. */
+  private _settle(state: WorkerState, result: WorkerResult): void {
+    state.busy = false;
+    const resolve = state.resolve;
+    delete state.resolve;
+    resolve?.(result);
+    this._drainPending();
+  }
+
+  /**
+   * A worker died. Settle whatever it was running, then replace it so the pool does not
+   * quietly shrink toward zero throughput.
+   */
+  private _onWorkerLost(state: WorkerState, reason: string): void {
+    if (this._terminated || state.dead) return;
+    this._settle(state, { success: false, error: reason });
+
+    if (++state.respawns > (this.opts.maxRespawns ?? 5)) {
+      // Respawning a worker whose script throws on load is an infinite loop. Mark the slot
+      // dead and keep serving from the rest; a pool with no live slots fails jobs fast
+      // rather than accumulating them.
+      state.dead = true;
+      frameworkLog("queue").error(`Worker slot gave up after ${state.respawns} respawns`, {
+        respawns: state.respawns,
+        reason,
+      });
+      this._drainPending();
+      return;
+    }
+
+    void this._spawn(state).catch(() => {
+      // The replacement failed to bootstrap; its own error/close listener handles the
+      // next attempt, so there is nothing to do here but not crash.
+    });
+  }
+
+  /**
+   * Send a job record to a free worker thread.
+   *
+   * When every thread is busy the job waits in memory, up to {@link WorkerPoolOptions.maxPending}.
+   * Beyond that it is reported as failed straight away: a waiting job is already claimed
+   * from the driver, and one that waits past its visibility timeout gets reclaimed and run
+   * a second time while the first copy is still queued here.
+   */
+  run(record: JobRecord): Promise<WorkerResult> {
+    const maxPending = this.opts.maxPending ?? this.opts.size * 64;
+    if (this._pending.length >= maxPending) {
+      return Promise.resolve({
+        success: false,
+        error: `WorkerPool backlog is full (${maxPending}) — job returned to the queue`,
+      });
+    }
+    if (this._workers.length > 0 && this._workers.every((w) => w.dead)) {
+      return Promise.resolve({
+        success: false,
+        error: "WorkerPool has no live workers — job returned to the queue",
+      });
+    }
+    return new Promise((resolve) => {
+      this._pending.push({ record, resolve });
+      this._drainPending();
+    });
+  }
+
+  /** Terminate all worker threads. Call after QueueManager.drain() completes. */
+  async terminate(): Promise<void> {
+    // Set first: terminate() closes every worker, and the close listener must not read
+    // that as a crash and respawn what we are shutting down.
+    this._terminated = true;
+
+    // Reject any callers still waiting for a worker — avoids hanging promises.
+    for (const { resolve } of this._pending) {
+      resolve({ success: false, error: "WorkerPool terminated before job could run" });
+    }
+    this._pending = [];
+
+    for (const state of this._workers) {
+      // Reject any job currently in-flight on this thread.
+      if (state.busy && state.resolve) {
+        state.resolve({ success: false, error: "WorkerPool terminated while job was running" });
+        delete state.resolve;
+      }
+      state.worker.terminate();
+    }
+    this._workers = [];
+  }
+
+  private _drainPending(): void {
+    while (this._pending.length > 0) {
+      const free = this._workers.find((w) => !w.busy && !w.dead);
+      if (!free) break;
+      const next = this._pending.shift()!;
+      free.busy = true;
+      free.resolve = next.resolve;
+      free.worker.postMessage({ type: "job", record: next.record });
+    }
+  }
+}