@zerotal/queue 1.0.0

package/src/drivers/SqliteDriver.ts

@@ -0,0 +1,366 @@
+import type { SQLInstance } from "@zerotal/orm";
+import type { QueueDriver, JobRecord, FailedRecord, BatchStatus } from "./QueueDriver.ts";
+import type { BatchRecord, BatchOptions } from "../Batch.ts";
+
+type JobRow = {
+  id: number;
+  queue: string;
+  class_name: string;
+  payload: string;
+  attempts: number;
+  max_attempts: number;
+  retry_delay: number;
+  available_at: number;
+  reserved_at: number | null;
+  created_at: string;
+  batch_id: string | null;
+};
+
+type FailedRow = {
+  id: number;
+  queue: string;
+  class_name: string;
+  payload: string;
+  attempts: number;
+  error: string;
+  failed_at: string;
+};
+
+type BatchRow = {
+  id: string;
+  name: string;
+  total_jobs: number;
+  pending_jobs: number;
+  failed_jobs: number;
+  failed_job_ids: string;
+  options: string;
+  created_at: string;
+  finished_at: string | null;
+};
+
+export class SqliteDriver implements QueueDriver {
+  private _sql: SQLInstance;
+  private _ready: Promise<void>;
+  /** Seconds a claimed job stays invisible before it is considered abandoned. */
+  private _visibilityTimeout: number;
+
+  constructor(sql: SQLInstance, options: { visibilityTimeout?: number } = {}) {
+    this._sql = sql;
+    // 90s is a sensible retry_after default. Must exceed the longest
+    // expected job runtime, or a slow (but alive) worker's job will be
+    // re-claimed by another worker.
+    this._visibilityTimeout = options.visibilityTimeout ?? 90;
+    this._ready = this._ensureTables();
+  }
+
+  private async _ensureTables(): Promise<void> {
+    await this._sql`
+      CREATE TABLE IF NOT EXISTS zerotal_jobs (
+        id           INTEGER PRIMARY KEY AUTOINCREMENT,
+        queue        TEXT    NOT NULL DEFAULT 'default',
+        class_name   TEXT    NOT NULL,
+        payload      TEXT    NOT NULL,
+        attempts     INTEGER NOT NULL DEFAULT 0,
+        max_attempts INTEGER NOT NULL DEFAULT 3,
+        retry_delay  INTEGER NOT NULL DEFAULT 1000,
+        available_at INTEGER NOT NULL,
+        reserved_at  INTEGER,
+        created_at   TEXT    NOT NULL,
+        batch_id     TEXT
+      )
+    `;
+    await this._sql`
+      CREATE TABLE IF NOT EXISTS zerotal_failed_jobs (
+        id         INTEGER PRIMARY KEY AUTOINCREMENT,
+        queue      TEXT    NOT NULL,
+        class_name TEXT    NOT NULL,
+        payload    TEXT    NOT NULL,
+        attempts   INTEGER NOT NULL,
+        error      TEXT    NOT NULL,
+        failed_at  TEXT    NOT NULL
+      )
+    `;
+    await this._sql`
+      CREATE TABLE IF NOT EXISTS zerotal_job_batches (
+        id             TEXT    PRIMARY KEY,
+        name           TEXT    NOT NULL DEFAULT '',
+        total_jobs     INTEGER NOT NULL DEFAULT 0,
+        pending_jobs   INTEGER NOT NULL DEFAULT 0,
+        failed_jobs    INTEGER NOT NULL DEFAULT 0,
+        failed_job_ids TEXT    NOT NULL DEFAULT '[]',
+        options        TEXT    NOT NULL DEFAULT '{}',
+        created_at     TEXT    NOT NULL,
+        finished_at    TEXT
+      )
+    `;
+    // Add batch_id column to existing databases that predate this migration.
+    try {
+      await this._sql`ALTER TABLE zerotal_jobs ADD COLUMN batch_id TEXT`;
+    } catch {
+      // Column already exists — safe to ignore.
+    }
+    // Add reserved_at column to existing databases that predate this migration.
+    try {
+      await this._sql`ALTER TABLE zerotal_jobs ADD COLUMN reserved_at INTEGER`;
+    } catch {
+      // Column already exists — safe to ignore.
+    }
+    await this._sql`
+      CREATE INDEX IF NOT EXISTS zerotal_jobs_pop_idx
+      ON zerotal_jobs (queue, available_at, reserved_at)
+    `;
+  }
+
+  private _mapRow(row: JobRow): JobRecord {
+    return {
+      id: row.id,
+      queue: row.queue,
+      className: row.class_name,
+      payload: row.payload,
+      attempts: row.attempts,
+      maxAttempts: row.max_attempts,
+      retryDelay: row.retry_delay,
+      availableAt: row.available_at,
+      createdAt: row.created_at,
+      batchId: row.batch_id ?? undefined,
+    };
+  }
+
+  private _mapBatchRow(row: BatchRow): BatchRecord {
+    return {
+      id: row.id,
+      name: row.name,
+      totalJobs: row.total_jobs,
+      pendingJobs: row.pending_jobs,
+      failedJobs: row.failed_jobs,
+      failedJobIds: JSON.parse(row.failed_job_ids) as number[],
+      options: JSON.parse(row.options) as BatchOptions,
+      createdAt: row.created_at,
+      finishedAt: row.finished_at ?? undefined,
+    };
+  }
+
+  async push(record: Omit<JobRecord, "id" | "createdAt">): Promise<void> {
+    await this._ready;
+    const now = new Date().toISOString();
+    const batchId = record.batchId ?? null;
+    await this._sql`
+      INSERT INTO zerotal_jobs
+        (queue, class_name, payload, attempts, max_attempts, retry_delay, available_at, created_at, batch_id)
+      VALUES
+        (${record.queue}, ${record.className}, ${record.payload},
+         ${record.attempts}, ${record.maxAttempts}, ${record.retryDelay},
+         ${record.availableAt}, ${now}, ${batchId})
+    `;
+  }
+
+  async pop(queue = "default"): Promise<JobRecord | null> {
+    await this._ready;
+    const now = Math.floor(Date.now() / 1000);
+    const staleBefore = now - this._visibilityTimeout;
+
+    // Claim exactly one due job with a single atomic UPDATE … RETURNING.
+    // A claimed row (reserved_at set) is invisible to other workers until the
+    // visibility timeout elapses, so concurrent `queue:work` processes can
+    // never double-process a job. If a worker crashes mid-job, reserved_at
+    // goes stale and the job becomes claimable again automatically.
+    // Statement-level atomicity in SQLite makes an explicit transaction
+    // unnecessary — the repeated reserved_at predicate on the outer UPDATE
+    // guards the SELECT-then-UPDATE window.
+    const rows = await this._sql<JobRow>`
+      UPDATE zerotal_jobs
+      SET    attempts    = attempts + 1,
+             reserved_at = ${now}
+      WHERE  id = (
+        SELECT id FROM zerotal_jobs
+        WHERE  queue = ${queue}
+          AND  available_at <= ${now}
+          AND  (reserved_at IS NULL OR reserved_at <= ${staleBefore})
+        ORDER BY id ASC
+        LIMIT 1
+      )
+      AND (reserved_at IS NULL OR reserved_at <= ${staleBefore})
+      RETURNING *
+    `;
+
+    const row = rows[0];
+    return row ? this._mapRow(row) : null;
+  }
+
+  async fail(record: JobRecord, error: string): Promise<void> {
+    await this._ready;
+    await this._sql`
+      INSERT INTO zerotal_failed_jobs
+        (queue, class_name, payload, attempts, error, failed_at)
+      VALUES
+        (${record.queue}, ${record.className}, ${record.payload},
+         ${record.attempts}, ${error}, ${new Date().toISOString()})
+    `;
+    await this._sql`DELETE FROM zerotal_jobs WHERE id = ${record.id}`;
+  }
+
+  async retry(record: JobRecord): Promise<void> {
+    await this._ready;
+    const nextAvailable = Math.floor(Date.now() / 1000) + Math.floor(record.retryDelay / 1000);
+    // Release the reservation so the job is claimable once its delay elapses.
+    await this._sql`
+      UPDATE zerotal_jobs
+      SET    available_at = ${nextAvailable},
+             reserved_at  = NULL
+      WHERE  id = ${record.id}
+    `;
+  }
+
+  async delete(record: JobRecord): Promise<void> {
+    await this._ready;
+    await this._sql`DELETE FROM zerotal_jobs WHERE id = ${record.id}`;
+  }
+
+  async size(queue = "default"): Promise<number> {
+    await this._ready;
+    const rows = await this._sql<{ count: number }>`
+      SELECT COUNT(*) as count FROM zerotal_jobs
+      WHERE queue = ${queue}
+    `;
+    return Number(rows[0]?.count ?? 0);
+  }
+
+  async flush(): Promise<void> {
+    await this._ready;
+    await this._sql`DELETE FROM zerotal_jobs`;
+    await this._sql`DELETE FROM zerotal_failed_jobs`;
+    await this._sql`DELETE FROM zerotal_job_batches`;
+  }
+
+  async listPending(queue?: string, limit = 50, offset = 0): Promise<JobRecord[]> {
+    await this._ready;
+    const rows: JobRow[] = queue
+      ? await this._sql<JobRow>`
+          SELECT * FROM zerotal_jobs WHERE queue = ${queue}
+          ORDER BY id ASC LIMIT ${limit} OFFSET ${offset}`
+      : await this._sql<JobRow>`
+          SELECT * FROM zerotal_jobs
+          ORDER BY id ASC LIMIT ${limit} OFFSET ${offset}`;
+    return rows.map((r) => this._mapRow(r));
+  }
+
+  async queues(): Promise<{ queue: string; pending: number }[]> {
+    await this._ready;
+    const rows = await this._sql<{ queue: string; count: number }>`
+      SELECT queue, COUNT(*) as count FROM zerotal_jobs GROUP BY queue ORDER BY queue`;
+    return rows.map((r) => ({ queue: r.queue, pending: Number(r.count) }));
+  }
+
+  async listFailed(queue?: string): Promise<FailedRecord[]> {
+    await this._ready;
+    const rows: FailedRow[] = queue
+      ? await this._sql<FailedRow>`
+          SELECT id, queue, class_name, payload, attempts, error, failed_at
+          FROM zerotal_failed_jobs WHERE queue = ${queue} ORDER BY failed_at DESC`
+      : await this._sql<FailedRow>`
+          SELECT id, queue, class_name, payload, attempts, error, failed_at
+          FROM zerotal_failed_jobs ORDER BY failed_at DESC`;
+
+    return rows.map((r) => ({
+      id: r.id,
+      queue: r.queue,
+      className: r.class_name,
+      payload: r.payload,
+      attempts: r.attempts,
+      error: r.error,
+      failedAt: r.failed_at,
+    }));
+  }
+
+  async deleteFailedRecord(id: number): Promise<void> {
+    await this._ready;
+    await this._sql`DELETE FROM zerotal_failed_jobs WHERE id = ${id}`;
+  }
+
+  async clearFailed(queue?: string): Promise<void> {
+    await this._ready;
+    if (queue) {
+      await this._sql`DELETE FROM zerotal_failed_jobs WHERE queue = ${queue}`;
+    } else {
+      await this._sql`DELETE FROM zerotal_failed_jobs`;
+    }
+  }
+
+  // ── Batch support ──────────────────────────────────────────────────────────
+
+  async createBatch(
+    id: string,
+    name: string,
+    totalJobs: number,
+    optionsJson: string,
+  ): Promise<void> {
+    await this._ready;
+    const now = new Date().toISOString();
+    await this._sql`
+      INSERT INTO zerotal_job_batches
+        (id, name, total_jobs, pending_jobs, failed_jobs, failed_job_ids, options, created_at)
+      VALUES
+        (${id}, ${name}, ${totalJobs}, ${totalJobs}, 0, '[]', ${optionsJson}, ${now})
+    `;
+  }
+
+  async recordBatchJobComplete(batchId: string): Promise<BatchStatus> {
+    await this._ready;
+    return this._sql.begin(async (tx: SQLInstance) => {
+      await tx`
+        UPDATE zerotal_job_batches
+        SET    pending_jobs = pending_jobs - 1
+        WHERE  id = ${batchId}
+      `;
+      const rows = await tx<{ pending_jobs: number; failed_jobs: number }>`
+        SELECT pending_jobs, failed_jobs FROM zerotal_job_batches WHERE id = ${batchId}
+      `;
+      const row = rows[0];
+      if (!row || row.pending_jobs > 0) return "pending";
+
+      await tx`
+        UPDATE zerotal_job_batches SET finished_at = ${new Date().toISOString()} WHERE id = ${batchId}
+      `;
+      return row.failed_jobs > 0 ? "failed" : "complete";
+    });
+  }
+
+  async recordBatchJobFailed(batchId: string, jobId: number): Promise<BatchStatus> {
+    await this._ready;
+    return this._sql.begin(async (tx: SQLInstance) => {
+      const current = await tx<{ failed_job_ids: string }>`
+        SELECT failed_job_ids FROM zerotal_job_batches WHERE id = ${batchId}
+      `;
+      if (!current[0]) return "pending";
+
+      const ids = JSON.parse(current[0].failed_job_ids) as number[];
+      ids.push(jobId);
+      const idsJson = JSON.stringify(ids);
+
+      await tx`
+        UPDATE zerotal_job_batches
+        SET    pending_jobs   = pending_jobs - 1,
+               failed_jobs    = failed_jobs + 1,
+               failed_job_ids = ${idsJson}
+        WHERE  id = ${batchId}
+      `;
+      const rows = await tx<{ pending_jobs: number }>`
+        SELECT pending_jobs FROM zerotal_job_batches WHERE id = ${batchId}
+      `;
+      if (!rows[0] || rows[0].pending_jobs > 0) return "pending";
+
+      await tx`
+        UPDATE zerotal_job_batches SET finished_at = ${new Date().toISOString()} WHERE id = ${batchId}
+      `;
+      return "failed";
+    });
+  }
+
+  async getBatch(batchId: string): Promise<BatchRecord | null> {
+    await this._ready;
+    const rows = await this._sql<BatchRow>`
+      SELECT * FROM zerotal_job_batches WHERE id = ${batchId}
+    `;
+    return rows[0] ? this._mapBatchRow(rows[0]) : null;
+  }
+}

package/src/drivers/SyncDriver.ts

@@ -0,0 +1,85 @@
+import type { QueueDriver, JobRecord, FailedRecord } from "./QueueDriver.ts";
+import type { Job } from "../Job.ts";
+
+type SyncQueueEntry = { record: JobRecord; job: Job | undefined };
+
+type FailedEntry = { record: JobRecord; error: string };
+
+export class SyncDriver implements QueueDriver {
+  private _queue: SyncQueueEntry[] = [];
+  private _failed: FailedEntry[] = [];
+
+  // Side channel — QueueManager sets this before calling push()
+  static _pendingJob: Job | undefined = undefined;
+
+  async push(record: Omit<JobRecord, "id" | "createdAt">): Promise<void> {
+    const entry: SyncQueueEntry = {
+      record: {
+        ...record,
+        id: this._queue.length + 1,
+        createdAt: new Date().toISOString(),
+      },
+      job: SyncDriver._pendingJob,
+    };
+    this._queue.push(entry);
+    SyncDriver._pendingJob = undefined;
+  }
+
+  async pop(): Promise<JobRecord | null> {
+    const item = this._queue.shift();
+    if (!item) return null;
+    item.record.attempts += 1;
+    return item.record;
+  }
+
+  async fail(record: JobRecord, error: string): Promise<void> {
+    this._failed.push({ record, error });
+  }
+
+  async retry(record: JobRecord): Promise<void> {
+    this._queue.unshift({ record, job: SyncDriver._pendingJob });
+    SyncDriver._pendingJob = undefined;
+  }
+
+  async delete(_record: JobRecord): Promise<void> {
+    // Already consumed by pop() — nothing to delete
+  }
+
+  async size(): Promise<number> {
+    return this._queue.length;
+  }
+
+  async flush(): Promise<void> {
+    this._queue = [];
+    this._failed = [];
+  }
+
+  get failedJobs(): FailedEntry[] {
+    return this._failed;
+  }
+
+  async listFailed(queue?: string): Promise<FailedRecord[]> {
+    const src = queue ? this._failed.filter((f) => f.record.queue === queue) : this._failed;
+    return src.map((f, i) => ({
+      id: i + 1,
+      queue: f.record.queue,
+      className: f.record.className,
+      payload: f.record.payload,
+      attempts: f.record.attempts,
+      error: f.error,
+      failedAt: new Date().toISOString(),
+    }));
+  }
+
+  async deleteFailedRecord(id: number): Promise<void> {
+    this._failed.splice(id - 1, 1);
+  }
+
+  async clearFailed(queue?: string): Promise<void> {
+    if (queue) {
+      this._failed = this._failed.filter((f) => f.record.queue !== queue);
+    } else {
+      this._failed = [];
+    }
+  }
+}

package/src/errors.ts

@@ -0,0 +1,42 @@
+import { ZerotalError } from "@zerotal/core";
+
+/** Base class for all @zerotal/queue errors. */
+export class QueueError extends ZerotalError {
+  constructor(message: string, code = "E_QUEUE", status = 500, context?: Record<string, unknown>) {
+    super(message, code, status, context);
+  }
+}
+
+/** Thrown when the Bus facade is used before QueueProvider is registered. */
+export class QueueNotInitializedError extends QueueError {
+  constructor() {
+    super(
+      "[Zerotal] Bus is not initialized. Register QueueProvider first.",
+      "E_QUEUE_NOT_INITIALIZED",
+      500,
+    );
+  }
+}
+
+/** Thrown when dispatching while the manager is shutting down. */
+export class QueueShuttingDownError extends QueueError {
+  constructor(className: string) {
+    super(
+      `Cannot dispatch ${className} — QueueManager is shutting down.`,
+      "E_QUEUE_SHUTTING_DOWN",
+      503,
+      { className },
+    );
+  }
+}
+
+/** Thrown when batching is requested on a driver that does not support it. */
+export class QueueBatchingUnsupportedError extends QueueError {
+  constructor() {
+    super(
+      "[Zerotal] The current queue driver does not support job batching. Use SqliteDriver.",
+      "E_QUEUE_BATCHING_UNSUPPORTED",
+      500,
+    );
+  }
+}

package/src/events.ts

@@ -0,0 +1,21 @@
+/**
+ * The queue package's framework events, emitted on core's {@link FrameworkEvents}
+ * bus. Observability packages subscribe to them by kind (their class name).
+ */
+
+/**
+ * Emitted at each stage of a background job's life — `status` is one of
+ * `dispatched`, `completed`, `failed`, or `retried`, with `error` set on
+ * failure. A single job fires several of these over its lifetime.
+ *
+ * @category Queue
+ */
+export class JobRan {
+  constructor(
+    readonly className: string,
+    readonly queue: string,
+    readonly status: "dispatched" | "completed" | "failed" | "retried",
+    readonly durationMs: number,
+    readonly error?: string,
+  ) {}
+}

package/src/facades/Queue.ts

@@ -0,0 +1,3 @@
+import { createFacade } from "@zerotal/core";
+
+export const Queue = createFacade("queue");

package/src/global.d.ts

@@ -0,0 +1,34 @@
+// Ambient declarations specific to this package.
+// Bun, Node (node:*), and bun:test types come from @types/bun (→ bun-types).
+// Only declarations bun-types does NOT provide are kept here.
+
+// ── Bun globals ───────────────────────────────────────────────────────────
+// Bun extends Request with native route params (e.g. /users/:id → { id: '42' })
+interface Request {
+  readonly params?: Record<string, string>;
+}
+
+// ── SQLInstance: callable tagged-template + DB methods ────────────────────
+interface SQLInstance {
+  <T = Record<string, unknown>>(
+    strings: TemplateStringsArray,
+    ...values: unknown[]
+  ): Promise<T[]>;
+  begin<T>(fn: (tx: SQLInstance) => Promise<T>): Promise<T>;
+  end(): Promise<void>;
+}
+
+// ── RedisInstance: minimal Bun.redis surface ──────────────────────────────
+interface RedisInstance {
+  get(key: string): Promise<string | null>;
+  set(key: string, value: string): Promise<void>;
+  set(key: string, value: string, options: { ex: number }): Promise<void>;
+  lpush(key: string, value: string): Promise<number>;
+  rpop(key: string): Promise<string | null>;
+  lrange(key: string, start: number, stop: number): Promise<string[]>;
+  llen(key: string): Promise<number>;
+  del(...keys: string[]): Promise<number>;
+  expire(key: string, seconds: number): Promise<number>;
+  keys(pattern: string): Promise<string[]>;
+  flushdb(): Promise<void>;
+}

package/src/index.ts

@@ -0,0 +1,29 @@
+export { Job } from "./Job.ts";
+export { JobRegistry } from "./JobRegistry.ts";
+export { QueueManager, queueStats } from "./QueueManager.ts";
+export type { JobStatus, QueueStats } from "./QueueManager.ts";
+export { WorkerPool } from "./WorkerPool.ts";
+export type { WorkerPoolOptions, WorkerResult } from "./WorkerPool.ts";
+export { QueueFake } from "./QueueFake.ts";
+export { QueueProvider } from "./provider/QueueProvider.ts";
+export { Queue } from "./facades/Queue.ts";
+export type { QueueDriver, JobRecord, BatchStatus } from "./drivers/QueueDriver.ts";
+export { SqliteDriver } from "./drivers/SqliteDriver.ts";
+export { SyncDriver } from "./drivers/SyncDriver.ts";
+export { RedisDriver } from "./drivers/RedisDriver.ts";
+
+// Batch & chain orchestration
+export { Bus } from "./Bus.ts";
+export { Batch } from "./Batch.ts";
+export { PendingBatch } from "./PendingBatch.ts";
+export type { BatchRecord, BatchOptions, SerializedJob } from "./Batch.ts";
+
+// Config factory
+export { QueueConfig } from "./config.ts";
+export type { QueueConfigShape } from "./config.ts";
+
+// Typed error vocabulary
+export * from "./errors.ts";
+
+// Framework instrumentation events (emitted on the core FrameworkEvents bus)
+export { JobRan } from "./events.ts";