@zerotal/queue 1.0.0

package/src/observability.ts

@@ -0,0 +1,131 @@
+/**
+ * Queue → observer bridges. The queue emits its own `JobRan` framework event on
+ * the core `FrameworkEvents` bus; this module forwards it to whichever observer
+ * packages are installed.
+ *
+ * Each observer's write surface is resolved from the container by binding key and
+ * typed through a local structural interface, so the queue depends on none of the
+ * observer packages — installing or removing an observer requires no change here.
+ */
+import { FrameworkEvents, RequestContext } from "@zerotal/core";
+import type { Application } from "@zerotal/core";
+import { JobRan } from "./events.ts";
+
+/** The subset of the telemetry tracer this bridge calls (bound as `telemetry`). */
+interface TelemetrySink {
+  recordCompleted(
+    name: string,
+    durationMs: number,
+    options?: {
+      kind?: "internal" | "server" | "client" | "producer" | "consumer";
+      attributes?: Record<string, string | number | boolean>;
+      status?: "ok" | "error";
+      errorMessage?: string;
+    },
+  ): unknown;
+}
+
+/** The subset of the monitor store this bridge calls (bound as `monitor.store`). */
+interface MonitorSink {
+  recordJob(job: {
+    status: string;
+    className?: string;
+    queue?: string;
+    ms?: number;
+    error?: string | null;
+  }): void;
+}
+
+/** The subset of the devtools trace sink this bridge calls (bound as `devtools.trace`). */
+interface DevtoolsSink {
+  bufferJob(
+    ctx: object,
+    j: { className: string; queue: string; status: string; durationMs: number; error?: string },
+  ): void;
+}
+
+/** The subset of the logger this bridge calls (bound as `log`). */
+interface LogSink {
+  warn(message: string, context?: Record<string, unknown>): void;
+  error(message: string, context?: Record<string, unknown>, error?: unknown): void;
+}
+
+/**
+ * Subscribe the queue's events to every installed observer. Returns a disposer
+ * that removes every subscription; call it from the queue provider's `onStopping()`.
+ */
+export function installQueueObservability(app: Application): () => void {
+  const unsubs: Array<() => void> = [];
+
+  const tracer = app.container.tryMake("telemetry" as never) as TelemetrySink | undefined;
+  if (tracer) {
+    unsubs.push(
+      FrameworkEvents.on(JobRan, (e) => {
+        if (e.status !== "completed" && e.status !== "failed") return;
+        void tracer.recordCompleted("queue.job", e.durationMs, {
+          kind: "consumer",
+          attributes: { "job.class": e.className, "job.queue": e.queue },
+          status: e.status === "failed" ? "error" : "ok",
+          ...(e.error ? { errorMessage: e.error } : {}),
+        });
+      }),
+    );
+  }
+
+  const store = app.container.tryMake("monitor.store" as never) as MonitorSink | undefined;
+  if (store) {
+    unsubs.push(
+      FrameworkEvents.on(JobRan, (e) =>
+        store.recordJob({
+          status: e.status,
+          className: e.className,
+          queue: e.queue,
+          ms: e.durationMs,
+          error: e.error ?? null,
+        }),
+      ),
+    );
+  }
+
+  const trace = app.container.tryMake("devtools.trace" as never) as DevtoolsSink | undefined;
+  if (trace) {
+    unsubs.push(
+      FrameworkEvents.on(JobRan, (e) => {
+        const ctx = RequestContext.tryGet();
+        if (!ctx) return;
+        trace.bufferJob(ctx, {
+          className: e.className,
+          queue: e.queue,
+          status: e.status,
+          durationMs: e.durationMs,
+          ...(e.error ? { error: e.error } : {}),
+        });
+      }),
+    );
+  }
+
+  const log = app.container.tryMake("log" as never) as LogSink | undefined;
+  if (log) {
+    unsubs.push(
+      FrameworkEvents.on(JobRan, (e) => {
+        if (e.status === "failed") {
+          log.error(
+            "Job failed",
+            { className: e.className, queue: e.queue, durationMs: e.durationMs },
+            new Error(e.error ?? "Job execution failed"),
+          );
+        } else if (e.status === "retried") {
+          log.warn("Job retried", {
+            className: e.className,
+            queue: e.queue,
+            durationMs: e.durationMs,
+          });
+        }
+      }),
+    );
+  }
+
+  return () => {
+    for (const unsub of unsubs) unsub();
+  };
+}

package/src/provider/QueueProvider.ts

@@ -0,0 +1,153 @@
+import { ServiceProvider } from "@zerotal/core";
+import type { AppEnvironment } from "@zerotal/core";
+import type { ConfigManager } from "@zerotal/core/config";
+import { _getConnection } from "@zerotal/orm";
+import { QueueManager } from "../QueueManager.ts";
+import { SqliteDriver } from "../drivers/SqliteDriver.ts";
+import { SyncDriver } from "../drivers/SyncDriver.ts";
+import { RedisDriver } from "../drivers/RedisDriver.ts";
+import { WorkerPool } from "../WorkerPool.ts";
+import { JobRegistry } from "../JobRegistry.ts";
+import { CallQueuedListener } from "@zerotal/core";
+import { Bus } from "../Bus.ts";
+import { installQueueObservability } from "../observability.ts";
+import { installQueueAdmin } from "../admin.ts";
+import { validateQueueConfig } from "../config.ts";
+import { frameworkLog } from "@zerotal/core/logger";
+
+declare module "@zerotal/core" {
+  interface ContainerBindings {
+    queue: QueueManager;
+  }
+}
+
+export class QueueProvider extends ServiceProvider {
+  static override provides = ["queue"] as const;
+  static override environments: AppEnvironment[] = ["web", "console", "worker", "test"];
+
+  private _workerInterval: ReturnType<typeof setInterval> | undefined = undefined;
+  private _workerPool: WorkerPool | undefined = undefined;
+  private _disposeObservability: (() => void) | undefined = undefined;
+
+  override onRegister(): void {
+    // Refuse a production boot on structural queue misconfiguration (unknown
+    // driver, workers without a bootstrap module). Runs in the boot-time config pass.
+    this.app.registerConfigValidator?.("queue", validateQueueConfig);
+
+    this.app.container.singleton("queue", () => {
+      const config = this.app.container.makeSync("config") as ConfigManager;
+      const driver = config.get<string>("queue.driver", "sqlite");
+
+      switch (driver) {
+        case "redis":
+          return new QueueManager(new RedisDriver());
+
+        case "sync":
+          return new QueueManager(new SyncDriver());
+
+        case "sqlite":
+        default:
+          return new QueueManager(new SqliteDriver(_getConnection()));
+      }
+    });
+  }
+
+  override async onBooting(): Promise<void> {
+    JobRegistry.register(CallQueuedListener as never);
+    const manager = (await this.app.container.make("queue")) as QueueManager;
+    Bus.setManager(manager);
+
+    // Offer the queue console to the admin panel, if one is installed. This runs
+    // in the booting phase (not `onBooted`) because the panel mounts its routes
+    // once booting finishes — and `onBooted` runs in parallel, so contributing
+    // there would race the mount.
+    installQueueAdmin(this.app);
+  }
+
+  override async onStarted(): Promise<void> {
+    const config = this.app.container.makeSync("config") as ConfigManager;
+    const workerCount = config.get<number>("queue.workers", 0);
+    const env = this.app._env;
+
+    // Resolve the Bun Worker bootstrap module path. When `queue.workerBootstrap`
+    // isn't configured the worker thread discovers jobs by convention (imports
+    // every app/jobs/*.ts) — same as the `jobs` convention concern, no codegen.
+    const workerBootstrap = config.get<string | undefined>("queue.workerBootstrap", undefined);
+
+    // Start polling when:
+    //   - running as a dedicated worker process (queue:work command), OR
+    //   - running as the HTTP server with Bun Worker threads configured
+    const usingBunWorkers = env === "web" && workerCount > 0;
+    const isWorkerProcess = env === "worker";
+    if (!usingBunWorkers && !isWorkerProcess) return;
+
+    const queue = this.app.container.makeSync("queue") as QueueManager;
+    const pollMs = config.get<number>("queue.pollInterval", 500);
+    const queues = config.get<string[]>("queue.queues", ["default"]);
+
+    if (usingBunWorkers) {
+      // Spawn N Bun OS threads — CPU-heavy jobs run there, not on the HTTP loop.
+      this._workerPool = new WorkerPool({ size: workerCount, bootstrapPath: workerBootstrap });
+      await this._workerPool.start();
+      queue.setWorkerPool(this._workerPool);
+      frameworkLog("queue").info(`Spawned ${workerCount} Bun Worker thread(s) for job execution`, {
+        workers: workerCount,
+      });
+    }
+
+    frameworkLog("queue").info(`Polling [${queues.join(", ")}] every ${pollMs}ms`, {
+      queues,
+      pollMs,
+    });
+
+    this._workerInterval = setInterval(async () => {
+      if (queue.isShuttingDown) return;
+      for (const q of queues) {
+        await queue.processNext(q).catch(console.error);
+      }
+    }, pollMs);
+  }
+
+  override async onStopping(): Promise<void> {
+    this._disposeObservability?.();
+    this._disposeObservability = undefined;
+
+    if (this._workerInterval !== undefined) {
+      clearInterval(this._workerInterval);
+      this._workerInterval = undefined;
+    }
+
+    const queue = this.app.container.tryMake("queue") as QueueManager | undefined;
+    if (queue) {
+      frameworkLog("queue").info("Draining in-flight jobs");
+      await queue.drain();
+      frameworkLog("queue").info("All jobs drained");
+    }
+
+    if (this._workerPool) {
+      await this._workerPool.terminate();
+      this._workerPool = undefined;
+      frameworkLog("queue").info("Bun Worker threads terminated");
+    }
+  }
+
+  override async onBooted(): Promise<void> {
+    this._disposeObservability = installQueueObservability(this.app);
+
+    const runner = this.app.container.tryMake("commands");
+    if (!runner) return;
+
+    runner.registerLazy("queue:work", () =>
+      import("../commands/QueueWorkCommand.ts").then((m) => m.QueueWorkCommand),
+    );
+    runner.registerLazy("queue:failed", () =>
+      import("../commands/QueueFailedCommand.ts").then((m) => m.QueueFailedCommand),
+    );
+    runner.registerLazy("queue:retry", () =>
+      import("../commands/QueueRetryCommand.ts").then((m) => m.QueueRetryCommand),
+    );
+    runner.registerLazy("queue:flush", () =>
+      import("../commands/QueueFlushCommand.ts").then((m) => m.QueueFlushCommand),
+    );
+  }
+}

package/src/queue-worker.ts

@@ -0,0 +1,98 @@
+/**
+ * Bun Worker thread script — runs in a separate OS thread.
+ *
+ * Protocol (messages from parent → worker):
+ *   { type: 'bootstrap', modulePath: string }
+ *     Dynamically import the module at `modulePath`. That module must register
+ *     all job classes via JobRegistry.register(). Replies { type: 'ready' } on
+ *     success or { type: 'bootstrap-error', message } on failure.
+ *
+ *   { type: 'job', record: JobRecord }
+ *     Instantiate and handle the job described by `record`. Replies
+ *     { type: 'done' } on success or { type: 'error', message } on failure.
+ *     The parent thread is responsible for delete / retry / fail storage ops.
+ */
+
+import { JobRegistry } from "./JobRegistry.ts";
+import type { JobRecord } from "./drivers/QueueDriver.ts";
+import type { Job } from "./Job.ts";
+
+let bootstrapped = false;
+
+self.onmessage = async (event: MessageEvent) => {
+  const msg = event.data as { type: string } & Record<string, unknown>;
+
+  if (msg.type === "bootstrap") {
+    try {
+      const modulePath = msg.modulePath as string | undefined;
+      if (modulePath) {
+        // Explicit bootstrap module (config: queue.workerBootstrap) — it must
+        // import/register every job class.
+        await import(modulePath);
+      } else {
+        // Convention: import every app/jobs/*.ts so each self-registers via
+        // JobRegistry.register(). Mirrors the `jobs` convention concern, run here
+        // because this bare worker thread does not boot a full Application.
+        // app/notifications/ comes along for the same reason: a queued
+        // notification names its class, and the class must have been imported
+        // here for SendNotificationJob to rebuild it.
+        const glob = new Bun.Glob("*.ts");
+        for (const dir of ["app/jobs", "app/notifications"]) {
+          try {
+            for await (const file of glob.scan({
+              cwd: `${process.cwd()}/${dir}`,
+              absolute: true,
+            })) {
+              await import(`file://${file.replace(/\\/g, "/")}`);
+            }
+          } catch {
+            /* directory doesn't exist — nothing to register */
+          }
+        }
+      }
+      bootstrapped = true;
+      self.postMessage({ type: "ready" });
+    } catch (err) {
+      self.postMessage({ type: "bootstrap-error", message: String(err) });
+    }
+    return;
+  }
+
+  if (msg.type === "job") {
+    if (!bootstrapped) {
+      self.postMessage({ type: "error", message: "Worker is not bootstrapped yet" });
+      return;
+    }
+
+    const record = msg.record as JobRecord;
+    try {
+      const JobClass = JobRegistry.resolve(record.className);
+      if (!JobClass) {
+        self.postMessage({ type: "error", message: `Unknown job class: ${record.className}` });
+        return;
+      }
+
+      const payload = JSON.parse(record.payload) as Record<string, unknown>;
+      let instance: Job;
+
+      if (
+        "fromPayload" in JobClass &&
+        typeof (JobClass as Record<string, unknown>).fromPayload === "function"
+      ) {
+        instance = (JobClass as { fromPayload(p: Record<string, unknown>): Job }).fromPayload(
+          payload,
+        );
+      } else {
+        instance = new (JobClass as new () => Job)();
+      }
+
+      await instance.handle();
+      self.postMessage({ type: "done" });
+    } catch (err) {
+      self.postMessage({
+        type: "error",
+        message: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+};