create-nexwire 4.3.0 → 4.4.0

package/fastify/src/framework/queue/queue.ts

@@ -0,0 +1,401 @@
+import { type Job, type JobsOptions, Queue, QueueEvents, Worker } from "bullmq";
+import { type DurableContext, type DurableJob, DurableWorker, RedisStateStore } from "bullmq-durable";
+import { queueConfig, redisConfig } from "@/config/index.js";
+import { discoverModuleFiles, importFile } from "@/framework/modules/discover.js";
+import { redisClientIfReady } from "@/framework/redis/client.js";
+import { logger } from "@/framework/support/logger.js";
+
+export type QueueJob = {
+  name: string;
+  data: any;
+};
+
+export type QueueHandler = (job: Job) => Promise<any>;
+export type DurableQueueHandler = (job: DurableJob, ctx: DurableContext) => Promise<any>;
+export type AnyQueueHandler = QueueHandler | DurableQueueHandler;
+
+const queues = new Map<string, Queue>();
+const events = new Map<string, QueueEvents>();
+const handlers = new Map<string, QueueHandler>();
+const durableHandlers = new Map<string, DurableQueueHandler>();
+const workers: Worker[] = [];
+/**
+ * Queue names this process already has a worker for. Each BullMQ `Worker` holds
+ * its own blocking Redis connection, so starting the runtime twice for one queue
+ * would silently double that connection for no benefit. Cleared by
+ * `stopQueueRuntime` so a later boot in the same process still works.
+ */
+const workedQueues = new Set<string>();
+let durableStateStore: RedisStateStore | null = null;
+
+/**
+ * Why: Jobs already reported as dropped, so that one unreachable queue cannot
+ *      flood the log from a request hot path.
+ * When: A call to queueJob finds no Redis.
+ * Where: Queue facade module scope.
+ * How: Keyed by queue + job name, because that pair is what identifies a queued
+ *      job - two queues running the same job name are two separate failures and
+ *      must each be reported once.
+ */
+const droppedJobs = new Set<string>();
+
+function queuePrefix() {
+  return queueConfig.prefix;
+}
+
+function key(queue: string, job: string) {
+  return `${queue}:${job}`;
+}
+
+/**
+ * Why: BullMQ manages its own connections. We pass the URL string,
+ * not the shared client, so BullMQ doesn't crash if the shared client dies.
+ */
+function connectionOptions() {
+  return { url: redisConfig.url };
+}
+
+/**
+ * Why: Registers a job handler for a queue + job name.
+ * When: Module job files are loaded during boot.
+ * Where: Module job definition files under `src/modules/<module>/jobs`.
+ * How: Stores handler in an in-memory lookup key. Pass `{ durable: true }` to
+ *      register a checkpointed handler `(job, ctx)` powered by bullmq-durable;
+ *      such a handler can split its work into `ctx.step(...)` checkpoints that
+ *      survive crashes/restarts instead of re-running from scratch.
+ */
+export function shouldQueue(job: string, queue: string, handler: AnyQueueHandler, options: { durable?: boolean } = {}) {
+  const target = key(queue || "default", job);
+  if (options.durable) {
+    durableHandlers.set(target, handler as DurableQueueHandler);
+  } else {
+    handlers.set(target, handler as QueueHandler);
+  }
+}
+
+/**
+ * Why: Gets or lazily creates a BullMQ queue instance.
+ * When: Enqueueing jobs or ensuring queue resources.
+ * Where: Queue facade and dispatcher.
+ * How: Reuses shared Redis connection and memoized queue map.
+ */
+export function getQueue(queue = "default") {
+  // Check if Redis is supposed to be enabled
+  if (!redisConfig.enabled) return null;
+
+  if (!queues.has(queue)) {
+    queues.set(
+      queue,
+      new Queue(queue, {
+        connection: connectionOptions(), // Let BullMQ handle its own connection
+        prefix: queuePrefix(),
+        defaultJobOptions: {
+          attempts: 3,
+          backoff: { type: "exponential", delay: 3000 },
+          removeOnComplete: 1000,
+          removeOnFail: 5000
+        }
+      })
+    );
+  }
+
+  return queues.get(queue)!;
+}
+
+/**
+ * Why: Provides the shared Redis-backed state store for durable queues.
+ * When: Durable queue/worker construction.
+ * Where: Queue facade internals.
+ * How: Creates one `RedisStateStore` reusing the shared Redis client so durable
+ *      state lives in Redis. Note the store opens its own client on first use -
+ *      bullmq-durable does this so that merely constructing a store costs no
+ *      connection - which is why `stopQueueRuntime` must call `close()` on it.
+ */
+function durableStateStoreIfReady() {
+  const client = redisClientIfReady();
+  if (!client) return null;
+
+  if (!durableStateStore) {
+    durableStateStore = new RedisStateStore({ connection: client as never, prefix: queueConfig.durablePrefix });
+  }
+  return durableStateStore;
+}
+
+/**
+ * Why: Facade-friendly alias for getQueue.
+ * When: Consumers want fluent queue access.
+ * Where: Framework facade/API usage.
+ * How: Delegates directly to getQueue.
+ */
+export function queue(queueName = "default") {
+  return getQueue(queueName);
+}
+
+/**
+ * Why: Ensures configured queues exist before UI/runtime usage.
+ * When: Queue dashboard setup or startup warmup.
+ * Where: Queue UI/bootstrap code.
+ * How: Iterates names and resolves queue instances.
+ */
+export function ensureQueues(queueNames: string[]) {
+  for (const queueName of queueNames) {
+    getQueue(queueName);
+  }
+}
+
+/**
+ * Why: Returns all created queue instances.
+ * When: Queue introspection and dashboard binding.
+ * Where: Queue dashboard setup code.
+ * How: Converts internal queue map to array.
+ */
+export function getAllQueues() {
+  return Array.from(queues.values());
+}
+
+/**
+ * Why: Explains that a job was lost instead of letting it vanish.
+ * When: queueJob is called and no Redis connection is ready.
+ * Where: Called from queueJob.
+ * How: Warns once per queue + job. The message names the job and its queue,
+ *      says plainly that it never ran, and gives both the Redis fix and the
+ *      in-process alternative, so a caller reading only this line knows what
+ *      to do without opening framework source.
+ */
+function warnDroppedJob(queueName: string, job: string, delay?: number) {
+  const id = key(queueName, job);
+  if (droppedJobs.has(id)) return;
+  droppedJobs.add(id);
+
+  const schedule = delay ? ` It was due in ${delay}s, so it was never even scheduled.` : "";
+  console.warn(
+    `[queue] Dropped job "${job}" on queue "${queueName}". Redis is not available, so this job was never queued and never ran.${schedule} ` +
+      `Enable REDIS in .env and run a queue worker, or call dispatchCommand("${job}", payload) without { async: true } to run it in-process now.`
+  );
+}
+
+/**
+ * Why: Adds a job to BullMQ with project defaults.
+ * When: Commands/events need background execution.
+ * Where: Dispatcher and feature modules.
+ * How: Resolves queue then calls add with retry/backoff options. When Redis is
+ *      absent there is nowhere to put the job, so it returns null and says so -
+ *      previously it returned null with no output at all, which meant silently
+ *      losing the work.
+ */
+export async function queueJob(
+  job: string,
+  data: any,
+  options: {
+    queue?: string;
+    delay?: number;
+    attempts?: number;
+    jobId?: string;
+    priority?: number;
+    removeOnComplete?: JobsOptions["removeOnComplete"];
+    removeOnFail?: JobsOptions["removeOnFail"];
+    backoff?: JobsOptions["backoff"];
+  } = {}
+) {
+  const queueName = options.queue || "default";
+  const queue = getQueue(queueName);
+  if (!queue) {
+    warnDroppedJob(queueName, job, options.delay);
+    return null;
+  }
+
+  try {
+    return await queue.add(job, data, {
+      delay: options.delay ? options.delay * 1000 : 0,
+      attempts: options.attempts ?? 3,
+      jobId: options.jobId,
+      priority: options.priority,
+      removeOnComplete: options.removeOnComplete ?? 1000,
+      removeOnFail: options.removeOnFail ?? 5000,
+      backoff: options.backoff ?? { type: "exponential", delay: 3000 }
+    });
+  } catch (error) {
+    logger.error(`Failed to queue job ${job}`, { error });
+    return null;
+  }
+}
+
+/**
+ * Why: Loads module job files so handlers are registered.
+ * When: Worker start and explicit queue boot flows.
+ * Where: Worker/runtime bootstrap.
+ * How: Discovers jobs glob and imports each file.
+ */
+export async function bootQueueJobs() {
+  const files = await discoverModuleFiles("**/jobs/*.{ts,js}");
+  for (const file of files) await importFile(file);
+  return files.length;
+}
+
+/**
+ * Why: Deletes phantom queues whose names are neither configured nor worked by
+ *      this process.
+ * When: Queue worker boot, after workers start.
+ * Where: Worker runtime bootstrap.
+ * How: Scans Redis for `{prefix}*:meta` keys, and for each queue name that is
+ *      not in queueConfig.queues nor in the queues this worker was launched
+ *      with (--queue), obliterates it only when it holds no live jobs
+ *      (waiting/active/delayed/paused/prioritized/waiting-children). Historical
+ *      completed/failed entries are ignored so leftover history cannot keep a
+ *      dead queue alive. Recreates nothing permanent: BullMQ re-creates a
+ *      queue's keys on demand if any code path still dispatches to it.
+ */
+export async function pruneStaleQueues(protectedQueues: string[] = []) {
+  if (!queueConfig.autoPruneQueues || !redisConfig.enabled) return;
+
+  const client = redisClientIfReady();
+  if (!client) return;
+
+  const configured = new Set([...queueConfig.queues, ...protectedQueues]);
+  const prefix = `${queuePrefix()}:`;
+  const candidates = new Set<string>();
+
+  try {
+    let cursor = "0";
+    do {
+      const [nextCursor, keys] = await client.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", 100);
+      cursor = nextCursor;
+      for (const key of keys) {
+        const name = key.slice(prefix.length, -5);
+        if (name && !configured.has(name)) candidates.add(name);
+      }
+    } while (cursor !== "0");
+
+    for (const name of candidates) {
+      try {
+        const queue = getQueue(name);
+        if (!queue) continue;
+
+        const counts = await queue.getJobCounts("waiting", "active", "delayed", "paused", "prioritized", "waiting-children");
+        const live = Object.values(counts).reduce((sum, n) => sum + (Number(n) || 0), 0);
+        if (live !== 0) continue;
+
+        await queue.obliterate({ force: true });
+        logger.info(`Pruned stale queue "${name}"`);
+      } catch (error: any) {
+        logger.warn(`Could not prune stale queue "${name}": ${error?.message || error}`);
+      }
+    }
+  } catch (error: any) {
+    logger.warn(`Could not scan queues for stale cleanup: ${error?.message || error}`);
+  }
+}
+
+/**
+ * Why: Starts BullMQ workers for selected queues.
+ * When: Queue worker process boots.
+ * Where: Worker runtime entrypoint.
+ * How: Boots handlers, creates QueueEvents, and creates workers.
+ */
+export async function startQueueWorker(queueNames = ["default"]) {
+  await bootQueueJobs();
+  if (!redisConfig.enabled) throw new Error("Redis is required for queue workers");
+
+  for (const queueName of queueNames) {
+    if (workedQueues.has(queueName)) continue;
+    workedQueues.add(queueName);
+
+    if (!events.has(queueName)) {
+      // Let BullMQ manage its own connection for events
+      events.set(queueName, new QueueEvents(queueName, { connection: connectionOptions(), prefix: queuePrefix() }));
+    }
+
+    const durable = [...durableHandlers.keys()].some((k) => k.startsWith(`${queueName}:`));
+
+    if (durable) {
+      // Durable worker still needs a shared client for its state store in this specific lib
+      const store = durableStateStoreIfReady();
+      if (!store) throw new Error("Redis is required for durable queue workers");
+
+      const map: Record<string, AnyQueueHandler> = {};
+      for (const [k, h] of handlers) {
+        if (k.startsWith(`${queueName}:`)) map[k.slice(queueName.length + 1)] = h;
+      }
+      for (const [k, h] of durableHandlers) {
+        if (k.startsWith(`${queueName}:`)) map[k.slice(queueName.length + 1)] = h;
+      }
+
+      const worker = new DurableWorker(queueName, map as never, {
+        connection: connectionOptions(),
+        prefix: queuePrefix(),
+        stateStore: store,
+        concurrency: 10
+      });
+
+      worker.on("completed", (job) => logger.info(`Processed: ${job.name} (${queueName})`));
+      worker.on("failed", (job, error) => logger.warn(`Failed: ${job?.name ?? "unknown"} (${queueName}) - ${error.message}`));
+
+      workers.push(worker as unknown as Worker);
+      continue;
+    }
+
+    const worker = new Worker(
+      queueName,
+      async (job) => {
+        logger.info(`Processing: ${job.name} (${queueName})`);
+        const handler = handlers.get(key(queueName, job.name));
+        if (!handler) throw new Error(`No handler registered for ${queueName}:${job.name}`);
+        return await handler(job);
+      },
+      { connection: connectionOptions(), prefix: queuePrefix(), concurrency: 10 }
+    );
+
+    worker.on("completed", (job) => logger.info(`Processed: ${job.name} (${queueName})`));
+    worker.on("failed", (job, error) => logger.warn(`Failed: ${job?.name ?? "unknown"} (${queueName}) - ${error.message}`));
+
+    workers.push(worker);
+  }
+}
+
+/**
+ * Why: Hard-clears ALL queue-system state for reset workflows.
+ * When: Local cleanup and maintenance commands.
+ * Where: `queue:clear` command.
+ * How: Wipes every BullMQ queue key (including jobs, job schedulers, and
+ *      repeat keys for queues not listed in `queues`, e.g. `demo`), plus the
+ *      durable key space, by scanning each prefix and deleting in batches.
+ *      Cache/session/rate-limit keys are intentionally left alone.
+ */
+export async function clearQueue() {
+  if (!redisConfig.enabled) return;
+  const client = redisClientIfReady();
+  if (!client) return;
+
+  for (const prefix of [queuePrefix(), queueConfig.durablePrefix]) {
+    const stream = client.scanStream({ match: `${prefix}*`, count: 100 });
+    for await (const keys of stream) {
+      if (keys.length) await client.del(keys);
+    }
+  }
+}
+
+/**
+ * Why: Gracefully stops all workers/events/queues.
+ * When: Process shutdown.
+ * Where: Server/worker lifecycle hooks.
+ * How: Closes resources with Promise.allSettled and clears registries. The
+ *      durable state store is closed too: it owns a Redis client of its own, and
+ *      dropping the reference without closing it would leak that connection on
+ *      every restart until Redis runs out of clients.
+ */
+export async function stopQueueRuntime() {
+  await Promise.allSettled(workers.map((worker) => worker.close()));
+  workers.length = 0;
+  workedQueues.clear();
+
+  await Promise.allSettled(Array.from(events.values()).map((queueEvents) => queueEvents.close()));
+  events.clear();
+
+  await Promise.allSettled(Array.from(queues.values()).map((queue) => queue.close()));
+  queues.clear();
+
+  if (durableStateStore) {
+    await Promise.allSettled([durableStateStore.close()]);
+    durableStateStore = null;
+  }
+}

package/fastify/src/framework/queue/ui.ts

@@ -0,0 +1,194 @@
+import { createBullBoard } from "@bull-board/api";
+import { BullMQAdapter } from "@bull-board/api/bullMQAdapter";
+import { FastifyAdapter } from "@bull-board/fastify";
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { queueConfig, redisConfig } from "@/config/index.js";
+import { ensureQueues, getQueue } from "@/framework/queue/queue.js";
+import { redisClientIfReady } from "@/framework/redis/client.js";
+import { cookie } from "@/framework/support/cookie.js";
+import { jwt } from "@/framework/support/jwt.js";
+import { parseCsvOrFallback } from "@/framework/support/lifecycle.js";
+
+const BASE_PATH = queueConfig.queueUi || "/queues";
+
+const POLL_INTERVAL_MS = 5_000;
+
+let pollTimer: ReturnType<typeof setInterval> | null = null;
+let syncInFlight = false;
+
+const dashboardQueueNames = new Set<string>();
+
+const serverAdapter = new FastifyAdapter();
+serverAdapter.setBasePath(BASE_PATH);
+
+const bullBoard = createBullBoard({
+  queues: [],
+  serverAdapter,
+  options: {
+    uiConfig: {
+      boardTitle: "Queue Dashboard"
+    }
+  }
+});
+
+export function stopQueueDashboard() {
+  if (pollTimer) {
+    clearInterval(pollTimer);
+    pollTimer = null;
+  }
+}
+
+async function refreshDashboardQueues() {
+  if (syncInFlight) return;
+  syncInFlight = true;
+
+  try {
+    const client = redisClientIfReady();
+    if (!client) return;
+
+    const discovered = (await discoverQueueNames()).sort();
+    ensureQueues(discovered);
+
+    for (const name of discovered) {
+      if (dashboardQueueNames.has(name)) continue;
+      const queue = getQueue(name);
+      if (!queue) continue;
+      bullBoard.addQueue(new BullMQAdapter(queue));
+      dashboardQueueNames.add(name);
+    }
+
+    for (const name of Array.from(dashboardQueueNames)) {
+      if (!discovered.includes(name)) {
+        bullBoard.removeQueue(name);
+        dashboardQueueNames.delete(name);
+      }
+    }
+  } catch {
+    // Keep the previous queue list if a refresh fails.
+  } finally {
+    syncInFlight = false;
+  }
+}
+
+function queuePrefix() {
+  return queueConfig.prefix;
+}
+
+function allowedQueueDashboardEmails() {
+  const emails = parseCsvOrFallback(queueConfig.allowedEmails, []);
+  return new Set(emails.map((email) => email.toLowerCase()));
+}
+
+function dashboardAuth() {
+  const allowedEmails = allowedQueueDashboardEmails();
+
+  return async (request: FastifyRequest, reply: FastifyReply) => {
+    /**
+     * Why: The email allowlist is the ONLY gate on the queue dashboard UI.
+     * When: No `allowedEmails` configured → the dashboard is open to everyone
+     *      in both dev and production (no login required). When configured,
+     *      only the listed email(s) pass — everyone else is blocked.
+     * Where: BullMQ dashboard route hook.
+     * How: An empty allowlist short-circuits straight through; otherwise the
+     *      requester's email (from the validated access token) must be in the
+     *      set. This keeps BullMQ workers (mail delivery etc.) always running
+     *      and gating strictly UI-only.
+     */
+    if (allowedEmails.size === 0) return;
+
+    const rawToken = await cookie.getAuth(request);
+    if (!rawToken) {
+      return reply.code(401).send({ message: "Unauthorized" });
+    }
+
+    const payload = await jwt.verifyToken(rawToken, "access");
+    if (!payload) {
+      return reply.code(401).send({ message: "Unauthorized" });
+    }
+
+    const normalized = payload as Record<string, unknown>;
+    const email = String(normalized.email ?? "")
+      .trim()
+      .toLowerCase();
+
+    if (!allowedEmails.has(email)) {
+      return reply.code(401).send({ message: "Unauthorized" });
+    }
+  };
+}
+
+async function discoverQueueNames(): Promise<string[]> {
+  const client = redisClientIfReady();
+  if (!client) return [];
+
+  const prefix = `${queuePrefix()}:`;
+  const names = new Set<string>();
+  let cursor = 0;
+
+  do {
+    const [nextCursor, keys] = await client.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", 100);
+    cursor = Number(nextCursor);
+    for (const key of keys) {
+      const name = key.slice(prefix.length, -5);
+      if (name) names.add(name);
+    }
+  } while (cursor !== 0);
+
+  return Array.from(names);
+}
+
+function unavailableHtml() {
+  return `
+    <!doctype html>
+    <html>
+      <head>
+        <title>Queue Dashboard Unavailable</title>
+        <style>
+          body { font-family: system-ui, sans-serif; padding: 48px; text-align: center; }
+          h1 { color: #dc2626; }
+        </style>
+      </head>
+      <body>
+        <h1>Queue Dashboard Unavailable</h1>
+        <p>Redis is not connected. Check the Redis URL in src/config/redis.ts and make sure Redis is running.</p>
+        <p>Redis URL: ${redisConfig.url}</p>
+      </body>
+    </html>
+  `;
+}
+
+/**
+ * Why: Mounts the BullMQ dashboard, or a fallback page when Redis is down.
+ * When: Kernel boots after queue jobs are registered.
+ * Where: Framework startup.
+ * How: Registers the BullBoard Fastify adapter at `BASE_PATH`, wrapped in a
+ *      scope that applies the email allowlist hook before any dashboard route.
+ */
+export async function setupQueueDashboard() {
+  if (!redisClientIfReady()) {
+    const route = (app: FastifyInstance) => {
+      const handler = async (_request: FastifyRequest, reply: FastifyReply) => reply.code(200).type("text/html").send(unavailableHtml());
+      app.get(BASE_PATH, { schema: { hide: true } }, handler);
+      app.get(`${BASE_PATH}/*`, { schema: { hide: true } }, handler);
+    };
+
+    return { basePath: BASE_PATH, enabled: false, route };
+  }
+
+  await refreshDashboardQueues();
+  if (!pollTimer) {
+    pollTimer = setInterval(refreshDashboardQueues, POLL_INTERVAL_MS);
+  }
+
+  const route = async (app: FastifyInstance) => {
+    await app.register(
+      async (scope) => {
+        scope.addHook("onRequest", dashboardAuth());
+        await scope.register(serverAdapter.registerPlugin());
+      },
+      { prefix: BASE_PATH }
+    );
+  };
+
+  return { basePath: BASE_PATH, enabled: true, route };
+}

package/fastify/src/framework/queue/worker.ts

@@ -0,0 +1,61 @@
+import { queueConfig } from "@/config/index.js";
+import { closeDatabase, initDatabase } from "@/framework/database/connection.js";
+import { pruneStaleQueues, startQueueWorker as startQueueWorkers, stopQueueRuntime } from "@/framework/queue/queue.js";
+import { closeRedis, initRedis, redisError, redisReady } from "@/framework/redis/client.js";
+import { parseCsvOrFallback, registerShutdownSignals, type ShutdownSignal } from "@/framework/support/lifecycle.js";
+import { logger } from "@/framework/support/logger.js";
+
+export interface WorkerHandle {
+  queues: string[];
+  shutdown: (signal: ShutdownSignal) => Promise<void>;
+}
+
+/**
+ * Why: Starts the queue worker and wires DB/Redis shutdown.
+ * When: CLI `queue:work` command or E2E test bootstrap.
+ * Where: Worker entrypoint.
+ * How: Parses --queue= args, inits DB/Redis, starts workers, registers signals.
+ *      When called as the main module, also prints the startup banner.
+ */
+export async function startQueueWorkerRuntime(queues: string[] = []): Promise<WorkerHandle> {
+  if (!queues.length) {
+    const queuesArg = process.argv.find((arg) => arg.startsWith("--queue="));
+    const queueNames = queuesArg?.split("=")[1];
+    queues = parseCsvOrFallback(queueNames, queueConfig.queues);
+  }
+
+  await initDatabase();
+  await initRedis();
+  if (!redisReady()) {
+    logger.error("Queue worker cannot start because Redis is unavailable", {
+      error: redisError()
+    });
+    process.exit(1);
+  }
+
+  await startQueueWorkers(queues);
+  await pruneStaleQueues(queues);
+  console.log(`Queue worker started: ${queues.join(", ")}`);
+
+  let shuttingDown = false;
+
+  async function shutdown(signal: ShutdownSignal) {
+    if (shuttingDown) return;
+    shuttingDown = true;
+
+    logger.info("Queue worker shutdown signal received", { signal });
+    await Promise.allSettled([stopQueueRuntime(), closeDatabase()]);
+    await closeRedis();
+
+    process.exit(0);
+  }
+
+  registerShutdownSignals(shutdown);
+
+  return { queues, shutdown };
+}
+
+const isMain = process.argv[1]?.endsWith("worker.ts") || process.argv[1]?.endsWith("worker.js");
+if (isMain) {
+  await startQueueWorkerRuntime();
+}