@podosoft/podokit 1.3.0 → 1.4.1

package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.spec.ts

@@ -1,5 +1,53 @@
 import { describe, expect, it } from "bun:test";
-import { externalJobId, parseExternalJobId } from "./bullmq-job.queue";
+import type { ConnectionOptions } from "bullmq";
+import { BullMqJobQueue, externalJobId, parseExternalJobId } from "./bullmq-job.queue";
+import { producerConnection, reconnectDelay, redisConnection } from "./queue";
+
+function producerAt(port: number): ConnectionOptions {
+  return producerConnection({
+    REDIS_HOST: "127.0.0.1",
+    REDIS_PORT: String(port),
+    REDIS_COMMAND_TIMEOUT_MS: "100",
+  });
+}
+
+function closedPort(): number {
+  const server = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
+  const port = server.port;
+  server.stop(true);
+  return port;
+}
+
+/**
+ * Answers the connection handshake and INFO like Redis 7, but stops answering a
+ * connection, without closing it, once it sends any other command.
+ */
+function stallingRedis(): { port: number; stop: () => void } {
+  const info = "# Server\r\nredis_version:7.2.0\r\nloading:0\r\n";
+  const stalled = new WeakSet<object>();
+  const server = Bun.listen({
+    hostname: "127.0.0.1",
+    port: 0,
+    socket: {
+      data(socket, data) {
+        // Each command is an array of bulk strings: "*<count>", "$<length>", "<name>", ...
+        const lines = data.toString().split("\r\n");
+        lines.forEach((line, index) => {
+          if (stalled.has(socket) || !line.startsWith("*")) return;
+          const command = lines[index + 2]?.toLowerCase();
+          if (command === "client") socket.write("+OK\r\n");
+          else if (command === "info") socket.write(`$${Buffer.byteLength(info)}\r\n${info}\r\n`);
+          else stalled.add(socket);
+        });
+      },
+    },
+  });
+  return { port: server.port, stop: () => server.stop(true) };
+}
+
+async function rejection(promise: Promise<unknown>): Promise<unknown> {
+  return promise.then(() => undefined, (error: unknown) => error);
+}
 
 describe("BullMQ job identifiers", () => {
   it("round-trips queue names without exposing an ambiguous id", () => {
@@ -8,3 +56,75 @@ describe("BullMQ job identifiers", () => {
     expect(parseExternalJobId("invalid")).toBeNull();
   });
 });
+
+describe("BullMQ connections", () => {
+  it("lets workers wait for replies and gives producers a bounded, supervised connection", () => {
+    expect(redisConnection({})).toMatchObject({ maxRetriesPerRequest: null });
+    const producer = producerConnection({ REDIS_COMMAND_TIMEOUT_MS: "750" });
+    expect(producer).toMatchObject({ enableOfflineQueue: false, commandTimeout: 750, lazyConnect: true });
+    expect("retryStrategy" in producer && producer.retryStrategy?.(1)).toBeNull();
+  });
+
+  it("backs off reconnects from 250 ms up to 5 seconds", () => {
+    expect([1, 2, 3, 4, 5, 6, 30].map(reconnectDelay)).toEqual([250, 500, 1_000, 2_000, 4_000, 5_000, 5_000]);
+  });
+});
+
+describe("BullMqJobQueue while Redis is unavailable", () => {
+  it("fails enqueues and reads at once before the producer has connected", async () => {
+    const queue = new BullMqJobQueue(producerAt(closedPort()));
+    try {
+      const started = Date.now();
+      expect(await rejection(queue.enqueue("demo", { text: "hello" })))
+        .toMatchObject({ code: "JOBS_UNAVAILABLE", statusCode: 503 });
+      expect(await rejection(queue.get(externalJobId("demo", "1"))))
+        .toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      expect(Date.now() - started).toBeLessThan(100);
+    } finally {
+      await queue.close();
+    }
+  });
+
+  it("reports a refused connection to the supervisor and keeps failing enqueues at once", async () => {
+    const queue = new BullMqJobQueue(producerAt(closedPort()));
+    const lost: unknown[] = [];
+    queue.onLost((reason) => { lost.push(reason); });
+    try {
+      await expect(queue.connect()).rejects.toThrow();
+      await expect(queue.ready()).rejects.toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      const started = Date.now();
+      expect(await rejection(queue.enqueue("demo", {}))).toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      expect(Date.now() - started).toBeLessThan(100);
+      expect(lost.length).toBeGreaterThan(0);
+    } finally {
+      await queue.close();
+    }
+  });
+
+  it("drops a connection Redis stops answering and fails at once until the supervisor reopens it", async () => {
+    const redis = stallingRedis();
+    const queue = new BullMqJobQueue(producerAt(redis.port));
+    const lost: unknown[] = [];
+    queue.onLost((reason) => { lost.push(reason); });
+    try {
+      await queue.connect();
+      let started = Date.now();
+      expect(await rejection(queue.enqueue("demo", { text: "hello" })))
+        .toMatchObject({ code: "JOBS_UNAVAILABLE", statusCode: 503 });
+      expect(Date.now() - started).toBeLessThan(1_000);
+      expect(lost.length).toBe(1);
+
+      started = Date.now();
+      expect(await rejection(queue.enqueue("demo", {}))).toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      expect(Date.now() - started).toBeLessThan(100);
+
+      await queue.connect();
+      await queue.ready();
+      await Bun.sleep(50);
+      expect(lost.length).toBe(1);
+    } finally {
+      await queue.close();
+      redis.stop();
+    }
+  });
+});

package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.ts

@@ -1,4 +1,5 @@
-import { Queue, Worker } from "bullmq";
+import { AppException } from "@podosoft/podokit-contracts";
+import { Queue, RedisConnection, Worker, type ConnectionOptions, type RedisClient } from "bullmq";
 import type {
   EnqueueOptions,
   JobHandler,
@@ -8,10 +9,15 @@ import type {
   JobStatus,
   Unsubscribe,
 } from "@podosoft/podokit-runtime";
-import { redisConnection } from "./queue";
+import type { SupervisedConnection } from "../core/services";
+import { producerConnection, redisConnection } from "./queue";
 
 const ID_SEPARATOR = "~";
 
+// BullMQ would otherwise wait for Redis and ask its version before the first
+// command, which keeps a call pending while Redis is down instead of failing it.
+const NO_STARTUP_ROUND_TRIP = { skipWaitingForReady: true, skipVersionCheck: true } as const;
+
 export function externalJobId(name: string, id: string): string {
   return `${encodeURIComponent(name)}${ID_SEPARATOR}${id}`;
 }
@@ -31,23 +37,57 @@ function statusOf(state: string): JobStatus {
   return "waiting";
 }
 
-export class BullMqJobQueue implements JobQueue {
+function isReplyTimeout(error: unknown): boolean {
+  return error instanceof Error && error.message === "Command timed out";
+}
+
+export function jobsUnavailable(): AppException {
+  return new AppException("JOBS_UNAVAILABLE", "Background jobs are temporarily unavailable.", 503);
+}
+
+/**
+ * Adds and reads jobs over one producer connection that the service registry
+ * supervises, and runs in-process handlers on BullMQ workers, which keep their
+ * own blocking connections and reconnect by themselves.
+ */
+export class BullMqJobQueue implements JobQueue, SupervisedConnection {
   private readonly queues = new Map<string, Queue>();
   private readonly workers = new Map<string, Worker>();
+  private producer: RedisConnection;
+  private lostListener?: (reason: unknown) => void;
+  private closed = false;
+
+  constructor(private readonly connection: ConnectionOptions = producerConnection()) {
+    this.producer = this.openProducer();
+  }
+
+  onLost(listener: (reason: unknown) => void): void {
+    this.lostListener = listener;
+  }
+
+  /** Opens the producer connection. */
+  async connect(): Promise<void> {
+    const client = await this.producer.client;
+    if (client.status !== "ready") await client.connect();
+  }
+
+  /** Rejects while Redis is unreachable or slower to answer than `REDIS_COMMAND_TIMEOUT_MS`. */
+  async ready(): Promise<void> {
+    await this.command((client) => client.info());
+  }
 
   async enqueue(
     name: string,
     payload: unknown,
     options: EnqueueOptions = {},
   ): Promise<JobReference> {
-    const queue = this.queue(name);
-    const job = await queue.add(name, payload, {
+    const job = await this.command((client) => this.queue(name, client).add(name, payload, {
       ...(options.delayMs === undefined ? {} : { delay: options.delayMs }),
       ...(options.attempts === undefined ? {} : { attempts: options.attempts }),
       ...(options.deduplicationKey === undefined
         ? {}
         : { deduplication: { id: options.deduplicationKey } }),
-    });
+    }));
     if (job.id === undefined) throw new Error("BullMQ did not assign a job id");
     return { id: externalJobId(name, job.id), name };
   }
@@ -55,12 +95,13 @@ export class BullMqJobQueue implements JobQueue {
   async get(externalId: string): Promise<JobSnapshot | null> {
     const parsed = parseExternalJobId(externalId);
     if (!parsed) return null;
-    const job = await this.queue(parsed.name).getJob(parsed.id);
+    const job = await this.command((client) => this.queue(parsed.name, client).getJob(parsed.id));
     if (!job) return null;
+    const state = await this.command(() => job.getState());
     return {
       id: externalId,
       name: parsed.name,
-      status: statusOf(await job.getState()),
+      status: statusOf(state),
       attempts: job.attemptsMade,
       createdAt: job.timestamp,
       updatedAt: job.finishedOn ?? job.processedOn ?? job.timestamp,
@@ -89,18 +130,75 @@ export class BullMqJobQueue implements JobQueue {
   }
 
   async close(): Promise<void> {
+    this.closed = true;
     await Promise.all([
       ...[...this.workers.values()].map((worker) => worker.close()),
       ...[...this.queues.values()].map((queue) => queue.close()),
     ]);
     this.workers.clear();
     this.queues.clear();
+    await this.producer.close(true);
+  }
+
+  /**
+   * Runs `send` on the producer connection and fails it with JOBS_UNAVAILABLE
+   * while the connection is closed or when Redis does not answer in time.
+   */
+  private async command<T>(send: (client: RedisClient) => Promise<T>): Promise<T> {
+    const producer = this.producer;
+    const client = await producer.client;
+    if (client.status !== "ready") throw jobsUnavailable();
+    try {
+      return await send(client);
+    } catch (error) {
+      const timedOut = isReplyTimeout(error);
+      if (timedOut && producer === this.producer) this.replaceProducer(error);
+      if (timedOut || producer !== this.producer || client.status !== "ready") throw jobsUnavailable();
+      throw error;
+    }
+  }
+
+  private openProducer(): RedisConnection {
+    const producer = new RedisConnection(this.connection, {
+      shared: false,
+      blocking: false,
+      ...NO_STARTUP_ROUND_TRIP,
+    });
+    // Connection errors reach callers as JOBS_UNAVAILABLE and the supervisor's log.
+    producer.on("error", () => undefined);
+    producer.on("close", () => {
+      if (producer === this.producer) this.report(new Error("Connection closed"));
+    });
+    return producer;
+  }
+
+  /**
+   * Redis answers in order, so after a reply times out every later command
+   * would wait behind it, and a server that stopped answering never completes
+   * the close either. The connection is dropped for a new one that the
+   * supervisor opens.
+   */
+  private replaceProducer(reason: unknown): void {
+    const stale = this.producer;
+    const queues = [...this.queues.values()];
+    this.queues.clear();
+    this.producer = this.openProducer();
+    void Promise.all(queues.map((queue) => queue.close()))
+      .then(() => stale.close(true))
+      .catch(() => undefined);
+    this.report(reason);
+  }
+
+  private report(reason: unknown): void {
+    if (!this.closed) this.lostListener?.(reason);
   }
 
-  private queue(name: string): Queue {
+  private queue(name: string, client: RedisClient): Queue {
     const current = this.queues.get(name);
     if (current) return current;
-    const queue = new Queue(name, { connection: redisConnection() });
+    const queue = new Queue(name, { connection: client, ...NO_STARTUP_ROUND_TRIP });
+    // It reports the producer connection's errors, which `openProducer` already handles.
+    queue.on("error", () => undefined);
     this.queues.set(name, queue);
     return queue;
   }

package/dist/templates/modules/bullmq/files/apps/api/src/jobs/jobs.module.ts

@@ -2,7 +2,7 @@ import { AppException } from "@podosoft/podokit-contracts";
 import { JOBS } from "@podosoft/podokit-runtime";
 import { Elysia, t } from "elysia";
 import { PROVIDERS } from "../config/providers";
-import type { AppPlugin, PodokitModule } from "../core/services";
+import { READINESS, type AppPlugin, type PodokitModule } from "../core/services";
 import { BullMqJobQueue } from "./bullmq-job.queue";
 
 const jobsPlugin: AppPlugin = ({ services }) => {
@@ -33,6 +33,8 @@ export const jobsModule: PodokitModule = {
     if (PROVIDERS.jobs !== "bullmq") return;
     const queue = new BullMqJobQueue();
     services.register(JOBS, queue, () => queue.close());
+    services.resolve(READINESS).register("jobs", () => queue.ready());
+    services.supervise("jobs", queue);
   },
   plugin: jobsPlugin,
 };

package/dist/templates/modules/bullmq/files/apps/api/src/jobs/queue.ts

@@ -1,7 +1,35 @@
-import { redisConnectionOptions, type RedisConnectionOptions } from "../config/redis.connection";
+import type { ConnectionOptions } from "bullmq";
+import { redisCommandTimeoutMs, redisConnectionOptions } from "../config/redis.connection";
 
 export const DEMO_QUEUE = "demo";
 
-export function redisConnection(): RedisConnectionOptions {
-  return redisConnectionOptions(process.env, { maxRetriesPerRequest: null });
+/** Wait before reconnect attempt `attempt`: 250 ms, doubling up to 5 seconds. */
+export function reconnectDelay(attempt: number): number {
+  return Math.min(250 * 2 ** Math.max(0, attempt - 1), 5_000);
+}
+
+/** Workers block on Redis, so BullMQ requires their commands to wait for a reply indefinitely. */
+export function redisConnection(env: NodeJS.ProcessEnv = process.env): ConnectionOptions {
+  return {
+    ...redisConnectionOptions(env, { maxRetriesPerRequest: null }),
+    retryStrategy: reconnectDelay,
+  };
+}
+
+/**
+ * The API's connection for adding and reading jobs. Commands fail at once while
+ * it is disconnected and wait at most `REDIS_COMMAND_TIMEOUT_MS` for a reply, so
+ * a request that enqueues a job cannot hang on Redis. It does not reconnect by
+ * itself: the service registry's supervisor reopens it.
+ */
+export function producerConnection(env: NodeJS.ProcessEnv = process.env): ConnectionOptions {
+  return {
+    ...redisConnectionOptions(env, {
+      lazyConnect: true,
+      enableOfflineQueue: false,
+      connectTimeout: 5_000,
+      commandTimeout: redisCommandTimeoutMs(env),
+    }),
+    retryStrategy: () => null,
+  };
 }

package/dist/templates/modules/bullmq/module.manifest.json

@@ -8,13 +8,17 @@
     "apps/api/src/config/providers.ts",
     "apps/api/src/config/redis.connection.ts"
   ],
+  "requiredProjectContents": {
+    "apps/api/src/core/services.ts": ["supervise(name: string, connection: SupervisedConnection): void {"],
+    "apps/api/src/config/redis.connection.ts": ["export function redisCommandTimeoutMs("]
+  },
   "managedGlobs": [
     "apps/api/src/jobs/**",
     "apps/api/src/main-worker.ts"
   ],
   "targetApp": "api",
   "dependencies": {
-    "bullmq": "^5.34.0"
+    "bullmq": "^5.57.0"
   },
   "scripts": {
     "start:worker": "bun dist/main-worker.js",

package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/events-redis.module.ts

@@ -8,9 +8,7 @@ export const eventsRedisModule: PodokitModule = {
   configure: (_env, services): void => {
     if (PROVIDERS.events !== "redis") return;
     const events = new RedisEventBus();
-    services.register(EVENTS, events, async () => {
-      await events.close();
-    });
-    services.onStart(() => events.ready());
+    services.register(EVENTS, events, () => events.close());
+    services.supervise("events", events);
   },
 };

package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/redis-event.bus.spec.ts

@@ -0,0 +1,224 @@
+import type { RedisClient } from "bun";
+import { describe, expect, it } from "bun:test";
+import { RedisCommandTimeoutError } from "../config/redis.connection";
+import { RedisEventBus } from "./redis-event.bus";
+
+type Listener = (message: string, channel: string) => void;
+
+/**
+ * One Redis server shared by fake clients: it can go down, and it drops every
+ * connection when it does. While paused it keeps connections open and answers nothing.
+ */
+class FakeRedis {
+  up = true;
+  paused = false;
+  readonly clients: FakeClient[] = [];
+
+  client(): RedisClient {
+    const client = new FakeClient(this);
+    this.clients.push(client);
+    return client as unknown as RedisClient;
+  }
+
+  stop(): void {
+    this.up = false;
+    for (const client of this.clients) client.drop();
+  }
+
+  deliver(channel: string, message: string): number {
+    let receivers = 0;
+    for (const client of this.clients) {
+      if (!client.connected || !client.channels.has(channel)) continue;
+      receivers += 1;
+      client.listener?.(message, channel);
+    }
+    return receivers;
+  }
+}
+
+class FakeClient {
+  connected = false;
+  onclose: ((error: Error) => void) | null = null;
+  readonly channels = new Set<string>();
+  listener?: Listener;
+  private readonly pending = new Set<(error: Error) => void>();
+
+  constructor(private readonly server: FakeRedis) {}
+
+  /** Never answers while the server is paused; closing the connection fails the command. */
+  private answer<T>(value: () => T): Promise<T> {
+    if (!this.server.paused) return Promise.resolve(value());
+    return new Promise<T>((_resolve, reject) => {
+      this.pending.add(reject);
+    });
+  }
+
+  connect(): Promise<void> {
+    if (!this.server.up) {
+      this.onclose?.(new Error("Connection closed"));
+      return Promise.reject(new Error("Connection closed"));
+    }
+    this.connected = true;
+    return Promise.resolve();
+  }
+
+  ping(): Promise<string> {
+    return this.connected ? this.answer(() => "PONG") : Promise.reject(new Error("Connection closed"));
+  }
+
+  publish(channel: string, message: string): Promise<number> {
+    if (!this.connected) {
+      return Promise.reject(new Error("Connection is closed and offline queue is disabled"));
+    }
+    return this.answer(() => this.server.deliver(channel, message));
+  }
+
+  subscribe(channel: string, listener: Listener): Promise<number> {
+    if (!this.connected) return Promise.reject(new Error("Connection closed"));
+    return this.answer(() => {
+      this.channels.add(channel);
+      this.listener = listener;
+      return this.channels.size;
+    });
+  }
+
+  drop(): void {
+    if (!this.connected) return;
+    // Bun forgets the server subscription but keeps its listener.
+    this.connected = false;
+    this.channels.clear();
+    this.onclose?.(new Error("Connection closed"));
+  }
+
+  close(): void {
+    if (!this.connected) return;
+    this.connected = false;
+    for (const reject of this.pending) reject(new Error("Connection closed"));
+    this.pending.clear();
+    this.onclose?.(new Error("Connection closed"));
+  }
+}
+
+function bus(server: FakeRedis): RedisEventBus {
+  return new RedisEventBus(
+    { EVENTS_REDIS_CHANNEL: "test:events", REDIS_COMMAND_TIMEOUT_MS: "50" },
+    () => server.client(),
+  );
+}
+
+describe("RedisEventBus reconnects", () => {
+  it("keeps handlers registered without Redis and delivers once the channel is subscribed", async () => {
+    const server = new FakeRedis();
+    server.up = false;
+    const events = bus(server);
+    let interruptions = 0;
+    events.onInterruption(() => { interruptions += 1; });
+    const received: unknown[] = [];
+    await events.subscribe("sse", (event) => { received.push(event); });
+
+    await expect(events.connect()).rejects.toThrow("Connection closed");
+    await expect(events.ready()).rejects.toThrow("not subscribed");
+    server.up = true;
+    await events.connect();
+    await events.ready();
+
+    await events.publish("sse", { id: 1 });
+    expect(received).toEqual([{ id: 1 }]);
+    // Streams opened before the first subscription may have missed events.
+    expect(interruptions).toBe(1);
+    events.close();
+  });
+
+  it("subscribes again after Redis drops the connections and reports one interruption", async () => {
+    const server = new FakeRedis();
+    const events = bus(server);
+    const lost: unknown[] = [];
+    events.onLost((reason) => { lost.push(reason); });
+    let interruptions = 0;
+    events.onInterruption(() => { interruptions += 1; });
+    const received: unknown[] = [];
+    await events.subscribe("config.changed", (event) => { received.push(event); });
+    await events.connect();
+    expect(interruptions).toBe(0);
+
+    server.stop();
+    expect(lost.length).toBeGreaterThan(0);
+    await expect(events.ready()).rejects.toThrow("not subscribed");
+    await expect(events.publish("config.changed", { origin: "a" })).rejects.toThrow("offline queue");
+    server.up = true;
+    await events.connect();
+
+    expect(interruptions).toBe(1);
+    await events.publish("config.changed", { origin: "b" });
+    expect(received).toEqual([{ origin: "b" }]);
+    events.close();
+  });
+
+  it("reports no interruption when only the publisher reconnects", async () => {
+    const server = new FakeRedis();
+    const events = bus(server);
+    let interruptions = 0;
+    events.onInterruption(() => { interruptions += 1; });
+    await events.connect();
+
+    server.clients[0]?.drop();
+    await events.connect();
+    expect(interruptions).toBe(0);
+    await events.ready();
+    events.close();
+  });
+
+  it("does not report connections it closes itself", async () => {
+    const server = new FakeRedis();
+    const events = bus(server);
+    const lost: unknown[] = [];
+    events.onLost((reason) => { lost.push(reason); });
+    await events.connect();
+
+    events.close();
+    expect(lost).toEqual([]);
+  });
+
+  it("fails a publish Redis does not answer and hands the publisher to the supervisor", async () => {
+    const server = new FakeRedis();
+    const events = bus(server);
+    const lost: unknown[] = [];
+    events.onLost((reason) => { lost.push(reason); });
+    let interruptions = 0;
+    events.onInterruption(() => { interruptions += 1; });
+    const received: unknown[] = [];
+    await events.subscribe("sse", (event) => { received.push(event); });
+    await events.connect();
+
+    server.paused = true;
+    const started = Date.now();
+    await expect(events.publish("sse", { id: 1 })).rejects.toBeInstanceOf(RedisCommandTimeoutError);
+    expect(Date.now() - started).toBeLessThan(1_000);
+    expect(lost.length).toBe(1);
+    await expect(events.ready()).rejects.toThrow("Connection closed");
+
+    server.paused = false;
+    await events.connect();
+    await events.publish("sse", { id: 2 });
+    expect(received).toEqual([{ id: 2 }]);
+    expect(interruptions).toBe(0);
+    events.close();
+  });
+
+  it("fails a subscription Redis does not confirm and subscribes on the next attempt", async () => {
+    const server = new FakeRedis();
+    server.paused = true;
+    const events = bus(server);
+    let interruptions = 0;
+    events.onInterruption(() => { interruptions += 1; });
+
+    await expect(events.connect()).rejects.toBeInstanceOf(RedisCommandTimeoutError);
+    await expect(events.ready()).rejects.toThrow("not subscribed");
+
+    server.paused = false;
+    await events.connect();
+    await events.ready();
+    expect(interruptions).toBe(1);
+    events.close();
+  });
+});