@podosoft/podokit 1.3.0 → 1.4.0

package/dist/templates/modules/auth/files/apps/api/src/auth/database-outage.spec.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it, mock } from "bun:test";
+import { APIError } from "better-auth/api";
+import { Pool } from "pg";
+import {
+  answersWithin,
+  checkAuthResponse,
+  rethrowAuthError,
+  sharedProbe,
+} from "./database-outage";
+
+const down = mock(async () => false);
+const up = mock(async () => true);
+
+async function rejection(promise: Promise<unknown>): Promise<unknown> {
+  return promise.then(() => undefined, (error: unknown) => error);
+}
+
+describe("checkAuthResponse", () => {
+  it("passes answers below 500 on without asking the database", async () => {
+    const probe = mock(async () => false);
+    for (const status of [200, 401, 403, 429]) {
+      const response = new Response(null, { status });
+      await expect(checkAuthResponse(response, probe)).resolves.toBe(response);
+    }
+    expect(probe).not.toHaveBeenCalled();
+  });
+
+  it("keeps a server error while the database answers", async () => {
+    const response = new Response(null, { status: 500 });
+    await expect(checkAuthResponse(response, up)).resolves.toBe(response);
+  });
+
+  it("reports a server error during a database outage as DATABASE_UNAVAILABLE", async () => {
+    const error = await rejection(checkAuthResponse(new Response(null, { status: 500 }), down));
+    expect(error).toMatchObject({ code: "DATABASE_UNAVAILABLE", statusCode: 503 });
+  });
+});
+
+describe("rethrowAuthError", () => {
+  it("rethrows client errors without asking the database", async () => {
+    const probe = mock(async () => false);
+    const unauthorized = new APIError("UNAUTHORIZED", { message: "Invalid session" });
+    await expect(rejection(rethrowAuthError(unauthorized, probe))).resolves.toBe(unauthorized);
+    expect(probe).not.toHaveBeenCalled();
+  });
+
+  it("reports a server error during a database outage as DATABASE_UNAVAILABLE", async () => {
+    const failed = APIError.from("INTERNAL_SERVER_ERROR", { message: "Failed to get session", code: "FAILED_TO_GET_SESSION" });
+    const error = await rejection(rethrowAuthError(failed, down));
+    expect(error).toMatchObject({ code: "DATABASE_UNAVAILABLE", statusCode: 503 });
+    expect(await rejection(rethrowAuthError(new Error("socket hang up"), down)))
+      .toMatchObject({ code: "DATABASE_UNAVAILABLE" });
+  });
+
+  it("rethrows other server errors unchanged while the database answers", async () => {
+    const failure = new Error("Unexpected");
+    await expect(rejection(rethrowAuthError(failure, up))).resolves.toBe(failure);
+  });
+});
+
+describe("answersWithin", () => {
+  it("reports success and failure of the query", async () => {
+    await expect(answersWithin(async () => [{ ok: 1 }], 100)).resolves.toBe(true);
+    await expect(answersWithin(async () => { throw new Error("refused"); }, 100)).resolves.toBe(false);
+  });
+
+  it("gives up on a query that does not answer", async () => {
+    const started = Date.now();
+    await expect(answersWithin(() => new Promise(() => undefined), 50)).resolves.toBe(false);
+    expect(Date.now() - started).toBeLessThan(1_000);
+  });
+
+  it("reports a PostgreSQL server that refuses or never answers as unreachable", async () => {
+    // Accepts the TCP connection and never answers, like a paused server.
+    const silent = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
+    const closed = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
+    closed.stop(true);
+    const pools = [silent.port, closed.port].map((port) =>
+      new Pool({
+        connectionString: `postgres://podokit:podokit@127.0.0.1:${port}/podokit`,
+        connectionTimeoutMillis: 2_000,
+      })
+    );
+    try {
+      for (const pool of pools) {
+        const started = Date.now();
+        await expect(answersWithin(() => pool.query("SELECT 1"), 200)).resolves.toBe(false);
+        expect(Date.now() - started).toBeLessThan(1_000);
+      }
+    } finally {
+      silent.stop(true);
+      await Promise.all(pools.map((pool) => pool.end().catch(() => undefined)));
+    }
+  });
+});
+
+describe("sharedProbe", () => {
+  it("lets concurrent callers share one probe and probes again afterwards", async () => {
+    let release: (value: boolean) => void = () => undefined;
+    const probe = mock(() => new Promise<boolean>((resolve) => { release = resolve; }));
+    const shared = sharedProbe(probe);
+
+    const first = shared();
+    const second = shared();
+    release(false);
+    await expect(Promise.all([first, second])).resolves.toEqual([false, false]);
+    expect(probe).toHaveBeenCalledTimes(1);
+
+    const third = shared();
+    release(true);
+    await expect(third).resolves.toBe(true);
+    expect(probe).toHaveBeenCalledTimes(2);
+  });
+});

package/dist/templates/modules/auth/files/apps/api/src/auth/database-outage.ts

@@ -0,0 +1,50 @@
+import { AppException } from "@podosoft/podokit-contracts";
+import { APIError } from "better-auth/api";
+
+/** Resolves whether the database answers now. */
+export type DatabaseProbe = () => Promise<boolean>;
+
+export function databaseUnavailable(): AppException {
+  return new AppException("DATABASE_UNAVAILABLE", "The database is temporarily unavailable", 503);
+}
+
+/** Whether `query` succeeds within `timeoutMs`; a query still waiting then counts as failed. */
+export async function answersWithin(query: () => Promise<unknown>, timeoutMs: number): Promise<boolean> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<boolean>((resolve) => {
+    timer = setTimeout(() => resolve(false), timeoutMs);
+  });
+  try {
+    return await Promise.race([query().then(() => true, () => false), deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Lets concurrent callers share one run of `probe`. */
+export function sharedProbe(probe: DatabaseProbe): DatabaseProbe {
+  let running: Promise<boolean> | undefined;
+  return () => {
+    running ??= probe().finally(() => {
+      running = undefined;
+    });
+    return running;
+  };
+}
+
+// Better Auth answers a failed query with a bare 500, and its session lookup
+// replaces the cause with INTERNAL_SERVER_ERROR. A server error is therefore
+// checked against the database itself before it is reported.
+
+/** Passes `response` on, unless it is a server error while the database does not answer. */
+export async function checkAuthResponse(response: Response, reachable: DatabaseProbe): Promise<Response> {
+  if (response.status < 500 || (await reachable())) return response;
+  throw databaseUnavailable();
+}
+
+/** Rethrows `error`, as DATABASE_UNAVAILABLE when it is a server error while the database does not answer. */
+export async function rethrowAuthError(error: unknown, reachable: DatabaseProbe): Promise<never> {
+  const clientError = error instanceof APIError && error.statusCode < 500;
+  if (!clientError && !(await reachable())) throw databaseUnavailable();
+  throw error;
+}

package/dist/templates/modules/auth/files/apps/api/src/auth/db.spec.ts

@@ -3,22 +3,25 @@ import type { DatabaseProviderName } from "@podosoft/podokit-runtime";
 import { validateEnv } from "../config/env.validation";
 import { PROVIDERS } from "../config/providers";
 import { postgresPool } from "./db";
+import { AuthPostgresPool } from "./postgres-pool";
 
 function databaseProvider(): DatabaseProviderName {
   return PROVIDERS.database;
 }
 
 describe("auth database pool", () => {
-  it("applies the pool limits from the environment", () => {
+  it("applies the pool limits and the query timeout from the environment", () => {
     if (databaseProvider() !== "postgres") {
       expect(postgresPool).toBeNull();
       return;
     }
     const env = validateEnv(process.env);
+    expect(postgresPool).toBeInstanceOf(AuthPostgresPool);
     expect(postgresPool?.options).toMatchObject({
       max: env.DB_AUTH_POOL_MAX,
       connectionTimeoutMillis: env.DB_CONNECT_TIMEOUT_MS,
       idleTimeoutMillis: env.DB_IDLE_TIMEOUT_MS,
+      query_timeout: env.DB_QUERY_TIMEOUT_MS,
     });
   });
 });

package/dist/templates/modules/auth/files/apps/api/src/auth/db.ts

@@ -1,11 +1,13 @@
 import { Database as SqliteDatabase } from "bun:sqlite";
 import type { SQL } from "bun";
-import { Pool } from "pg";
 import type { AuthConfigRow, ConfigVersions } from "@podosoft/podokit-auth";
 import type { DatabaseProviderName } from "@podosoft/podokit-runtime";
 import { validateEnv } from "../config/env.validation";
 import { PROVIDERS } from "../config/providers";
 import { databaseUrl, sqliteDatabasePath } from "../database/database";
+import { answersWithin, sharedProbe, type DatabaseProbe } from "./database-outage";
+import { isMaintenanceCommand } from "./maintenance";
+import { AuthPostgresPool } from "./postgres-pool";
 
 const env = validateEnv(process.env);
 const url = databaseUrl(env);
@@ -15,11 +17,12 @@ function databaseProvider(): DatabaseProviderName {
 const provider = databaseProvider();
 
 export const postgresPool = provider === "postgres"
-  ? new Pool({
+  ? new AuthPostgresPool({
       connectionString: url,
       max: env.DB_AUTH_POOL_MAX,
       connectionTimeoutMillis: env.DB_CONNECT_TIMEOUT_MS,
       idleTimeoutMillis: env.DB_IDLE_TIMEOUT_MS,
+      ...(isMaintenanceCommand() ? {} : { query_timeout: env.DB_QUERY_TIMEOUT_MS }),
     })
   : null;
 
@@ -30,6 +33,12 @@ export const sqliteDatabase = provider === "sqlite"
 export const authDatabase = postgresPool ?? sqliteDatabase;
 if (!authDatabase) throw new Error("No authentication database is configured");
 
+/** Whether the authentication database answers `SELECT 1` within the readiness check timeout. */
+export const authDatabaseReachable: DatabaseProbe = sharedProbe(async () => {
+  if (!postgresPool) return true;
+  return answersWithin(() => postgresPool.query("SELECT 1"), env.READINESS_CHECK_TIMEOUT_MS);
+});
+
 let queryDatabase: SQL | undefined;
 
 export function configureAuthQueryDatabase(sql: SQL): void {

package/dist/templates/modules/auth/files/apps/api/src/auth/maintenance-command.ts

@@ -0,0 +1,5 @@
+// Imported first by maintenance commands, before ./auth creates the Better Auth
+// instance and ./db opens its pool.
+import { startMaintenanceCommand } from "./maintenance";
+
+startMaintenanceCommand();

package/dist/templates/modules/auth/files/apps/api/src/auth/maintenance.ts

@@ -0,0 +1,14 @@
+// Servers validate Better Auth's database schema at startup and bound every auth
+// query with DB_QUERY_TIMEOUT_MS. Maintenance commands, such as the migration
+// runner and the admin bootstrap, build the same instance before the schema they
+// change exists or matches, and may run statements for longer, so they opt out
+// for their process.
+let maintenance = false;
+
+export function startMaintenanceCommand(): void {
+  maintenance = true;
+}
+
+export function isMaintenanceCommand(): boolean {
+  return maintenance;
+}

package/dist/templates/modules/auth/files/apps/api/src/auth/postgres-pool.spec.ts

@@ -0,0 +1,136 @@
+import { describe, expect, it } from "bun:test";
+import type { Socket } from "bun";
+import type { PoolClient } from "pg";
+import { AuthPostgresPool } from "./postgres-pool";
+
+interface FakePostgres {
+  port: number;
+  /** While false, the server keeps every connection open and ignores queries, like a paused one. */
+  answering: boolean;
+  accepted: number;
+  closed: number;
+  stop: () => void;
+}
+
+function message(type: string, body: Buffer): Buffer {
+  const header = Buffer.alloc(5);
+  header.write(type, 0, "latin1");
+  header.writeInt32BE(body.length + 4, 1);
+  return Buffer.concat([header, body]);
+}
+
+const AUTHENTICATION_OK = message("R", Buffer.from([0, 0, 0, 0]));
+const READY_FOR_QUERY = message("Z", Buffer.from("I"));
+const SELECT_DONE = Buffer.concat([message("C", Buffer.from("SELECT 0\0")), READY_FOR_QUERY]);
+
+/** Speaks enough of the PostgreSQL protocol for pg to connect and run simple queries. */
+function fakePostgres(): FakePostgres {
+  const started = new WeakSet<Socket<undefined>>();
+  const server: FakePostgres = { port: 0, answering: true, accepted: 0, closed: 0, stop: () => undefined };
+  const listener = Bun.listen({
+    hostname: "127.0.0.1",
+    port: 0,
+    socket: {
+      open() {
+        server.accepted += 1;
+      },
+      close() {
+        server.closed += 1;
+      },
+      data(socket, data) {
+        if (!started.has(socket)) {
+          started.add(socket);
+          socket.write(Buffer.concat([AUTHENTICATION_OK, READY_FOR_QUERY]));
+          return;
+        }
+        if (server.answering && data[0] === "Q".charCodeAt(0)) socket.write(SELECT_DONE);
+      },
+    },
+  });
+  server.port = listener.port;
+  server.stop = () => listener.stop(true);
+  return server;
+}
+
+function poolFor(server: FakePostgres): AuthPostgresPool {
+  return new AuthPostgresPool({
+    host: "127.0.0.1",
+    port: server.port,
+    user: "podokit",
+    database: "podokit",
+    max: 1,
+    query_timeout: 50,
+  });
+}
+
+async function until(condition: () => boolean): Promise<void> {
+  const deadline = Date.now() + 2_000;
+  while (!condition()) {
+    if (Date.now() > deadline) throw new Error("Condition not met in time");
+    await new Promise((resolve) => setTimeout(resolve, 5));
+  }
+}
+
+describe("AuthPostgresPool", () => {
+  it("keeps a client whose queries were answered", async () => {
+    const server = fakePostgres();
+    const pool = poolFor(server);
+    try {
+      for (let round = 0; round < 2; round += 1) {
+        const client = await pool.connect();
+        await client.query("SELECT 1");
+        client.release();
+      }
+      expect(server.accepted).toBe(1);
+      expect(pool.idleCount).toBe(1);
+    } finally {
+      await pool.end();
+      server.stop();
+    }
+  });
+
+  it("discards a client released while its query still waits for the reply", async () => {
+    const server = fakePostgres();
+    const pool = poolFor(server);
+    try {
+      const client = await pool.connect();
+      server.answering = false;
+      await expect(client.query("SELECT 1")).rejects.toThrow("Query read timeout");
+      client.release();
+
+      expect(pool.totalCount).toBe(0);
+      await until(() => server.closed === 1);
+
+      server.answering = true;
+      const next = await pool.connect();
+      await next.query("SELECT 1");
+      next.release();
+      expect(server.accepted).toBe(2);
+      expect(pool.idleCount).toBe(1);
+    } finally {
+      await pool.end();
+      server.stop();
+    }
+  });
+
+  it("discards the client through the callback form and Pool.query", async () => {
+    const server = fakePostgres();
+    const pool = poolFor(server);
+    try {
+      server.answering = false;
+      await expect(pool.query("SELECT 1")).rejects.toThrow("Query read timeout");
+      expect(pool.totalCount).toBe(0);
+
+      const client = await new Promise<PoolClient>((resolve, reject) => {
+        pool.connect((error, connected) => (connected ? resolve(connected) : reject(error)));
+      });
+      await expect(client.query("SELECT 1")).rejects.toThrow("Query read timeout");
+      client.release();
+      expect(pool.totalCount).toBe(0);
+      await until(() => server.closed === 2);
+    } finally {
+      await pool.end();
+      server.stop();
+    }
+  });
+});

package/dist/templates/modules/auth/files/apps/api/src/auth/postgres-pool.ts

@@ -0,0 +1,40 @@
+import { Pool, type PoolClient } from "pg";
+
+type ConnectCallback = Parameters<Pool["connect"]>[0];
+
+/** Whether `client` still waits for the reply to a query; pg sets `readyForQuery` once the server is ready for the next one. */
+function awaitingReply(client: PoolClient): boolean {
+  return Reflect.get(client, "readyForQuery") === false;
+}
+
+function discardWhileAwaitingReply(client: PoolClient): PoolClient {
+  const release = client.release;
+  client.release = (error?: Error | boolean): void => {
+    release.call(
+      client,
+      error || (awaitingReply(client) ? new Error("Released while waiting for a query reply") : undefined),
+    );
+  };
+  return client;
+}
+
+/**
+ * pg fails a query after `query_timeout` but leaves the connection waiting for
+ * the reply. `Pool.query` discards that client, but Kysely, which Better Auth
+ * runs on, releases it as healthy, so the pool would hand it out again and
+ * every later query on it would queue behind the missing reply. This pool
+ * discards a client released while it still waits for a reply: pg closes its
+ * socket, and the next query gets a new connection.
+ */
+export class AuthPostgresPool extends Pool {
+  override connect(): Promise<PoolClient>;
+  override connect(callback: ConnectCallback): void;
+  override connect(callback?: ConnectCallback): Promise<PoolClient> | void {
+    if (!callback) return super.connect().then(discardWhileAwaitingReply);
+    super.connect((error, client, done) => {
+      if (!client) return callback(error, client, done);
+      const guarded = discardWhileAwaitingReply(client);
+      callback(error, guarded, guarded.release);
+    });
+  }
+}

package/dist/templates/modules/auth/files/apps/api/src/migrate.ts

@@ -1,6 +1,7 @@
 // Keep this first: Better Auth checks the schema when ./auth/auth creates its
-// instance, and the migrations below are what make that schema match.
-import "./auth/skip-schema-validation";
+// instance, the migrations below are what make that schema match, and they may
+// run longer than the auth pool's query timeout.
+import "./auth/maintenance-command";
 import { getMigrations } from "better-auth/db/migration";
 import { auth } from "./auth/auth";
 import { closeAuthDatabase, postgresPool, sqliteDatabase } from "./auth/db";

package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.spec.ts

@@ -1,5 +1,53 @@
 import { describe, expect, it } from "bun:test";
-import { externalJobId, parseExternalJobId } from "./bullmq-job.queue";
+import type { ConnectionOptions } from "bullmq";
+import { BullMqJobQueue, externalJobId, parseExternalJobId } from "./bullmq-job.queue";
+import { producerConnection, reconnectDelay, redisConnection } from "./queue";
+
+function producerAt(port: number): ConnectionOptions {
+  return producerConnection({
+    REDIS_HOST: "127.0.0.1",
+    REDIS_PORT: String(port),
+    REDIS_COMMAND_TIMEOUT_MS: "100",
+  });
+}
+
+function closedPort(): number {
+  const server = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
+  const port = server.port;
+  server.stop(true);
+  return port;
+}
+
+/**
+ * Answers the connection handshake and INFO like Redis 7, but stops answering a
+ * connection, without closing it, once it sends any other command.
+ */
+function stallingRedis(): { port: number; stop: () => void } {
+  const info = "# Server\r\nredis_version:7.2.0\r\nloading:0\r\n";
+  const stalled = new WeakSet<object>();
+  const server = Bun.listen({
+    hostname: "127.0.0.1",
+    port: 0,
+    socket: {
+      data(socket, data) {
+        // Each command is an array of bulk strings: "*<count>", "$<length>", "<name>", ...
+        const lines = data.toString().split("\r\n");
+        lines.forEach((line, index) => {
+          if (stalled.has(socket) || !line.startsWith("*")) return;
+          const command = lines[index + 2]?.toLowerCase();
+          if (command === "client") socket.write("+OK\r\n");
+          else if (command === "info") socket.write(`$${Buffer.byteLength(info)}\r\n${info}\r\n`);
+          else stalled.add(socket);
+        });
+      },
+    },
+  });
+  return { port: server.port, stop: () => server.stop(true) };
+}
+
+async function rejection(promise: Promise<unknown>): Promise<unknown> {
+  return promise.then(() => undefined, (error: unknown) => error);
+}
 
 describe("BullMQ job identifiers", () => {
   it("round-trips queue names without exposing an ambiguous id", () => {
@@ -8,3 +56,75 @@ describe("BullMQ job identifiers", () => {
     expect(parseExternalJobId("invalid")).toBeNull();
   });
 });
+
+describe("BullMQ connections", () => {
+  it("lets workers wait for replies and gives producers a bounded, supervised connection", () => {
+    expect(redisConnection({})).toMatchObject({ maxRetriesPerRequest: null });
+    const producer = producerConnection({ REDIS_COMMAND_TIMEOUT_MS: "750" });
+    expect(producer).toMatchObject({ enableOfflineQueue: false, commandTimeout: 750, lazyConnect: true });
+    expect("retryStrategy" in producer && producer.retryStrategy?.(1)).toBeNull();
+  });
+
+  it("backs off reconnects from 250 ms up to 5 seconds", () => {
+    expect([1, 2, 3, 4, 5, 6, 30].map(reconnectDelay)).toEqual([250, 500, 1_000, 2_000, 4_000, 5_000, 5_000]);
+  });
+});
+
+describe("BullMqJobQueue while Redis is unavailable", () => {
+  it("fails enqueues and reads at once before the producer has connected", async () => {
+    const queue = new BullMqJobQueue(producerAt(closedPort()));
+    try {
+      const started = Date.now();
+      expect(await rejection(queue.enqueue("demo", { text: "hello" })))
+        .toMatchObject({ code: "JOBS_UNAVAILABLE", statusCode: 503 });
+      expect(await rejection(queue.get(externalJobId("demo", "1"))))
+        .toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      expect(Date.now() - started).toBeLessThan(100);
+    } finally {
+      await queue.close();
+    }
+  });
+
+  it("reports a refused connection to the supervisor and keeps failing enqueues at once", async () => {
+    const queue = new BullMqJobQueue(producerAt(closedPort()));
+    const lost: unknown[] = [];
+    queue.onLost((reason) => { lost.push(reason); });
+    try {
+      await expect(queue.connect()).rejects.toThrow();
+      await expect(queue.ready()).rejects.toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      const started = Date.now();
+      expect(await rejection(queue.enqueue("demo", {}))).toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      expect(Date.now() - started).toBeLessThan(100);
+      expect(lost.length).toBeGreaterThan(0);
+    } finally {
+      await queue.close();
+    }
+  });
+
+  it("drops a connection Redis stops answering and fails at once until the supervisor reopens it", async () => {
+    const redis = stallingRedis();
+    const queue = new BullMqJobQueue(producerAt(redis.port));
+    const lost: unknown[] = [];
+    queue.onLost((reason) => { lost.push(reason); });
+    try {
+      await queue.connect();
+      let started = Date.now();
+      expect(await rejection(queue.enqueue("demo", { text: "hello" })))
+        .toMatchObject({ code: "JOBS_UNAVAILABLE", statusCode: 503 });
+      expect(Date.now() - started).toBeLessThan(1_000);
+      expect(lost.length).toBe(1);
+
+      started = Date.now();
+      expect(await rejection(queue.enqueue("demo", {}))).toMatchObject({ code: "JOBS_UNAVAILABLE" });
+      expect(Date.now() - started).toBeLessThan(100);
+
+      await queue.connect();
+      await queue.ready();
+      await Bun.sleep(50);
+      expect(lost.length).toBe(1);
+    } finally {
+      await queue.close();
+      redis.stop();
+    }
+  });
+});