@podosoft/podokit 1.3.0 → 1.4.0

package/dist/templates/fullstack/apps/api/src/core/api-contract.spec.ts

@@ -16,6 +16,7 @@ describe("API contract inventory", () => {
     expect(expected).toContain("POST /api/auth/sign-in/email");
     expect(expected).toContain("PUT /cache/{key}");
     expect(expected).toContain("POST /files");
+    expect(expected).toContain("PUT /files/uploads/{id}/parts/{partNumber}");
   });
 
   it("uses one structural parameter name for every public blog post route", () => {

package/dist/templates/fullstack/apps/api/src/core/api-contract.ts

@@ -71,7 +71,15 @@ const MODULE_ROUTES: Record<string, string[]> = {
     "DELETE /admin/blog/{id}",
   ],
   bullmq: ["POST /jobs", "GET /jobs/{id}"],
-  "file-upload": ["POST /files", "GET /files/{key}/url"],
+  "file-upload": [
+    "POST /files",
+    "GET /files/{key}/url",
+    "POST /files/uploads",
+    "GET /files/uploads/{id}",
+    "PUT /files/uploads/{id}/parts/{partNumber}",
+    "POST /files/uploads/{id}/complete",
+    "DELETE /files/uploads/{id}",
+  ],
   "job-progress": ["POST /progress"],
   "object-storage-local": [
     "GET /files/content/*",

package/dist/templates/fullstack/apps/api/src/core/services.spec.ts

@@ -4,11 +4,15 @@ import { validateEnv } from "../config/env.validation";
 import { PROVIDERS } from "../config/providers";
 import {
   AccessPolicy,
+  ConnectionSupervisor,
   createCoreServices,
   OpenApiRegistry,
   REQUEST_IDENTITY,
   RequestGuardRegistry,
   ServiceRegistry,
+  type ConnectionEvents,
+  type RetryTimer,
+  type SupervisedConnection,
 } from "./services";
 
 function databaseProvider(): DatabaseProviderName {
@@ -50,6 +54,180 @@ describe("ServiceRegistry", () => {
   });
 });
 
+describe("ServiceRegistry supervision", () => {
+  test("starts and runs start hooks while a supervised dependency is down", async () => {
+    const registry = new ServiceRegistry();
+    let attempts = 0;
+    let hookRan = false;
+    registry.supervise("redis", {
+      connect: () => {
+        attempts += 1;
+        return Promise.reject(new Error("Connection closed"));
+      },
+    });
+    registry.onStart(() => { hookRan = true; });
+    registry.freeze();
+
+    await registry.start();
+    expect(hookRan).toBe(true);
+    expect(attempts).toBe(1);
+    await registry.close();
+    await Bun.sleep(400);
+    expect(attempts).toBe(1);
+  });
+
+  test("connects a reachable dependency before the start hooks run", async () => {
+    const registry = new ServiceRegistry();
+    const order: string[] = [];
+    registry.supervise("database", {
+      connect: async () => {
+        await Bun.sleep(5);
+        order.push("connected");
+      },
+    });
+    registry.onStart(() => { order.push("hook"); });
+    registry.freeze();
+
+    await registry.start();
+    expect(order).toEqual(["connected", "hook"]);
+    await registry.close();
+  });
+});
+
+interface FakeTimer extends RetryTimer {
+  delays: number[];
+  pending(): number;
+  fire(): Promise<void>;
+}
+
+function fakeTimer(): FakeTimer {
+  const tasks: Array<{ task: () => void; cancelled: boolean }> = [];
+  const delays: number[] = [];
+  return {
+    delays,
+    after: (delayMs, task) => {
+      const entry = { task, cancelled: false };
+      delays.push(delayMs);
+      tasks.push(entry);
+      return () => { entry.cancelled = true; };
+    },
+    pending: () => tasks.filter((entry) => !entry.cancelled).length,
+    fire: async () => {
+      const entry = tasks.shift();
+      if (entry && !entry.cancelled) entry.task();
+      await Bun.sleep(0);
+    },
+  };
+}
+
+function recordingEvents(): ConnectionEvents & { log: string[] } {
+  const log: string[] = [];
+  return {
+    log,
+    unavailable: (name, reason) => { log.push(`unavailable ${name}: ${reason}`); },
+    restored: (name) => { log.push(`restored ${name}`); },
+  };
+}
+
+class FlakyConnection implements SupervisedConnection {
+  attempts = 0;
+  failuresLeft = 0;
+  private listener?: (reason: unknown) => void;
+
+  connect(): Promise<void> {
+    this.attempts += 1;
+    if (this.failuresLeft > 0) {
+      this.failuresLeft -= 1;
+      return Promise.reject(new Error("Connection closed"));
+    }
+    return Promise.resolve();
+  }
+
+  onLost(listener: (reason: unknown) => void): void {
+    this.listener = listener;
+  }
+
+  drop(): void {
+    this.listener?.(new Error("Connection reset"));
+  }
+}
+
+describe("ConnectionSupervisor", () => {
+  const policy = { initialDelayMs: 100, maxDelayMs: 400 };
+
+  test("retries with doubling delays up to the cap and reports each state change once", async () => {
+    const connection = new FlakyConnection();
+    connection.failuresLeft = 5;
+    const timer = fakeTimer();
+    const events = recordingEvents();
+    const supervisor = new ConnectionSupervisor("redis", connection, events, { policy, timer });
+
+    await supervisor.start();
+    expect(supervisor.connected).toBe(false);
+    while (timer.pending() > 0) await timer.fire();
+
+    expect(connection.attempts).toBe(6);
+    expect(timer.delays).toEqual([100, 200, 400, 400, 400]);
+    expect(events.log).toEqual(["unavailable redis: Connection closed", "restored redis"]);
+    expect(supervisor.connected).toBe(true);
+  });
+
+  test("reopens a lost connection and starts the backoff over", async () => {
+    const connection = new FlakyConnection();
+    const timer = fakeTimer();
+    const events = recordingEvents();
+    const supervisor = new ConnectionSupervisor("redis", connection, events, { policy, timer });
+    await supervisor.start();
+    expect(supervisor.connected).toBe(true);
+
+    connection.failuresLeft = 1;
+    connection.drop();
+    connection.drop();
+    expect(supervisor.connected).toBe(false);
+    expect(timer.pending()).toBe(1);
+    while (timer.pending() > 0) await timer.fire();
+
+    expect(timer.delays).toEqual([100, 200]);
+    expect(connection.attempts).toBe(3);
+    expect(events.log).toEqual(["unavailable redis: Connection reset", "restored redis"]);
+    expect(supervisor.connected).toBe(true);
+  });
+
+  test("tries again when the connection drops while an attempt runs", async () => {
+    const connection = new FlakyConnection();
+    const timer = fakeTimer();
+    const supervisor = new ConnectionSupervisor("events", {
+      connect: async () => {
+        await connection.connect();
+        if (connection.attempts === 1) connection.drop();
+      },
+      onLost: (listener) => connection.onLost(listener),
+    }, recordingEvents(), { policy, timer });
+
+    await supervisor.start();
+    expect(supervisor.connected).toBe(false);
+    await timer.fire();
+    expect(connection.attempts).toBe(2);
+    expect(supervisor.connected).toBe(true);
+  });
+
+  test("stops retrying once stopped", async () => {
+    const connection = new FlakyConnection();
+    connection.failuresLeft = 10;
+    const timer = fakeTimer();
+    const supervisor = new ConnectionSupervisor("redis", connection, recordingEvents(), { policy, timer });
+    await supervisor.start();
+    expect(timer.pending()).toBe(1);
+
+    supervisor.stop();
+    expect(timer.pending()).toBe(0);
+    connection.drop();
+    await timer.fire();
+    expect(connection.attempts).toBe(1);
+    expect(timer.pending()).toBe(0);
+  });
+});
+
 describe("AccessPolicy", () => {
   test("defaults to session access", () => {
     const policy = new AccessPolicy();
@@ -66,6 +244,24 @@ describe("AccessPolicy", () => {
     expect(policy.resolve(new Request("http://localhost/api/auth/sign-in/email"))).toBe("public");
   });
 
+  test("applies GET rules to HEAD requests and leaves other methods alone", () => {
+    const policy = new AccessPolicy();
+    policy.register("GET", "/profile-images/:fileName", "public");
+    policy.register("POST", "/uploads", "public");
+    policy.register("GET", "/reports/:id", "api-key");
+
+    const head = (path: string): Request => new Request(`http://localhost${path}`, { method: "HEAD" });
+    expect(policy.resolve(head("/profile-images/a.png"))).toBe("public");
+    expect(policy.resolve(head("/reports/1"))).toBe("api-key");
+    expect(policy.resolve(head("/uploads"))).toBe("session");
+    expect(policy.resolve(head("/todos"))).toBe("session");
+    expect(policy.resolve(new Request("http://localhost/profile-images/a.png", { method: "DELETE" }))).toBe("session");
+
+    policy.register("HEAD", "/profile-images/:fileName", "session");
+    expect(policy.resolve(head("/profile-images/a.png"))).toBe("session");
+    expect(policy.resolve(new Request("http://localhost/profile-images/a.png"))).toBe("public");
+  });
+
   test("uses the most recently registered matching rule", () => {
     const policy = new AccessPolicy();
     policy.register("*", "/content/*", "public");

package/dist/templates/fullstack/apps/api/src/core/services.ts

@@ -10,9 +10,151 @@ import { DATABASE_READINESS_CHECK, ReadinessService } from "../health/readiness.
 
 export type { ServiceKey } from "@podosoft/podokit-runtime";
 
+/** A dependency that `ServiceRegistry.supervise` keeps connected. */
+export interface SupervisedConnection {
+  /**
+   * Opens whatever is closed and restores what depends on it, such as
+   * subscriptions. Rejects while the dependency is unreachable.
+   */
+  connect(): Promise<void>;
+  /** Takes the callback to run whenever an open connection drops. */
+  onLost?(listener: (reason: unknown) => void): void;
+}
+
+export interface ReconnectPolicy {
+  /** Delay before the first retry. Each further failure doubles it. */
+  initialDelayMs: number;
+  /** Longest delay between two attempts. */
+  maxDelayMs: number;
+}
+
+export const DEFAULT_RECONNECT_POLICY: ReconnectPolicy = { initialDelayMs: 250, maxDelayMs: 5_000 };
+
+export interface ConnectionEvents {
+  unavailable(name: string, reason: string): void;
+  restored(name: string, downtimeMs: number): void;
+}
+
+export interface RetryTimer {
+  /** Runs `task` once after `delayMs`; the returned function cancels it. */
+  after(delayMs: number, task: () => void): () => void;
+}
+
+const unrefTimer: RetryTimer = {
+  after: (delayMs, task) => {
+    const timer = setTimeout(task, delayMs);
+    timer.unref?.();
+    return () => clearTimeout(timer);
+  },
+};
+
+export interface SupervisorOptions {
+  policy?: ReconnectPolicy;
+  timer?: RetryTimer;
+}
+
+function reasonOf(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/**
+ * Connects in the background and keeps the connection open: a failed attempt is
+ * retried with capped exponential backoff, and a lost connection is reopened.
+ * Events fire when the state changes, not on every attempt.
+ */
+export class ConnectionSupervisor {
+  private up = false;
+  private stopped = false;
+  private failures = 0;
+  private drops = 0;
+  private lastDrop: unknown;
+  private downSince?: number;
+  private cancelRetry?: () => void;
+  private firstAttempt?: Promise<void>;
+  private readonly policy: ReconnectPolicy;
+  private readonly timer: RetryTimer;
+
+  constructor(
+    readonly name: string,
+    private readonly connection: SupervisedConnection,
+    private readonly events: ConnectionEvents,
+    options: SupervisorOptions = {},
+  ) {
+    this.policy = options.policy ?? DEFAULT_RECONNECT_POLICY;
+    this.timer = options.timer ?? unrefTimer;
+    connection.onLost?.((reason) => this.lost(reason));
+  }
+
+  get connected(): boolean {
+    return this.up;
+  }
+
+  /** Resolves after the first attempt, whether or not it connected. */
+  start(): Promise<void> {
+    this.firstAttempt ??= this.attempt();
+    return this.firstAttempt;
+  }
+
+  stop(): void {
+    this.stopped = true;
+    this.up = false;
+    this.cancelRetry?.();
+    this.cancelRetry = undefined;
+  }
+
+  private lost(reason: unknown): void {
+    if (this.stopped) return;
+    this.drops += 1;
+    this.lastDrop = reason;
+    // While an attempt is pending it sees the changed count and tries again.
+    if (!this.up) return;
+    this.up = false;
+    this.retry(reason);
+  }
+
+  private async attempt(): Promise<void> {
+    this.cancelRetry = undefined;
+    if (this.stopped) return;
+    const drops = this.drops;
+    try {
+      await this.connection.connect();
+    } catch (error) {
+      if (!this.stopped) this.retry(error);
+      return;
+    }
+    if (this.stopped) return;
+    // Bun reports a failed reconnect through the same close callback, so a drop
+    // during the attempt can mean a part of the connection is closed again.
+    if (drops !== this.drops) {
+      this.retry(this.lastDrop);
+      return;
+    }
+    this.up = true;
+    this.failures = 0;
+    if (this.downSince !== undefined) {
+      this.events.restored(this.name, Date.now() - this.downSince);
+      this.downSince = undefined;
+    }
+  }
+
+  private retry(reason: unknown): void {
+    if (this.downSince === undefined) {
+      this.downSince = Date.now();
+      this.events.unavailable(this.name, reasonOf(reason));
+    }
+    const delay = Math.min(
+      this.policy.maxDelayMs,
+      this.policy.initialDelayMs * 2 ** Math.min(this.failures, 30),
+    );
+    this.failures += 1;
+    this.cancelRetry = this.timer.after(delay, () => void this.attempt());
+  }
+}
+
 export class ServiceRegistry {
   private readonly values = new Map<symbol, unknown>();
   private readonly starters: Array<() => void | Promise<void>> = [];
+  private readonly supervisors: ConnectionSupervisor[] = [];
   private readonly closers: Array<() => void | Promise<void>> = [];
   private startPromise?: Promise<void>;
   private closePromise?: Promise<void>;
@@ -41,17 +183,44 @@ export class ServiceRegistry {
     return this.values.get(key) as T | undefined;
   }
 
+  /** Runs `start` before the server listens; a rejection fails startup. */
   onStart(start: () => void | Promise<void>): void {
     if (this.frozen) throw new Error("Service registry is frozen");
     this.starters.push(start);
   }
 
+  /**
+   * Connects `connection` when the registry starts and keeps it connected.
+   * Startup waits only for the first attempt, so the server listens while the
+   * dependency is down; register a readiness check that reports it.
+   */
+  supervise(name: string, connection: SupervisedConnection): void {
+    if (this.frozen) throw new Error("Service registry is frozen");
+    this.supervisors.push(new ConnectionSupervisor(name, connection, {
+      // Resolved on use: a module such as logging may replace the logger.
+      unavailable: (dependency, reason) => {
+        this.tryResolve(LOGGER)?.warn(
+          { dependency, reason },
+          "Dependency connection failed; retrying in the background",
+        );
+      },
+      restored: (dependency, downtimeMs) => {
+        this.tryResolve(LOGGER)?.info({ dependency, downtimeMs }, "Dependency connection restored");
+      },
+    }));
+  }
+
   freeze(): void {
     this.frozen = true;
   }
 
+  /**
+   * Makes a first connection attempt for every supervised dependency at once,
+   * so those that are up are connected before the start hooks run in order.
+   */
   start(): Promise<void> {
     this.startPromise ??= (async () => {
+      await Promise.all(this.supervisors.map((supervisor) => supervisor.start()));
       for (const start of this.starters) await start();
     })();
     return this.startPromise;
@@ -59,6 +228,7 @@ export class ServiceRegistry {
 
   close(): Promise<void> {
     this.closePromise ??= (async () => {
+      for (const supervisor of this.supervisors) supervisor.stop();
       for (const close of [...this.closers].reverse()) await close();
     })();
     return this.closePromise;
@@ -90,13 +260,19 @@ export class AccessPolicy {
     this.rules.push({ method: method.toUpperCase(), matcher: routeMatcher(pattern), level });
   }
 
+  /**
+   * The level of the most recently registered matching rule. A HEAD request
+   * also matches GET rules, because Elysia answers it with the GET handler.
+   */
   resolve(request: Request): AccessLevel {
     const path = new URL(request.url).pathname;
     const method = request.method.toUpperCase();
     const rule = [...this.rules]
       .reverse()
       .find((candidate) =>
-        (candidate.method === "*" || candidate.method === method) && candidate.matcher.test(path)
+        (candidate.method === "*" || candidate.method === method ||
+          (method === "HEAD" && candidate.method === "GET")) &&
+        candidate.matcher.test(path)
       );
     return rule?.level ?? "session";
   }
@@ -199,7 +375,10 @@ export function createCoreServices(env: AppEnv): ServiceRegistry {
   accessPolicy.register("*", "/api-docs-json", "public");
   accessPolicy.register("*", "/api-docs-elysia-json", "public");
   services.register(DATABASE, database, () => database.close());
-  services.onStart(() => database.connect());
+  services.supervise(DATABASE_READINESS_CHECK, {
+    connect: () => services.resolve(DATABASE).connect(),
+    onLost: (listener) => database.onLost(listener),
+  });
   services.register(READINESS, readiness);
   readiness.register(DATABASE_READINESS_CHECK, () => services.resolve(DATABASE).ping());
   services.register(LOGGER, logger);

package/dist/templates/fullstack/apps/api/src/database/database.spec.ts

@@ -153,6 +153,12 @@ describe("PostgreSQL pool settings", () => {
     expect(() => validateEnv({ DB_POOL_MAX: "0" })).toThrow("DB_POOL_MAX");
     expect(() => validateEnv({ DB_AUTH_POOL_MAX: "-1" })).toThrow("DB_AUTH_POOL_MAX");
     expect(() => validateEnv({ DB_CONNECT_TIMEOUT_MS: "0" })).toThrow("DB_CONNECT_TIMEOUT_MS");
+    expect(() => validateEnv({ DB_QUERY_TIMEOUT_MS: "0" })).toThrow("DB_QUERY_TIMEOUT_MS");
+  });
+
+  it("bounds a query on a server that stopped answering after ten seconds by default", () => {
+    expect(env.DB_QUERY_TIMEOUT_MS).toBe(10_000);
+    expect(validateEnv({ DB_QUERY_TIMEOUT_MS: "2500" }).DB_QUERY_TIMEOUT_MS).toBe(2_500);
   });
 });
 
@@ -168,11 +174,12 @@ describe("isDatabaseUnavailable", () => {
     }
   });
 
-  it("recognizes pg pool and connection timeouts, also when wrapped", () => {
+  it("recognizes pg pool, connection, and query timeouts, also when wrapped", () => {
     const acquire = new Error("timeout exceeded when trying to connect");
     const connect = new Error("Connection terminated due to connection timeout");
     expect(isDatabaseUnavailable(acquire)).toBe(true);
     expect(isDatabaseUnavailable(connect)).toBe(true);
+    expect(isDatabaseUnavailable(new Error("Query read timeout"))).toBe(true);
     expect(isDatabaseUnavailable(new Error("Failed to get session", { cause: acquire }))).toBe(true);
   });
 

package/dist/templates/fullstack/apps/api/src/database/database.ts

@@ -4,6 +4,7 @@ import { mkdirSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import type { AppEnv } from "../config/env.validation";
 import { PROVIDERS } from "../config/providers";
+import { answersOnNewConnection, PoolGuard } from "./pool-guard";
 
 function isSqliteUrl(url: string): boolean {
   return url === ":memory:" || /^(?:sqlite|file):/.test(url);
@@ -62,8 +63,9 @@ export function postgresPoolOptions(env: AppEnv): PostgresPoolOptions {
   };
 }
 
-// Bun SQL and pg report a database that cannot be reached, or a connection that
-// could not be opened or taken from a full pool in time, with these shapes.
+// Bun SQL and pg report a database that cannot be reached, a connection that
+// could not be opened or taken from a full pool in time, or a query the server
+// did not answer in time, with these shapes.
 const UNAVAILABLE_SQL_CODES = new Set([
   "ERR_POSTGRES_CONNECTION_TIMEOUT",
   "ERR_POSTGRES_CONNECTION_REFUSED",
@@ -73,6 +75,7 @@ const UNAVAILABLE_SQL_CODES = new Set([
 const UNAVAILABLE_PG_MESSAGES = new Set([
   "timeout exceeded when trying to connect",
   "Connection terminated due to connection timeout",
+  "Query read timeout",
 ]);
 
 /** Whether an error, or one it wraps, means the database is unavailable rather than the request being wrong. */
@@ -86,20 +89,36 @@ export function isDatabaseUnavailable(error: unknown): boolean {
   return false;
 }
 
-function createSql(provider: DatabaseProviderName, env: AppEnv): SQL {
-  const url = databaseUrl(env);
-  return provider === "sqlite"
-    ? new SQL(url, { adapter: "sqlite", create: true, readwrite: true, strict: true })
-    : new SQL(url, postgresPoolOptions(env));
-}
-
 export class Database {
   readonly provider: DatabaseProviderName = selectedDatabaseProvider();
   readonly sql: SQL;
+  private readonly guard?: PoolGuard;
   private ready?: Promise<void>;
 
   constructor(env: AppEnv, sql?: SQL) {
-    this.sql = sql ?? createSql(this.provider, env);
+    if (sql) {
+      this.sql = sql;
+      return;
+    }
+    const url = databaseUrl(env);
+    if (this.provider === "sqlite") {
+      this.sql = new SQL(url, { adapter: "sqlite", create: true, readwrite: true, strict: true });
+      return;
+    }
+    this.guard = new PoolGuard(() => new SQL(url, postgresPoolOptions(env)), {
+      stallTimeoutMs: env.DB_QUERY_TIMEOUT_MS,
+      serverAnswers: () => answersOnNewConnection(url, env.READINESS_CHECK_TIMEOUT_MS),
+    });
+    // The supervisor's next connect() then checks the new pool.
+    this.guard.onLost(() => {
+      this.ready = undefined;
+    });
+    this.sql = this.guard.sql;
+  }
+
+  /** Takes the callback to run when the pool is replaced because PostgreSQL stopped answering. */
+  onLost(listener: (reason: unknown) => void): void {
+    this.guard?.onLost(listener);
   }
 
   connect(): Promise<void> {