@cosmicdrift/kumiko-framework 0.327.0 → 0.328.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.327.0",
+  "version": "0.328.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -202,8 +202,8 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-http": "0.327.0",
-    "@cosmicdrift/kumiko-types": "0.327.0",
+    "@cosmicdrift/kumiko-http": "0.328.0",
+    "@cosmicdrift/kumiko-types": "0.328.0",
     "bullmq": "^5.76.7",
     "bun-types": "^1.3.13",
     "hono": "^4.13.1",
@@ -219,7 +219,7 @@
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.327.0",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.328.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"
   },

package/src/bun-db/__tests__/closed-connection-retry.integration.test.ts

@@ -6,7 +6,11 @@ import postgres from "postgres";
 import { constraintOf, extractPgError, isUniqueViolation } from "../../db/pg-error";
 import { testDatabaseUrl } from "../../testing/closed-connection-error";
 import { waitFor } from "../../testing/wait-for";
-import { isClosedConnectionError, unsafeReadRetrying } from "../query";
+import {
+  describeClosedConnectionFailure,
+  isClosedConnectionError,
+  unsafeReadRetrying,
+} from "../query";
 
 const DATABASE_URL = testDatabaseUrl();
 
@@ -157,3 +161,92 @@ describe.each(drivers)("extractPgError — %s unique-violation SQLSTATE", (kind)
     }
   });
 });
+
+describe("describeClosedConnectionFailure", () => {
+  test("describes a real postgres-js pool error without SQL text or params", async () => {
+    const applicationName = randomAppName();
+    const pool = postgres(DATABASE_URL, {
+      max: 2,
+      connection: { application_name: applicationName },
+    });
+    try {
+      await Promise.all([pool.unsafe("select 1"), pool.unsafe("select 1")]);
+      const secretParam = `secret-${crypto.randomUUID()}`;
+      const pending = pool
+        .unsafe("select $1::text as x from pg_sleep(0.3)", [secretParam])
+        .catch((e: unknown) => e);
+      const pid = await pollForActivePid(applicationName, "%pg_sleep(0.3)%");
+      await terminateBackendPid(pid);
+      const caught = await pending;
+
+      const described = describeClosedConnectionFailure(pool, caught, 3);
+      expect(described).toMatchObject({
+        driver: "postgres-js",
+        pooled: true,
+        attempts: 3,
+        poolMax: 2,
+      });
+      expect(described?.["errorCode"]).toBe(extractPgError(caught)?.code);
+      const serialized = JSON.stringify(described);
+      expect(serialized).not.toContain(secretParam);
+      expect(serialized).not.toContain("pg_sleep");
+    } finally {
+      await pool.end({ timeout: 0 });
+    }
+  });
+
+  test("describes a real postgres-js transaction handle as not pooled", async () => {
+    const applicationName = randomAppName();
+    const pool = postgres(DATABASE_URL, {
+      max: 2,
+      connection: { application_name: applicationName },
+    });
+    try {
+      let described: Record<string, unknown> | undefined;
+      await pool
+        .begin(async (tx) => {
+          const pending = tx.unsafe("select 1 from pg_sleep(0.3)").catch((e: unknown) => e);
+          const pid = await pollForActivePid(applicationName, "%pg_sleep(0.3)%");
+          await terminateBackendPid(pid);
+          described = describeClosedConnectionFailure(tx, await pending, 1);
+        })
+        .catch(() => undefined);
+      expect(described).toMatchObject({ driver: "postgres-js", pooled: false, attempts: 1 });
+    } finally {
+      await pool.end({ timeout: 0 });
+    }
+  });
+
+  test("describes a real Bun.SQL closed-pool error", async () => {
+    const pool = new Bun.SQL({ url: DATABASE_URL, max: 1, idleTimeout: 5, maxLifetime: 10 });
+    await pool.unsafe("select 1");
+    await pool.close();
+    const caught = await pool.unsafe("select 1").catch((e: unknown) => e);
+
+    expect(describeClosedConnectionFailure(pool, caught, 1)).toMatchObject({
+      driver: "bun-sql",
+      pooled: true,
+      poolMax: 1,
+      idleTimeoutMs: 5000,
+      maxLifetimeMs: 10000,
+      errorCode: "ERR_POSTGRES_CONNECTION_CLOSED",
+    });
+  });
+
+  test("returns undefined for an unrelated error", () => {
+    expect(describeClosedConnectionFailure({}, new Error("boom"), 1)).toBeUndefined();
+    expect(describeClosedConnectionFailure({}, "not an error", 1)).toBeUndefined();
+  });
+
+  test("matches the driver's AbortError for a closed connection", () => {
+    const abort = new DOMException("The connection was closed.", "AbortError");
+    expect(describeClosedConnectionFailure({}, abort, 2)).toMatchObject({
+      driver: "unknown",
+      attempts: 2,
+      errorName: "AbortError",
+    });
+    expect(
+      describeClosedConnectionFailure({}, new DOMException("other", "AbortError"), 1),
+    ).toBeUndefined();
+  });
+});

package/src/bun-db/query.ts

@@ -28,6 +28,7 @@ import type {
 // Static polyfill import (not getTemporal()): coercion results stay on one
 // Temporal implementation repo-wide, even when Bun exposes a global (#1480).
 import { Temporal } from "temporal-polyfill";
+import { requestContext } from "../api/request-context";
 import { computeBlindIndex, configuredBlindIndexKey } from "../crypto/blind-index";
 import { SQL_EXPR_BRAND } from "../db/dialect";
 import type { EntityTableMeta } from "../db/entity-table-meta";
@@ -35,6 +36,7 @@ import { extractPgError } from "../db/pg-error";
 import { type NotExecutorOnly, toSnakeCase } from "../db/table-builder";
 import { camelCase as envCamelCase } from "../env";
 import { InternalError } from "../errors";
+import { createFallbackLogger } from "../logging";
 import { parseJsonSafe } from "../utils/safe-json";
 
 // Idempotent snake_case → camelCase. `env.camelCase` always lowercases first
@@ -755,6 +757,120 @@ function isPooledClient(raw: unknown): boolean {
   );
 }
 
+const closedConnectionLog = createFallbackLogger("bun-db");
+
+// The driver rejects in-flight queries with this AbortError (no code) when the
+// pool closes under them, so isClosedConnectionError can't see it.
+const DRIVER_CLOSED_ABORT_MESSAGE = "The connection was closed.";
+
+function isDriverClosedAbort(err: unknown): boolean {
+  return (
+    err instanceof Error && err.name === "AbortError" && err.message === DRIVER_CLOSED_ABORT_MESSAGE
+  );
+}
+
+function isHandleLike(raw: unknown): raw is Record<string, unknown> {
+  return raw !== null && (typeof raw === "object" || typeof raw === "function");
+}
+
+type ClosedConnectionDriver = "bun-sql" | "postgres-js" | "unknown";
+
+// Bun.SQL is not instanceof-checkable (Bun.SQL.prototype is invalid) and its tx
+// handles are plain function objects, so discriminate by members only one driver
+// has: `flush` on Bun.SQL (pool and tx), `typed` on postgres-js (pool and tx).
+function driverOf(raw: unknown): ClosedConnectionDriver {
+  if (!isHandleLike(raw)) return "unknown";
+  if (typeof raw["flush"] === "function") return "bun-sql";
+  if (typeof raw["typed"] === "function") return "postgres-js";
+  return "unknown";
+}
+
+function positiveNumber(v: unknown): number | undefined {
+  return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : undefined;
+}
+
+// postgres-js configures timeouts in seconds, Bun.SQL in milliseconds.
+function timeoutOptionMs(
+  options: Record<string, unknown>,
+  driver: ClosedConnectionDriver,
+  bunName: string,
+  postgresJsName: string,
+): number | undefined {
+  if (driver === "postgres-js") {
+    const seconds = positiveNumber(options[postgresJsName]);
+    return seconds === undefined ? undefined : seconds * 1000;
+  }
+  return positiveNumber(options[bunName]);
+}
+
+function errorProp(err: unknown, name: string): unknown {
+  return isHandleLike(err) ? err[name] : undefined;
+}
+
+// Deliberately excludes SQL text and parameters: app SQL can carry PII.
+export function describeClosedConnectionFailure(
+  raw: unknown,
+  err: unknown,
+  attempts: number,
+): Record<string, unknown> | undefined {
+  if (!isClosedConnectionError(err) && !isDriverClosedAbort(err)) return undefined;
+  const driver = driverOf(raw);
+  const rawOptions = isHandleLike(raw) ? raw["options"] : undefined;
+  const options = isHandleLike(rawOptions) ? rawOptions : {};
+  return {
+    driver,
+    pooled: isPooledClient(raw),
+    attempts,
+    poolMax: positiveNumber(options["max"]),
+    idleTimeoutMs: timeoutOptionMs(options, driver, "idleTimeout", "idle_timeout"),
+    maxLifetimeMs: timeoutOptionMs(options, driver, "maxLifetime", "max_lifetime"),
+    connectionTimeoutMs: timeoutOptionMs(options, driver, "connectionTimeout", "connect_timeout"),
+    errorName: errorProp(err, "name"),
+    errorCode: extractPgError(err)?.code ?? errorProp(err, "code"),
+    errorErrno: errorProp(err, "errno"),
+    errorMessage: errorProp(err, "message"),
+  };
+}
+
+type ClosedConnectionContext = { readonly operation: string; readonly table?: string };
+
+function logClosedConnection(
+  raw: unknown,
+  err: unknown,
+  attempts: number,
+  context?: ClosedConnectionContext,
+): void {
+  const failure = describeClosedConnectionFailure(raw, err, attempts);
+  // skip: not a closed-connection error, nothing to diagnose
+  if (!failure) return;
+  // Request identity lets this line be joined to the route's `handler failed` log.
+  const request = requestContext.get();
+  closedConnectionLog.warn("closed connection surfaced", {
+    ...failure,
+    ...context,
+    requestId: request?.requestId,
+    correlationId: request?.correlationId,
+    handler: request?.handler,
+  });
+}
+
+// Write and other non-retried paths: log when a closed-connection error leaves
+// bun-db, rethrow unchanged.
+async function unsafeLoggingClosedConnection(
+  db: AnyDb,
+  sqlText: string,
+  params: readonly unknown[],
+  context: ClosedConnectionContext,
+): Promise<readonly unknown[]> {
+  const raw = asRawClient(db);
+  try {
+    return await raw.unsafe(sqlText, params);
+  } catch (err) {
+    logClosedConnection(raw, err, 1, context);
+    throw err;
+  }
+}
+
 function poolMaxOf(raw: unknown): number {
   // @cast-boundary driver pool options — both postgres-js and Bun.SQL expose options.max
   const max = (raw as { options?: { max?: unknown } }).options?.max;
@@ -774,7 +890,10 @@ export async function unsafeReadRetrying<TRow>(
   try {
     return (await raw.unsafe(sqlText, params)) as readonly TRow[];
   } catch (err) {
-    if (!isPooledClient(raw) || !isClosedConnectionError(err)) throw err;
+    if (!isPooledClient(raw) || !isClosedConnectionError(err)) {
+      logClosedConnection(raw, err, 1);
+      throw err;
+    }
     const maxAttempts = poolMaxOf(raw) + 1;
     let lastErr: unknown = err;
     for (let attempt = 0; attempt < maxAttempts; attempt++) {
@@ -782,9 +901,13 @@ export async function unsafeReadRetrying<TRow>(
         return (await raw.unsafe(sqlText, params)) as readonly TRow[];
       } catch (retryErr) {
         lastErr = retryErr;
-        if (!isClosedConnectionError(retryErr)) throw retryErr;
+        if (!isClosedConnectionError(retryErr)) {
+          logClosedConnection(raw, retryErr, attempt + 2);
+          throw retryErr;
+        }
       }
     }
+    logClosedConnection(raw, lastErr, maxAttempts + 1);
     throw lastErr;
   }
 }
@@ -876,7 +999,10 @@ export async function insertMany<TRow = any>(
     valuesClauses.push(`(${placeholders.join(", ")})`);
   }
   const sqlText = `INSERT INTO ${quoteIdent(info.name)} (${cols}) VALUES ${valuesClauses.join(", ")} RETURNING *`;
-  const raw = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<string, unknown>[];
+  const raw = (await unsafeLoggingClosedConnection(db, sqlText, params, {
+    operation: "insertMany",
+    table: info.name,
+  })) as readonly Record<string, unknown>[];
   return coerceRows(raw, info) as readonly TRow[];
 }
 
@@ -909,10 +1035,10 @@ export async function insertOne<TRow = any>(
     })
     .join(", ");
   const sqlText = `INSERT INTO ${quoteIdent(info.name)} (${cols}) VALUES (${placeholders}) RETURNING *`;
-  const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<
-    string,
-    unknown
-  >[];
+  const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
+    operation: "insertOne",
+    table: info.name,
+  })) as readonly Record<string, unknown>[];
   const first = rows[0];
   if (!first) return undefined;
   return coerceRow(first, info) as TRow;
@@ -956,7 +1082,10 @@ export async function updateMany<TRow = any>(
     for (const v of w.values) values.push(v);
   }
   sqlText += " RETURNING *";
-  const raw = (await asRawClient(db).unsafe(sqlText, values)) as readonly Record<string, unknown>[];
+  const raw = (await unsafeLoggingClosedConnection(db, sqlText, values, {
+    operation: "updateMany",
+    table: info.name,
+  })) as readonly Record<string, unknown>[];
   return coerceRows(raw, info) as readonly TRow[];
 }
 
@@ -973,7 +1102,10 @@ export async function deleteMany(
   const w = buildWhereClause(info, where, 1);
   let sqlText = `DELETE FROM ${quoteIdent(info.name)}`;
   if (w.sqlText) sqlText += ` WHERE ${w.sqlText}`;
-  await asRawClient(db).unsafe(sqlText, w.values);
+  await unsafeLoggingClosedConnection(db, sqlText, w.values, {
+    operation: "deleteMany",
+    table: info.name,
+  });
 }
 
 type InsertEntry = {
@@ -1106,10 +1238,10 @@ export async function upsertOnConflict<TRow = any>(
 
   const conflictList = conflictCols.map((c) => quoteIdent(c)).join(", ");
   const sqlText = `${sqlPrefix} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateParts.join(", ")} RETURNING *`;
-  const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<
-    string,
-    unknown
-  >[];
+  const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
+    operation: "upsertOnConflict",
+    table: info.name,
+  })) as readonly Record<string, unknown>[];
   const first = rows[0];
   if (!first) return undefined;
   return coerceRow(first, info) as TRow;
@@ -1188,10 +1320,10 @@ export async function incrementCounter<TRow = any>(
 
   const conflictList = conflictCols.map((c) => quoteIdent(c)).join(", ");
   const sqlText = `${sqlPrefix} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateParts.join(", ")} RETURNING *`;
-  const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<
-    string,
-    unknown
-  >[];
+  const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
+    operation: "incrementCounter",
+    table: info.name,
+  })) as readonly Record<string, unknown>[];
   const first = rows[0];
   if (!first) return undefined;
   return coerceRow(first, info) as TRow;
@@ -1227,7 +1359,10 @@ export async function deleteManyBatched(
     const sqlText = `DELETE FROM ${tableQ} WHERE ${idQ} IN (
       SELECT ${idQ} FROM ${tableQ} WHERE ${w.sqlText} LIMIT $${limitIdx}
     ) RETURNING ${idQ}`;
-    const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly unknown[];
+    const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
+      operation: "deleteManyBatched",
+      table: info.name,
+    })) as readonly unknown[];
     batches++;
     deleted += rows.length;
     if (rows.length < limit) break;

package/src/changes.json

@@ -1,4 +1,40 @@
 [
+  {
+    "version": "0.328.0",
+    "type": "improvement",
+    "title": "bun-db logs a diagnostic warning when a closed-connection error surfaces",
+    "detail": "When a closed-connection error leaves bun-db (retries exhausted, a transaction/reserved handle, or a write path), a `[bun-db] closed connection surfaced` warning records driver, pooled vs. transaction handle, attempts, pool size, timeouts and the error code. It never includes SQL text or parameters. Behaviour, retries and timeouts are unchanged."
+  },
+  {
+    "version": "0.328.0",
+    "type": "improvement",
+    "title": "runProdApp and runWorkerApp re-encrypt plaintext events after a field gains a PII annotation",
+    "detail": "When an event field is annotated as PII after events were already stored, the stored payloads stay plaintext because a projection rebuild copies them unchanged. After start, `runProdApp` and `runWorkerApp` now run `runPiiEventBackfill` in the background: it encrypts those events in place once per annotation state, queues the affected projections (including multi-stream projections with their own table) in `kumiko_pending_rebuilds` and rebuilds them, so ciphertext and blind-index columns get materialized. Later boots only run a cheap catch-up over the newest events. Replicas are serialized by an advisory lock, each batch commits on its own, and failed events are logged and retried on the next boot. Progress lives in the new table `kumiko_pii_backfill_state`, created on first run. Behavior change: the first boot after the bump rewrites `kumiko_events` payloads for annotated fields; set `KUMIKO_SKIP_PII_BACKFILL=1` to opt out. Calling `backfillEventPiiEncryption` by hand stays necessary for `eraseUnresolvableSubjects`, which the boot run never sets. `backfillEventPiiEncryptionBatch` is the new single-batch primitive both build on. `runPendingRebuilds` (the schema-apply queue drain) now also rebuilds queued multi-stream projection tables instead of failing on them; a failed multi-stream rebuild in either path stays queued and leaves the live consumer running."
+  },
+  {
+    "version": "0.328.0",
+    "type": "fix",
+    "title": "The projection-rebuild job also rebuilds multi-stream projections",
+    "detail": "`enqueueProjectionRebuild` and the `jobs:job:projection-rebuild` worker used to call the single-stream rebuild only, so a multi-stream projection name failed with \"not registered\". Both now go through `rebuildProjectionOrMultiStream` (exported from `@cosmicdrift/kumiko-framework/migrations`), the same helper the pending-rebuild queue uses. For multi-stream projections the skip mode comes from the projection's declared `errorMode`; the job payload's `skipApplyErrors` only applies to single-stream projections."
+  },
+  {
+    "version": "0.328.0",
+    "type": "improvement",
+    "title": "SearchAdapter gains an optional dropAllIndexes() for rebuilds on a persistent index",
+    "detail": "`dropAllIndexes()` deletes every tenant index the adapter owns and returns the count. The in-memory adapter clears all tenants; the Meilisearch adapter deletes all indexes under its prefix (and refuses an empty prefix) and resets its configured-tenant state so the next write recreates a filterable index."
+  },
+  {
+    "version": "0.328.0",
+    "type": "improvement",
+    "title": "setupTestStack can wire job run-logger callbacks, and setupTestStackFromFeatures does so by default",
+    "detail": "`TestStackOptions.jobs.runLogger` receives `{ registry, db }` and returns `onJobStart`/`onJobComplete`/`onJobFailed`. The stack awaits them before drainJobs() observes the outcome, so a log row is always written before drainJobs() resolves or rejects. `setupTestStackFromFeatures` (and with it `setupAppTestStack`) defaults to the same `jobRunLoggerCallbacks` prod uses whenever `jobs` is set and the jobs feature is mounted, so `tenantVisibleFailure` rows and `jobs:query:failures` work in tests without a hand-built harness. Such suites now write `job_runs` and tenant failure rows for every job run; a suite that counts those rows, or that wants the old behavior, passes its own `jobs.runLogger` (for example one returning `undefined`)."
+  },
+  {
+    "version": "0.328.0",
+    "type": "improvement",
+    "title": "build-app-schema carries optionTones and collects the new screen i18n keys",
+    "detail": "Select `optionTones` reach the client schema. `createLabel`, `searchPlaceholder`, `summary`, related list `description` and `itemNoun` are collected as i18n keys of the screen, so they are translated like other screen text."
+  },
   {
     "version": "0.327.0",
     "type": "improvement",