@cosmicdrift/kumiko-framework 0.308.0 → 0.310.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.308.0",
+  "version": "0.310.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -198,8 +198,8 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-http": "0.308.0",
-    "@cosmicdrift/kumiko-types": "0.308.0",
+    "@cosmicdrift/kumiko-http": "0.310.0",
+    "@cosmicdrift/kumiko-types": "0.310.0",
     "bullmq": "^5.76.7",
     "bun-types": "^1.3.13",
     "hono": "^4.13.1",
@@ -208,14 +208,14 @@
     "jose": "^6.2.3",
     "meilisearch": "^0.58.0",
     "pino": "^10.3.1",
-    "postgres": "^3.4.9",
+    "postgres": "npm:@bender0oo0/postgres@3.4.9-kumiko.1",
     "temporal-polyfill": "^0.3.2",
     "ts-morph": "^28.0.0",
     "uuid": "^14.0.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.308.0",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.310.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"
   },

package/src/bun-db/__tests__/postgres-closed-socket-write.integration.test.ts

@@ -0,0 +1,87 @@
+// Guards the nextWrite null-socket backport (porsager/postgres#1209) on the
+// @bender0oo0/postgres fork pinned in package.json: reserve a connection, kill
+// the backend socket mid-flight, then send another query on the reserved
+// connection. Upstream postgres 3.4.9 crashes the process with an
+// uncaughtException ("null is not an object (evaluating 'socket.write')")
+// instead of rejecting; the fork rejects with CONNECTION_CLOSED. Must stay
+// green after switching back to an upstream release (kumiko-framework#3248).
+
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import net from "node:net";
+import postgres from "postgres";
+import { testDatabaseUrl } from "../../testing/closed-connection-error";
+
+const DATABASE_URL = testDatabaseUrl();
+const dbUrl = new URL(DATABASE_URL);
+
+let uncaught: unknown;
+function onUncaught(err: unknown): void {
+  uncaught = err;
+}
+
+beforeEach(() => {
+  uncaught = undefined;
+  process.on("uncaughtException", onUncaught);
+});
+
+afterEach(() => {
+  process.off("uncaughtException", onUncaught);
+});
+
+test("reserved connection rejects with CONNECTION_CLOSED instead of crashing on a dead socket", async () => {
+  let downstream: net.Socket | undefined;
+  const proxy = net.createServer((socket) => {
+    downstream = socket;
+    const upstream = net.connect(Number(dbUrl.port), dbUrl.hostname);
+    socket.pipe(upstream).pipe(socket);
+    socket.on("error", () => upstream.destroy());
+    upstream.on("error", () => socket.destroy());
+  });
+  await new Promise<void>((resolve) => proxy.listen(0, resolve));
+  const address = proxy.address();
+  if (address === null || typeof address === "string") {
+    throw new Error("proxy did not bind to a TCP port");
+  }
+
+  const sql = postgres({
+    host: "127.0.0.1",
+    port: address.port,
+    user: dbUrl.username,
+    password: dbUrl.password,
+    database: dbUrl.pathname.slice(1),
+    max: 1,
+  });
+
+  try {
+    const reserved = await sql.reserve();
+    try {
+      await reserved`select 1`;
+      downstream?.end();
+      await new Promise((resolve) => setTimeout(resolve, 50));
+
+      const HANG = Symbol("hang");
+      const result = await Promise.race([
+        reserved`select 1`.then(
+          () => {
+            throw new Error("expected the query to reject with CONNECTION_CLOSED, but it resolved");
+          },
+          (err: unknown) => err,
+        ),
+        new Promise((resolve) => setTimeout(() => resolve(HANG), 3000)),
+      ]);
+
+      if (result === HANG) {
+        throw new Error("query on the dead-socket connection hung instead of rejecting");
+      }
+      expect(result).toMatchObject({ code: "CONNECTION_CLOSED" });
+      // Give the process a tick to surface an uncaughtException if the guard regressed.
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      expect(uncaught).toBeUndefined();
+    } finally {
+      reserved.release();
+    }
+  } finally {
+    proxy.close();
+    await sql.end({ timeout: 1 });
+  }
+});

package/src/bun-db/connection.ts

@@ -54,6 +54,8 @@ export function createBunDbConnection(
   if (options.connectTimeoutSeconds !== undefined) {
     pgOpts.connect_timeout = options.connectTimeoutSeconds;
   }
+  // ponytail: "postgres" resolves to the @bender0oo0/postgres fork (nextWrite
+  // null-socket guard, porsager/postgres#1209); switch back per kumiko-framework#3248.
   const listenClient = postgres(url, pgOpts);
 
   return {

package/src/changes.json

@@ -1,4 +1,34 @@
 [
+  {
+    "version": "0.310.0",
+    "type": "improvement",
+    "title": "Boot validator checks the target tab of navigate actions",
+    "detail": "A navigate action with `tab` fails boot when its target is not a\nprojectionDetail with layout.mode \"tabs\" or has no section with that id.\nThe check covers list rowActions, relatedList rowActions, screen actions,\nsection actions and relatedList emptyState actions."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "LISTEN gauge drops to 0 on a DB outage",
+    "detail": "`kumiko_event_dispatcher_listen_connected` stayed at 1 when the LISTEN\nconnection died during a DB outage, although delivery had already fallen\nback to polling. The dispatcher now sets it to 0 when the idle pre-check\ndetects the outage. postgres.js re-LISTENs on its own once the DB is back\n(delayed by its connect backoff, measured 10-30s) and sets it back to 1."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "event-dispatcher logs when the DB comes back after an outage (fw#3243)",
+    "detail": "A DB outage already kept the dispatcher process alive (the postgres pool\nreconnects per query and the dispatcher keeps polling from its cursor) and\nlogged `idle pre-check failed` once per outage, but recovery was silent:\nops had no signal that delivery was back to normal without restarting the\nprocess. The idle pre-check now logs `idle pre-check recovered, database\nreachable again` once the next pass succeeds after a logged failure, no\nrestart needed."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "Pin postgres to a fork carrying the nextWrite null-socket fix (fw#3243)",
+    "detail": "The framework now depends on @bender0oo0/postgres@3.4.9-kumiko.1, an exact pin of\npostgres 3.4.9 plus a backport of porsager/postgres#1209. Without the backport, a\nclosed connection can crash the whole process with an uncaughtException\n(\"null is not an object (evaluating 'socket.write')\") instead of rejecting the\nin-flight query with CONNECTION_CLOSED. Apps get the fix automatically with this\nframework bump, no app-side changes needed. The pin is temporary and tracked for\nremoval in kumiko-framework#3248 once the fix lands upstream."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "setupTestStack creates r.entity() tables — same table list as kumiko schema generate (fw#3102)",
+    "detail": "Since 0.290.0 (fw#2881), every authenticated request reads `read_tenants` (rejectIfTenantTeardown) as soon as tenant-lifecycle is mounted. Test stacks needed a manually created tenant table for that to work, because setupTestStack pushed projection/multiStreamProjection/storeTable sources but never r.entity() tables. setupTestStack now creates every entity's backing table itself via collectTableMetas, the same table list `kumiko schema generate` uses, so a feature that only declares r.entity() no longer 42P01s on first write in a test stack. App-side wrappers like `unsafeEnsureEntityTable(db, tenantEntity, \"tenant\")` right after setupTestStack can be removed. Callers that create/migrate entity tables themselves can opt out via `entityTables: false` (used internally by `setupAppTestStack`'s `registryTables: false`), and setupTestStack now drops the ephemeral test DB/Redis if the table DDL fails instead of leaking them."
+  },
   {
     "version": "0.308.0",
     "type": "fix",

package/src/db/__tests__/feature-table-sources.test.ts

@@ -1,6 +1,6 @@
 // Regression test for #255-class drift: storeTables must appear in
-// enumerateFeatureTableSources so setupTestStack's auto-push and
-// collectTableMetas see the exact same table set. Before this fix,
+// enumerateFeatureTableSources so collectTableMetas (shared by setupTestStack's
+// auto-push and `kumiko schema generate`) sees the exact same table set. Before this fix,
 // unmanaged tables were only handled by collectTableMetas directly, so any
 // app relying on setupTestStack's ephemeral DB (Playwright/e2e) never got
 // its store tables created — e.g. the bundled "sessions" feature's

package/src/db/collect-table-metas.ts

@@ -1,9 +1,6 @@
-// collectTableMetas — kanonische ENTITY_METAS-Quelle für `kumiko schema
-// generate`. Erfasst dieselben Tabellen-Quellen wie der setupTestStack-
-// auto-push (entities, storeTables, projections, multiStreamProjections) —
-// die frühere Template-Variante sammelte nur entities + storeTables, wodurch
-// projection-only-Tabellen (z.B. billing-foundation read_subscriptions) nie
-// in Migrations landeten und der erste Prod-Write crashte (#255).
+// Canonical table source for both `kumiko schema generate` and setupTestStack
+// (entities, storeTables, projections, multiStreamProjections). One shared
+// source so the test-DB push and migrations cannot drift apart again (#255, #3102).
 
 import type { FeatureDefinition } from "../engine/types";
 import { compareByCodepoint } from "../utils";

package/src/db/feature-table-sources.ts

@@ -1,9 +1,8 @@
 // Single enumeration of every table-bearing registration on a feature
 // (r.projection, r.multiStreamProjection with table, r.storeTable).
-// Consumed by BOTH the setupTestStack auto-push and collectTableMetas —
-// one list, so test-DB-push and `kumiko schema generate` cannot drift
-// apart again (#255).
-// A new table-bearing registrar must be added HERE, not in the consumers.
+// Consumed by collectTableMetas's Pass 2 (`kumiko schema generate` and
+// setupTestStack both go through collectTableMetas — see there).
+// A new table-bearing registrar must be added HERE, not in the consumer.
 
 import type { FeatureDefinition } from "../engine/types";
 

package/src/derivatives/delete-stored-file.ts

@@ -0,0 +1,86 @@
+// Shared binary-deletion helper: deletes an original storage key plus every
+// derivative rendered from it (thumbnails/resized variants — see
+// derivatives-context.ts). Derivatives are never tracked anywhere but the
+// storage layer: they're rendered under a deterministic key (deriveKey +
+// variantSuffix) computed from the original's key, on demand, and nothing
+// writes that key back onto a FileRef row. Listing the original's key-prefix
+// and keeping only candidates whose suffix matches the derivative grammar,
+// anchored to the original's own basename AND extension, means a
+// same-directory sibling (a different file, possibly another tenant's) shares
+// the list prefix but fails the extension/suffix check and is never touched.
+//
+// Used by both the user-data-rights-defaults forget hook and the
+// data-retention hardDelete cleanup — moved here so neither copies the logic.
+
+import { assertSafeStorageKey } from "../files/types";
+import { derivativeListPrefix, isDerivativeKeyOf } from "./variant-key";
+
+export type StoredFileStore = {
+  list(prefix: string): Promise<readonly string[]>;
+  delete(key: string): Promise<void>;
+};
+
+// Wrap rather than let a raw provider error (e.g. S3 AccessDenied) surface
+// unexplained — list() is a requirement on top of delete(); a bucket policy
+// granting only s3:DeleteObject would otherwise fail every run instead of
+// just leaving derivatives behind. Name the likely cause so an operator
+// doesn't have to guess. `errorContext` names the caller for the message
+// prefix (e.g. "user-data-rights-defaults:fileRef", "data-retention:hardDelete").
+export async function listDerivativeKeys(
+  originalKey: string,
+  store: Pick<StoredFileStore, "list">,
+  errorContext: string,
+): Promise<readonly string[]> {
+  let candidates: readonly string[];
+  try {
+    candidates = await store.list(derivativeListPrefix(originalKey));
+  } catch (err) {
+    throw new Error(
+      `[${errorContext}] provider.list() failed while looking up derivatives of key=${originalKey} — forget/erasure requires list permission on the storage bucket (e.g. s3:ListBucket), not just delete. Original error: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  const derivativeKeys: string[] = [];
+  for (const candidate of candidates) {
+    if (!isDerivativeKeyOf(originalKey, candidate)) continue;
+    try {
+      assertSafeStorageKey(candidate);
+    } catch {
+      // ponytail: a listing hit that fails the storage-key grammar can't be
+      // a real derivative (every derivative is written via a validated
+      // write()) — skip rather than pass a malformed key to delete().
+      continue;
+    }
+    derivativeKeys.push(candidate);
+  }
+  return derivativeKeys;
+}
+
+// Deletes the original key + every derivative. Returns the keys whose delete
+// threw — callers fail closed on a non-empty list (delete is idempotent, so a
+// retry on the next run converges). assertSafeStorageKey guards the original
+// AND every derivative right before the delete call, not just at listing time.
+export async function deleteStoredFileAndDerivatives(
+  originalKey: string,
+  store: StoredFileStore,
+  errorContext: string,
+): Promise<readonly string[]> {
+  assertSafeStorageKey(originalKey);
+  const keysToDelete = [
+    originalKey,
+    ...(await listDerivativeKeys(originalKey, store, errorContext)),
+  ];
+  const failedKeys: string[] = [];
+  for (const key of keysToDelete) {
+    try {
+      assertSafeStorageKey(key);
+      await store.delete(key);
+    } catch (err) {
+      failedKeys.push(key);
+      // biome-ignore lint/suspicious/noConsole: operator-visibility for binary-cleanup-failure
+      console.warn(
+        `[${errorContext}] storage delete failed key=${key} err=${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+  return failedKeys;
+}

package/src/derivatives/index.ts

@@ -1,3 +1,5 @@
+export type { StoredFileStore } from "./delete-stored-file";
+export { deleteStoredFileAndDerivatives, listDerivativeKeys } from "./delete-stored-file";
 export type { DerivativesContextDeps } from "./derivatives-context";
 export { createDerivativesContext, resolveRenderer } from "./derivatives-context";
 export { resolveFieldVariant } from "./field-variants";

package/src/engine/__tests__/row-action-navigate-tab.test.ts

@@ -0,0 +1,107 @@
+import { describe, expect, test } from "bun:test";
+import * as z from "zod";
+import { withBootValidatorFixture } from "../../testing/boot-validator-fixture";
+import { validateBoot as validateBootRaw } from "../boot-validator";
+import { defineFeature } from "../define-feature";
+import { createEntity, createTextField } from "../factories";
+
+function validateBoot(features: Parameters<typeof validateBootRaw>[0]): void {
+  validateBootRaw(withBootValidatorFixture(features));
+}
+
+function buildFeature(options: {
+  readonly detailTabs: boolean;
+  readonly tab: string;
+  readonly via: "entityEditSection" | "entityListRowAction";
+}) {
+  return defineFeature("app", (r) => {
+    r.entity(
+      "vehicle",
+      createEntity({
+        fields: { name: createTextField({ personal: false, reason: "test_fixture" }) },
+      }),
+    );
+    r.queryHandler("vehicle:detail", z.object({}), async () => ({ name: "x" }), {
+      access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
+    });
+    r.screen({
+      id: "vehicle-detail",
+      type: "projectionDetail",
+      query: "app:query:vehicle:detail",
+      layout: options.detailTabs
+        ? {
+            mode: "tabs",
+            sections: [
+              { id: "overview", title: "Overview", fields: ["name"] },
+              { id: "channels", title: "Channels", fields: ["name"] },
+            ],
+          }
+        : { sections: [{ title: "Overview", fields: ["name"] }] },
+    });
+    const navigateAction = {
+      kind: "navigate" as const,
+      id: "view-channels",
+      label: "app.actions.viewChannels",
+      screen: "vehicle-detail",
+      tab: options.tab,
+    };
+    if (options.via === "entityEditSection") {
+      r.screen({
+        id: "vehicle-edit",
+        type: "entityEdit",
+        entity: "vehicle",
+        layout: {
+          sections: [
+            {
+              id: "main",
+              title: "Main",
+              columns: 1,
+              fields: ["name"],
+              actions: [{ ...navigateAction, entityId: "vehicleId" }],
+            },
+          ],
+        },
+      });
+    } else {
+      r.screen({
+        id: "vehicle-list",
+        type: "entityList",
+        entity: "vehicle",
+        columns: ["name"],
+        rowActions: [navigateAction],
+      });
+    }
+  });
+}
+
+describe("validateBoot — RowActionNavigate.tab", () => {
+  test("entityEdit section action navigates to a known tab boots cleanly", () => {
+    const feature = buildFeature({ detailTabs: true, tab: "channels", via: "entityEditSection" });
+    expect(() => validateBoot([feature])).not.toThrow();
+  });
+
+  test("entityEdit section action navigates to an unknown tab throws", () => {
+    const feature = buildFeature({ detailTabs: true, tab: "unknown", via: "entityEditSection" });
+    expect(() => validateBoot([feature])).toThrow(
+      /navigates to tab "unknown", which is not a section id on target screen/,
+    );
+  });
+
+  test("entityEdit section action navigating to a non-tabs projectionDetail throws", () => {
+    const feature = buildFeature({ detailTabs: false, tab: "overview", via: "entityEditSection" });
+    expect(() => validateBoot([feature])).toThrow(
+      /is not a projectionDetail with layout.mode "tabs"/,
+    );
+  });
+
+  test("entityList rowAction navigating to an unknown tab throws (central resolution path)", () => {
+    const feature = buildFeature({
+      detailTabs: true,
+      tab: "unknown",
+      via: "entityListRowAction",
+    });
+    expect(() => validateBoot([feature])).toThrow(
+      /navigates to tab "unknown", which is not a section id on target screen/,
+    );
+  });
+});

package/src/engine/boot-validator/screens.ts

@@ -542,11 +542,48 @@ type RowActionNavigateRuntime = RowActionNavigateBase & {
   readonly entity?: string;
 };
 
+// The target can be any screen, so this checks the resolved target's
+// sections, not the host's.
+function validateRowActionNavigateTab(
+  featureName: string,
+  screenId: string,
+  screenType: "entityList" | "projectionList" | "projectionDetail" | "entityEdit",
+  actionLabel: string,
+  action: RowActionNavigateRuntime,
+  target: { readonly featureName: string; readonly screen: ScreenDefinition } | undefined,
+): void {
+  // skip: no tab declared — nothing to check against the target's sections
+  if (action.tab === undefined) return;
+  if (target === undefined) {
+    throw new Error(
+      `[Feature ${featureName}] Screen "${screenId}" (${screenType}) ${actionLabel} "${action.id}" ` +
+        `sets tab "${action.tab}", but its navigate-target could not be resolved to a screen.`,
+    );
+  }
+  if (target.screen.type !== "projectionDetail" || target.screen.layout.mode !== "tabs") {
+    throw new Error(
+      `[Feature ${featureName}] Screen "${screenId}" (${screenType}) ${actionLabel} "${action.id}" ` +
+        `sets tab "${action.tab}", but navigate-target screen "${target.screen.id}" is not a ` +
+        `projectionDetail with layout.mode "tabs" — only those read the tab param.`,
+    );
+  }
+  const sectionIds = target.screen.layout.sections
+    .map((section) => section.id)
+    .filter((id): id is string => id !== undefined);
+  if (!sectionIds.includes(action.tab)) {
+    throw new Error(
+      `[Feature ${featureName}] Screen "${screenId}" (${screenType}) ${actionLabel} "${action.id}" ` +
+        `navigates to tab "${action.tab}", which is not a section id on target screen ` +
+        `"${target.screen.id}". Available: ${sectionIds.join(", ") || "(none)"}.`,
+    );
+  }
+}
+
 function resolveRowActionNavigateTarget(
   featureName: string,
   screenId: string,
   screenType: "entityList" | "projectionList" | "projectionDetail" | "entityEdit",
-  actionLabel: "rowAction" | "action",
+  actionLabel: string,
   action: RowActionNavigateRuntime,
   allScreenQns: ReadonlySet<string>,
   navTargetShortIds: ReadonlySet<string>,
@@ -586,6 +623,7 @@ function resolveRowActionNavigateTarget(
           `detailFor: "${action.entity}".`,
       );
     }
+    validateRowActionNavigateTab(featureName, screenId, screenType, actionLabel, action, detail);
     return detail;
   }
   if (action.screen === undefined) {
@@ -601,7 +639,9 @@ function resolveRowActionNavigateTarget(
         `navigate-target "${action.screen}" does not resolve to a registered screen in any feature.`,
     );
   }
-  return screensByShortId.get(action.screen)?.[0];
+  const target = screensByShortId.get(action.screen)?.[0];
+  validateRowActionNavigateTab(featureName, screenId, screenType, actionLabel, action, target);
+  return target;
 }
 
 // Shared by the mint step's own handler (screen.handler) and its optional
@@ -1225,6 +1265,19 @@ export function validateScreens(
               `section "${sectionLabel}" action`,
               action,
             );
+            if (action.kind === "navigate" && action.tab !== undefined) {
+              resolveRowActionNavigateTarget(
+                feature.name,
+                screenId,
+                "projectionDetail",
+                `section "${sectionLabel}" action`,
+                action,
+                allScreenQns,
+                navTargetShortIds,
+                screensByShortId,
+                detailForScreens,
+              );
+            }
           }
         }
         if (section.kind === "relatedList" && section.emptyState?.action !== undefined) {
@@ -1235,6 +1288,22 @@ export function validateScreens(
             `section "${sectionLabel}" emptyState action`,
             section.emptyState.action,
           );
+          if (
+            section.emptyState.action.kind === "navigate" &&
+            section.emptyState.action.tab !== undefined
+          ) {
+            resolveRowActionNavigateTarget(
+              feature.name,
+              screenId,
+              "projectionDetail",
+              `section "${sectionLabel}" emptyState action`,
+              section.emptyState.action,
+              allScreenQns,
+              navTargetShortIds,
+              screensByShortId,
+              detailForScreens,
+            );
+          }
         }
         if (isExtensionEditSection(section)) {
           // projectionDetail is read-only (no composed form submit) — an
@@ -1983,6 +2052,19 @@ export function validateScreens(
               `section "${sectionLabel}" action`,
               action,
             );
+            if (action.kind === "navigate" && action.tab !== undefined) {
+              resolveRowActionNavigateTarget(
+                feature.name,
+                screenId,
+                "entityEdit",
+                `section "${sectionLabel}" action`,
+                action,
+                allScreenQns,
+                navTargetShortIds,
+                screensByShortId,
+                detailForScreens,
+              );
+            }
           }
         }
         if (isExtensionEditSection(section)) {

package/src/pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts

@@ -0,0 +1,192 @@
+// kumiko-framework#3243: a DB outage must not crash the dispatcher and must
+// not stay silent forever. The idle pre-check's error log (doPass) fires once
+// per outage, and a matching "recovered" line must fire once the DB answers
+// again, so ops can see the return without restarting the process. Routes the
+// dispatcher's own db connection through a TCP proxy (same pattern as
+// bun-db/__tests__/postgres-closed-socket-write.integration.test.ts) so the
+// outage can be simulated without touching the shared test DB other suites
+// depend on.
+
+import { afterEach, beforeAll, describe, expect, test } from "bun:test";
+import net from "node:net";
+import postgres from "postgres";
+import { createDbConnection } from "../../db/connection";
+import { createEventStoreExecutor } from "../../db/event-store-executor";
+import { createTenantDb, type TenantDb } from "../../db/tenant-db";
+import { defineFeature } from "../../engine";
+import type { StoredEvent } from "../../event-store";
+import type { Logger } from "../../logging/types";
+import { type MetricEvent, RecordingMeter, registerStandardMetrics } from "../../observability";
+import { createEventDispatcher, type EventConsumer, type EventDispatcher } from "../../pipeline";
+import {
+  resetEventStore,
+  setupTestStack,
+  type TestStack,
+  TestUsers,
+  unsafeCreateEntityTable,
+} from "../../stack";
+import { sharedWidgetEntity, sharedWidgetTable, waitFor } from "../../testing";
+import { testDatabaseUrl } from "../../testing/closed-connection-error";
+
+const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
+  entityName: "widget",
+});
+
+const feature = defineFeature("dboutage", (r) => {
+  r.entity("widget", sharedWidgetEntity);
+});
+
+const admin = TestUsers.admin;
+let stack: TestStack;
+let tdb: TenantDb;
+// setupTestStack creates its own ephemeral kumiko_test_<random> DB and never
+// exposes its name/URL. Passing an explicit name here so the proxy below
+// can build the exact connection string setupTestStack itself would have.
+const testDbName = `kumiko_test_dboutage_${crypto.randomUUID().slice(0, 8)}`;
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [feature], systemHooks: [], dbName: testDbName });
+  await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
+  tdb = createTenantDb(stack.db, admin.tenantId);
+});
+
+afterEach(async () => {
+  await resetEventStore(stack, ["read_widgets"]);
+});
+
+async function appendWidget(name: string): Promise<void> {
+  await executor.create({ name }, admin, tdb);
+}
+
+function recordingLogger(): Logger & { readonly lines: string[] } {
+  const lines: string[] = [];
+  const logger: Logger & { lines: string[] } = {
+    lines,
+    info: (msg) => lines.push(`info:${msg}`),
+    warn: (msg) => lines.push(`warn:${msg}`),
+    error: (msg) => lines.push(`error:${msg}`),
+    debug: () => {},
+    child: () => logger,
+  };
+  return logger;
+}
+
+// A TCP passthrough to the real test DB. Stopping it (server.close() +
+// destroying every live socket) simulates an outage without touching the
+// shared test DB; restarting on the SAME port simulates the DB coming back
+// without the dispatcher's client needing a new connection string.
+function startDbProxy(dbUrl: URL): { server: net.Server; stop: () => Promise<void> } {
+  const sockets = new Set<net.Socket>();
+  const server = net.createServer((downstream) => {
+    sockets.add(downstream);
+    const upstream = net.connect(Number(dbUrl.port), dbUrl.hostname);
+    sockets.add(upstream);
+    downstream.pipe(upstream).pipe(downstream);
+    downstream.on("error", () => upstream.destroy());
+    upstream.on("error", () => downstream.destroy());
+    downstream.on("close", () => sockets.delete(downstream));
+    upstream.on("close", () => sockets.delete(upstream));
+  });
+  return {
+    server,
+    stop: async () => {
+      for (const socket of sockets) socket.destroy();
+      await new Promise<void>((resolve) => server.close(() => resolve()));
+    },
+  };
+}
+
+async function listenOn(server: net.Server, port: number): Promise<void> {
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, resolve);
+  });
+}
+
+describe("E: dispatcher survives a DB outage and logs the recovery", () => {
+  test("idle pre-check logs the outage once and the recovery once, then resumes delivery", async () => {
+    const dbUrl = new URL(testDatabaseUrl().replace(/\/[^/]+$/, `/${testDbName}`));
+    const proxy1 = startDbProxy(dbUrl);
+    await listenOn(proxy1.server, 0);
+    const address = proxy1.server.address();
+    if (address === null || typeof address === "string") {
+      throw new Error("proxy did not bind to a TCP port");
+    }
+    const proxyPort = address.port;
+
+    const proxiedUrl = new URL(dbUrl.toString());
+    proxiedUrl.hostname = "127.0.0.1";
+    proxiedUrl.port = String(proxyPort);
+    const proxiedDb = createDbConnection(proxiedUrl.toString(), { maxConnections: 1 });
+
+    const observed: string[] = [];
+    const consumer: EventConsumer = {
+      name: "dboutage:consumer",
+      handler: async (event: StoredEvent) => {
+        observed.push(String(event.payload["name"]));
+      },
+    };
+
+    const logger = recordingLogger();
+    const metricEvents: MetricEvent[] = [];
+    const lastListenGauge = (): number | undefined =>
+      metricEvents
+        .filter(
+          (e) => e.type === "gauge.set" && e.name === "kumiko_event_dispatcher_listen_connected",
+        )
+        .pop()?.value;
+    const meter = new RecordingMeter((e) => metricEvents.push(e));
+    registerStandardMetrics(meter);
+    const listenClient = postgres(proxiedUrl.toString(), { max: 1 });
+    const dispatcher: EventDispatcher = createEventDispatcher({
+      db: proxiedDb.db,
+      consumers: [consumer],
+      context: { db: proxiedDb.db, log: logger },
+      pollIntervalMs: 50,
+      pgClient: listenClient,
+      meter,
+    });
+
+    try {
+      await dispatcher.start();
+      expect(lastListenGauge()).toBe(1);
+
+      await appendWidget("before-outage");
+      await waitFor(() => observed.includes("before-outage"));
+      // Delivery is at-least-once, so killing the proxy mid-commit would
+      // redeliver "before-outage" after recovery. Drain first so the turn's
+      // TX has already committed.
+      await dispatcher.drain();
+
+      await proxy1.stop();
+
+      await waitFor(() => logger.lines.some((line) => line.includes("idle pre-check failed")));
+      expect(lastListenGauge()).toBe(0);
+
+      const proxy2 = startDbProxy(dbUrl);
+      await listenOn(proxy2.server, proxyPort);
+      try {
+        await waitFor(() => logger.lines.some((line) => line.includes("idle pre-check recovered")));
+
+        await appendWidget("after-recovery");
+        await waitFor(() => observed.includes("after-recovery"));
+      } finally {
+        await proxy2.stop();
+      }
+
+      const failedCount = logger.lines.filter((line) =>
+        line.includes("idle pre-check failed"),
+      ).length;
+      const recoveredCount = logger.lines.filter((line) =>
+        line.includes("idle pre-check recovered"),
+      ).length;
+      expect(failedCount).toBe(1);
+      expect(recoveredCount).toBe(1);
+      expect(observed).toEqual(["before-outage", "after-recovery"]);
+    } finally {
+      await dispatcher.stop();
+      await listenClient.end({ timeout: 1 });
+      await proxiedDb.close();
+    }
+  });
+});

package/src/pipeline/event-dispatcher.ts

@@ -434,7 +434,16 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
           instanceId: consumerInstanceId(consumer, options.instanceId),
         })),
       );
-      idlePreCheckErrorLogged = false;
+      if (idlePreCheckErrorLogged) {
+        idlePreCheckErrorLogged = false;
+        const recoveredMsg =
+          "[event-dispatcher] idle pre-check recovered, database reachable again";
+        if (context.log) {
+          context.log.info(recoveredMsg);
+        } else {
+          console.log(recoveredMsg);
+        }
+      }
     } catch (e) {
       idleKeys = new Set();
       // Log once per outage, not once per poll tick — see
@@ -443,6 +452,9 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
       // signal for ops, not a delivery failure.
       if (!idlePreCheckErrorLogged) {
         idlePreCheckErrorLogged = true;
+        // The LISTEN peer talks to the same DB, so it is gone too; only a
+        // later onlisten may flip the gauge back to 1.
+        emitEventDispatcherListenConnected(meter, false);
         const msg = e instanceof Error ? e.message : String(e);
         const logMsg = `[event-dispatcher] idle pre-check failed, falling back to per-consumer locking every tick until it recovers: ${msg}`;
         if (context.log) {
@@ -697,10 +709,11 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
       // fallback (dropped subscriptions, missed commits under load).
       //
       // Observability: the gauge kumiko_event_dispatcher_listen_connected
-      // flips to 1 on initial subscribe AND on every postgres.js silent
-      // reconnect (via the onlisten callback). A drop to 0 while running
-      // means delivery latency regressed from TCP-round-trip to
-      // pollIntervalMs — ops-visible.
+      // flips to 1 on initial subscribe and again on every reconnect that
+      // succeeds (onlisten callback). It drops to 0 on the initial connect
+      // failure, on stop() and when the idle pre-check detects a DB outage.
+      // An outage shorter than one poll tick is not detected; the gauge then
+      // stays at 1 until postgres.js's delayed re-LISTEN lands.
       emitEventDispatcherListenConnected(meter, false);
       if (options.pgClient) {
         try {
@@ -712,10 +725,14 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
               });
             },
             () => {
-              // Fires on initial connect AND on each reconnect. postgres.js
-              // reconnects transparently if the TCP connection drops, so the
-              // only way to see the recovery window is to flip the gauge
-              // every time this callback lands.
+              // Fires on initial connect and again when postgres.js's own
+              // re-LISTEN lands after a drop; during an outage that re-LISTEN
+              // waits in its connection backoff (measured 10-30s after the DB
+              // returns), polling covers delivery meanwhile. Never unlisten()
+              // + listen() by hand: that leaves duplicate listeners.
+              // ponytail: no own LISTEN reconnect, the backoff gap is polled;
+              // add a dedicated LISTEN connection if prod wires a pgClient
+              // and that latency matters.
               emitEventDispatcherListenConnected(meter, true);
             },
           );

package/src/stack/__tests__/setup-test-stack-entity-tables.integration.test.ts

@@ -0,0 +1,65 @@
+// #3102: setupTestStack pushed projection/MSP/storeTable sources but never
+// r.entity() tables, so any feature relying purely on entities crashed on
+// first write with 42P01 unless the test manually created the table.
+
+import { afterAll, describe, expect, test } from "bun:test";
+import { asRawClient } from "../../db/query";
+import { createEntity, createTextField, defineFeature } from "../../engine";
+import { setupTestStack, type TestStack } from "../test-stack";
+
+const widgetEntity = createEntity({
+  table: "widgets",
+  fields: {
+    name: createTextField({ personal: false, reason: "test_fixture", required: true }),
+  },
+});
+
+const gadgetEntity = createEntity({
+  table: "gadgets",
+  fields: {
+    label: createTextField({ personal: false, reason: "test_fixture", required: true }),
+  },
+});
+
+const widgetFeature = defineFeature("widget", (r) => {
+  r.entity("widget", widgetEntity);
+});
+
+const twoEntityFeature = defineFeature("gadgetbox", (r) => {
+  r.entity("widget", widgetEntity);
+  r.entity("gadget", gadgetEntity);
+});
+
+async function tableExistsInInformationSchema(
+  db: TestStack["db"],
+  tableName: string,
+): Promise<boolean> {
+  const rows = await asRawClient(db).unsafe<{ table_name: string }>(
+    `SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1`,
+    [tableName],
+  );
+  return rows.length === 1;
+}
+
+describe("setupTestStack — r.entity() tables", () => {
+  let stack: TestStack | undefined;
+
+  afterAll(async () => {
+    await stack?.cleanup();
+  });
+
+  test("creates the backing table for a plain r.entity() without manual table setup", async () => {
+    stack = await setupTestStack({ features: [widgetFeature] });
+    expect(await tableExistsInInformationSchema(stack.db, "widgets")).toBe(true);
+  });
+
+  test("creates every entity's backing table when a feature registers more than one", async () => {
+    const twoEntityStack = await setupTestStack({ features: [twoEntityFeature] });
+    try {
+      expect(await tableExistsInInformationSchema(twoEntityStack.db, "widgets")).toBe(true);
+      expect(await tableExistsInInformationSchema(twoEntityStack.db, "gadgets")).toBe(true);
+    } finally {
+      await twoEntityStack.cleanup();
+    }
+  });
+});

package/src/stack/test-stack.ts

@@ -4,7 +4,6 @@ import type { JwtHelper } from "../api/jwt";
 import { buildServer } from "../api/server";
 import { createSseBroker, type SseBroker } from "../api/sse-broker";
 import type { PgClient } from "../db/connection";
-import { extractTableInfo } from "../db/query";
 import { validateOwnershipBoot } from "../engine/boot-validator/ownership";
 import { createRegistry } from "../engine/registry";
 import type { AppContext, FeatureDefinition, JobRunIn, Registry, TenantId } from "../engine/types";
@@ -181,6 +180,10 @@ export type TestStackOptions = {
    *  out (e.g. 60_000) so the polling timer can't land inside the
    *  assertion window and mask a dead subscription — see E.4 (#2042). */
   eventDispatcherPollIntervalMs?: number;
+  /** Opt out of auto-created r.entity() tables for callers that create/migrate
+   *  them themselves. Default true — false pushes only projection/MSP/
+   *  storeTable sources, same as before fw#3102. */
+  entityTables?: boolean;
 };
 
 const DEFAULT_JWT_SECRET = "test-stack-secret-minimum-32-characters!!";
@@ -215,68 +218,72 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
     createTestRedis(),
   ]);
 
-  // Every ES-entity writes events via createEventStoreExecutor in the
-  // feature's write handlers. Auto-create the events table so every
-  // setupTestStack call is ready for writes without needing a manual
-  // createEventsTable().
-  await createEventsTable(testDb.db);
-  // Archive-stream metadata — needed by ctx.appendEvent's archive guard and
-  // loadAggregate's default-skip. Idempotent, so production boot running
-  // the same call is fine.
-  await createArchivedStreamsTable(testDb.db);
+  // Any throw in this DDL region must still drop the just-created ephemeral
+  // DB/Redis — the caller never gets the stack object back to call
+  // cleanup() itself (same "leaked ephemeral DB" failure class as the later
+  // try/catch below, but for setup that runs before that try starts).
+  try {
+    // Every ES-entity writes events via createEventStoreExecutor in the
+    // feature's write handlers. Auto-create the events table so every
+    // setupTestStack call is ready for writes without needing a manual
+    // createEventsTable().
+    await createEventsTable(testDb.db);
+    // Archive-stream metadata — needed by ctx.appendEvent's archive guard and
+    // loadAggregate's default-skip. Idempotent, so production boot running
+    // the same call is fine.
+    await createArchivedStreamsTable(testDb.db);
 
-  // Framework state for projection rebuild/status + event-consumer cursors.
-  // Idempotent — production boot flows run the same calls.
-  const { createProjectionStateTable, createEventConsumerStateTable } = await import("../pipeline");
-  await createProjectionStateTable(testDb.db);
-  await createEventConsumerStateTable(testDb.db);
+    // Framework state for projection rebuild/status + event-consumer cursors.
+    // Idempotent — production boot flows run the same calls.
+    const { createProjectionStateTable, createEventConsumerStateTable } = await import(
+      "../pipeline"
+    );
+    await createProjectionStateTable(testDb.db);
+    await createEventConsumerStateTable(testDb.db);
 
-  // Files support: when a provider is registered, the fileRefs table must
-  // exist before the first upload. Skipped when no provider — the table
-  // stays off tenant test DBs that never touch files.
-  if (options.files) {
-    const { fileRefsTable } = await import("../files");
-    await unsafePushTables(testDb.db, { fileRefsTable });
-  }
-
-  // Projection-/MSP-/raw-tables: the executor (or async dispatcher) writes
-  // into them as soon as the first matching event flows, so the DDL must
-  // exist before setupTestStack returns. The source list is shared with
-  // collectTableMetas (`kumiko schema generate`) — divergence between the
-  // two was exactly the #255 prod-crash. Two registrations backed by the
-  // same physical table (e.g. an alternative apply-shape for the same
-  // read-model in a test feature) are deduped by table reference so we
-  // emit only one CREATE TABLE per physical table.
-  const { enumerateFeatureTableSources } = await import("../db/feature-table-sources");
-  const projectionTables: Record<string, unknown> = {};
-  // Dedup by NAME, matching collectTableMetas — by-reference alone let two
-  // distinct table objects with the same name slip through as a double
-  // CREATE TABLE while schema-generate emitted only one meta (silent
-  // test-vs-schema divergence).
-  const seenTableNames = new Set<string>();
-  for (const feature of options.features) {
-    for (const { table, origin } of enumerateFeatureTableSources(feature)) {
-      const name = extractTableInfo(table).name;
-      if (seenTableNames.has(name)) continue;
-      seenTableNames.add(name);
-      projectionTables[origin] = table;
+    // Files support: when a provider is registered, the fileRefs table must
+    // exist before the first upload. Skipped when no provider — the table
+    // stays off tenant test DBs that never touch files.
+    if (options.files) {
+      const { fileRefsTable } = await import("../files");
+      await unsafePushTables(testDb.db, { fileRefsTable });
     }
-  }
-  if (Object.keys(projectionTables).length > 0) {
-    // unsafePushTables emits raw CREATE TABLE — fine for ephemeral test DBs but
-    // collides on re-boot against a persistent DB whose projection tables
-    // were created during a previous run. Filter out the ones that already
-    // exist so the re-boot doesn't fail on duplicate CREATE TABLE.
+
+    // Same table list as `kumiko schema generate` (collectTableMetas) — divergence
+    // was the #255 prod-crash and the #3102 missing r.entity() tables.
+    const { collectTableMetas } = await import("../db/collect-table-metas");
     const { tableExists } = await import("../db/schema-inspection");
+    let metas = collectTableMetas(options.features);
+    if (options.entityTables === false) {
+      const { enumerateFeatureTableSources } = await import("../db/feature-table-sources");
+      const { extractTableInfo } = await import("../db/query");
+      const nonEntityNames = new Set(
+        options.features.flatMap((f) =>
+          enumerateFeatureTableSources(f).map((s) => extractTableInfo(s.table).name),
+        ),
+      );
+      metas = metas.filter((m) => nonEntityNames.has(m.tableName));
+    }
     const missing: Record<string, unknown> = {};
-    for (const [key, tbl] of Object.entries(projectionTables)) {
-      const physical = extractTableInfo(tbl).name;
-      if (await tableExists(testDb.db, `public.${physical}`)) continue;
-      missing[key] = tbl;
+    for (const meta of metas) {
+      if (await tableExists(testDb.db, `public.${meta.tableName}`)) continue;
+      missing[meta.tableName] = meta;
     }
     if (Object.keys(missing).length > 0) {
       await unsafePushTables(testDb.db, missing);
     }
+  } catch (error) {
+    try {
+      await testDb.cleanup();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    try {
+      await testRedis.cleanup();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    throw error;
   }
 
   const searchAdapter = createInMemorySearchAdapter();

package/src/testing/__tests__/real-providers.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { requireRealProviders } from "../real-providers";
+import { isRealProviderRun, REAL_PROVIDERS_ENV, requireRealProviders } from "../real-providers";
 
 const CI = "CI";
 const FLAG = "KUMIKO_REAL_PROVIDERS";
@@ -52,3 +52,11 @@ describe("requireRealProviders", () => {
     expect(() => requireRealProviders()).not.toThrow();
   });
 });
+
+describe("isRealProviderRun", () => {
+  test("reads the exported flag name from an explicit env map, defaulting to process.env", () => {
+    expect(isRealProviderRun({ [REAL_PROVIDERS_ENV]: "1" })).toBe(true);
+    expect(isRealProviderRun({ [REAL_PROVIDERS_ENV]: "true" })).toBe(false);
+    expect(isRealProviderRun({})).toBe(false);
+  });
+});

package/src/testing/index.ts

@@ -45,7 +45,7 @@ export {
   createRecordingProvider,
   type RecordingProvider,
 } from "./observability-recorder";
-export { requireRealProviders } from "./real-providers";
+export { isRealProviderRun, REAL_PROVIDERS_ENV, requireRealProviders } from "./real-providers";
 export { deleteRows, seedRow, seedRows, updateRows } from "./seed";
 export {
   sharedItemEntity,

package/src/testing/real-providers.ts

@@ -1,4 +1,4 @@
-const REAL_PROVIDERS_FLAG = "KUMIKO_REAL_PROVIDERS";
+export const REAL_PROVIDERS_ENV = "KUMIKO_REAL_PROVIDERS";
 const CI_FLAG = "CI";
 
 function isCi(): boolean {
@@ -6,15 +6,23 @@ function isCi(): boolean {
   return ci !== undefined && ci !== "" && ci !== "0" && ci.toLowerCase() !== "false";
 }
 
+// Apps and the e2e/bunfig templates gate real-provider runs on this same
+// check instead of re-reading KUMIKO_REAL_PROVIDERS themselves.
+export function isRealProviderRun(
+  env: Readonly<Record<string, string | undefined>> = process.env,
+): boolean {
+  return env[REAL_PROVIDERS_ENV] === "1";
+}
+
 export function requireRealProviders(): void {
   if (isCi()) {
     throw new Error(
-      `Real-provider tests never run in CI (CI is set). Run them locally with ${REAL_PROVIDERS_FLAG}=1 via the test:real / e2e:real script.`,
+      `Real-provider tests never run in CI (CI is set). Run them locally with ${REAL_PROVIDERS_ENV}=1 via the test:real / e2e:real script.`,
     );
   }
-  if (process.env[REAL_PROVIDERS_FLAG] !== "1") {
+  if (!isRealProviderRun()) {
     throw new Error(
-      `Real-provider test needs ${REAL_PROVIDERS_FLAG}=1 (an API key alone never enables it). Run it via the test:real / e2e:real script.`,
+      `Real-provider test needs ${REAL_PROVIDERS_ENV}=1 (an API key alone never enables it). Run it via the test:real / e2e:real script.`,
     );
   }
 }