@cosmicdrift/kumiko-framework 0.307.0 → 0.309.0

package/src/api/server.ts

@@ -50,6 +50,7 @@ import {
   type GlobalIpRateLimitOptions,
   globalIpRateLimit,
 } from "../rate-limit";
+import { deriveSearchAdapterConfig } from "../search/derive-search-adapter-config";
 import type { SearchAdapter } from "../search/types";
 import { assertUnreachable, generateId } from "../utils";
 import { NO_ROUTE_MATCH_HEADER_NAME, PUBLIC_API_PATHS } from "./api-constants";
@@ -488,6 +489,16 @@ export function buildServer(options: ServerOptions): KumikoServer {
   // @cast-boundary engine-bridge — searchAdapter is an optional context-extension
   const searchAdapter = (contextWithObservability as { searchAdapter?: SearchAdapter })
     .searchAdapter;
+  // Adapter is built before the registry exists (app boot order) — hand it
+  // the registry-derived default config here so it can lazily configure
+  // each tenant index on first access instead of requiring per-tenant app
+  // wiring.
+  const derivedSearchConfig = searchAdapter
+    ? deriveSearchAdapterConfig(options.registry)
+    : undefined;
+  if (searchAdapter && derivedSearchConfig) {
+    searchAdapter.setDefaultConfig?.(derivedSearchConfig);
+  }
 
   const sseConsumerEnabled = options.eventDispatcher?.systemConsumers?.sse ?? true;
   const searchConsumerEnabled = options.eventDispatcher?.systemConsumers?.search ?? true;

package/src/api/sse-broker.ts

@@ -12,6 +12,26 @@ export type SseEvent = {
   data: Record<string, unknown>;
 };
 
+// What a publish invalidates. "user" is the broadest (and the historical
+// default — every scope narrower than this is an opt-in from a caller that
+// knows exactly which credential(s) it revoked).
+export type AccessInvalidationScope =
+  | { readonly kind: "user" }
+  | { readonly kind: "all-except-session"; readonly keptSessionId: string }
+  | { readonly kind: "sessions"; readonly sessionIds: readonly string[] }
+  | { readonly kind: "pat-tokens"; readonly tokenIds: readonly string[] };
+
+// What a stream's own listener is authenticated as — used to decide whether
+// a narrow scope (sessions/pat-tokens) applies to it. A listener with
+// neither field (should not normally happen — dispatch-stream.ts always
+// passes at least one) is treated as unidentifiable and closed by every
+// narrow scope, same fail-closed stance as a PAT stream facing a
+// sessions-scope.
+export type AccessInvalidationCredential = {
+  readonly sid?: string;
+  readonly patTokenId?: string;
+};
+
 export type SseBroker = {
   addClient(channel: string, send: (event: SseEvent) => void, close: () => void): string;
   removeClient(channel: string, clientId: string): void;
@@ -27,22 +47,37 @@ export type SseBroker = {
   subscribeAccessInvalidation(
     userId: string,
     onInvalidate: () => void,
-    ownSid?: string,
+    credential?: AccessInvalidationCredential,
   ): () => void;
-  // `keptSessionId` spares exactly one stream: the caller's own session on a
-  // "revoke all others" write. It is a keep-list, not a list of revoked
-  // sessions, so a stream of a session already revoked without an event
-  // (plain logout) still closes. Every other reason stays userwide.
-  publishAccessInvalidation(userId: string, keptSessionId?: string): void;
+  // `scope` narrows which credential(s) close; omitted = userwide (every
+  // listener), the historical default and still what unrelated
+  // reasons (role change, membership removal) use.
+  publishAccessInvalidation(userId: string, scope?: AccessInvalidationScope): void;
 };
 
-// Fail-closed: without a kept sid, or for a listener without its own sid
-// (PAT/bearer), nothing is spared.
-function isSparedByKeptSessionId(
-  ownSid: string | undefined,
-  keptSessionId: string | undefined,
+// Fail-closed: a narrow scope only spares/limits a listener it can positively
+// match against its own credential. A listener with neither sid nor
+// patTokenId of its own can't be matched by "sessions"/"pat-tokens", so it
+// closes — same reasoning "all-except-session" already used for a sidless
+// (PAT/bearer) listener.
+export function shouldInvalidateListener(
+  credential: AccessInvalidationCredential,
+  scope: AccessInvalidationScope,
 ): boolean {
-  return keptSessionId !== undefined && ownSid === keptSessionId;
+  switch (scope.kind) {
+    case "user":
+      return true;
+    case "all-except-session":
+      return credential.sid !== scope.keptSessionId;
+    case "sessions":
+      return credential.sid !== undefined
+        ? scope.sessionIds.includes(credential.sid)
+        : credential.patTokenId === undefined;
+    case "pat-tokens":
+      return credential.patTokenId !== undefined
+        ? scope.tokenIds.includes(credential.patTokenId)
+        : credential.sid === undefined;
+  }
 }
 
 export function createSseBroker(): SseBroker {
@@ -51,11 +86,14 @@ export function createSseBroker(): SseBroker {
   // makes pushToChannel/publishAccessInvalidation reach every replica's
   // clients — this reference implementation stays single-process only.
   const channels = new Map<string, Map<string, SseClient>>();
-  // Keyed by callback reference, value is the subscriber's own sid. Every
-  // subscriber must pass a distinct closure (dispatch-stream.ts does, one per
-  // stream). Two subscribes with the SAME reference for the same user
-  // collapse into one listener, and the first unsubscribe kills both.
-  const accessInvalidationListeners = new Map<string, Map<() => void, string | undefined>>();
+  // Keyed by callback reference, value is the subscriber's own credential.
+  // Every subscriber must pass a distinct closure (dispatch-stream.ts does,
+  // one per stream). Two subscribes with the SAME reference for the same
+  // user collapse into one listener, and the first unsubscribe kills both.
+  const accessInvalidationListeners = new Map<
+    string,
+    Map<() => void, AccessInvalidationCredential>
+  >();
 
   function getOrCreateChannel(channel: string): Map<string, SseClient> {
     let clients = channels.get(channel);
@@ -103,14 +141,14 @@ export function createSseBroker(): SseBroker {
       return total;
     },
 
-    subscribeAccessInvalidation(userId, onInvalidate, ownSid) {
+    subscribeAccessInvalidation(userId, onInvalidate, credential) {
       const channel = userAccessChannel(userId);
       let listeners = accessInvalidationListeners.get(channel);
       if (!listeners) {
         listeners = new Map();
         accessInvalidationListeners.set(channel, listeners);
       }
-      listeners.set(onInvalidate, ownSid);
+      listeners.set(onInvalidate, credential ?? {});
       return () => {
         const current = accessInvalidationListeners.get(channel);
         // skip: already unsubscribed (e.g. stream ended after a publish already fired)
@@ -120,15 +158,16 @@ export function createSseBroker(): SseBroker {
       };
     },
 
-    publishAccessInvalidation(userId, keptSessionId) {
+    publishAccessInvalidation(userId, scope) {
       const channel = userAccessChannel(userId);
       const listeners = accessInvalidationListeners.get(channel);
       // skip: no live stream is watching this user right now
       if (!listeners) return;
+      const resolvedScope: AccessInvalidationScope = scope ?? { kind: "user" };
       // Snapshot before iterating — a fired listener unsubscribes itself,
       // which would mutate `listeners` mid-iteration otherwise.
-      for (const [onInvalidate, ownSid] of [...listeners]) {
-        if (isSparedByKeptSessionId(ownSid, keptSessionId)) continue;
+      for (const [onInvalidate, credential] of [...listeners]) {
+        if (!shouldInvalidateListener(credential, resolvedScope)) continue;
         onInvalidate();
       }
     },

package/src/bun-db/__tests__/postgres-closed-socket-write.integration.test.ts

@@ -0,0 +1,87 @@
+// Guards the nextWrite null-socket backport (porsager/postgres#1209) on the
+// @bender0oo0/postgres fork pinned in package.json: reserve a connection, kill
+// the backend socket mid-flight, then send another query on the reserved
+// connection. Upstream postgres 3.4.9 crashes the process with an
+// uncaughtException ("null is not an object (evaluating 'socket.write')")
+// instead of rejecting; the fork rejects with CONNECTION_CLOSED. Must stay
+// green after switching back to an upstream release (kumiko-framework#3248).
+
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import net from "node:net";
+import postgres from "postgres";
+import { testDatabaseUrl } from "../../testing/closed-connection-error";
+
+const DATABASE_URL = testDatabaseUrl();
+const dbUrl = new URL(DATABASE_URL);
+
+let uncaught: unknown;
+function onUncaught(err: unknown): void {
+  uncaught = err;
+}
+
+beforeEach(() => {
+  uncaught = undefined;
+  process.on("uncaughtException", onUncaught);
+});
+
+afterEach(() => {
+  process.off("uncaughtException", onUncaught);
+});
+
+test("reserved connection rejects with CONNECTION_CLOSED instead of crashing on a dead socket", async () => {
+  let downstream: net.Socket | undefined;
+  const proxy = net.createServer((socket) => {
+    downstream = socket;
+    const upstream = net.connect(Number(dbUrl.port), dbUrl.hostname);
+    socket.pipe(upstream).pipe(socket);
+    socket.on("error", () => upstream.destroy());
+    upstream.on("error", () => socket.destroy());
+  });
+  await new Promise<void>((resolve) => proxy.listen(0, resolve));
+  const address = proxy.address();
+  if (address === null || typeof address === "string") {
+    throw new Error("proxy did not bind to a TCP port");
+  }
+
+  const sql = postgres({
+    host: "127.0.0.1",
+    port: address.port,
+    user: dbUrl.username,
+    password: dbUrl.password,
+    database: dbUrl.pathname.slice(1),
+    max: 1,
+  });
+
+  try {
+    const reserved = await sql.reserve();
+    try {
+      await reserved`select 1`;
+      downstream?.end();
+      await new Promise((resolve) => setTimeout(resolve, 50));
+
+      const HANG = Symbol("hang");
+      const result = await Promise.race([
+        reserved`select 1`.then(
+          () => {
+            throw new Error("expected the query to reject with CONNECTION_CLOSED, but it resolved");
+          },
+          (err: unknown) => err,
+        ),
+        new Promise((resolve) => setTimeout(() => resolve(HANG), 3000)),
+      ]);
+
+      if (result === HANG) {
+        throw new Error("query on the dead-socket connection hung instead of rejecting");
+      }
+      expect(result).toMatchObject({ code: "CONNECTION_CLOSED" });
+      // Give the process a tick to surface an uncaughtException if the guard regressed.
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      expect(uncaught).toBeUndefined();
+    } finally {
+      reserved.release();
+    }
+  } finally {
+    proxy.close();
+    await sql.end({ timeout: 1 });
+  }
+});

package/src/bun-db/connection.ts

@@ -54,6 +54,8 @@ export function createBunDbConnection(
   if (options.connectTimeoutSeconds !== undefined) {
     pgOpts.connect_timeout = options.connectTimeoutSeconds;
   }
+  // ponytail: "postgres" resolves to the @bender0oo0/postgres fork (nextWrite
+  // null-socket guard, porsager/postgres#1209); switch back per kumiko-framework#3248.
   const listenClient = postgres(url, pgOpts);
 
   return {

package/src/changes.json

@@ -1,4 +1,65 @@
 [
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "LISTEN gauge drops to 0 on a DB outage",
+    "detail": "`kumiko_event_dispatcher_listen_connected` stayed at 1 when the LISTEN\nconnection died during a DB outage, although delivery had already fallen\nback to polling. The dispatcher now sets it to 0 when the idle pre-check\ndetects the outage. postgres.js re-LISTENs on its own once the DB is back\n(delayed by its connect backoff, measured 10-30s) and sets it back to 1."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "event-dispatcher logs when the DB comes back after an outage (fw#3243)",
+    "detail": "A DB outage already kept the dispatcher process alive (the postgres pool\nreconnects per query and the dispatcher keeps polling from its cursor) and\nlogged `idle pre-check failed` once per outage, but recovery was silent:\nops had no signal that delivery was back to normal without restarting the\nprocess. The idle pre-check now logs `idle pre-check recovered, database\nreachable again` once the next pass succeeds after a logged failure, no\nrestart needed."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "Pin postgres to a fork carrying the nextWrite null-socket fix (fw#3243)",
+    "detail": "The framework now depends on @bender0oo0/postgres@3.4.9-kumiko.1, an exact pin of\npostgres 3.4.9 plus a backport of porsager/postgres#1209. Without the backport, a\nclosed connection can crash the whole process with an uncaughtException\n(\"null is not an object (evaluating 'socket.write')\") instead of rejecting the\nin-flight query with CONNECTION_CLOSED. Apps get the fix automatically with this\nframework bump, no app-side changes needed. The pin is temporary and tracked for\nremoval in kumiko-framework#3248 once the fix lands upstream."
+  },
+  {
+    "version": "0.309.0",
+    "type": "fix",
+    "title": "setupTestStack creates r.entity() tables — same table list as kumiko schema generate (fw#3102)",
+    "detail": "Since 0.290.0 (fw#2881), every authenticated request reads `read_tenants` (rejectIfTenantTeardown) as soon as tenant-lifecycle is mounted. Test stacks needed a manually created tenant table for that to work, because setupTestStack pushed projection/multiStreamProjection/storeTable sources but never r.entity() tables. setupTestStack now creates every entity's backing table itself via collectTableMetas, the same table list `kumiko schema generate` uses, so a feature that only declares r.entity() no longer 42P01s on first write in a test stack. App-side wrappers like `unsafeEnsureEntityTable(db, tenantEntity, \"tenant\")` right after setupTestStack can be removed. Callers that create/migrate entity tables themselves can opt out via `entityTables: false` (used internally by `setupAppTestStack`'s `registryTables: false`), and setupTestStack now drops the ephemeral test DB/Redis if the table DDL fails instead of leaking them."
+  },
+  {
+    "version": "0.308.0",
+    "type": "fix",
+    "title": "startFrom \"now\" and MSP rebuild hand-off no longer lose out-of-order commits"
+  },
+  {
+    "version": "0.308.0",
+    "type": "fix",
+    "title": "Idle event-dispatcher passes no longer lock consumer rows or write WAL"
+  },
+  {
+    "version": "0.308.0",
+    "type": "breaking",
+    "title": "Boot validator rejects a projectionDetail screen/section/emptyState action with no resolvable icon (fw#3234)",
+    "migration": "Every screen.actions / section.actions / section.emptyState.action entry on a projectionDetail screen, and every section.actions entry on an entityEdit screen, must resolve an icon: either an explicit `icon` on the action, or an id the shared action-icon map (@cosmicdrift/kumiko-types resolveActionIcon) already derives from the full id or its last/first kebab segment. entityEdit's own screen-level `actions` are not checked (unchanged, out of scope for this rule). An action that fails this now fails boot instead of rendering without an icon; give it an explicit `icon` or rename it to an id the map covers."
+  },
+  {
+    "version": "0.308.0",
+    "type": "improvement",
+    "title": "EventConsumer.batchHandler — turn-level batching with per-event fallback, used by the search consumer"
+  },
+  {
+    "version": "0.308.0",
+    "type": "fix",
+    "title": "Meilisearch tenant indexes are configured automatically on first access"
+  },
+  {
+    "version": "0.308.0",
+    "type": "breaking",
+    "title": "Session and PAT revoke now close their own credential's open SSE stream, not just other reasons' userwide invalidations",
+    "migration": "An app-injected SseBroker implementation must update its subscribeAccessInvalidation/publishAccessInvalidation signatures: the third parameter of subscribeAccessInvalidation is now an AccessInvalidationCredential object (`{ sid?, patTokenId? }`) instead of a bare sid string, and the second parameter of publishAccessInvalidation is now an AccessInvalidationScope object (`{ kind: \"user\" } | { kind: \"all-except-session\"; keptSessionId } | { kind: \"sessions\"; sessionIds } | { kind: \"pat-tokens\"; tokenIds }`) instead of a bare keptSessionId string. Example: `subscribeAccessInvalidation(userId, cb, sid)` becomes `subscribeAccessInvalidation(userId, cb, { sid })`, and `publishAccessInvalidation(userId, keptSessionId)` becomes `publishAccessInvalidation(userId, { kind: \"all-except-session\", keptSessionId })`."
+  },
+  {
+    "version": "0.308.0",
+    "type": "fix",
+    "title": "zod namespace import lets client bundles tree-shake unused locales"
+  },
   {
     "version": "0.307.0",
     "type": "fix",

package/src/compliance/override-schema.ts

@@ -17,7 +17,7 @@
 // Schema — wer die overriden will, würde die Profile-Identitaet
 // zerstören.
 
-import { z } from "zod";
+import * as z from "zod";
 
 // DurationSpec: { days } | { hours } — strict bedeutet beide Forms
 // muessen exakt 1 property haben (kein "{ days: 30, hours: 1 }").

package/src/crypto/__tests__/event-pii.test.ts

@@ -5,7 +5,7 @@
 
 import { afterEach, describe, expect, test } from "bun:test";
 import { normalizeEventPiiSubject } from "@cosmicdrift/kumiko-types/handlers";
-import { z } from "zod";
+import * as z from "zod";
 import { createRegistry, defineFeature } from "../../engine";
 import type { TenantId } from "../../engine/types/identifiers";
 import {

package/src/crypto/kms-adapter.ts

@@ -3,7 +3,7 @@ import {
   type SubjectId,
   subjectIdToKey,
 } from "@cosmicdrift/kumiko-types/kms-adapter-types";
-import { z } from "zod";
+import * as z from "zod";
 import type { TenantId } from "../engine/types/identifiers";
 
 export * from "@cosmicdrift/kumiko-types/kms-adapter-types";

package/src/db/__tests__/feature-table-sources.test.ts

@@ -1,6 +1,6 @@
 // Regression test for #255-class drift: storeTables must appear in
-// enumerateFeatureTableSources so setupTestStack's auto-push and
-// collectTableMetas see the exact same table set. Before this fix,
+// enumerateFeatureTableSources so collectTableMetas (shared by setupTestStack's
+// auto-push and `kumiko schema generate`) sees the exact same table set. Before this fix,
 // unmanaged tables were only handled by collectTableMetas directly, so any
 // app relying on setupTestStack's ephemeral DB (Playwright/e2e) never got
 // its store tables created — e.g. the bundled "sessions" feature's

package/src/db/__tests__/tenant-db-where-merge.integration.test.ts

@@ -6,7 +6,7 @@
 
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
 import { updateRows } from "@cosmicdrift/kumiko-framework/testing";
-import { z } from "zod";
+import * as z from "zod";
 import { selectMany } from "../../bun-db";
 import {
   createEntity,

package/src/db/collect-table-metas.ts

@@ -1,9 +1,6 @@
-// collectTableMetas — kanonische ENTITY_METAS-Quelle für `kumiko schema
-// generate`. Erfasst dieselben Tabellen-Quellen wie der setupTestStack-
-// auto-push (entities, storeTables, projections, multiStreamProjections) —
-// die frühere Template-Variante sammelte nur entities + storeTables, wodurch
-// projection-only-Tabellen (z.B. billing-foundation read_subscriptions) nie
-// in Migrations landeten und der erste Prod-Write crashte (#255).
+// Canonical table source for both `kumiko schema generate` and setupTestStack
+// (entities, storeTables, projections, multiStreamProjections). One shared
+// source so the test-DB push and migrations cannot drift apart again (#255, #3102).
 
 import type { FeatureDefinition } from "../engine/types";
 import { compareByCodepoint } from "../utils";

package/src/db/feature-table-sources.ts

@@ -1,9 +1,8 @@
 // Single enumeration of every table-bearing registration on a feature
 // (r.projection, r.multiStreamProjection with table, r.storeTable).
-// Consumed by BOTH the setupTestStack auto-push and collectTableMetas —
-// one list, so test-DB-push and `kumiko schema generate` cannot drift
-// apart again (#255).
-// A new table-bearing registrar must be added HERE, not in the consumers.
+// Consumed by collectTableMetas's Pass 2 (`kumiko schema generate` and
+// setupTestStack both go through collectTableMetas — see there).
+// A new table-bearing registrar must be added HERE, not in the consumer.
 
 import type { FeatureDefinition } from "../engine/types";
 

package/src/db/queries/event-consumer.ts

@@ -30,9 +30,60 @@ export async function lockEventConsumersShareMode(db: AnyDb): Promise<void> {
   await asRawClient(db).unsafe(`LOCK TABLE "kumiko_event_consumers" IN SHARE MODE`);
 }
 
+// The MAX(id) horizon plus every currently-open id gap below it, read from a
+// single snapshot (READ COMMITTED gives each statement its own). A gap can
+// only later fill in if some transaction holding a lower id was still
+// in-flight in that same snapshot — pg_snapshot_xip is empty otherwise, so
+// the gap CTEs short-circuit and the window scan over kumiko_events never
+// runs. Used both to seed startFrom "now" (#3231) and to seed an MSP
+// rebuild's handoff to the live dispatcher (msp-rebuild.ts).
+export async function selectEventIdHorizonWithMissingRanges(
+  db: AnyDb,
+): Promise<{ readonly horizon: bigint; readonly gaps: PendingGapEntry[] }> {
+  const rows = (await asRawClient(db).unsafe(
+    `WITH b AS (
+       SELECT COALESCE(MAX("id"), 0) AS horizon, MIN("id") AS min_id FROM "kumiko_events"
+     ),
+     r AS (
+       SELECT 1::bigint AS f, b.min_id - 1 AS t FROM b
+        WHERE b.min_id > 1 AND EXISTS (SELECT 1 FROM pg_snapshot_xip(pg_current_snapshot()))
+       UNION ALL
+       SELECT w."id" + 1, w.next_id - 1
+         FROM (SELECT "id", lead("id") OVER (ORDER BY "id") AS next_id FROM "kumiko_events") w
+        WHERE w.next_id > w."id" + 1 AND EXISTS (SELECT 1 FROM pg_snapshot_xip(pg_current_snapshot()))
+     )
+     SELECT b.horizon::text AS horizon,
+            pg_snapshot_xmax(pg_current_snapshot())::text AS xmax,
+            r.f::text AS gap_from,
+            r.t::text AS gap_to
+       FROM b LEFT JOIN r ON true
+      ORDER BY r.f`,
+  )) as ReadonlyArray<{
+    horizon: string;
+    xmax: string;
+    gap_from: string | null;
+    gap_to: string | null;
+  }>;
+  const first = rows[0];
+  if (first === undefined)
+    throw new Error("selectEventIdHorizonWithMissingRanges: no row returned");
+  const gaps: PendingGapEntry[] = [];
+  for (const row of rows) {
+    if (row.gap_from === null || row.gap_to === null) continue;
+    gaps.push({ from: row.gap_from, to: row.gap_to, xmax: row.xmax });
+  }
+  return { horizon: BigInt(first.horizon), gaps };
+}
+
 // startFrom "now" seeds the FIRST-registration cursor at the current
 // MAX(events.id) instead of the column default 0 — mounting a consumer into
-// an existing app then skips the historical log instead of replaying it.
+// an existing app then skips the historical log instead of replaying it. The
+// horizon and pending_gaps are read from one snapshot (selectEventIdHorizon
+// WithMissingRanges): a transaction holding a lower id can still be
+// in-flight at boot, and without seeding those gaps its event would commit
+// after the cursor already sits above it, so `id > cursor` would never
+// deliver it (#3231). The cheap existence check skips the scan entirely for
+// the (overwhelmingly common) case of an already-registered consumer.
 // ON CONFLICT DO NOTHING keeps both branches safe for an existing row: the
 // subquery only ever affects the row this statement inserts.
 export async function insertConsumerIfAbsent(
@@ -42,11 +93,18 @@ export async function insertConsumerIfAbsent(
   startFrom: "beginning" | "now" = "beginning",
 ): Promise<void> {
   if (startFrom === "now") {
+    const existing = (await asRawClient(db).unsafe(
+      `SELECT 1 FROM "kumiko_event_consumers" WHERE "name" = $1 AND "instance_id" = $2`,
+      [name, instanceId],
+    )) as ReadonlyArray<unknown>;
+    // skip: already registered — an existing cursor is never re-seeded.
+    if (existing.length > 0) return;
+    const { horizon, gaps } = await selectEventIdHorizonWithMissingRanges(db);
     await asRawClient(db).unsafe(
-      `INSERT INTO "kumiko_event_consumers" ("name", "instance_id", "status", "last_processed_event_id")
-       VALUES ($1, $2, 'idle', COALESCE((SELECT MAX("id") FROM "kumiko_events"), 0))
+      `INSERT INTO "kumiko_event_consumers" ("name", "instance_id", "status", "last_processed_event_id", "pending_gaps")
+       VALUES ($1, $2, 'idle', $3, $4::text::jsonb)
        ON CONFLICT ("name", "instance_id") DO NOTHING`,
-      [name, instanceId],
+      [name, instanceId, horizon, JSON.stringify(gaps)],
     );
   } else {
     await asRawClient(db).unsafe(
@@ -83,6 +141,37 @@ export async function deleteOrphanedPerInstanceConsumerRows(
   );
 }
 
+// Lock-free pre-check for doPass: which (name, instance_id) pairs are
+// provably idle — no locking, no xid, no WAL. A pair only qualifies when its
+// row exists, its status isn't dead (dead must still go through
+// acquireConsumerState for auto-rearm), it has no pending_gaps, and no event
+// exists past its cursor. Paired via unnest so the two positional arrays
+// zip element-wise into rows instead of a cartesian product.
+//
+// `deadStatus` is passed in rather than imported from event-consumer-state.ts
+// (which defines ConsumerStatuses) — that module already imports from this
+// one, so importing back would be a require cycle (same pattern as
+// deleteOrphanedPerInstanceConsumerRows above).
+export async function selectProvablyIdleConsumerPairs(
+  db: AnyDb,
+  names: readonly string[],
+  instanceIds: readonly string[],
+  deadStatus: string,
+): Promise<ReadonlyArray<{ readonly name: string; readonly instanceId: string }>> {
+  const rows = (await asRawClient(db).unsafe(
+    `SELECT c."name" AS name, c."instance_id" AS instance_id
+     FROM "kumiko_event_consumers" c
+     WHERE ("name", "instance_id") = ANY (SELECT * FROM unnest($1::text[], $2::text[]))
+       AND "status" != $3
+       AND "pending_gaps" = '[]'::jsonb
+       AND NOT EXISTS (
+         SELECT 1 FROM "kumiko_events" e WHERE e."id" > c."last_processed_event_id"
+       )`,
+    [names, instanceIds, deadStatus],
+  )) as ReadonlyArray<{ name: string; instance_id: string }>;
+  return rows.map((r) => ({ name: r.name, instanceId: r.instance_id }));
+}
+
 export async function selectConsumerForUpdateSkipLocked(
   db: AnyDb,
   name: string,
@@ -249,6 +338,7 @@ export async function updateConsumerRebuildCursor(
   name: string,
   instanceId: string,
   lastProcessedEventId: bigint,
+  pendingGaps: readonly PendingGapEntry[],
 ): Promise<void> {
   await asRawClient(db).unsafe(
     `UPDATE "kumiko_event_consumers" SET
@@ -256,9 +346,11 @@ export async function updateConsumerRebuildCursor(
        "status" = 'idle',
        "attempts" = 0,
        "last_error" = NULL,
+       -- text param + cast: a JS string bound straight to ::jsonb double-encodes under Bun.SQL
+       "pending_gaps" = $2::text::jsonb,
        "updated_at" = now()
-     WHERE "name" = $2 AND "instance_id" = $3`,
-    [lastProcessedEventId, name, instanceId],
+     WHERE "name" = $3 AND "instance_id" = $4`,
+    [lastProcessedEventId, JSON.stringify(pendingGaps), name, instanceId],
   );
 }
 

package/src/derivatives/__tests__/derivatives-context.integration.test.ts

@@ -6,7 +6,7 @@
 
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
 import type { DerivativeRendererPlugin } from "@cosmicdrift/kumiko-types/derivatives-types";
-import { z } from "zod";
+import * as z from "zod";
 import { defineFeature, EXT_DERIVATIVE_RENDERER } from "../../engine";
 import { InternalError, NotFoundError, writeFailure } from "../../errors";
 import { createFilesFeature } from "../../files/feature";

package/src/derivatives/delete-stored-file.ts

@@ -0,0 +1,86 @@
+// Shared binary-deletion helper: deletes an original storage key plus every
+// derivative rendered from it (thumbnails/resized variants — see
+// derivatives-context.ts). Derivatives are never tracked anywhere but the
+// storage layer: they're rendered under a deterministic key (deriveKey +
+// variantSuffix) computed from the original's key, on demand, and nothing
+// writes that key back onto a FileRef row. Listing the original's key-prefix
+// and keeping only candidates whose suffix matches the derivative grammar,
+// anchored to the original's own basename AND extension, means a
+// same-directory sibling (a different file, possibly another tenant's) shares
+// the list prefix but fails the extension/suffix check and is never touched.
+//
+// Used by both the user-data-rights-defaults forget hook and the
+// data-retention hardDelete cleanup — moved here so neither copies the logic.
+
+import { assertSafeStorageKey } from "../files/types";
+import { derivativeListPrefix, isDerivativeKeyOf } from "./variant-key";
+
+export type StoredFileStore = {
+  list(prefix: string): Promise<readonly string[]>;
+  delete(key: string): Promise<void>;
+};
+
+// Wrap rather than let a raw provider error (e.g. S3 AccessDenied) surface
+// unexplained — list() is a requirement on top of delete(); a bucket policy
+// granting only s3:DeleteObject would otherwise fail every run instead of
+// just leaving derivatives behind. Name the likely cause so an operator
+// doesn't have to guess. `errorContext` names the caller for the message
+// prefix (e.g. "user-data-rights-defaults:fileRef", "data-retention:hardDelete").
+export async function listDerivativeKeys(
+  originalKey: string,
+  store: Pick<StoredFileStore, "list">,
+  errorContext: string,
+): Promise<readonly string[]> {
+  let candidates: readonly string[];
+  try {
+    candidates = await store.list(derivativeListPrefix(originalKey));
+  } catch (err) {
+    throw new Error(
+      `[${errorContext}] provider.list() failed while looking up derivatives of key=${originalKey} — forget/erasure requires list permission on the storage bucket (e.g. s3:ListBucket), not just delete. Original error: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  const derivativeKeys: string[] = [];
+  for (const candidate of candidates) {
+    if (!isDerivativeKeyOf(originalKey, candidate)) continue;
+    try {
+      assertSafeStorageKey(candidate);
+    } catch {
+      // ponytail: a listing hit that fails the storage-key grammar can't be
+      // a real derivative (every derivative is written via a validated
+      // write()) — skip rather than pass a malformed key to delete().
+      continue;
+    }
+    derivativeKeys.push(candidate);
+  }
+  return derivativeKeys;
+}
+
+// Deletes the original key + every derivative. Returns the keys whose delete
+// threw — callers fail closed on a non-empty list (delete is idempotent, so a
+// retry on the next run converges). assertSafeStorageKey guards the original
+// AND every derivative right before the delete call, not just at listing time.
+export async function deleteStoredFileAndDerivatives(
+  originalKey: string,
+  store: StoredFileStore,
+  errorContext: string,
+): Promise<readonly string[]> {
+  assertSafeStorageKey(originalKey);
+  const keysToDelete = [
+    originalKey,
+    ...(await listDerivativeKeys(originalKey, store, errorContext)),
+  ];
+  const failedKeys: string[] = [];
+  for (const key of keysToDelete) {
+    try {
+      assertSafeStorageKey(key);
+      await store.delete(key);
+    } catch (err) {
+      failedKeys.push(key);
+      // biome-ignore lint/suspicious/noConsole: operator-visibility for binary-cleanup-failure
+      console.warn(
+        `[${errorContext}] storage delete failed key=${key} err=${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+  return failedKeys;
+}

package/src/derivatives/index.ts

@@ -1,3 +1,5 @@
+export type { StoredFileStore } from "./delete-stored-file";
+export { deleteStoredFileAndDerivatives, listDerivativeKeys } from "./delete-stored-file";
 export type { DerivativesContextDeps } from "./derivatives-context";
 export { createDerivativesContext, resolveRenderer } from "./derivatives-context";
 export { resolveFieldVariant } from "./field-variants";