@cosmicdrift/kumiko-framework 0.305.0 → 0.307.0

package/src/pipeline/dispatcher.ts

@@ -1,3 +1,5 @@
+import type { WriteOrigin } from "@cosmicdrift/kumiko-types/event-store-types";
+import { runWithWriteOrigin } from "../api/request-context";
 import type { SseBroker } from "../api/sse-broker";
 import type { buildEntityTable } from "../db/table-builder";
 import {
@@ -18,7 +20,7 @@ import type {
   WriteResult,
 } from "../engine/types";
 import type { TenantId } from "../engine/types/identifiers";
-import { reraiseAsKumikoError } from "../errors";
+import { InternalError, reraiseAsKumikoError } from "../errors";
 import { getFallbackMeter, getFallbackTracer, registerStandardMetrics } from "../observability";
 import { createEscapeHatchReportWindow } from "../observability/escape-hatch-report";
 import { INTERACTIVE_SIGN_IN_POLICY, resolveActiveMembershipFn } from "./active-membership";
@@ -32,7 +34,7 @@ import type { IdempotencyGuard } from "./idempotency";
 import type { LifecycleHooks } from "./lifecycle-pipeline";
 import { createMemberReaderFn } from "./member-reader";
 import { createTenantTimezoneCache } from "./tenant-timezone-cache";
-import { rootWriteOrigin } from "./write-origin";
+import { effectiveWriteOrigin, isPersonalDataGated, rootWriteOrigin } from "./write-origin";
 
 // Re-export for callers that reach for dispatcher-adjacent types (tests,
 // HTTP-layer stubs) — dispatch consumes these, grouping the type-surface
@@ -111,12 +113,55 @@ export type Dispatcher = {
   createMemberReader(tenantId: TenantId): MemberReader;
 };
 
+// Kept off the public Dispatcher type: only a job may pass an inherited origin.
+type DispatcherInternals = {
+  writeWithOrigin: (
+    type: string,
+    payload: unknown,
+    user: SessionUser,
+    inheritedOrigin: WriteOrigin,
+  ) => Promise<WriteResult>;
+  queryWithOrigin: (
+    type: string,
+    payload: unknown,
+    user: SessionUser,
+    inheritedOrigin: WriteOrigin,
+  ) => Promise<unknown>;
+};
+
+const dispatcherInternals = new WeakMap<Dispatcher, DispatcherInternals>();
+
 // Adapts Dispatcher's (type, payload, user) call shape to DispatchWriteRef's
 // (user, qn, payload) — JobRunner.attachDispatcher needs the latter.
 export function dispatcherToWriteRef(dispatcher: Dispatcher): DispatchWriteRef {
+  const internals = dispatcherInternals.get(dispatcher);
   return {
-    write: (user, qn, payload) => dispatcher.write(qn, payload, user),
-    queryAs: (user, qn, payload) => dispatcher.query(qn, payload, user),
+    write: (user, qn, payload, inheritedOrigin) => {
+      if (inheritedOrigin && isPersonalDataGated(inheritedOrigin)) {
+        if (!internals) {
+          throw new InternalError({
+            message:
+              `JobContext.write("${qn}") carries a gated origin but this dispatcher has no ` +
+              "registered origin-aware internals — refusing to fall open to the ungated path.",
+          });
+        }
+        return internals.writeWithOrigin(qn, payload, user, inheritedOrigin);
+      }
+      return dispatcher.write(qn, payload, user);
+    },
+    queryAs: (user, qn, payload, inheritedOrigin) => {
+      if (inheritedOrigin && isPersonalDataGated(inheritedOrigin)) {
+        if (!internals) {
+          throw new InternalError({
+            message:
+              `JobContext.queryAs("${qn}") carries a gated origin but this dispatcher has no ` +
+              "registered origin-aware internals — refusing to fall open to the ungated path.",
+          });
+        }
+        return internals.queryWithOrigin(qn, payload, user, inheritedOrigin);
+      }
+      return dispatcher.query(qn, payload, user);
+    },
     createMemberReader: (tenantId) => dispatcher.createMemberReader(tenantId),
   };
 }
@@ -173,7 +218,7 @@ export function createDispatcher(
     membershipQuery,
   };
 
-  return {
+  const dispatcher: Dispatcher = {
     async write(typeOrRef, payload, user, requestId?) {
       const type = resolveType(typeOrRef);
       // Idempotency handled inside runBatch (caches BatchResult under requestId).
@@ -185,7 +230,8 @@ export function createDispatcher(
 
     query: (typeOrRef, payload, user) => {
       const type = resolveType(typeOrRef);
-      return executeQuery(ctx, type, payload, user, rootWriteOrigin(registry, type, user));
+      const origin = rootWriteOrigin(registry, type, user);
+      return runWithWriteOrigin(origin, () => executeQuery(ctx, type, payload, user, origin));
     },
 
     stream: (typeOrRef, payload, user) => {
@@ -210,4 +256,23 @@ export function createDispatcher(
 
     createMemberReader: (tenantId) => createMemberReaderFn(ctx, tenantId),
   };
+
+  dispatcherInternals.set(dispatcher, {
+    writeWithOrigin: async (type, payload, user, inheritedOrigin) => {
+      const batchResult = await runBatch(
+        ctx,
+        [{ type, payload }],
+        user,
+        undefined,
+        inheritedOrigin,
+      );
+      return unwrapSingle(batchResult);
+    },
+    queryWithOrigin: (type, payload, user, inheritedOrigin) => {
+      const origin = effectiveWriteOrigin(rootWriteOrigin(registry, type, user), inheritedOrigin);
+      return runWithWriteOrigin(origin, () => executeQuery(ctx, type, payload, user, origin));
+    },
+  });
+
+  return dispatcher;
 }

package/src/pipeline/event-consumer-state.ts

@@ -5,6 +5,7 @@ import {
   index,
   instant,
   integer,
+  jsonb,
   table as pgTable,
   primaryKey,
   sql,
@@ -64,6 +65,21 @@ export const SHARED_INSTANCE_SENTINEL = "__shared__";
 // The default(sql`0`) on lastProcessedEventId mirrors projection-state.ts:
 // drizzle-kit's JSON snapshot generator can't serialise a bigint literal, so
 // the server-side default is specified as raw SQL instead of .default(0n).
+
+// A contiguous run of ids below the cursor invisible on some earlier turn,
+// not yet resolved (delivered) or proven burnt (see event-dispatcher.ts's
+// processConsumer). Ranges, not individual ids — a large historical gap
+// (retention prune, or a new consumer starting "beginning" over pruned
+// history) then costs O(1) entries instead of one per missing id. xmax is
+// the pg_snapshot_xmax() captured when the range was recorded: once a later
+// turn's xmin passes it, every xact that could still produce a row in this
+// range has finished, so it's provably rolled back, not just slow.
+// bigint/xid8 travel as strings — JS bigint doesn't round-trip through jsonb.
+export type PendingGapEntry = {
+  readonly from: string;
+  readonly to: string;
+  readonly xmax: string;
+};
 export const eventConsumerStateTable = pgTable(
   "kumiko_event_consumers",
   {
@@ -82,6 +98,7 @@ export const eventConsumerStateTable = pgTable(
     // poisoned), as does a manual restartConsumer()/enableConsumer()/
     // skipPoisonEvent() — an operator vouching the consumer is healthy.
     rearmCount: integer("rearm_count").notNull().default(0),
+    pendingGaps: jsonb("pending_gaps").$type<PendingGapEntry[]>().default([]).notNull(),
     lastError: text("last_error"),
     updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
   },
@@ -142,6 +159,15 @@ export async function createEventConsumerStateTable(db: DbConnection): Promise<v
       " NOT NULL",
       /* ifNotExists */ true,
     );
+    await alterTableAddColumn(
+      db,
+      "kumiko_event_consumers",
+      "pending_gaps",
+      "jsonb",
+      " DEFAULT '[]'::jsonb",
+      " NOT NULL",
+      /* ifNotExists */ true,
+    );
     // Runs on every boot, including mid-rolling-deploy: a new pod can delete
     // an old pod's still-in-use per-instance row out from under it. That old
     // pod's next acquireConsumer then finds no row, delivers no more SSE/

package/src/pipeline/event-dispatcher-admin.ts

@@ -1,13 +1,15 @@
 import type { DbConnection, DbTx } from "../db/connection";
 import {
   advanceConsumerPastEventReturning,
+  removePendingGapReturning,
   updateConsumerStatusReturning,
 } from "../db/queries/event-consumer";
-import { selectNextEventIdAfter } from "../db/queries/event-store";
+import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges } from "../db/queries/event-store";
 import { coerceRow, extractTableInfo, selectMany } from "../db/query";
 import { getEventsHighWaterMark } from "../event-store";
 import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL } from "./event-consumer-state";
 import type { ConsumerStateRow, ConsumerStateRowShape } from "./event-dispatcher-delivery";
+import { rangeContainsId, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges";
 
 // --- Ops recovery surface ---
 //
@@ -129,10 +131,13 @@ export async function enableConsumer(
   return applyConsumerStatusTransition(db, name, instanceId, "idle");
 }
 
-// skipPoisonEvent advances the cursor past the first event after the
-// current cursor. Single TX so concurrent dispatcher passes can't double-
-// advance. If no event exists past the cursor, there is nothing to skip —
-// treat as idempotent no-op (cursor already at head).
+// skipPoisonEvent advances past whatever is currently blocking the consumer.
+// If the smallest visible id inside pending_gaps exists, THAT'S the poison —
+// halt-on-poison walks gap ids before any id past the cursor — so it's split
+// out of its range directly; the cursor stays put (it's already above this
+// id). Otherwise the poison is the usual next event after the cursor. Single
+// TX so concurrent dispatcher passes can't double-advance. Neither exists →
+// idempotent no-op.
 export async function skipPoisonEvent(
   db: DbConnection,
   name: string,
@@ -140,6 +145,28 @@ export async function skipPoisonEvent(
 ): Promise<ConsumerRecoveryState & { readonly skippedEventId: bigint | null }> {
   const before = await requireConsumerRow(db, name, instanceId);
   return db.begin(async (tx: DbTx) => {
+    const pendingGaps = before.pendingGaps;
+    const smallestVisiblePending = await selectSmallestVisibleIdInRanges(
+      tx,
+      toIdRanges(pendingGaps),
+    );
+
+    if (smallestVisiblePending !== null) {
+      const newPendingGaps = pendingGaps.flatMap((gap) =>
+        rangeContainsId(gap, smallestVisiblePending)
+          ? splitRangeExcludingIds(gap, [smallestVisiblePending])
+          : [gap],
+      );
+      const raw = await removePendingGapReturning(tx, name, instanceId, newPendingGaps);
+      const updated =
+        raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
+      if (!updated)
+        throw new Error(
+          `Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`,
+        );
+      return { ...normalizeConsumerState(updated), skippedEventId: smallestVisiblePending };
+    }
+
     const poisonId = await selectNextEventIdAfter(tx, before.lastProcessedEventId);
     if (poisonId === null) {
       const [unchanged] = await selectMany<ConsumerStateRow>(tx, eventConsumerStateTable, {

package/src/pipeline/event-dispatcher-delivery.ts

@@ -12,8 +12,12 @@ import {
   selectConsumerForUpdateSkipLocked,
   updateConsumerDeliveryOutcome,
 } from "../db/queries/event-consumer";
-import { selectEventsHeadId } from "../db/queries/event-store";
-import { coerceRow, extractTableInfo, selectMany } from "../db/query";
+import {
+  type PendingIdRange,
+  selectEventsHeadId,
+  selectPendingAndNewEventRows,
+} from "../db/queries/event-store";
+import { coerceRow, extractTableInfo } from "../db/query";
 import { qnScope } from "../engine/qualified-name";
 import type { AppContext } from "../engine/types";
 import { eventsTable, toStoredEvent as rowToStoredEvent } from "../event-store";
@@ -26,9 +30,18 @@ import {
 import {
   ConsumerStatuses,
   eventConsumerStateTable,
+  type PendingGapEntry,
   SHARED_INSTANCE_SENTINEL,
 } from "./event-consumer-state";
 import type { EventConsumer } from "./event-dispatcher";
+import { parseWriteOrigin } from "./write-origin";
+
+// Fails closed without throwing: a throw would poison the event for every consumer.
+const UNPARSEABLE_STORED_WRITE_ORIGIN = {
+  rootHandler: "<unknown>",
+  anonymousRoot: true,
+  publicIntake: false,
+} as const;
 
 // Per-consumer pass mechanics: acquire the state row, fetch pending events,
 // hand them to the consumer's handler in order, persist the outcome. Split
@@ -46,6 +59,7 @@ export type ConsumerStateRowShape = {
   readonly status: string;
   readonly attempts: number;
   readonly rearmCount: number;
+  readonly pendingGaps: readonly PendingGapEntry[];
   readonly lastError: string | null;
   readonly updatedAt: Temporal.Instant;
 };
@@ -186,17 +200,20 @@ export async function markProcessing(tx: DbTx, name: string, instanceId: string)
   await markConsumerProcessing(tx, name, instanceId);
 }
 
+// `pendingRanges` are id ranges below `cursor` the consumer is still
+// watching as gaps (invisible on an earlier turn — see event-dispatcher.ts's
+// processConsumer). Fetching them alongside the plain `id > cursor` window
+// means a row that committed late becomes visible and deliverable the next
+// time this consumer's turn runs, instead of being permanently skipped.
 export async function fetchPendingEvents(
   tx: DbTx,
   cursor: bigint,
   batchSize: number,
+  pendingRanges: readonly PendingIdRange[] = [],
 ): Promise<ReadonlyArray<StoredEventRow>> {
-  return (await selectMany(
-    tx,
-    eventsTable,
-    { id: { gt: cursor } },
-    { orderBy: { col: "id", direction: "asc" }, limit: batchSize },
-  )) as ReadonlyArray<StoredEventRow>; // @cast-boundary db-row
+  const rawRows = await selectPendingAndNewEventRows(tx, cursor, pendingRanges, batchSize);
+  const info = extractTableInfo(eventsTable);
+  return rawRows.map((row) => coerceRow(row, info) as StoredEventRow); // @cast-boundary db-row
 }
 
 export type DeliveryOutcome = {
@@ -206,6 +223,11 @@ export type DeliveryOutcome = {
   readonly deadLettered: boolean;
   readonly processed: number;
   readonly failed: number;
+  // Which of the *pending* ids in `events` (id <= the cursor this delivery
+  // started from) got resolved this pass — delivered or skip-applied. The
+  // caller (event-dispatcher.ts) splits exactly these out of pending_gaps;
+  // everything else in the input batch was a "new" row past the old cursor.
+  readonly resolvedPendingIds: readonly bigint[];
 };
 
 // Deliver events to the consumer's handler in events.id order. Halt-on-
@@ -221,83 +243,113 @@ export async function deliverEvents(
   maxAttempts: number,
   state: ConsumerStateRow,
 ): Promise<DeliveryOutcome> {
-  let cursor = state.lastProcessedEventId;
+  const startCursor = state.lastProcessedEventId;
+  let cursor = startCursor;
   let attempts = state.attempts;
   let lastError: string | null = state.lastError ?? null;
   let deadLettered = false;
   const effectiveMaxAttempts = consumer.errorPolicy?.maxAttempts ?? maxAttempts;
   let processed = 0;
   let failed = 0;
+  const resolvedPendingIds: bigint[] = [];
+
+  // A pending row sits below startCursor: it resolves its gap but never moves
+  // the cursor backward. ORDER BY id walks all pending rows first.
+  const resolve = (id: bigint): void => {
+    if (id > cursor) cursor = id;
+    if (id <= startCursor) resolvedPendingIds.push(id);
+    attempts = 0;
+    lastError = null;
+  };
 
   for (const row of events) {
     try {
-      // Propagate causation: if the handler calls ctx.appendEvent, the new
-      // event should record THIS event as its cause. correlationId is
-      // inherited unchanged — it survives the hop across streams by design.
-      // requestId falls back to a fresh id because the dispatcher runs
-      // outside any HTTP request (background poll), and a stable log-
-      // correlation handle is still useful for debugging.
-      const stored = rowToStoredEvent(row);
-      const correlationId = stored.metadata.correlationId ?? requestContext.generateId();
-      const causationId = String(stored.id);
-      const requestId = requestContext.generateId();
-      // #3043 — an event this apply writes is attributed to the consumer, not
-      // to whatever wrote the triggering event; causationId already links back.
-      await requestContext.run(
-        {
-          requestId,
-          correlationId,
-          causationId,
-          handler: consumer.name,
-          feature: consumer.featureName ?? qnScope(consumer.name),
-        },
-        async () => {
-          await consumer.handler(stored, context);
-        },
-      );
-      cursor = row.id;
-      attempts = 0;
-      lastError = null;
+      await applyEvent(consumer, row, context);
+      resolve(row.id);
       processed += 1;
     } catch (e) {
       const errMessage = e instanceof Error ? e.message : String(e);
+      failed += 1;
       if (consumer.errorPolicy?.skipApplyErrors) {
-        // Best-effort mode: record the error on the skip counter so ops
-        // can alert on a spike of skipped events, advance the cursor past
-        // the bad event, keep going. The consumer stays "idle", not "dead".
-        // Also emit a warn-level log line — the metric tells ops THAT events
-        // are being dropped, the log tells them WHICH events. Without this
-        // a poisoned-then-skipped event is invisible to forensic search.
-        const errorClass = e instanceof Error ? e.constructor.name : "UnknownError";
-        emitDispatcherError(context.meter ?? getFallbackMeter(), {
-          handler: consumer.name,
-          errorClass,
-        });
-        context.log?.warn(
-          `event-dispatcher: ${consumer.name} skipped event ${row.id} (${errorClass}): ${errMessage}`,
-        );
-        cursor = row.id;
-        attempts = 0;
-        lastError = null;
-        failed += 1;
+        reportSkippedEvent(consumer, row.id, e, errMessage, context);
+        resolve(row.id);
         continue;
       }
       attempts += 1;
       lastError = errMessage;
-      failed += 1;
       if (attempts >= effectiveMaxAttempts) deadLettered = true;
       break;
     }
   }
 
-  return { cursor, attempts, lastError, deadLettered, processed, failed };
+  return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
+}
+
+async function applyEvent(
+  consumer: EventConsumer,
+  row: StoredEventRow,
+  context: AppContext,
+): Promise<void> {
+  // Propagate causation: if the handler calls ctx.appendEvent, the new
+  // event should record THIS event as its cause. correlationId is
+  // inherited unchanged — it survives the hop across streams by design.
+  // requestId falls back to a fresh id because the dispatcher runs
+  // outside any HTTP request (background poll), and a stable log-
+  // correlation handle is still useful for debugging.
+  const stored = rowToStoredEvent(row);
+  const correlationId = stored.metadata.correlationId ?? requestContext.generateId();
+  const causationId = String(stored.id);
+  const requestId = requestContext.generateId();
+  // The job-trigger consumer's handleEvent stamps event-triggered jobs from this.
+  const rawStoredWriteOrigin = stored.metadata.writeOrigin;
+  const writeOrigin =
+    rawStoredWriteOrigin === undefined
+      ? undefined
+      : (parseWriteOrigin(rawStoredWriteOrigin) ?? UNPARSEABLE_STORED_WRITE_ORIGIN);
+  // #3043 — an event this apply writes is attributed to the consumer, not
+  // to whatever wrote the triggering event; causationId already links back.
+  await requestContext.run(
+    {
+      requestId,
+      correlationId,
+      causationId,
+      handler: consumer.name,
+      feature: consumer.featureName ?? qnScope(consumer.name),
+      writeOrigin,
+    },
+    async () => {
+      await consumer.handler(stored, context);
+    },
+  );
+}
+
+// Best-effort mode: record the error on the skip counter so ops can alert on
+// a spike of skipped events; the consumer stays "idle", not "dead". The
+// warn-level log line tells them WHICH events — without it a
+// poisoned-then-skipped event is invisible to forensic search.
+function reportSkippedEvent(
+  consumer: EventConsumer,
+  eventId: bigint,
+  e: unknown,
+  errMessage: string,
+  context: AppContext,
+): void {
+  const errorClass = e instanceof Error ? e.constructor.name : "UnknownError";
+  emitDispatcherError(context.meter ?? getFallbackMeter(), { handler: consumer.name, errorClass });
+  context.log?.warn(
+    `event-dispatcher: ${consumer.name} skipped event ${eventId} (${errorClass}): ${errMessage}`,
+  );
 }
 
+export type PersistedConsumerOutcome = DeliveryOutcome & {
+  readonly pendingGaps: readonly PendingGapEntry[];
+};
+
 export async function persistConsumerOutcome(
   tx: DbTx,
   name: string,
   instanceId: string,
-  outcome: DeliveryOutcome,
+  outcome: PersistedConsumerOutcome,
 ): Promise<void> {
   await updateConsumerDeliveryOutcome(tx, name, instanceId, outcome);
 }