@cosmicdrift/kumiko-framework 0.306.0 → 0.308.0

package/src/pipeline/__tests__/unsafe-raw-escape-hatch.integration.test.ts

@@ -4,7 +4,7 @@
 // escape-hatch.integration.test.ts (fw#2855).
 
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import { z } from "zod";
+import * as z from "zod";
 import { executeRawQuery } from "../../db/queries/raw-sql";
 import { defineFeature } from "../../engine";
 import { createTestUser, setupTestStack, type TestStack } from "../../stack";

package/src/pipeline/dispatch-batch.ts

@@ -1,4 +1,5 @@
-import { requestContext } from "../api/request-context";
+import type { WriteOrigin } from "@cosmicdrift/kumiko-types/event-store-types";
+import { requestContext, runWithWriteOrigin } from "../api/request-context";
 import type { DbConnection } from "../db/connection";
 import { transaction } from "../db/query";
 import type { DeleteContext, SaveContext, SessionUser, WriteResult } from "../engine/types";
@@ -14,7 +15,20 @@ import {
   isLifecycleResult,
   wrapToKumiko,
 } from "./dispatcher-utils";
-import { rootWriteOrigin } from "./write-origin";
+import { effectiveWriteOrigin, isPersonalDataGated, rootWriteOrigin } from "./write-origin";
+
+// afterCommit hooks fire in flushAfterCommit, outside the command's scope.
+function rewrapHooksWithOrigin(
+  afterCommitHooks: AfterCommitHook[],
+  fromIndex: number,
+  origin: WriteOrigin,
+): void {
+  for (let i = fromIndex; i < afterCommitHooks.length; i++) {
+    const original = afterCommitHooks[i];
+    if (!original) continue;
+    afterCommitHooks[i] = () => runWithWriteOrigin(origin, original);
+  }
+}
 
 // Core batch logic extracted so write() and command() can reuse it
 // (a single write = batch of one, running in its own transaction).
@@ -23,15 +37,18 @@ export async function runBatch(
   commands: readonly BatchCommand[],
   user: SessionUser,
   requestId?: string,
+  inheritedOrigin?: WriteOrigin,
 ): Promise<BatchResult> {
   const current = requestContext.get();
   if (!current?.signal) {
-    return runBatchBody(ctx, commands, user, requestId);
+    return runBatchBody(ctx, commands, user, requestId, inheritedOrigin);
   }
   // Strip the signal: a disconnect would roll back the tx, idempotency would
   // cache a 500 for the uncommitted write and afterCommit effects would be lost.
   const { signal: _signal, ...withoutSignal } = current;
-  return requestContext.run(withoutSignal, () => runBatchBody(ctx, commands, user, requestId));
+  return requestContext.run(withoutSignal, () =>
+    runBatchBody(ctx, commands, user, requestId, inheritedOrigin),
+  );
 }
 
 async function runBatchBody(
@@ -39,6 +56,7 @@ async function runBatchBody(
   commands: readonly BatchCommand[],
   user: SessionUser,
   requestId?: string,
+  inheritedOrigin?: WriteOrigin,
 ): Promise<BatchResult> {
   const { idempotency, lifecycle, appContext: context } = ctx;
   if (commands.length === 0) {
@@ -109,10 +127,12 @@ async function runBatchBody(
     }
   };
 
+  const origins: WriteOrigin[] = [];
+
   // Fires the batch-level system hooks with every successful save/delete
   // context from this run. Called after flushAfterCommit so per-save hooks
   // have all completed first; errors are isolated inside lifecycleHooks.
-  const flushBatchHooks = async () => {
+  const flushBatchHooksInner = async () => {
     try {
       const saves: SaveContext[] = [];
       const deletes: DeleteContext[] = [];
@@ -133,6 +153,14 @@ async function runBatchBody(
     }
   };
 
+  // Batch hooks see every command's saves, so they run under the strictest origin.
+  const flushBatchHooks = async () => {
+    const strictest = origins.find(isPersonalDataGated) ?? origins[0];
+    // skip: no command ran, so batch hooks have no saves or deletes to see
+    if (!strictest) return;
+    await runWithWriteOrigin(strictest, flushBatchHooksInner);
+  };
+
   // batch() opens its own outer transaction — needs the top-level
   // connection's `.begin()` (TransactionSql exposes only `.savepoint()`).
   const db = resolveDbSource(ctx, undefined) as DbConnection | undefined;
@@ -143,15 +171,16 @@ async function runBatchBody(
     for (let i = 0; i < commands.length; i++) {
       const cmd = commands[i];
       if (!cmd) continue;
-      const res = await executeNestedWrite(
-        ctx,
-        cmd.type,
-        cmd.payload,
-        user,
+      const origin = effectiveWriteOrigin(
         rootWriteOrigin(ctx.registry, cmd.type, user),
-        undefined,
-        afterCommitHooks,
+        inheritedOrigin,
       );
+      origins.push(origin);
+      const hookStart = afterCommitHooks.length;
+      const res = await runWithWriteOrigin(origin, () =>
+        executeNestedWrite(ctx, cmd.type, cmd.payload, user, origin, undefined, afterCommitHooks),
+      );
+      rewrapHooksWithOrigin(afterCommitHooks, hookStart, origin);
       results.push(res);
       if (!res.isSuccess) {
         // No tx means no rollback — but we still drop afterCommit hooks,
@@ -170,15 +199,16 @@ async function runBatchBody(
       for (let i = 0; i < commands.length; i++) {
         const cmd = commands[i];
         if (!cmd) continue;
-        const res = await executeNestedWrite(
-          ctx,
-          cmd.type,
-          cmd.payload,
-          user,
+        const origin = effectiveWriteOrigin(
           rootWriteOrigin(ctx.registry, cmd.type, user),
-          tx,
-          afterCommitHooks,
+          inheritedOrigin,
+        );
+        origins.push(origin);
+        const hookStart = afterCommitHooks.length;
+        const res = await runWithWriteOrigin(origin, () =>
+          executeNestedWrite(ctx, cmd.type, cmd.payload, user, origin, tx, afterCommitHooks),
         );
+        rewrapHooksWithOrigin(afterCommitHooks, hookStart, origin);
         results.push(res);
         if (!res.isSuccess) {
           throw new BatchRollback(i, res.error);

package/src/pipeline/dispatch-stream.ts

@@ -1,3 +1,4 @@
+import type { AccessInvalidationCredential } from "../api/sse-broker";
 import { hasAccess } from "../engine/access";
 import type { SessionUser } from "../engine/types";
 import {
@@ -74,9 +75,17 @@ async function* executeStreamInner(
   const invalidated = new Promise<void>((resolve) => {
     resolveInvalidated = resolve;
   });
-  const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(user.id, () => {
-    resolveInvalidated?.();
-  });
+  const streamCredential: AccessInvalidationCredential = {
+    ...(user.sid !== undefined && { sid: user.sid }),
+    ...(user.pat?.tokenId !== undefined && { patTokenId: user.pat.tokenId }),
+  };
+  const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(
+    user.id,
+    () => {
+      resolveInvalidated?.();
+    },
+    streamCredential,
+  );
 
   let iterator: AsyncIterator<unknown> | undefined;
   // When access is revoked mid-pull, `iterator.next()` is still in flight.

package/src/pipeline/dispatcher-utils.ts

@@ -6,7 +6,14 @@ import type {
   SessionUser,
   WriteResult,
 } from "../engine/types";
-import { type FieldIssue, toKumikoError, type WriteErrorInfo } from "../errors";
+import {
+  type FieldIssue,
+  type KumikoError,
+  toKumikoError,
+  VersionConflictError,
+  type WriteErrorInfo,
+} from "../errors";
+import { VersionConflictError as EventStoreVersionConflictError } from "../event-store/errors";
 
 export type FailedWriteResult = Extract<WriteResult, { isSuccess: false }>;
 
@@ -177,4 +184,16 @@ export function resolveType(type: HandlerType): string {
   return typeof type === "string" ? type : type.name;
 }
 
-export const wrapToKumiko = toKumikoError;
+// A custom write losing an append race (ctx.appendEvent, stream.append) is a
+// retryable 409 like the CRUD executor's, not an internal error. Mapped here
+// because errors/ is client code and must not import the event store.
+// currentVersion -1 is the executor's "not looked up" sentinel.
+export function wrapToKumiko(e: unknown): KumikoError {
+  if (e instanceof EventStoreVersionConflictError) {
+    return new VersionConflictError(
+      { entityId: e.aggregateId, expectedVersion: e.expectedVersion, currentVersion: -1 },
+      { cause: e },
+    );
+  }
+  return toKumikoError(e);
+}

package/src/pipeline/dispatcher.ts

@@ -1,3 +1,5 @@
+import type { WriteOrigin } from "@cosmicdrift/kumiko-types/event-store-types";
+import { runWithWriteOrigin } from "../api/request-context";
 import type { SseBroker } from "../api/sse-broker";
 import type { buildEntityTable } from "../db/table-builder";
 import {
@@ -18,7 +20,7 @@ import type {
   WriteResult,
 } from "../engine/types";
 import type { TenantId } from "../engine/types/identifiers";
-import { reraiseAsKumikoError } from "../errors";
+import { InternalError, reraiseAsKumikoError } from "../errors";
 import { getFallbackMeter, getFallbackTracer, registerStandardMetrics } from "../observability";
 import { createEscapeHatchReportWindow } from "../observability/escape-hatch-report";
 import { INTERACTIVE_SIGN_IN_POLICY, resolveActiveMembershipFn } from "./active-membership";
@@ -32,7 +34,7 @@ import type { IdempotencyGuard } from "./idempotency";
 import type { LifecycleHooks } from "./lifecycle-pipeline";
 import { createMemberReaderFn } from "./member-reader";
 import { createTenantTimezoneCache } from "./tenant-timezone-cache";
-import { rootWriteOrigin } from "./write-origin";
+import { effectiveWriteOrigin, isPersonalDataGated, rootWriteOrigin } from "./write-origin";
 
 // Re-export for callers that reach for dispatcher-adjacent types (tests,
 // HTTP-layer stubs) — dispatch consumes these, grouping the type-surface
@@ -111,12 +113,55 @@ export type Dispatcher = {
   createMemberReader(tenantId: TenantId): MemberReader;
 };
 
+// Kept off the public Dispatcher type: only a job may pass an inherited origin.
+type DispatcherInternals = {
+  writeWithOrigin: (
+    type: string,
+    payload: unknown,
+    user: SessionUser,
+    inheritedOrigin: WriteOrigin,
+  ) => Promise<WriteResult>;
+  queryWithOrigin: (
+    type: string,
+    payload: unknown,
+    user: SessionUser,
+    inheritedOrigin: WriteOrigin,
+  ) => Promise<unknown>;
+};
+
+const dispatcherInternals = new WeakMap<Dispatcher, DispatcherInternals>();
+
 // Adapts Dispatcher's (type, payload, user) call shape to DispatchWriteRef's
 // (user, qn, payload) — JobRunner.attachDispatcher needs the latter.
 export function dispatcherToWriteRef(dispatcher: Dispatcher): DispatchWriteRef {
+  const internals = dispatcherInternals.get(dispatcher);
   return {
-    write: (user, qn, payload) => dispatcher.write(qn, payload, user),
-    queryAs: (user, qn, payload) => dispatcher.query(qn, payload, user),
+    write: (user, qn, payload, inheritedOrigin) => {
+      if (inheritedOrigin && isPersonalDataGated(inheritedOrigin)) {
+        if (!internals) {
+          throw new InternalError({
+            message:
+              `JobContext.write("${qn}") carries a gated origin but this dispatcher has no ` +
+              "registered origin-aware internals — refusing to fall open to the ungated path.",
+          });
+        }
+        return internals.writeWithOrigin(qn, payload, user, inheritedOrigin);
+      }
+      return dispatcher.write(qn, payload, user);
+    },
+    queryAs: (user, qn, payload, inheritedOrigin) => {
+      if (inheritedOrigin && isPersonalDataGated(inheritedOrigin)) {
+        if (!internals) {
+          throw new InternalError({
+            message:
+              `JobContext.queryAs("${qn}") carries a gated origin but this dispatcher has no ` +
+              "registered origin-aware internals — refusing to fall open to the ungated path.",
+          });
+        }
+        return internals.queryWithOrigin(qn, payload, user, inheritedOrigin);
+      }
+      return dispatcher.query(qn, payload, user);
+    },
     createMemberReader: (tenantId) => dispatcher.createMemberReader(tenantId),
   };
 }
@@ -173,7 +218,7 @@ export function createDispatcher(
     membershipQuery,
   };
 
-  return {
+  const dispatcher: Dispatcher = {
     async write(typeOrRef, payload, user, requestId?) {
       const type = resolveType(typeOrRef);
       // Idempotency handled inside runBatch (caches BatchResult under requestId).
@@ -185,7 +230,8 @@ export function createDispatcher(
 
     query: (typeOrRef, payload, user) => {
       const type = resolveType(typeOrRef);
-      return executeQuery(ctx, type, payload, user, rootWriteOrigin(registry, type, user));
+      const origin = rootWriteOrigin(registry, type, user);
+      return runWithWriteOrigin(origin, () => executeQuery(ctx, type, payload, user, origin));
     },
 
     stream: (typeOrRef, payload, user) => {
@@ -210,4 +256,23 @@ export function createDispatcher(
 
     createMemberReader: (tenantId) => createMemberReaderFn(ctx, tenantId),
   };
+
+  dispatcherInternals.set(dispatcher, {
+    writeWithOrigin: async (type, payload, user, inheritedOrigin) => {
+      const batchResult = await runBatch(
+        ctx,
+        [{ type, payload }],
+        user,
+        undefined,
+        inheritedOrigin,
+      );
+      return unwrapSingle(batchResult);
+    },
+    queryWithOrigin: (type, payload, user, inheritedOrigin) => {
+      const origin = effectiveWriteOrigin(rootWriteOrigin(registry, type, user), inheritedOrigin);
+      return runWithWriteOrigin(origin, () => executeQuery(ctx, type, payload, user, origin));
+    },
+  });
+
+  return dispatcher;
 }

package/src/pipeline/event-consumer-state.ts

@@ -5,6 +5,7 @@ import {
   index,
   instant,
   integer,
+  jsonb,
   table as pgTable,
   primaryKey,
   sql,
@@ -64,6 +65,21 @@ export const SHARED_INSTANCE_SENTINEL = "__shared__";
 // The default(sql`0`) on lastProcessedEventId mirrors projection-state.ts:
 // drizzle-kit's JSON snapshot generator can't serialise a bigint literal, so
 // the server-side default is specified as raw SQL instead of .default(0n).
+
+// A contiguous run of ids below the cursor invisible on some earlier turn,
+// not yet resolved (delivered) or proven burnt (see event-dispatcher.ts's
+// processConsumer). Ranges, not individual ids — a large historical gap
+// (retention prune, or a new consumer starting "beginning" over pruned
+// history) then costs O(1) entries instead of one per missing id. xmax is
+// the pg_snapshot_xmax() captured when the range was recorded: once a later
+// turn's xmin passes it, every xact that could still produce a row in this
+// range has finished, so it's provably rolled back, not just slow.
+// bigint/xid8 travel as strings — JS bigint doesn't round-trip through jsonb.
+export type PendingGapEntry = {
+  readonly from: string;
+  readonly to: string;
+  readonly xmax: string;
+};
 export const eventConsumerStateTable = pgTable(
   "kumiko_event_consumers",
   {
@@ -82,6 +98,7 @@ export const eventConsumerStateTable = pgTable(
     // poisoned), as does a manual restartConsumer()/enableConsumer()/
     // skipPoisonEvent() — an operator vouching the consumer is healthy.
     rearmCount: integer("rearm_count").notNull().default(0),
+    pendingGaps: jsonb("pending_gaps").$type<PendingGapEntry[]>().default([]).notNull(),
     lastError: text("last_error"),
     updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
   },
@@ -142,6 +159,15 @@ export async function createEventConsumerStateTable(db: DbConnection): Promise<v
       " NOT NULL",
       /* ifNotExists */ true,
     );
+    await alterTableAddColumn(
+      db,
+      "kumiko_event_consumers",
+      "pending_gaps",
+      "jsonb",
+      " DEFAULT '[]'::jsonb",
+      " NOT NULL",
+      /* ifNotExists */ true,
+    );
     // Runs on every boot, including mid-rolling-deploy: a new pod can delete
     // an old pod's still-in-use per-instance row out from under it. That old
     // pod's next acquireConsumer then finds no row, delivers no more SSE/

package/src/pipeline/event-dispatcher-admin.ts

@@ -1,13 +1,15 @@
 import type { DbConnection, DbTx } from "../db/connection";
 import {
   advanceConsumerPastEventReturning,
+  removePendingGapReturning,
   updateConsumerStatusReturning,
 } from "../db/queries/event-consumer";
-import { selectNextEventIdAfter } from "../db/queries/event-store";
+import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges } from "../db/queries/event-store";
 import { coerceRow, extractTableInfo, selectMany } from "../db/query";
 import { getEventsHighWaterMark } from "../event-store";
 import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL } from "./event-consumer-state";
 import type { ConsumerStateRow, ConsumerStateRowShape } from "./event-dispatcher-delivery";
+import { rangeContainsId, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges";
 
 // --- Ops recovery surface ---
 //
@@ -129,10 +131,13 @@ export async function enableConsumer(
   return applyConsumerStatusTransition(db, name, instanceId, "idle");
 }
 
-// skipPoisonEvent advances the cursor past the first event after the
-// current cursor. Single TX so concurrent dispatcher passes can't double-
-// advance. If no event exists past the cursor, there is nothing to skip —
-// treat as idempotent no-op (cursor already at head).
+// skipPoisonEvent advances past whatever is currently blocking the consumer.
+// If the smallest visible id inside pending_gaps exists, THAT'S the poison —
+// halt-on-poison walks gap ids before any id past the cursor — so it's split
+// out of its range directly; the cursor stays put (it's already above this
+// id). Otherwise the poison is the usual next event after the cursor. Single
+// TX so concurrent dispatcher passes can't double-advance. Neither exists →
+// idempotent no-op.
 export async function skipPoisonEvent(
   db: DbConnection,
   name: string,
@@ -140,6 +145,28 @@ export async function skipPoisonEvent(
 ): Promise<ConsumerRecoveryState & { readonly skippedEventId: bigint | null }> {
   const before = await requireConsumerRow(db, name, instanceId);
   return db.begin(async (tx: DbTx) => {
+    const pendingGaps = before.pendingGaps;
+    const smallestVisiblePending = await selectSmallestVisibleIdInRanges(
+      tx,
+      toIdRanges(pendingGaps),
+    );
+
+    if (smallestVisiblePending !== null) {
+      const newPendingGaps = pendingGaps.flatMap((gap) =>
+        rangeContainsId(gap, smallestVisiblePending)
+          ? splitRangeExcludingIds(gap, [smallestVisiblePending])
+          : [gap],
+      );
+      const raw = await removePendingGapReturning(tx, name, instanceId, newPendingGaps);
+      const updated =
+        raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
+      if (!updated)
+        throw new Error(
+          `Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`,
+        );
+      return { ...normalizeConsumerState(updated), skippedEventId: smallestVisiblePending };
+    }
+
     const poisonId = await selectNextEventIdAfter(tx, before.lastProcessedEventId);
     if (poisonId === null) {
       const [unchanged] = await selectMany<ConsumerStateRow>(tx, eventConsumerStateTable, {