@cosmicdrift/kumiko-framework 0.165.0 → 2.0.0

package/src/pipeline/__tests__/lifecycle-pipeline.test.ts

@@ -202,74 +202,78 @@ describe("runPostSave", () => {
   test("postSave errors don't throw — logged and continued", async () => {
     const calls: string[] = [];
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-
-    const registry = makeRegistry({
-      postSave: [
-        async () => {
-          throw new Error("feature-fail");
-        },
-      ],
-    });
-
-    const systemHooks: SystemHooks = {
-      postSave: [
-        {
-          name: "search",
-          priority: 1000,
-          fn: async () => {
-            calls.push("search-ran");
+    try {
+      const registry = makeRegistry({
+        postSave: [
+          async () => {
+            throw new Error("feature-fail");
           },
-        },
-      ],
-    };
-
-    const pipeline = createLifecycleHooks(registry, systemHooks);
-    // Should not throw
-    await pipeline.runPostSave("test:write:user", savectx, {});
+        ],
+      });
 
-    // System hook still ran despite feature hook failure
-    expect(calls).toContain("search-ran");
-    expect(consoleSpy).toHaveBeenCalled();
-    consoleSpy.mockRestore();
+      const systemHooks: SystemHooks = {
+        postSave: [
+          {
+            name: "search",
+            priority: 1000,
+            fn: async () => {
+              calls.push("search-ran");
+            },
+          },
+        ],
+      };
+
+      const pipeline = createLifecycleHooks(registry, systemHooks);
+      // Should not throw
+      await pipeline.runPostSave("test:write:user", savectx, {});
+
+      // System hook still ran despite feature hook failure
+      expect(calls).toContain("search-ran");
+      expect(consoleSpy).toHaveBeenCalled();
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 
   test("system hook failure doesn't block other system hooks", async () => {
     const calls: string[] = [];
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-
-    const registry = makeRegistry();
-
-    const systemHooks: SystemHooks = {
-      postSave: [
-        {
-          name: "search",
-          priority: 1000,
-          fn: async () => {
-            throw new Error("meili-down");
+    try {
+      const registry = makeRegistry();
+
+      const systemHooks: SystemHooks = {
+        postSave: [
+          {
+            name: "search",
+            priority: 1000,
+            fn: async () => {
+              throw new Error("meili-down");
+            },
           },
-        },
-        {
-          name: "sse",
-          priority: 1001,
-          fn: async () => {
-            calls.push("sse-ran");
+          {
+            name: "sse",
+            priority: 1001,
+            fn: async () => {
+              calls.push("sse-ran");
+            },
           },
-        },
-        {
-          name: "audit",
-          priority: 1002,
-          fn: async () => {
-            calls.push("audit-ran");
+          {
+            name: "audit",
+            priority: 1002,
+            fn: async () => {
+              calls.push("audit-ran");
+            },
           },
-        },
-      ],
-    };
+        ],
+      };
 
-    const pipeline = createLifecycleHooks(registry, systemHooks);
-    await pipeline.runPostSave("test:write:user", savectx, {});
+      const pipeline = createLifecycleHooks(registry, systemHooks);
+      await pipeline.runPostSave("test:write:user", savectx, {});
 
-    expect(calls).toEqual(["sse-ran", "audit-ran"]);
-    consoleSpy.mockRestore();
+      expect(calls).toEqual(["sse-ran", "audit-ran"]);
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 });
 
@@ -336,29 +340,32 @@ describe("runPostSave phase routing", () => {
 
   test("afterCommit phase: hook errors are logged, never thrown", async () => {
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-    const afterRan: string[] = [];
-    const feature = defineFeature("phases", (r) => {
-      r.entity("user", createEntity({ table: "Users", fields: {} }));
-      r.writeHandler("user", z.object({}), async () => ({ isSuccess: true, data: null }), {
-        access: { openToAll: true },
-      });
-      r.hook("postSave", "user", async () => {
-        throw new Error("afterCommit-boom");
+    try {
+      const afterRan: string[] = [];
+      const feature = defineFeature("phases", (r) => {
+        r.entity("user", createEntity({ table: "Users", fields: {} }));
+        r.writeHandler("user", z.object({}), async () => ({ isSuccess: true, data: null }), {
+          access: { openToAll: true },
+        });
+        r.hook("postSave", "user", async () => {
+          throw new Error("afterCommit-boom");
+        });
+        r.hook("postSave", "user", async () => {
+          afterRan.push("second");
+        });
       });
-      r.hook("postSave", "user", async () => {
-        afterRan.push("second");
-      });
-    });
-    const registry = createRegistry([feature]);
-    const pipeline = createLifecycleHooks(registry);
+      const registry = createRegistry([feature]);
+      const pipeline = createLifecycleHooks(registry);
 
-    // Must not throw — errors are swallowed + logged
-    await pipeline.runPostSave("phases:write:user", savectx, {}, "afterCommit");
+      // Must not throw — errors are swallowed + logged
+      await pipeline.runPostSave("phases:write:user", savectx, {}, "afterCommit");
 
-    // Subsequent hooks still fire (failure in one hook doesn't block the rest)
-    expect(afterRan).toEqual(["second"]);
-    expect(consoleSpy).toHaveBeenCalled();
-    consoleSpy.mockRestore();
+      // Subsequent hooks still fire (failure in one hook doesn't block the rest)
+      expect(afterRan).toEqual(["second"]);
+      expect(consoleSpy).toHaveBeenCalled();
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 
   test("system hooks respect their phase setting", async () => {
@@ -724,28 +731,31 @@ describe("runPostSaveBatch / runPostDeleteBatch", () => {
     ],
   ])("one %s hook throwing doesn't stop the others (Promise.allSettled) — logged, never thrown", async (_name, buildHooks, run) => {
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-    const calls: string[] = [];
-    const systemHooks = buildHooks([
-      {
-        name: "failing",
-        priority: 1000,
-        fn: async () => {
-          throw new Error("batch-hook-boom");
+    try {
+      const calls: string[] = [];
+      const systemHooks = buildHooks([
+        {
+          name: "failing",
+          priority: 1000,
+          fn: async () => {
+            throw new Error("batch-hook-boom");
+          },
         },
-      },
-      {
-        name: "ok",
-        priority: 1001,
-        fn: async () => {
-          calls.push("ok-ran");
+        {
+          name: "ok",
+          priority: 1001,
+          fn: async () => {
+            calls.push("ok-ran");
+          },
         },
-      },
-    ]);
-    const pipeline = createLifecycleHooks(makeRegistry(), systemHooks);
-    // Must not throw.
-    await run(pipeline);
-    expect(calls).toEqual(["ok-ran"]);
-    expect(consoleSpy).toHaveBeenCalled();
-    consoleSpy.mockRestore();
+      ]);
+      const pipeline = createLifecycleHooks(makeRegistry(), systemHooks);
+      // Must not throw.
+      await run(pipeline);
+      expect(calls).toEqual(["ok-ran"]);
+      expect(consoleSpy).toHaveBeenCalled();
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 });

package/src/pipeline/dispatch-shared.ts

@@ -1,4 +1,5 @@
 import { requestContext } from "../api/request-context";
+import type { SseBroker } from "../api/sse-broker";
 import type { DbConnection, DbRunner, DbTx } from "../db/connection";
 import { runInSavepoint, selectMany } from "../db/query";
 import type { buildEntityTable } from "../db/table-builder";
@@ -30,7 +31,10 @@ import {
   isStreamArchived,
   restoreStream as restoreStreamHelper,
 } from "../event-store/archive";
-import { VersionConflictError as EventStoreVersionConflictError } from "../event-store/errors";
+import {
+  IdempotentAppendConflictError as EventStoreIdempotentAppendConflictError,
+  VersionConflictError as EventStoreVersionConflictError,
+} from "../event-store/errors";
 import {
   getStreamVersion,
   loadAggregate,
@@ -53,6 +57,7 @@ import {
   emitDispatcherHandler,
   type getFallbackMeter,
   getFallbackTracer,
+  observabilityContext,
 } from "../observability";
 import { buildBucketKey } from "../rate-limit";
 import { createTzContext } from "../time";
@@ -92,6 +97,7 @@ export type DispatchContext = {
   lifecycle: LifecycleHooks | undefined;
   jobRunner: JobRunnerRef | undefined;
   effectiveFeatures: EffectiveFeaturesResolver | undefined;
+  sseBroker: SseBroker | undefined;
   tableCache: Map<string, ReturnType<typeof buildEntityTable>>;
   transitionCache: Map<string, ReturnType<typeof defineTransitions>>;
   tracer: ReturnType<typeof getFallbackTracer>;
@@ -208,6 +214,9 @@ export function buildHandlerContext(
   // `queryAs` / `writeAs` let a handler explicitly switch identity
   // (e.g. system-privileged lookups that bypass field-access read filters).
   const bridgeSink = afterCommitHooks ?? [];
+  const scheduleAfterCommit = (hook: AfterCommitHook): void => {
+    bridgeSink.push(hook);
+  };
   const bridge = {
     query: (targetType: string, payload: unknown) =>
       executeQuery(ctx, targetType, payload, user, tx), // @wrapper-known semantic-alias
@@ -263,6 +272,9 @@ export function buildHandlerContext(
         if (e instanceof EventStoreVersionConflictError) {
           return { ok: false as const, conflict: e };
         }
+        if (e instanceof EventStoreIdempotentAppendConflictError) {
+          return { ok: false as const, conflict: e };
+        }
         throw e;
       }
     },
@@ -542,6 +554,7 @@ export function buildHandlerContext(
     _userId: user.id,
     _tenantId: user.tenantId,
     _handlerType: type,
+    scheduleAfterCommit,
     ...(includeDeleted && { includeDeleted: true }),
     ...bridge,
   } as HandlerContext; // @cast-boundary engine-bridge
@@ -611,9 +624,13 @@ export async function runHandlerInstrumented<T>(
 
 // Generator-native counterpart to runHandlerInstrumented — a stream's
 // lifetime spans every `for await` pull the caller makes, so the span
-// can't be scoped via withSpan's single-callback shape. startSpan/end
-// bracket the whole yield* instead; metrics land in the same finally
-// path so success/failure/throw all hit one emit, like the Promise path.
+// can't be scoped via withSpan's single-callback shape. Drive the inner
+// generator manually (not yield*): each pull must run inside
+// observabilityContext.run({ activeSpan: span }) so handler-side
+// startSpan() calls parent onto the dispatcher span. finally forwards
+// .return() to the inner generator (yield* did that for free). Metrics
+// land in the same finally path so success/failure/throw/abort all hit
+// one emit, like the Promise path.
 export async function* runStreamInstrumented<T>(
   ctx: DispatchContext,
   type: string,
@@ -623,25 +640,61 @@ export async function* runStreamInstrumented<T>(
   const { tracer: dispatcherTracer, meter: dispatcherMeter, registry } = ctx;
   const start = performance.now();
   let success = true;
+  // Set only once `inner()` has drained on its own — NOT set when the
+  // consumer walks away early (generator.return(), the normal SSE-abort
+  // path in api/routes.ts). yield* forwards that .return() straight through
+  // without throwing, so without this flag an aborted stream would fall
+  // through to the success-metric branch below and get counted as a clean
+  // completion.
+  let completedNormally = false;
   let errorClass: string | undefined;
   const span = dispatcherTracer.startSpan("kumiko.dispatcher.handler", {
     attributes: dispatcherSpanAttributes(type, "stream", user, registry.getHandlerFeature(type)),
   });
+  const it = inner();
   try {
-    yield* inner();
+    let next = await observabilityContext.run({ activeSpan: span }, () => it.next());
+    while (!next.done) {
+      yield next.value;
+      next = await observabilityContext.run({ activeSpan: span }, () => it.next());
+    }
+    completedNormally = true;
+    return next.value;
   } catch (error) {
     success = false;
     errorClass = error instanceof Error && error.name ? error.name : "UnknownError";
+    if (error instanceof Error) span.recordException(error);
     span.setStatus("error", errorClass);
     throw error;
   } finally {
+    // forward close so inner()'s finally still fires — yield* did this for free
+    try {
+      await observabilityContext.run({ activeSpan: span }, () => it.return?.(undefined));
+    } catch (closeError) {
+      // Only fold this in when nothing has already failed — a close-time
+      // error while an earlier error is in flight would mask the real
+      // cause reported above.
+      if (success) {
+        success = false;
+        errorClass =
+          closeError instanceof Error && closeError.name ? closeError.name : "UnknownError";
+        span.setStatus("error", errorClass);
+      }
+    }
     span.end();
     if (!success && errorClass) {
       emitDispatcherError(dispatcherMeter, { handler: type, errorClass });
     }
+    // Consumer-abort (generator.return(), the normal SSE Tab-close path)
+    // is not a handler failure — label it outcome:"aborted" so live-stream
+    // success rates stay meaningful. True handler throws keep success:false.
     emitDispatcherHandler(
       dispatcherMeter,
-      { handler: type, success },
+      {
+        handler: type,
+        success,
+        outcome: completedNormally ? "completed" : "aborted",
+      },
       (performance.now() - start) / 1000,
     );
   }

package/src/pipeline/dispatch-stream.ts

@@ -53,17 +53,38 @@ async function* executeStreamInner(
     throw validationErrorFromZod(parsed.error);
   }
 
-  const handlerContext = buildHandlerContext(ctx, type, user);
-  const chunks = handler.handler({ type, payload: parsed.data, user }, handlerContext);
+  // Mid-stream access revocation: subscribed for the stream's lifetime, not
+  // just checked once like hasAccess above. A role/session/tenant change
+  // fires this via ctx.sseBroker.publishAccessInvalidation(userId) (session-
+  // revoke / tenant-membership consumers — issue #1559/#1560); the flag is
+  // read at the same per-chunk cadence as ensureFeatureEnabled below, so a
+  // stream cuts before its next chunk rather than running indefinitely.
+  let accessInvalidated = false;
+  const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(user.id, () => {
+    accessInvalidated = true;
+  });
 
-  // Consumer-driven pull (for await) is the backpressure mechanism — the
-  // handler generator only advances once the caller reads the previous
-  // chunk, no explicit buffering/throttling needed on either side.
-  for await (const chunk of chunks) {
-    // Re-checked per chunk, not just at stream-start: a feature disabled
-    // mid-stream must cut an already-open stream, not just block new ones.
-    await ensureFeatureEnabled(ctx, type, user.tenantId);
-    assertNoSecretLeak(chunk);
-    yield chunk;
+  try {
+    const handlerContext = buildHandlerContext(ctx, type, user);
+    const chunks = handler.handler({ type, payload: parsed.data, user }, handlerContext);
+
+    // Consumer-driven pull (for await) is the backpressure mechanism — the
+    // handler generator only advances once the caller reads the previous
+    // chunk, no explicit buffering/throttling needed on either side.
+    for await (const chunk of chunks) {
+      // Re-checked per chunk, not just at stream-start: a feature disabled
+      // mid-stream must cut an already-open stream, not just block new ones.
+      await ensureFeatureEnabled(ctx, type, user.tenantId);
+      if (accessInvalidated) {
+        throw new AccessDeniedError({
+          message: `access revoked mid-stream for ${type}`,
+          details: { handler: type },
+        });
+      }
+      assertNoSecretLeak(chunk);
+      yield chunk;
+    }
+  } finally {
+    unsubscribeAccessInvalidation?.();
   }
 }

package/src/pipeline/dispatcher.ts

@@ -1,3 +1,4 @@
+import type { SseBroker } from "../api/sse-broker";
 import type { buildEntityTable } from "../db/table-builder";
 import type { defineTransitions } from "../engine/state-machine";
 import type { EffectiveFeaturesResolver } from "../engine/tier-resolver-extension";
@@ -43,6 +44,10 @@ export type DispatcherOptions = {
   // event-skips and a confusing operator-UI — the framework cannot
   // enforce this contract, but the recipe-test pins the convention.
   effectiveFeatures?: EffectiveFeaturesResolver;
+  // In-memory SSE broker — dispatch-stream.ts subscribes to a stream's
+  // user-scoped access-invalidation channel on it. Absent in setups without
+  // SSE wired up (dispatch-stream then just skips the subscription).
+  sseBroker?: SseBroker;
 };
 
 export type Dispatcher = {
@@ -82,7 +87,7 @@ export function createDispatcher(
   context: AppContext,
   options: DispatcherOptions = {},
 ): Dispatcher {
-  const { idempotency, lifecycle, jobRunner, effectiveFeatures } = options;
+  const { idempotency, lifecycle, jobRunner, effectiveFeatures, sseBroker } = options;
 
   // Pre-build tables and transition maps for auto-guard (avoid per-request allocation)
   const tableCache = new Map<string, ReturnType<typeof buildEntityTable>>();
@@ -101,6 +106,7 @@ export function createDispatcher(
     lifecycle,
     jobRunner,
     effectiveFeatures,
+    sseBroker,
     tableCache,
     transitionCache,
     tracer: dispatcherTracer,

package/src/pipeline/event-consumer-state.ts

@@ -10,8 +10,8 @@ import {
   sql,
   text,
 } from "../db/dialect";
-import { alterTableAddColumn } from "../db/queries/test-stack";
-import { columnNamesOf, tableExists } from "../db/schema-inspection";
+import { alterTableAddColumn } from "../db/queries/ddl";
+import { tableExists } from "../db/schema-inspection";
 import { unsafePushTables } from "../stack";
 
 // Reserved sentinel used in the instance_id column for consumers whose
@@ -112,17 +112,20 @@ export async function createEventConsumerStateTable(db: DbConnection): Promise<v
   // Still check for columns added after the table's first deploy (e.g.
   // rearm_count) so an older DB catches up without a dedicated migration.
   if (await tableExists(db, "public.kumiko_event_consumers")) {
-    const cols = await columnNamesOf(db, "kumiko_event_consumers");
-    if (!cols.has("rearm_count")) {
-      await alterTableAddColumn(
-        db,
-        "kumiko_event_consumers",
-        "rearm_count",
-        "integer",
-        " DEFAULT 0",
-        " NOT NULL",
-      );
-    }
+    // ADD COLUMN IF NOT EXISTS is idempotent and race-safe on its own —
+    // no need to pre-check columnNamesOf first. Two instances booting
+    // concurrently against the same DB (this table is explicitly
+    // multi-instance) both ALTER, neither crashes on "column already
+    // exists" (#1362).
+    await alterTableAddColumn(
+      db,
+      "kumiko_event_consumers",
+      "rearm_count",
+      "integer",
+      " DEFAULT 0",
+      " NOT NULL",
+      /* ifNotExists */ true,
+    );
     // skip: table (+ any missing column) is already up to date
     return;
   }

package/src/pipeline/event-dispatcher-delivery.ts

@@ -1,3 +1,7 @@
+// Value-only import, aliased to avoid shadowing the ambient global
+// `Temporal` TYPE that ConsumerStateRow.updatedAt/StoredEventRow.createdAt
+// resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
+import { Temporal as TemporalPolyfill } from "temporal-polyfill";
 import { requestContext } from "../api/request-context";
 import type { DbConnection, DbTx } from "../db/connection";
 import {
@@ -111,8 +115,18 @@ export async function acquireConsumerState(
     // a prior re-arm) gates the retry; maxRearmCount stops a poison event
     // from looping forever (re-arm → same event fails → dead → re-arm →
     // ...) — after the cap it stays dead until a human intervenes.
-    const cooldownDeadline = Temporal.Now.instant().subtract({ milliseconds: rearmCooldownMs });
-    const cooldownElapsed = Temporal.Instant.compare(state.updatedAt, cooldownDeadline) <= 0;
+    const cooldownDeadline = TemporalPolyfill.Now.instant().subtract({
+      milliseconds: rearmCooldownMs,
+    });
+    // @cast-boundary temporal-polyfill-vs-ambient: same TC39 Temporal.Instant
+    // at runtime — state.updatedAt is DB-row-typed against the ambient
+    // global, two distinct nominal types across the two .d.ts sources (see
+    // event-store.ts).
+    const cooldownElapsed =
+      TemporalPolyfill.Instant.compare(
+        state.updatedAt as unknown as InstanceType<typeof TemporalPolyfill.Instant>,
+        cooldownDeadline,
+      ) <= 0;
     if (cooldownElapsed && state.rearmCount < maxRearmCount) {
       const rearmed = await rearmDeadConsumer(tx, name, instanceId);
       const rearmedState =
@@ -120,10 +134,10 @@ export async function acquireConsumerState(
         (coerceRow(rearmed, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
       if (rearmedState) return { state: rearmedState, skip: null };
     }
-    // ponytail: no log/metric fires when the rearm budget is exhausted here
-    // (the exact "braucht manuellen Eingriff" moment) — queryable via
-    // getConsumerState but silent otherwise. Add an emitDispatcherError-style
-    // signal if ops needs a push instead of a dead+lag poll.
+    // Caller (event-dispatcher.ts's processConsumer) emits
+    // kumiko_event_consumer_rearm_exhausted_total once per (consumer,
+    // instance) transition into this branch — it has the process-lifetime
+    // state to dedupe across poll passes that this pure function doesn't.
     return { state: null, skip: "dead" };
   }
   return { state, skip: null };

package/src/pipeline/event-dispatcher.ts

@@ -4,6 +4,7 @@ import { SYSTEM_TENANT_ID } from "../engine/types/identifiers";
 import { EVENTS_PUBSUB_CHANNEL, type StoredEvent } from "../event-store";
 import {
   emitEventConsumerPassOutcome,
+  emitEventConsumerRearmExhausted,
   emitEventDispatcherListenConnected,
   getFallbackMeter,
   getFallbackTracer,
@@ -201,6 +202,14 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
   }
   const tracer: Tracer = options.tracer ?? getFallbackTracer();
   const meter: Meter = options.meter ?? getFallbackMeter();
+  // Tracks which (consumer, instanceId) pairs already fired
+  // kumiko_event_consumer_rearm_exhausted_total, so a consumer stuck dead
+  // across many poll passes emits the ops-signal once, not every pass
+  // (that would be log/metric spam for a state that hasn't changed).
+  // Process-lifetime only — restarting the dispatcher re-arms reporting,
+  // which is fine: a fresh process re-observing a still-dead consumer is
+  // exactly the "still needs a human" signal ops wants.
+  const reportedDeadConsumers = new Set<string>();
 
   let running = false;
   // Separate from `running` on purpose: pre-registration of consumer state
@@ -304,8 +313,21 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
         // disabled/dead. Nothing to deliver this pass.
         if (acquired.skip !== null) {
           span.setAttribute("consumer.skip_reason", acquired.skip);
+          if (acquired.skip === "dead") {
+            const reportKey = `${consumer.name}:${instanceId}`;
+            if (!reportedDeadConsumers.has(reportKey)) {
+              reportedDeadConsumers.add(reportKey);
+              emitEventConsumerRearmExhausted(meter, { consumer: consumer.name, instanceId });
+            }
+          }
+          // skip: skip reason already recorded on the span above (and, for
+          // "dead", already emitted as a metric) — nothing left to deliver.
           return;
         }
+        // Acquired normally (including via a successful auto-rearm) — clear
+        // any prior dead-report so a future exhaustion re-emits instead of
+        // staying permanently suppressed by this process's Set.
+        reportedDeadConsumers.delete(`${consumer.name}:${instanceId}`);
 
         const events = await fetchPendingEvents(tx, acquired.state.lastProcessedEventId, batchSize);
         // skip: nothing to deliver — no markProcessing/persistConsumerOutcome write,

package/src/pipeline/index.ts

@@ -57,6 +57,8 @@ export {
 } from "./projection-state";
 export { runProjectionsForEvent } from "./projections-runner";
 export {
+  ACCESS_INVALIDATION_CONSUMER_NAME,
+  createAccessInvalidationEventConsumer,
   createSearchEventConsumer,
   createSseBroadcastEventConsumer,
   SEARCH_CONSUMER_NAME,