@cosmicdrift/kumiko-framework 0.165.0 → 0.165.1

package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts

@@ -11,6 +11,13 @@ import { createEventStoreExecutor } from "../../db/event-store-executor";
 import { asRawClient } from "../../db/query";
 import { createTenantDb, type TenantDb } from "../../db/tenant-db";
 import { defineFeature } from "../../engine";
+import {
+  DEFAULT_SENSITIVE_CONFIG,
+  type MetricEvent,
+  type ObservabilityProvider,
+  RecordingMeter,
+  RecordingTracer,
+} from "../../observability";
 import { getConsumerState } from "../../pipeline";
 import {
   resetEventStore,
@@ -20,6 +27,7 @@ import {
   unsafeCreateEntityTable,
 } from "../../stack";
 import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
+import { withoutAmbientTemporal } from "../../testing/without-ambient-temporal";
 import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
 
 const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
@@ -151,6 +159,23 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(deadAgain?.rearmCount).toBe(0);
   });
 
+  test("kumiko-framework#1525: cooldown-elapsed re-arm check works without relying on globalThis.Temporal", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+    await backdateUpdatedAt(10);
+
+    // ensureTemporalPolyfill re-installs if the ambient global was torn down,
+    // so runOnce stays safe if a nested path calls it during this window.
+    await withoutAmbientTemporal(async () => {
+      await stack.eventDispatcher?.runOnce();
+    });
+
+    const revived = await getConsumerState(stack.db, qn);
+    expect(revived?.status).toBe("idle");
+    expect(revived?.rearmCount).toBe(1);
+  });
+
   test("permanently dead after maxRearmCount cycles, even once cooldown elapses again", async () => {
     poisonNames.add("poison");
     await appendWidget("poison");
@@ -177,4 +202,62 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(stillExhausted?.status).toBe("dead");
     expect(stillExhausted?.rearmCount).toBe(3);
   });
+
+  test("kumiko_event_consumer_rearm_exhausted_total fires once, not per still-dead pass (#1362)", async () => {
+    // Dedicated stack with a RecordingMeter — module-level `stack` above is
+    // shared across every test in this file and has no metric capture.
+    const metricEvents: MetricEvent[] = [];
+    const meter = new RecordingMeter((e) => metricEvents.push(e));
+    const tracer = new RecordingTracer({
+      sensitiveConfig: DEFAULT_SENSITIVE_CONFIG,
+      onSpanEnd: () => {},
+    });
+    const recordingProvider: ObservabilityProvider = {
+      name: "recording",
+      meter,
+      tracer,
+      shutdown: async () => {},
+    };
+
+    const recStack = await setupTestStack({
+      features: [rearmFeature],
+      systemHooks: [],
+      observability: recordingProvider,
+    });
+    try {
+      await unsafeCreateEntityTable(recStack.db, sharedWidgetEntity, "widget");
+      const recTdb = createTenantDb(recStack.db, admin.tenantId);
+
+      poisonNames.add("poison-metric");
+      await executor.create({ name: "poison-metric" }, admin, recTdb);
+      for (let i = 0; i < 10; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      // The 10th pass is the one whose failed delivery flips status to
+      // "dead" (persisted for the NEXT pass to see) — acquireConsumerState's
+      // dead-branch (where the exhausted-metric fires) only runs starting
+      // from the pass AFTER that transition, same as the "stays dead within
+      // cooldown window" test above needing one extra runOnce().
+      await recStack.eventDispatcher?.runOnce();
+
+      const exhaustedEvents = () =>
+        metricEvents.filter(
+          (e) =>
+            e.type === "counter.inc" && e.name === "kumiko_event_consumer_rearm_exhausted_total",
+        );
+      expect(exhaustedEvents().length).toBe(1);
+
+      // Several more passes while it's still dead — must stay at 1, not
+      // re-fire on every pass that observes "still dead" (log/metric spam
+      // the finding flagged).
+      for (let i = 0; i < 5; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      expect(exhaustedEvents().length).toBe(1);
+      expect(exhaustedEvents()[0]?.labels?.["consumer"]).toBe(qn);
+    } finally {
+      await resetEventStore(recStack, ["read_widgets"]);
+      await recStack.cleanup();
+    }
+  });
 });

package/src/pipeline/__tests__/lifecycle-pipeline.test.ts

@@ -202,74 +202,78 @@ describe("runPostSave", () => {
   test("postSave errors don't throw — logged and continued", async () => {
     const calls: string[] = [];
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-
-    const registry = makeRegistry({
-      postSave: [
-        async () => {
-          throw new Error("feature-fail");
-        },
-      ],
-    });
-
-    const systemHooks: SystemHooks = {
-      postSave: [
-        {
-          name: "search",
-          priority: 1000,
-          fn: async () => {
-            calls.push("search-ran");
+    try {
+      const registry = makeRegistry({
+        postSave: [
+          async () => {
+            throw new Error("feature-fail");
           },
-        },
-      ],
-    };
-
-    const pipeline = createLifecycleHooks(registry, systemHooks);
-    // Should not throw
-    await pipeline.runPostSave("test:write:user", savectx, {});
+        ],
+      });
 
-    // System hook still ran despite feature hook failure
-    expect(calls).toContain("search-ran");
-    expect(consoleSpy).toHaveBeenCalled();
-    consoleSpy.mockRestore();
+      const systemHooks: SystemHooks = {
+        postSave: [
+          {
+            name: "search",
+            priority: 1000,
+            fn: async () => {
+              calls.push("search-ran");
+            },
+          },
+        ],
+      };
+
+      const pipeline = createLifecycleHooks(registry, systemHooks);
+      // Should not throw
+      await pipeline.runPostSave("test:write:user", savectx, {});
+
+      // System hook still ran despite feature hook failure
+      expect(calls).toContain("search-ran");
+      expect(consoleSpy).toHaveBeenCalled();
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 
   test("system hook failure doesn't block other system hooks", async () => {
     const calls: string[] = [];
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-
-    const registry = makeRegistry();
-
-    const systemHooks: SystemHooks = {
-      postSave: [
-        {
-          name: "search",
-          priority: 1000,
-          fn: async () => {
-            throw new Error("meili-down");
+    try {
+      const registry = makeRegistry();
+
+      const systemHooks: SystemHooks = {
+        postSave: [
+          {
+            name: "search",
+            priority: 1000,
+            fn: async () => {
+              throw new Error("meili-down");
+            },
           },
-        },
-        {
-          name: "sse",
-          priority: 1001,
-          fn: async () => {
-            calls.push("sse-ran");
+          {
+            name: "sse",
+            priority: 1001,
+            fn: async () => {
+              calls.push("sse-ran");
+            },
           },
-        },
-        {
-          name: "audit",
-          priority: 1002,
-          fn: async () => {
-            calls.push("audit-ran");
+          {
+            name: "audit",
+            priority: 1002,
+            fn: async () => {
+              calls.push("audit-ran");
+            },
           },
-        },
-      ],
-    };
+        ],
+      };
 
-    const pipeline = createLifecycleHooks(registry, systemHooks);
-    await pipeline.runPostSave("test:write:user", savectx, {});
+      const pipeline = createLifecycleHooks(registry, systemHooks);
+      await pipeline.runPostSave("test:write:user", savectx, {});
 
-    expect(calls).toEqual(["sse-ran", "audit-ran"]);
-    consoleSpy.mockRestore();
+      expect(calls).toEqual(["sse-ran", "audit-ran"]);
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 });
 
@@ -336,29 +340,32 @@ describe("runPostSave phase routing", () => {
 
   test("afterCommit phase: hook errors are logged, never thrown", async () => {
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-    const afterRan: string[] = [];
-    const feature = defineFeature("phases", (r) => {
-      r.entity("user", createEntity({ table: "Users", fields: {} }));
-      r.writeHandler("user", z.object({}), async () => ({ isSuccess: true, data: null }), {
-        access: { openToAll: true },
-      });
-      r.hook("postSave", "user", async () => {
-        throw new Error("afterCommit-boom");
+    try {
+      const afterRan: string[] = [];
+      const feature = defineFeature("phases", (r) => {
+        r.entity("user", createEntity({ table: "Users", fields: {} }));
+        r.writeHandler("user", z.object({}), async () => ({ isSuccess: true, data: null }), {
+          access: { openToAll: true },
+        });
+        r.hook("postSave", "user", async () => {
+          throw new Error("afterCommit-boom");
+        });
+        r.hook("postSave", "user", async () => {
+          afterRan.push("second");
+        });
       });
-      r.hook("postSave", "user", async () => {
-        afterRan.push("second");
-      });
-    });
-    const registry = createRegistry([feature]);
-    const pipeline = createLifecycleHooks(registry);
+      const registry = createRegistry([feature]);
+      const pipeline = createLifecycleHooks(registry);
 
-    // Must not throw — errors are swallowed + logged
-    await pipeline.runPostSave("phases:write:user", savectx, {}, "afterCommit");
+      // Must not throw — errors are swallowed + logged
+      await pipeline.runPostSave("phases:write:user", savectx, {}, "afterCommit");
 
-    // Subsequent hooks still fire (failure in one hook doesn't block the rest)
-    expect(afterRan).toEqual(["second"]);
-    expect(consoleSpy).toHaveBeenCalled();
-    consoleSpy.mockRestore();
+      // Subsequent hooks still fire (failure in one hook doesn't block the rest)
+      expect(afterRan).toEqual(["second"]);
+      expect(consoleSpy).toHaveBeenCalled();
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 
   test("system hooks respect their phase setting", async () => {
@@ -724,28 +731,31 @@ describe("runPostSaveBatch / runPostDeleteBatch", () => {
     ],
   ])("one %s hook throwing doesn't stop the others (Promise.allSettled) — logged, never thrown", async (_name, buildHooks, run) => {
     const consoleSpy = spyOn(console, "error").mockImplementation(() => {});
-    const calls: string[] = [];
-    const systemHooks = buildHooks([
-      {
-        name: "failing",
-        priority: 1000,
-        fn: async () => {
-          throw new Error("batch-hook-boom");
+    try {
+      const calls: string[] = [];
+      const systemHooks = buildHooks([
+        {
+          name: "failing",
+          priority: 1000,
+          fn: async () => {
+            throw new Error("batch-hook-boom");
+          },
         },
-      },
-      {
-        name: "ok",
-        priority: 1001,
-        fn: async () => {
-          calls.push("ok-ran");
+        {
+          name: "ok",
+          priority: 1001,
+          fn: async () => {
+            calls.push("ok-ran");
+          },
         },
-      },
-    ]);
-    const pipeline = createLifecycleHooks(makeRegistry(), systemHooks);
-    // Must not throw.
-    await run(pipeline);
-    expect(calls).toEqual(["ok-ran"]);
-    expect(consoleSpy).toHaveBeenCalled();
-    consoleSpy.mockRestore();
+      ]);
+      const pipeline = createLifecycleHooks(makeRegistry(), systemHooks);
+      // Must not throw.
+      await run(pipeline);
+      expect(calls).toEqual(["ok-ran"]);
+      expect(consoleSpy).toHaveBeenCalled();
+    } finally {
+      consoleSpy.mockRestore();
+    }
   });
 });

package/src/pipeline/dispatch-shared.ts

@@ -1,4 +1,5 @@
 import { requestContext } from "../api/request-context";
+import type { SseBroker } from "../api/sse-broker";
 import type { DbConnection, DbRunner, DbTx } from "../db/connection";
 import { runInSavepoint, selectMany } from "../db/query";
 import type { buildEntityTable } from "../db/table-builder";
@@ -30,7 +31,10 @@ import {
   isStreamArchived,
   restoreStream as restoreStreamHelper,
 } from "../event-store/archive";
-import { VersionConflictError as EventStoreVersionConflictError } from "../event-store/errors";
+import {
+  IdempotentAppendConflictError as EventStoreIdempotentAppendConflictError,
+  VersionConflictError as EventStoreVersionConflictError,
+} from "../event-store/errors";
 import {
   getStreamVersion,
   loadAggregate,
@@ -53,6 +57,7 @@ import {
   emitDispatcherHandler,
   type getFallbackMeter,
   getFallbackTracer,
+  observabilityContext,
 } from "../observability";
 import { buildBucketKey } from "../rate-limit";
 import { createTzContext } from "../time";
@@ -92,6 +97,7 @@ export type DispatchContext = {
   lifecycle: LifecycleHooks | undefined;
   jobRunner: JobRunnerRef | undefined;
   effectiveFeatures: EffectiveFeaturesResolver | undefined;
+  sseBroker: SseBroker | undefined;
   tableCache: Map<string, ReturnType<typeof buildEntityTable>>;
   transitionCache: Map<string, ReturnType<typeof defineTransitions>>;
   tracer: ReturnType<typeof getFallbackTracer>;
@@ -208,6 +214,9 @@ export function buildHandlerContext(
   // `queryAs` / `writeAs` let a handler explicitly switch identity
   // (e.g. system-privileged lookups that bypass field-access read filters).
   const bridgeSink = afterCommitHooks ?? [];
+  const scheduleAfterCommit = (hook: AfterCommitHook): void => {
+    bridgeSink.push(hook);
+  };
   const bridge = {
     query: (targetType: string, payload: unknown) =>
       executeQuery(ctx, targetType, payload, user, tx), // @wrapper-known semantic-alias
@@ -263,6 +272,9 @@ export function buildHandlerContext(
         if (e instanceof EventStoreVersionConflictError) {
           return { ok: false as const, conflict: e };
         }
+        if (e instanceof EventStoreIdempotentAppendConflictError) {
+          return { ok: false as const, conflict: e };
+        }
         throw e;
       }
     },
@@ -542,6 +554,7 @@ export function buildHandlerContext(
     _userId: user.id,
     _tenantId: user.tenantId,
     _handlerType: type,
+    scheduleAfterCommit,
     ...(includeDeleted && { includeDeleted: true }),
     ...bridge,
   } as HandlerContext; // @cast-boundary engine-bridge
@@ -611,9 +624,13 @@ export async function runHandlerInstrumented<T>(
 
 // Generator-native counterpart to runHandlerInstrumented — a stream's
 // lifetime spans every `for await` pull the caller makes, so the span
-// can't be scoped via withSpan's single-callback shape. startSpan/end
-// bracket the whole yield* instead; metrics land in the same finally
-// path so success/failure/throw all hit one emit, like the Promise path.
+// can't be scoped via withSpan's single-callback shape. Drive the inner
+// generator manually (not yield*): each pull must run inside
+// observabilityContext.run({ activeSpan: span }) so handler-side
+// startSpan() calls parent onto the dispatcher span. finally forwards
+// .return() to the inner generator (yield* did that for free). Metrics
+// land in the same finally path so success/failure/throw/abort all hit
+// one emit, like the Promise path.
 export async function* runStreamInstrumented<T>(
   ctx: DispatchContext,
   type: string,
@@ -623,25 +640,61 @@ export async function* runStreamInstrumented<T>(
   const { tracer: dispatcherTracer, meter: dispatcherMeter, registry } = ctx;
   const start = performance.now();
   let success = true;
+  // Set only once `inner()` has drained on its own — NOT set when the
+  // consumer walks away early (generator.return(), the normal SSE-abort
+  // path in api/routes.ts). yield* forwards that .return() straight through
+  // without throwing, so without this flag an aborted stream would fall
+  // through to the success-metric branch below and get counted as a clean
+  // completion.
+  let completedNormally = false;
   let errorClass: string | undefined;
   const span = dispatcherTracer.startSpan("kumiko.dispatcher.handler", {
     attributes: dispatcherSpanAttributes(type, "stream", user, registry.getHandlerFeature(type)),
   });
+  const it = inner();
   try {
-    yield* inner();
+    let next = await observabilityContext.run({ activeSpan: span }, () => it.next());
+    while (!next.done) {
+      yield next.value;
+      next = await observabilityContext.run({ activeSpan: span }, () => it.next());
+    }
+    completedNormally = true;
+    return next.value;
   } catch (error) {
     success = false;
     errorClass = error instanceof Error && error.name ? error.name : "UnknownError";
+    if (error instanceof Error) span.recordException(error);
     span.setStatus("error", errorClass);
     throw error;
   } finally {
+    // forward close so inner()'s finally still fires — yield* did this for free
+    try {
+      await observabilityContext.run({ activeSpan: span }, () => it.return?.(undefined));
+    } catch (closeError) {
+      // Only fold this in when nothing has already failed — a close-time
+      // error while an earlier error is in flight would mask the real
+      // cause reported above.
+      if (success) {
+        success = false;
+        errorClass =
+          closeError instanceof Error && closeError.name ? closeError.name : "UnknownError";
+        span.setStatus("error", errorClass);
+      }
+    }
     span.end();
     if (!success && errorClass) {
       emitDispatcherError(dispatcherMeter, { handler: type, errorClass });
     }
+    // Consumer-abort (generator.return(), the normal SSE Tab-close path)
+    // is not a handler failure — label it outcome:"aborted" so live-stream
+    // success rates stay meaningful. True handler throws keep success:false.
     emitDispatcherHandler(
       dispatcherMeter,
-      { handler: type, success },
+      {
+        handler: type,
+        success,
+        outcome: completedNormally ? "completed" : "aborted",
+      },
       (performance.now() - start) / 1000,
     );
   }

package/src/pipeline/dispatch-stream.ts

@@ -53,17 +53,49 @@ async function* executeStreamInner(
     throw validationErrorFromZod(parsed.error);
   }
 
-  const handlerContext = buildHandlerContext(ctx, type, user);
-  const chunks = handler.handler({ type, payload: parsed.data, user }, handlerContext);
+  // Mid-stream access revocation must also cut *idle* SSE streams (heartbeat
+  // only — no chunk). A boolean flag read only after the next chunk would
+  // leave revoked sessions open indefinitely (fw#1563). Race each pull
+  // against an invalidated Deferred instead.
+  let resolveInvalidated: (() => void) | undefined;
+  const invalidated = new Promise<void>((resolve) => {
+    resolveInvalidated = resolve;
+  });
+  const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(user.id, () => {
+    resolveInvalidated?.();
+  });
 
-  // Consumer-driven pull (for await) is the backpressure mechanism — the
-  // handler generator only advances once the caller reads the previous
-  // chunk, no explicit buffering/throttling needed on either side.
-  for await (const chunk of chunks) {
-    // Re-checked per chunk, not just at stream-start: a feature disabled
-    // mid-stream must cut an already-open stream, not just block new ones.
-    await ensureFeatureEnabled(ctx, type, user.tenantId);
-    assertNoSecretLeak(chunk);
-    yield chunk;
+  let iterator: AsyncIterator<unknown> | undefined;
+  try {
+    const handlerContext = buildHandlerContext(ctx, type, user);
+    const chunks = handler.handler({ type, payload: parsed.data, user }, handlerContext);
+    iterator = chunks[Symbol.asyncIterator]();
+
+    while (true) {
+      const nextPull = iterator.next();
+      const outcome = await Promise.race([
+        nextPull.then((result) => ({ kind: "chunk" as const, result })),
+        invalidated.then(() => ({ kind: "invalidated" as const })),
+      ]);
+      if (outcome.kind === "invalidated") {
+        throw new AccessDeniedError({
+          message: `access revoked mid-stream for ${type}`,
+          details: { handler: type },
+        });
+      }
+      if (outcome.result.done) break;
+      await ensureFeatureEnabled(ctx, type, user.tenantId);
+      assertNoSecretLeak(outcome.result.value);
+      yield outcome.result.value;
+    }
+  } finally {
+    unsubscribeAccessInvalidation?.();
+    // Consumer break / access revoke / throw — always close the handler
+    // generator so its finally (cleanup) runs (for-await would do this).
+    // Do NOT swallow return() errors — close-time cleanup failures must
+    // surface to runStreamInstrumented (#1543).
+    if (iterator !== undefined) {
+      await iterator.return?.(undefined);
+    }
   }
 }

package/src/pipeline/dispatcher.ts

@@ -1,3 +1,4 @@
+import type { SseBroker } from "../api/sse-broker";
 import type { buildEntityTable } from "../db/table-builder";
 import type { defineTransitions } from "../engine/state-machine";
 import type { EffectiveFeaturesResolver } from "../engine/tier-resolver-extension";
@@ -43,6 +44,10 @@ export type DispatcherOptions = {
   // event-skips and a confusing operator-UI — the framework cannot
   // enforce this contract, but the recipe-test pins the convention.
   effectiveFeatures?: EffectiveFeaturesResolver;
+  // In-memory SSE broker — dispatch-stream.ts subscribes to a stream's
+  // user-scoped access-invalidation channel on it. Absent in setups without
+  // SSE wired up (dispatch-stream then just skips the subscription).
+  sseBroker?: SseBroker;
 };
 
 export type Dispatcher = {
@@ -82,7 +87,7 @@ export function createDispatcher(
   context: AppContext,
   options: DispatcherOptions = {},
 ): Dispatcher {
-  const { idempotency, lifecycle, jobRunner, effectiveFeatures } = options;
+  const { idempotency, lifecycle, jobRunner, effectiveFeatures, sseBroker } = options;
 
   // Pre-build tables and transition maps for auto-guard (avoid per-request allocation)
   const tableCache = new Map<string, ReturnType<typeof buildEntityTable>>();
@@ -101,6 +106,7 @@ export function createDispatcher(
     lifecycle,
     jobRunner,
     effectiveFeatures,
+    sseBroker,
     tableCache,
     transitionCache,
     tracer: dispatcherTracer,

package/src/pipeline/event-consumer-state.ts

@@ -10,8 +10,8 @@ import {
   sql,
   text,
 } from "../db/dialect";
-import { alterTableAddColumn } from "../db/queries/test-stack";
-import { columnNamesOf, tableExists } from "../db/schema-inspection";
+import { alterTableAddColumn } from "../db/queries/ddl";
+import { tableExists } from "../db/schema-inspection";
 import { unsafePushTables } from "../stack";
 
 // Reserved sentinel used in the instance_id column for consumers whose
@@ -112,17 +112,20 @@ export async function createEventConsumerStateTable(db: DbConnection): Promise<v
   // Still check for columns added after the table's first deploy (e.g.
   // rearm_count) so an older DB catches up without a dedicated migration.
   if (await tableExists(db, "public.kumiko_event_consumers")) {
-    const cols = await columnNamesOf(db, "kumiko_event_consumers");
-    if (!cols.has("rearm_count")) {
-      await alterTableAddColumn(
-        db,
-        "kumiko_event_consumers",
-        "rearm_count",
-        "integer",
-        " DEFAULT 0",
-        " NOT NULL",
-      );
-    }
+    // ADD COLUMN IF NOT EXISTS is idempotent and race-safe on its own —
+    // no need to pre-check columnNamesOf first. Two instances booting
+    // concurrently against the same DB (this table is explicitly
+    // multi-instance) both ALTER, neither crashes on "column already
+    // exists" (#1362).
+    await alterTableAddColumn(
+      db,
+      "kumiko_event_consumers",
+      "rearm_count",
+      "integer",
+      " DEFAULT 0",
+      " NOT NULL",
+      /* ifNotExists */ true,
+    );
     // skip: table (+ any missing column) is already up to date
     return;
   }