@cosmicdrift/kumiko-framework 0.165.0 → 0.165.2

package/src/migrations/__tests__/kumiko-drift.integration.test.ts

@@ -8,7 +8,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { type BunTestDb, createTestDb } from "../../bun-db/__tests__/bun-test-db";
-import { buildEntityTableMeta } from "../../db/entity-table-meta";
+import { deriveEntityTableMeta } from "../../db/entity-table-meta";
 import { generateMigration, writeSnapshotJson } from "../../db/migrate-generator";
 import {
   baselineMigrations,
@@ -200,7 +200,7 @@ describe("kumiko-drift end-to-end (generate → apply → gate)", () => {
       table: "kdrift_gen",
       fields: { name: createTextField({ required: true }) },
     });
-    const meta = buildEntityTableMeta("kdriftGen", entity);
+    const meta = deriveEntityTableMeta("kdriftGen", entity);
     const result = generateMigration({
       metas: [meta],
       prevSnapshot: null,

package/src/migrations/projection-table-index.ts

@@ -3,7 +3,7 @@
 // rebuild-Marker → mappt Tabellen auf Projektionen → rebuildProjection).
 //
 // Drizzle-frei: der Tabellen-Name kommt aus dem kumiko-Symbol das
-// buildEntityTable/buildEntityTableMeta an die Table-Definition hängt.
+// buildEntityTable/deriveEntityTableMeta an die Table-Definition hängt.
 
 import { extractTableName } from "../db";
 import type { Registry } from "../engine/types/feature";

package/src/observability/index.ts

@@ -50,6 +50,7 @@ export {
   emitDispatcherHandler,
   emitEventConsumerLag,
   emitEventConsumerPassOutcome,
+  emitEventConsumerRearmExhausted,
   emitEventDispatcherListenConnected,
   emitHttpRequest,
   emitJobQueueDepth,

package/src/observability/standard-metrics.ts

@@ -23,7 +23,7 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
     name: "kumiko_dispatcher_handler_duration_seconds",
     type: "histogram",
     description: "Dispatcher handler latency in seconds.",
-    labels: ["handler", "success"],
+    labels: ["handler", "success", "outcome"],
     buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5],
   },
   {
@@ -83,6 +83,20 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
       "Event deliveries that threw. Repeated failures on the same event lead to dead-letter.",
     labels: ["consumer", "instance_id"],
   },
+  // Fires once per (consumer, instance) transition into "stuck dead" —
+  // the rearm-cooldown/rearm-budget in event-dispatcher-delivery.ts gave up
+  // and the consumer now needs an operator (restartConsumer()) to recover.
+  // Without this, "dead + auto-rearm exhausted" is only visible via a
+  // getConsumerState poll — ops has no push signal that a consumer crossed
+  // from "will self-heal" to "needs a human". Deliberately a counter, not a
+  // gauge: emitted once per transition (event-dispatcher.ts tracks which
+  // consumers already fired this), not once per still-dead poll pass.
+  {
+    name: "kumiko_event_consumer_rearm_exhausted_total",
+    type: "counter",
+    description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
+    labels: ["consumer", "instance_id"],
+  },
   // LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
   // the dispatcher is running signal that delivery latency has regressed
   // from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
@@ -141,12 +155,18 @@ export function emitHttpRequest(
 
 export function emitDispatcherHandler(
   meter: Meter,
-  labels: { readonly handler: string; readonly success: boolean },
+  labels: {
+    readonly handler: string;
+    readonly success: boolean;
+    /** completed = drained normally; aborted = consumer walked away (SSE). */
+    readonly outcome?: "completed" | "aborted";
+  },
   durationSeconds: number,
 ): void {
   meter.histogram("kumiko_dispatcher_handler_duration_seconds").observe(durationSeconds, {
     handler: labels.handler,
     success: String(labels.success),
+    outcome: labels.outcome ?? "completed",
   });
 }
 
@@ -203,6 +223,19 @@ export function emitEventConsumerLag(
   });
 }
 
+// Caller (event-dispatcher.ts) is responsible for calling this exactly
+// once per (consumer, instanceId) transition into exhausted-dead — not
+// once per poll pass while it stays dead.
+export function emitEventConsumerRearmExhausted(
+  meter: Meter,
+  labels: { readonly consumer: string; readonly instanceId: string },
+): void {
+  meter.counter("kumiko_event_consumer_rearm_exhausted_total").inc(1, {
+    consumer: labels.consumer,
+    instance_id: labels.instanceId,
+  });
+}
+
 export function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void {
   meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
 }

package/src/observability/types/index.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability";
+export type * from "@cosmicdrift/kumiko-types/observability";

package/src/observability/types/metric.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/metric";
+export type * from "@cosmicdrift/kumiko-types/observability/metric";

package/src/observability/types/provider.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/provider";
+export type * from "@cosmicdrift/kumiko-types/observability/provider";

package/src/observability/types/span.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/span";
+export type * from "@cosmicdrift/kumiko-types/observability/span";

package/src/pipeline/__tests__/dispatcher.test.ts

@@ -1,9 +1,11 @@
 import { describe, expect, test } from "bun:test";
 import { z } from "zod";
+import type { SseBroker } from "../../api/sse-broker";
 import { createEntity, createRegistry, createTextField, defineFeature } from "../../engine";
 import type { TenantId } from "../../engine/types/identifiers";
 import { createSecret } from "../../secrets/types";
 import { createTestUser } from "../../stack";
+import { createRecordingProvider } from "../../testing";
 import { createDispatcher } from "../dispatcher";
 
 const streamCleanupState = { closed: false };
@@ -291,6 +293,66 @@ describe("dispatcher.stream", () => {
     ).rejects.toMatchObject({ code: "access_denied" });
   });
 
+  test("access invalidation mid-stream rejects with AccessDeniedError and unsubscribes", async () => {
+    let onInvalidate: (() => void) | undefined;
+    let unsubscribeCalls = 0;
+    const broker: SseBroker = {
+      addClient() {
+        return "c";
+      },
+      removeClient() {},
+      pushToChannel() {},
+      getClientCount() {
+        return 0;
+      },
+      getTotalClientCount() {
+        return 0;
+      },
+      subscribeAccessInvalidation(_userId, cb) {
+        onInvalidate = cb;
+        return () => {
+          unsubscribeCalls++;
+        };
+      },
+      publishAccessInvalidation() {},
+    };
+
+    let releaseHang: (() => void) | undefined;
+    const hang = new Promise<void>((resolve) => {
+      releaseHang = resolve;
+    });
+
+    const revokeFeature = defineFeature("revoke", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          await hang;
+          yield { i: 1 };
+          releaseHang?.();
+        },
+        { access: { roles: ["Admin"] } },
+      );
+    });
+
+    const user = createTestUser({ roles: ["Admin"] });
+    const dispatcher = createDispatcher(createRegistry([revokeFeature]), {}, { sseBroker: broker });
+    const gen = dispatcher.stream("revoke:stream:tail", {}, user);
+    const first = await gen.next();
+    expect(first.value).toEqual({ i: 0 });
+    expect(onInvalidate).toBeDefined();
+    // Start the idle second pull, then revoke — mirrors heartbeat-only SSE
+    // streams that must die without waiting for the next chunk (#1563).
+    const second = gen.next();
+    onInvalidate?.();
+    await expect(second).rejects.toMatchObject({
+      code: "access_denied",
+      message: expect.stringContaining("access revoked mid-stream"),
+    });
+    expect(unsubscribeCalls).toBe(1);
+  });
+
   test("throws for unknown stream handler", async () => {
     const dispatcher = createTestDispatcher();
 
@@ -324,6 +386,156 @@ describe("dispatcher.stream", () => {
 
     expect(streamCleanupState.closed).toBe(true);
   });
+
+  test("activates the dispatcher span in observabilityContext for every pull, not just the first", async () => {
+    // Regression for #1516: runStreamInstrumented used to start the span but
+    // never run the stream inside observabilityContext.run(...), so a
+    // handler-side startSpan() picked up whatever ALS context happened to be
+    // active at that pull (or none) instead of the stream's dispatcher span.
+    const provider = createRecordingProvider();
+    const nestedFeature = defineFeature("nested", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({ count: z.number().int().min(0) }),
+        async function* (query, handlerCtx) {
+          for (let i = 0; i < query.payload.count; i++) {
+            const child = handlerCtx.tracer.startSpan("nested.child");
+            child.end();
+            yield { i };
+          }
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([nestedFeature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    await collect(dispatcher.stream("nested:stream:tail", { count: 3 }, createTestUser()));
+
+    const dispatcherSpan = provider.spansByName("kumiko.dispatcher.handler")[0]!;
+    const childSpans = provider.spansByName("nested.child");
+    expect(childSpans).toHaveLength(3);
+    for (const child of childSpans) {
+      expect(child.parentSpanId).toBe(dispatcherSpan.spanId);
+    }
+  });
+
+  test("a close-time error on an otherwise-clean stream is reported, not swallowed", async () => {
+    // Regression for #1543: runStreamInstrumented's finally used to
+    // catch+discard any error from generator.return() unconditionally, even
+    // when nothing else had failed — an abandoned stream whose handler's
+    // own cleanup throws (closing a cursor/unsubscribing) was invisible to
+    // both the error-counter and the span status.
+    const provider = createRecordingProvider();
+    const closeFailFeature = defineFeature("closefail", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          try {
+            yield { i: 0 };
+            yield { i: 1 };
+          } finally {
+            // biome-ignore lint/correctness/noUnsafeFinally: deliberately simulating a handler whose close-time cleanup fails
+            throw new Error("close-boom");
+          }
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([closeFailFeature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    const gen = dispatcher.stream("closefail:stream:tail", {}, createTestUser());
+    for await (const _chunk of gen) {
+      break;
+    }
+
+    const errorCounters = provider.metricEvents.filter(
+      (e) => e.type === "counter.inc" && e.name === "kumiko_dispatcher_handler_errors_total",
+    );
+    expect(errorCounters).toHaveLength(1);
+    const dispatcherSpan = provider.spansByName("kumiko.dispatcher.handler")[0]!;
+    expect(dispatcherSpan.status).toBe("error");
+  });
+
+  test("draining a stream to completion emits outcome:completed", async () => {
+    // completedNormally=true only when the inner generator finishes on its
+    // own — pins the success-path label so a refactor that drops the flag
+    // cannot silently reclassify clean streams as aborted (#1551).
+    const provider = createRecordingProvider();
+    const feature = defineFeature("complete", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          yield { i: 1 };
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([feature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    await collect(dispatcher.stream("complete:stream:tail", {}, createTestUser()));
+
+    const duration = provider.metricEvents.find(
+      (e) =>
+        e.type === "histogram.observe" && e.name === "kumiko_dispatcher_handler_duration_seconds",
+    );
+    expect(duration).toBeDefined();
+    expect(duration?.labels).toMatchObject({
+      handler: "complete:stream:tail",
+      success: "true",
+      outcome: "completed",
+    });
+  });
+
+  test("generator.return() mid-stream emits outcome:aborted, not completed", async () => {
+    // Consumer walk-away (SSE tab-close) must not count as a clean
+    // completion — completedNormally stays false when return() interrupts
+    // the loop (#1551).
+    const provider = createRecordingProvider();
+    const feature = defineFeature("abort", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          yield { i: 1 };
+          yield { i: 2 };
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([feature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    const gen = dispatcher.stream("abort:stream:tail", {}, createTestUser());
+    const first = await gen.next();
+    expect(first.value).toEqual({ i: 0 });
+    await gen.return(undefined);
+
+    const duration = provider.metricEvents.find(
+      (e) =>
+        e.type === "histogram.observe" && e.name === "kumiko_dispatcher_handler_duration_seconds",
+    );
+    expect(duration).toBeDefined();
+    expect(duration?.labels).toMatchObject({
+      handler: "abort:stream:tail",
+      success: "true",
+      outcome: "aborted",
+    });
+  });
 });
 
 // --- postQuery hooks on standalone (entity-less) queries ---

package/src/pipeline/__tests__/event-consumer-state.integration.test.ts

@@ -0,0 +1,31 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { createTestDb, type TestDb } from "../../stack";
+import { createEventConsumerStateTable } from "../event-consumer-state";
+
+// #1362: two dispatcher instances booting concurrently against the same DB
+// both see kumiko_event_consumers already present but rearm_count missing,
+// and both ALTER TABLE. Without IF NOT EXISTS the loser crashes on boot
+// with "column already exists" — a real TOCTOU since this table is
+// explicitly multi-instance.
+describe("createEventConsumerStateTable — concurrent boot", () => {
+  let testDb: TestDb;
+
+  beforeAll(async () => {
+    testDb = await createTestDb();
+  });
+
+  afterAll(async () => {
+    await testDb.cleanup();
+  });
+
+  test("running it twice concurrently does not throw", async () => {
+    await createEventConsumerStateTable(testDb.db);
+
+    await expect(
+      Promise.all([
+        createEventConsumerStateTable(testDb.db),
+        createEventConsumerStateTable(testDb.db),
+      ]),
+    ).resolves.toBeDefined();
+  });
+});

package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts

@@ -11,6 +11,13 @@ import { createEventStoreExecutor } from "../../db/event-store-executor";
 import { asRawClient } from "../../db/query";
 import { createTenantDb, type TenantDb } from "../../db/tenant-db";
 import { defineFeature } from "../../engine";
+import {
+  DEFAULT_SENSITIVE_CONFIG,
+  type MetricEvent,
+  type ObservabilityProvider,
+  RecordingMeter,
+  RecordingTracer,
+} from "../../observability";
 import { getConsumerState } from "../../pipeline";
 import {
   resetEventStore,
@@ -20,6 +27,7 @@ import {
   unsafeCreateEntityTable,
 } from "../../stack";
 import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
+import { withoutAmbientTemporal } from "../../testing/without-ambient-temporal";
 import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
 
 const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
@@ -151,6 +159,23 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(deadAgain?.rearmCount).toBe(0);
   });
 
+  test("kumiko-framework#1525: cooldown-elapsed re-arm check works without relying on globalThis.Temporal", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+    await backdateUpdatedAt(10);
+
+    // ensureTemporalPolyfill re-installs if the ambient global was torn down,
+    // so runOnce stays safe if a nested path calls it during this window.
+    await withoutAmbientTemporal(async () => {
+      await stack.eventDispatcher?.runOnce();
+    });
+
+    const revived = await getConsumerState(stack.db, qn);
+    expect(revived?.status).toBe("idle");
+    expect(revived?.rearmCount).toBe(1);
+  });
+
   test("permanently dead after maxRearmCount cycles, even once cooldown elapses again", async () => {
     poisonNames.add("poison");
     await appendWidget("poison");
@@ -177,4 +202,62 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(stillExhausted?.status).toBe("dead");
     expect(stillExhausted?.rearmCount).toBe(3);
   });
+
+  test("kumiko_event_consumer_rearm_exhausted_total fires once, not per still-dead pass (#1362)", async () => {
+    // Dedicated stack with a RecordingMeter — module-level `stack` above is
+    // shared across every test in this file and has no metric capture.
+    const metricEvents: MetricEvent[] = [];
+    const meter = new RecordingMeter((e) => metricEvents.push(e));
+    const tracer = new RecordingTracer({
+      sensitiveConfig: DEFAULT_SENSITIVE_CONFIG,
+      onSpanEnd: () => {},
+    });
+    const recordingProvider: ObservabilityProvider = {
+      name: "recording",
+      meter,
+      tracer,
+      shutdown: async () => {},
+    };
+
+    const recStack = await setupTestStack({
+      features: [rearmFeature],
+      systemHooks: [],
+      observability: recordingProvider,
+    });
+    try {
+      await unsafeCreateEntityTable(recStack.db, sharedWidgetEntity, "widget");
+      const recTdb = createTenantDb(recStack.db, admin.tenantId);
+
+      poisonNames.add("poison-metric");
+      await executor.create({ name: "poison-metric" }, admin, recTdb);
+      for (let i = 0; i < 10; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      // The 10th pass is the one whose failed delivery flips status to
+      // "dead" (persisted for the NEXT pass to see) — acquireConsumerState's
+      // dead-branch (where the exhausted-metric fires) only runs starting
+      // from the pass AFTER that transition, same as the "stays dead within
+      // cooldown window" test above needing one extra runOnce().
+      await recStack.eventDispatcher?.runOnce();
+
+      const exhaustedEvents = () =>
+        metricEvents.filter(
+          (e) =>
+            e.type === "counter.inc" && e.name === "kumiko_event_consumer_rearm_exhausted_total",
+        );
+      expect(exhaustedEvents().length).toBe(1);
+
+      // Several more passes while it's still dead — must stay at 1, not
+      // re-fire on every pass that observes "still dead" (log/metric spam
+      // the finding flagged).
+      for (let i = 0; i < 5; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      expect(exhaustedEvents().length).toBe(1);
+      expect(exhaustedEvents()[0]?.labels?.["consumer"]).toBe(qn);
+    } finally {
+      await resetEventStore(recStack, ["read_widgets"]);
+      await recStack.cleanup();
+    }
+  });
 });