@cosmicdrift/kumiko-framework 0.164.0 → 0.165.1

package/src/observability/standard-metrics.ts

@@ -23,7 +23,7 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
     name: "kumiko_dispatcher_handler_duration_seconds",
     type: "histogram",
     description: "Dispatcher handler latency in seconds.",
-    labels: ["handler", "success"],
+    labels: ["handler", "success", "outcome"],
     buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5],
   },
   {
@@ -83,6 +83,20 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
       "Event deliveries that threw. Repeated failures on the same event lead to dead-letter.",
     labels: ["consumer", "instance_id"],
   },
+  // Fires once per (consumer, instance) transition into "stuck dead" —
+  // the rearm-cooldown/rearm-budget in event-dispatcher-delivery.ts gave up
+  // and the consumer now needs an operator (restartConsumer()) to recover.
+  // Without this, "dead + auto-rearm exhausted" is only visible via a
+  // getConsumerState poll — ops has no push signal that a consumer crossed
+  // from "will self-heal" to "needs a human". Deliberately a counter, not a
+  // gauge: emitted once per transition (event-dispatcher.ts tracks which
+  // consumers already fired this), not once per still-dead poll pass.
+  {
+    name: "kumiko_event_consumer_rearm_exhausted_total",
+    type: "counter",
+    description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
+    labels: ["consumer", "instance_id"],
+  },
   // LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
   // the dispatcher is running signal that delivery latency has regressed
   // from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
@@ -141,12 +155,18 @@ export function emitHttpRequest(
 
 export function emitDispatcherHandler(
   meter: Meter,
-  labels: { readonly handler: string; readonly success: boolean },
+  labels: {
+    readonly handler: string;
+    readonly success: boolean;
+    /** completed = drained normally; aborted = consumer walked away (SSE). */
+    readonly outcome?: "completed" | "aborted";
+  },
   durationSeconds: number,
 ): void {
   meter.histogram("kumiko_dispatcher_handler_duration_seconds").observe(durationSeconds, {
     handler: labels.handler,
     success: String(labels.success),
+    outcome: labels.outcome ?? "completed",
   });
 }
 
@@ -203,6 +223,19 @@ export function emitEventConsumerLag(
   });
 }
 
+// Caller (event-dispatcher.ts) is responsible for calling this exactly
+// once per (consumer, instanceId) transition into exhausted-dead — not
+// once per poll pass while it stays dead.
+export function emitEventConsumerRearmExhausted(
+  meter: Meter,
+  labels: { readonly consumer: string; readonly instanceId: string },
+): void {
+  meter.counter("kumiko_event_consumer_rearm_exhausted_total").inc(1, {
+    consumer: labels.consumer,
+    instance_id: labels.instanceId,
+  });
+}
+
 export function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void {
   meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
 }

package/src/observability/types/index.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability";
+export type * from "@cosmicdrift/kumiko-types/observability";

package/src/observability/types/metric.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/metric";
+export type * from "@cosmicdrift/kumiko-types/observability/metric";

package/src/observability/types/provider.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/provider";
+export type * from "@cosmicdrift/kumiko-types/observability/provider";

package/src/observability/types/span.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/span";
+export type * from "@cosmicdrift/kumiko-types/observability/span";

package/src/pipeline/__tests__/dispatcher.test.ts

@@ -2,9 +2,13 @@ import { describe, expect, test } from "bun:test";
 import { z } from "zod";
 import { createEntity, createRegistry, createTextField, defineFeature } from "../../engine";
 import type { TenantId } from "../../engine/types/identifiers";
+import { createSecret } from "../../secrets/types";
 import { createTestUser } from "../../stack";
+import { createRecordingProvider } from "../../testing";
 import { createDispatcher } from "../dispatcher";
 
+const streamCleanupState = { closed: false };
+
 const echoFeature = defineFeature("echo", (r) => {
   r.entity("item", createEntity({ table: "Items", fields: { name: createTextField() } }));
 
@@ -36,6 +40,29 @@ const echoFeature = defineFeature("echo", (r) => {
     { access: { roles: ["Admin"] } },
   );
 
+  r.streamHandler(
+    "item:tail-leak",
+    z.object({}),
+    async function* () {
+      yield { apiKey: createSecret("leak-me") };
+    },
+    { access: { roles: ["Admin"] } },
+  );
+
+  r.streamHandler(
+    "item:tail-cleanup",
+    z.object({}),
+    async function* () {
+      try {
+        yield { i: 0 };
+        yield { i: 1 };
+      } finally {
+        streamCleanupState.closed = true;
+      }
+    },
+    { access: { roles: ["Admin"] } },
+  );
+
   r.hook("validation", "item:create", (data) => {
     if (data["name"] === "forbidden") return [{ field: "name", error: "forbidden_name" }];
     return null;
@@ -272,6 +299,182 @@ describe("dispatcher.stream", () => {
       collect(dispatcher.stream("nonexistent", {}, createTestUser({ roles: ["Admin"] }))),
     ).rejects.toMatchObject({ code: "not_found", httpStatus: 404 });
   });
+
+  test("a chunk containing a Secret<> value aborts the stream instead of leaking it", async () => {
+    const dispatcher = createTestDispatcher();
+
+    await expect(
+      collect(
+        dispatcher.stream("echo:stream:item:tail-leak", {}, createTestUser({ roles: ["Admin"] })),
+      ),
+    ).rejects.toMatchObject({ message: expect.stringContaining("Secret<> leaked") });
+  });
+
+  test("consumer breaking out of for-await runs the handler generator's finally block", async () => {
+    streamCleanupState.closed = false;
+    const dispatcher = createTestDispatcher();
+    const gen = dispatcher.stream(
+      "echo:stream:item:tail-cleanup",
+      {},
+      createTestUser({ roles: ["Admin"] }),
+    );
+
+    for await (const _chunk of gen) {
+      break;
+    }
+
+    expect(streamCleanupState.closed).toBe(true);
+  });
+
+  test("activates the dispatcher span in observabilityContext for every pull, not just the first", async () => {
+    // Regression for #1516: runStreamInstrumented used to start the span but
+    // never run the stream inside observabilityContext.run(...), so a
+    // handler-side startSpan() picked up whatever ALS context happened to be
+    // active at that pull (or none) instead of the stream's dispatcher span.
+    const provider = createRecordingProvider();
+    const nestedFeature = defineFeature("nested", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({ count: z.number().int().min(0) }),
+        async function* (query, handlerCtx) {
+          for (let i = 0; i < query.payload.count; i++) {
+            const child = handlerCtx.tracer.startSpan("nested.child");
+            child.end();
+            yield { i };
+          }
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([nestedFeature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    await collect(dispatcher.stream("nested:stream:tail", { count: 3 }, createTestUser()));
+
+    const dispatcherSpan = provider.spansByName("kumiko.dispatcher.handler")[0]!;
+    const childSpans = provider.spansByName("nested.child");
+    expect(childSpans).toHaveLength(3);
+    for (const child of childSpans) {
+      expect(child.parentSpanId).toBe(dispatcherSpan.spanId);
+    }
+  });
+
+  test("a close-time error on an otherwise-clean stream is reported, not swallowed", async () => {
+    // Regression for #1543: runStreamInstrumented's finally used to
+    // catch+discard any error from generator.return() unconditionally, even
+    // when nothing else had failed — an abandoned stream whose handler's
+    // own cleanup throws (closing a cursor/unsubscribing) was invisible to
+    // both the error-counter and the span status.
+    const provider = createRecordingProvider();
+    const closeFailFeature = defineFeature("closefail", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          try {
+            yield { i: 0 };
+            yield { i: 1 };
+          } finally {
+            // biome-ignore lint/correctness/noUnsafeFinally: deliberately simulating a handler whose close-time cleanup fails
+            throw new Error("close-boom");
+          }
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([closeFailFeature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    const gen = dispatcher.stream("closefail:stream:tail", {}, createTestUser());
+    for await (const _chunk of gen) {
+      break;
+    }
+
+    const errorCounters = provider.metricEvents.filter(
+      (e) => e.type === "counter.inc" && e.name === "kumiko_dispatcher_handler_errors_total",
+    );
+    expect(errorCounters).toHaveLength(1);
+    const dispatcherSpan = provider.spansByName("kumiko.dispatcher.handler")[0]!;
+    expect(dispatcherSpan.status).toBe("error");
+  });
+
+  test("draining a stream to completion emits outcome:completed", async () => {
+    // completedNormally=true only when the inner generator finishes on its
+    // own — pins the success-path label so a refactor that drops the flag
+    // cannot silently reclassify clean streams as aborted (#1551).
+    const provider = createRecordingProvider();
+    const feature = defineFeature("complete", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          yield { i: 1 };
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([feature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    await collect(dispatcher.stream("complete:stream:tail", {}, createTestUser()));
+
+    const duration = provider.metricEvents.find(
+      (e) =>
+        e.type === "histogram.observe" && e.name === "kumiko_dispatcher_handler_duration_seconds",
+    );
+    expect(duration).toBeDefined();
+    expect(duration?.labels).toMatchObject({
+      handler: "complete:stream:tail",
+      success: "true",
+      outcome: "completed",
+    });
+  });
+
+  test("generator.return() mid-stream emits outcome:aborted, not completed", async () => {
+    // Consumer walk-away (SSE tab-close) must not count as a clean
+    // completion — completedNormally stays false when return() interrupts
+    // the loop (#1551).
+    const provider = createRecordingProvider();
+    const feature = defineFeature("abort", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          yield { i: 1 };
+          yield { i: 2 };
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([feature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    const gen = dispatcher.stream("abort:stream:tail", {}, createTestUser());
+    const first = await gen.next();
+    expect(first.value).toEqual({ i: 0 });
+    await gen.return(undefined);
+
+    const duration = provider.metricEvents.find(
+      (e) =>
+        e.type === "histogram.observe" && e.name === "kumiko_dispatcher_handler_duration_seconds",
+    );
+    expect(duration).toBeDefined();
+    expect(duration?.labels).toMatchObject({
+      handler: "abort:stream:tail",
+      success: "true",
+      outcome: "aborted",
+    });
+  });
 });
 
 // --- postQuery hooks on standalone (entity-less) queries ---

package/src/pipeline/__tests__/event-consumer-state.integration.test.ts

@@ -0,0 +1,31 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { createTestDb, type TestDb } from "../../stack";
+import { createEventConsumerStateTable } from "../event-consumer-state";
+
+// #1362: two dispatcher instances booting concurrently against the same DB
+// both see kumiko_event_consumers already present but rearm_count missing,
+// and both ALTER TABLE. Without IF NOT EXISTS the loser crashes on boot
+// with "column already exists" — a real TOCTOU since this table is
+// explicitly multi-instance.
+describe("createEventConsumerStateTable — concurrent boot", () => {
+  let testDb: TestDb;
+
+  beforeAll(async () => {
+    testDb = await createTestDb();
+  });
+
+  afterAll(async () => {
+    await testDb.cleanup();
+  });
+
+  test("running it twice concurrently does not throw", async () => {
+    await createEventConsumerStateTable(testDb.db);
+
+    await expect(
+      Promise.all([
+        createEventConsumerStateTable(testDb.db),
+        createEventConsumerStateTable(testDb.db),
+      ]),
+    ).resolves.toBeDefined();
+  });
+});

package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts

@@ -11,6 +11,13 @@ import { createEventStoreExecutor } from "../../db/event-store-executor";
 import { asRawClient } from "../../db/query";
 import { createTenantDb, type TenantDb } from "../../db/tenant-db";
 import { defineFeature } from "../../engine";
+import {
+  DEFAULT_SENSITIVE_CONFIG,
+  type MetricEvent,
+  type ObservabilityProvider,
+  RecordingMeter,
+  RecordingTracer,
+} from "../../observability";
 import { getConsumerState } from "../../pipeline";
 import {
   resetEventStore,
@@ -20,6 +27,7 @@ import {
   unsafeCreateEntityTable,
 } from "../../stack";
 import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
+import { withoutAmbientTemporal } from "../../testing/without-ambient-temporal";
 import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
 
 const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
@@ -151,6 +159,23 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(deadAgain?.rearmCount).toBe(0);
   });
 
+  test("kumiko-framework#1525: cooldown-elapsed re-arm check works without relying on globalThis.Temporal", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+    await backdateUpdatedAt(10);
+
+    // ensureTemporalPolyfill re-installs if the ambient global was torn down,
+    // so runOnce stays safe if a nested path calls it during this window.
+    await withoutAmbientTemporal(async () => {
+      await stack.eventDispatcher?.runOnce();
+    });
+
+    const revived = await getConsumerState(stack.db, qn);
+    expect(revived?.status).toBe("idle");
+    expect(revived?.rearmCount).toBe(1);
+  });
+
   test("permanently dead after maxRearmCount cycles, even once cooldown elapses again", async () => {
     poisonNames.add("poison");
     await appendWidget("poison");
@@ -177,4 +202,62 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(stillExhausted?.status).toBe("dead");
     expect(stillExhausted?.rearmCount).toBe(3);
   });
+
+  test("kumiko_event_consumer_rearm_exhausted_total fires once, not per still-dead pass (#1362)", async () => {
+    // Dedicated stack with a RecordingMeter — module-level `stack` above is
+    // shared across every test in this file and has no metric capture.
+    const metricEvents: MetricEvent[] = [];
+    const meter = new RecordingMeter((e) => metricEvents.push(e));
+    const tracer = new RecordingTracer({
+      sensitiveConfig: DEFAULT_SENSITIVE_CONFIG,
+      onSpanEnd: () => {},
+    });
+    const recordingProvider: ObservabilityProvider = {
+      name: "recording",
+      meter,
+      tracer,
+      shutdown: async () => {},
+    };
+
+    const recStack = await setupTestStack({
+      features: [rearmFeature],
+      systemHooks: [],
+      observability: recordingProvider,
+    });
+    try {
+      await unsafeCreateEntityTable(recStack.db, sharedWidgetEntity, "widget");
+      const recTdb = createTenantDb(recStack.db, admin.tenantId);
+
+      poisonNames.add("poison-metric");
+      await executor.create({ name: "poison-metric" }, admin, recTdb);
+      for (let i = 0; i < 10; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      // The 10th pass is the one whose failed delivery flips status to
+      // "dead" (persisted for the NEXT pass to see) — acquireConsumerState's
+      // dead-branch (where the exhausted-metric fires) only runs starting
+      // from the pass AFTER that transition, same as the "stays dead within
+      // cooldown window" test above needing one extra runOnce().
+      await recStack.eventDispatcher?.runOnce();
+
+      const exhaustedEvents = () =>
+        metricEvents.filter(
+          (e) =>
+            e.type === "counter.inc" && e.name === "kumiko_event_consumer_rearm_exhausted_total",
+        );
+      expect(exhaustedEvents().length).toBe(1);
+
+      // Several more passes while it's still dead — must stay at 1, not
+      // re-fire on every pass that observes "still dead" (log/metric spam
+      // the finding flagged).
+      for (let i = 0; i < 5; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      expect(exhaustedEvents().length).toBe(1);
+      expect(exhaustedEvents()[0]?.labels?.["consumer"]).toBe(qn);
+    } finally {
+      await resetEventStore(recStack, ["read_widgets"]);
+      await recStack.cleanup();
+    }
+  });
 });

package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts

@@ -0,0 +1,106 @@
+// createJobTriggerEventConsumer — proves r.job's trigger.on can fire on an
+// r.defineEvent QN appended by a multiStreamProjection's unsafeAppendEvent
+// (kumiko-framework#1505). Mirrors document-ingest-foundation's actual
+// request-ingest MSP (upload → fileRef.created → an owned defineEvent),
+// the motivating case for this fix — fileRef.created itself never reaches
+// jobRunner.handleEvent because the upload route appends it via the raw
+// event-store executor, not a write-handler dispatch (see #1505).
+//
+// Not covered here: a job triggered on a write/query-handler QN still
+// firing exactly once (unaffected by the new consumer). The full suite
+// stays green (e.g. the lane-routing sample), but that's not a positive
+// test of the partition guard — no stored event's `type` is ever a
+// handler QN in practice (entity events are "entity.verb"; custom
+// write-handlers like lane-routing's don't append to the store at all),
+// so `getWriteHandler`/`getQueryHandler` in the new consumer's handler is
+// defense-in-depth for an input shape the framework doesn't currently
+// produce, not something exercised end-to-end by any test today.
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { z } from "zod";
+import { entityEventName } from "../../db";
+import { defineFeature } from "../../engine";
+import { createInMemoryFileProvider, type InMemoryFileProvider } from "../../files";
+import { setupTestStack, type TestStack, TestUsers } from "../../stack";
+import { waitFor } from "../../testing";
+
+const ITEM_REQUESTED_EVENT_QN = "job-trigger-fixture:event:item-requested";
+const FILE_REF_CREATED = entityEventName("fileRef", "created");
+
+const processedItems: Array<{ readonly fileRefId: string }> = [];
+
+const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
+  r.defineEvent("item-requested", z.object({ fileRefId: z.string().min(1) }));
+
+  // Mirrors document-ingest-foundation's request-ingest MSP exactly: reacts
+  // to fileRef.created, appends a NEW event via unsafeAppendEvent — no
+  // write-handler behind the appended event itself.
+  r.multiStreamProjection({
+    name: "request-item",
+    apply: {
+      [FILE_REF_CREATED]: async (event, _tx, ctx) => {
+        await ctx.unsafeAppendEvent({
+          aggregateId: event.aggregateId,
+          aggregateType: "job-trigger-fixture-request",
+          type: ITEM_REQUESTED_EVENT_QN,
+          payload: { fileRefId: event.aggregateId },
+        });
+      },
+    },
+  });
+
+  // Under test: only reachable via createJobTriggerEventConsumer, since
+  // ITEM_REQUESTED_EVENT_QN is an r.defineEvent QN, not a handler QN.
+  r.job(
+    "process-item",
+    { trigger: { on: ITEM_REQUESTED_EVENT_QN }, runIn: "worker" },
+    async (payload) => {
+      processedItems.push({ fileRefId: payload["fileRefId"] as string });
+    },
+  );
+});
+
+let stack: TestStack;
+let provider: InMemoryFileProvider;
+
+beforeAll(async () => {
+  provider = createInMemoryFileProvider();
+  stack = await setupTestStack({
+    features: [jobTriggerFixtureFeature],
+    files: { storageProvider: provider },
+    jobs: { consumerLane: "worker" },
+  });
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+beforeEach(() => {
+  processedItems.length = 0;
+  provider.clear();
+});
+
+describe("job-trigger event consumer", () => {
+  test("a job triggers on an r.defineEvent QN appended by an MSP's unsafeAppendEvent", async () => {
+    const token = await stack.jwt.sign(TestUsers.admin);
+    const formData = new FormData();
+    formData.append("file", new File([Buffer.from("hello")], "note.txt", { type: "text/plain" }));
+    const res = await stack.app.request("/api/files", {
+      method: "POST",
+      headers: { Authorization: `Bearer ${token}` },
+      body: formData,
+    });
+    expect(res.status).toBe(201);
+
+    await waitFor(async () => {
+      // Drives both the MSP (appends item-requested off fileRef.created)
+      // and the new job-trigger consumer (reacts to it) — may need more
+      // than one pass since the MSP's append happens mid-drain.
+      await stack.eventDispatcher?.runOnce();
+      expect(processedItems).toHaveLength(1);
+    });
+
+    expect(processedItems[0]?.fileRefId).toBeTruthy();
+  });
+});