@cosmicdrift/kumiko-framework 0.165.0 → 2.0.0

package/src/jobs/job-runner.ts

@@ -1,7 +1,8 @@
 import { type Job, Queue, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context";
-import type { DbRow } from "../db/connection";
+import type { DbConnection, DbRow } from "../db/connection";
+import { createTenantDb } from "../db/tenant-db";
 import { createSystemUser } from "../engine/system-user";
 import {
   type AppContext,
@@ -9,7 +10,9 @@ import {
   type Registry,
   type SessionUser,
   SYSTEM_TENANT_ID,
+  type TenantId,
 } from "../engine/types";
+import { createFileContext } from "../files/file-handle";
 import type { Logger } from "../logging/types";
 import {
   emitJobQueueDepth,
@@ -37,9 +40,10 @@ export type JobLogEntry = {
   timestamp: Temporal.Instant;
 };
 
-function createJobLogger(logs: JobLogEntry[]): Logger {
+function createJobLogger(logs: JobLogEntry[], baseContext: Record<string, unknown> = {}): Logger {
   function push(level: "info" | "warn" | "error", msg: string, data?: Record<string, unknown>) {
-    const message = data ? `${msg} ${JSON.stringify(data)}` : msg;
+    const merged = { ...baseContext, ...data };
+    const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
     logs.push({ level, message, timestamp: Temporal.Now.instant() });
   }
   const logger: Logger = {
@@ -52,9 +56,13 @@ function createJobLogger(logs: JobLogEntry[]): Logger {
     error(msg, data) {
       push("error", msg, data);
     },
+    // No-op by design, not missing coverage: JobLogEntry.level (and the
+    // persisted run-completed/-failed event schema in bundled-features/
+    // jobs/events.ts) only has info|warn|error — a job's debug() calls have
+    // nowhere durable to land.
     debug() {},
-    child() {
-      return logger;
+    child(context) {
+      return createJobLogger(logs, { ...baseContext, ...context });
     },
   };
   return logger;
@@ -100,7 +108,7 @@ export type JobRunnerOptions = {
   // Tests set a unique prefix (e.g. `"test-${Date.now()}"`) for isolation —
   // two parallel test-runners never see each other's jobs.
   queueNamePrefix?: string | undefined;
-  getActiveTenantIds?: () => Promise<number[]>;
+  getActiveTenantIds?: () => Promise<TenantId[]>;
   onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
   onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[]) => void;
   onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[]) => void;
@@ -331,13 +339,35 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
       SYSTEM_TENANT_ID;
     const triggeredById = (rawData["_triggeredById"] as string | undefined) ?? null; // @cast-boundary dynamic-key
 
-    // _triggerName aus rawData übernehmen falls gesetzt — handleEvent
-    // packt das beim Multi-Trigger-Dispatch rein (siehe unten). Über
-    // jobContext.triggerName freigegeben damit der Handler nicht selbst
-    // im rohen Payload kramen muss.
+    // Carry `_triggerName` from rawData when set — handleEvent injects it on
+    // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
+    // don't dig through the raw payload themselves.
     const triggerName = rawData["_triggerName"] as string | undefined; // @cast-boundary dynamic-key
+    // Mirror dispatch-shared.ts buildHandlerContext: ctx.files must resolve
+    // through the same _fileProviderResolver for jobs as for write-handlers,
+    // otherwise event-triggered jobs silently get an unresolved ctx.files.
+    const fileResolver = context._fileProviderResolver;
+    const files = fileResolver ? createFileContext(() => fileResolver(tenantId)) : context.files;
+    const jobSystemUser = createSystemUser(tenantId);
+    // Same buildHandlerContext parity as ctx.files above: ctx.notify/ctx.config
+    // must resolve for jobs the same way they do for write-handlers, or an
+    // app-author job calling ctx.notify(...)/ctx.config(...) hits a TypeError
+    // at runtime the write-handler path never would (framework#1532).
+    const notify = context._notifyFactory?.(jobSystemUser, tenantId);
+    const configDb = context.db as DbConnection | undefined; // @cast-boundary db-operator
+    const config =
+      context._configAccessorFactory && configDb
+        ? context._configAccessorFactory({
+            user: { id: jobSystemUser.id, tenantId },
+            db: createTenantDb(configDb, tenantId, "system"),
+            secrets: context.secrets,
+          })
+        : undefined;
     const jobContext: AppContext = {
       ...context,
+      files,
+      ...(notify !== undefined && { notify }),
+      ...(config !== undefined && { config }),
       // The runner owns the registry it resolved this job from — expose it so
       // workers can reach projections/jobs without the app author duplicating
       // it into `context` (the JobContext contract guarantees `registry`).
@@ -345,7 +375,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
       // Expose the runner so handlers can chain a follow-up job. Symmetric with
       // how the command-dispatcher hands write-handlers their jobRunner.
       ...(selfRunner !== undefined && { jobRunner: selfRunner }),
-      systemUser: createSystemUser(tenantId),
+      systemUser: jobSystemUser,
       triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
       log: createJobLogger(logs),
       ...(triggerName !== undefined && { triggerName }),

package/src/logging/types.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/logger";
+export type * from "@cosmicdrift/kumiko-types/logger";

package/src/observability/index.ts

@@ -50,6 +50,7 @@ export {
   emitDispatcherHandler,
   emitEventConsumerLag,
   emitEventConsumerPassOutcome,
+  emitEventConsumerRearmExhausted,
   emitEventDispatcherListenConnected,
   emitHttpRequest,
   emitJobQueueDepth,

package/src/observability/standard-metrics.ts

@@ -23,7 +23,7 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
     name: "kumiko_dispatcher_handler_duration_seconds",
     type: "histogram",
     description: "Dispatcher handler latency in seconds.",
-    labels: ["handler", "success"],
+    labels: ["handler", "success", "outcome"],
     buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5],
   },
   {
@@ -83,6 +83,20 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
       "Event deliveries that threw. Repeated failures on the same event lead to dead-letter.",
     labels: ["consumer", "instance_id"],
   },
+  // Fires once per (consumer, instance) transition into "stuck dead" —
+  // the rearm-cooldown/rearm-budget in event-dispatcher-delivery.ts gave up
+  // and the consumer now needs an operator (restartConsumer()) to recover.
+  // Without this, "dead + auto-rearm exhausted" is only visible via a
+  // getConsumerState poll — ops has no push signal that a consumer crossed
+  // from "will self-heal" to "needs a human". Deliberately a counter, not a
+  // gauge: emitted once per transition (event-dispatcher.ts tracks which
+  // consumers already fired this), not once per still-dead poll pass.
+  {
+    name: "kumiko_event_consumer_rearm_exhausted_total",
+    type: "counter",
+    description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
+    labels: ["consumer", "instance_id"],
+  },
   // LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
   // the dispatcher is running signal that delivery latency has regressed
   // from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
@@ -141,12 +155,18 @@ export function emitHttpRequest(
 
 export function emitDispatcherHandler(
   meter: Meter,
-  labels: { readonly handler: string; readonly success: boolean },
+  labels: {
+    readonly handler: string;
+    readonly success: boolean;
+    /** completed = drained normally; aborted = consumer walked away (SSE). */
+    readonly outcome?: "completed" | "aborted";
+  },
   durationSeconds: number,
 ): void {
   meter.histogram("kumiko_dispatcher_handler_duration_seconds").observe(durationSeconds, {
     handler: labels.handler,
     success: String(labels.success),
+    outcome: labels.outcome ?? "completed",
   });
 }
 
@@ -203,6 +223,19 @@ export function emitEventConsumerLag(
   });
 }
 
+// Caller (event-dispatcher.ts) is responsible for calling this exactly
+// once per (consumer, instanceId) transition into exhausted-dead — not
+// once per poll pass while it stays dead.
+export function emitEventConsumerRearmExhausted(
+  meter: Meter,
+  labels: { readonly consumer: string; readonly instanceId: string },
+): void {
+  meter.counter("kumiko_event_consumer_rearm_exhausted_total").inc(1, {
+    consumer: labels.consumer,
+    instance_id: labels.instanceId,
+  });
+}
+
 export function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void {
   meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
 }

package/src/observability/types/index.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability";
+export type * from "@cosmicdrift/kumiko-types/observability";

package/src/observability/types/metric.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/metric";
+export type * from "@cosmicdrift/kumiko-types/observability/metric";

package/src/observability/types/provider.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/provider";
+export type * from "@cosmicdrift/kumiko-types/observability/provider";

package/src/observability/types/span.ts

@@ -1 +1 @@
-export * from "@cosmicdrift/kumiko-types/observability/span";
+export type * from "@cosmicdrift/kumiko-types/observability/span";

package/src/pipeline/__tests__/dispatcher.test.ts

@@ -4,6 +4,7 @@ import { createEntity, createRegistry, createTextField, defineFeature } from "..
 import type { TenantId } from "../../engine/types/identifiers";
 import { createSecret } from "../../secrets/types";
 import { createTestUser } from "../../stack";
+import { createRecordingProvider } from "../../testing";
 import { createDispatcher } from "../dispatcher";
 
 const streamCleanupState = { closed: false };
@@ -324,6 +325,156 @@ describe("dispatcher.stream", () => {
 
     expect(streamCleanupState.closed).toBe(true);
   });
+
+  test("activates the dispatcher span in observabilityContext for every pull, not just the first", async () => {
+    // Regression for #1516: runStreamInstrumented used to start the span but
+    // never run the stream inside observabilityContext.run(...), so a
+    // handler-side startSpan() picked up whatever ALS context happened to be
+    // active at that pull (or none) instead of the stream's dispatcher span.
+    const provider = createRecordingProvider();
+    const nestedFeature = defineFeature("nested", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({ count: z.number().int().min(0) }),
+        async function* (query, handlerCtx) {
+          for (let i = 0; i < query.payload.count; i++) {
+            const child = handlerCtx.tracer.startSpan("nested.child");
+            child.end();
+            yield { i };
+          }
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([nestedFeature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    await collect(dispatcher.stream("nested:stream:tail", { count: 3 }, createTestUser()));
+
+    const dispatcherSpan = provider.spansByName("kumiko.dispatcher.handler")[0]!;
+    const childSpans = provider.spansByName("nested.child");
+    expect(childSpans).toHaveLength(3);
+    for (const child of childSpans) {
+      expect(child.parentSpanId).toBe(dispatcherSpan.spanId);
+    }
+  });
+
+  test("a close-time error on an otherwise-clean stream is reported, not swallowed", async () => {
+    // Regression for #1543: runStreamInstrumented's finally used to
+    // catch+discard any error from generator.return() unconditionally, even
+    // when nothing else had failed — an abandoned stream whose handler's
+    // own cleanup throws (closing a cursor/unsubscribing) was invisible to
+    // both the error-counter and the span status.
+    const provider = createRecordingProvider();
+    const closeFailFeature = defineFeature("closefail", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          try {
+            yield { i: 0 };
+            yield { i: 1 };
+          } finally {
+            // biome-ignore lint/correctness/noUnsafeFinally: deliberately simulating a handler whose close-time cleanup fails
+            throw new Error("close-boom");
+          }
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([closeFailFeature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    const gen = dispatcher.stream("closefail:stream:tail", {}, createTestUser());
+    for await (const _chunk of gen) {
+      break;
+    }
+
+    const errorCounters = provider.metricEvents.filter(
+      (e) => e.type === "counter.inc" && e.name === "kumiko_dispatcher_handler_errors_total",
+    );
+    expect(errorCounters).toHaveLength(1);
+    const dispatcherSpan = provider.spansByName("kumiko.dispatcher.handler")[0]!;
+    expect(dispatcherSpan.status).toBe("error");
+  });
+
+  test("draining a stream to completion emits outcome:completed", async () => {
+    // completedNormally=true only when the inner generator finishes on its
+    // own — pins the success-path label so a refactor that drops the flag
+    // cannot silently reclassify clean streams as aborted (#1551).
+    const provider = createRecordingProvider();
+    const feature = defineFeature("complete", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          yield { i: 1 };
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([feature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    await collect(dispatcher.stream("complete:stream:tail", {}, createTestUser()));
+
+    const duration = provider.metricEvents.find(
+      (e) =>
+        e.type === "histogram.observe" && e.name === "kumiko_dispatcher_handler_duration_seconds",
+    );
+    expect(duration).toBeDefined();
+    expect(duration?.labels).toMatchObject({
+      handler: "complete:stream:tail",
+      success: "true",
+      outcome: "completed",
+    });
+  });
+
+  test("generator.return() mid-stream emits outcome:aborted, not completed", async () => {
+    // Consumer walk-away (SSE tab-close) must not count as a clean
+    // completion — completedNormally stays false when return() interrupts
+    // the loop (#1551).
+    const provider = createRecordingProvider();
+    const feature = defineFeature("abort", (r) => {
+      r.streamHandler(
+        "tail",
+        z.object({}),
+        async function* () {
+          yield { i: 0 };
+          yield { i: 1 };
+          yield { i: 2 };
+        },
+        { access: { openToAll: true } },
+      );
+    });
+    const dispatcher = createDispatcher(createRegistry([feature]), {
+      tracer: provider.tracer,
+      meter: provider.meter,
+    });
+
+    const gen = dispatcher.stream("abort:stream:tail", {}, createTestUser());
+    const first = await gen.next();
+    expect(first.value).toEqual({ i: 0 });
+    await gen.return(undefined);
+
+    const duration = provider.metricEvents.find(
+      (e) =>
+        e.type === "histogram.observe" && e.name === "kumiko_dispatcher_handler_duration_seconds",
+    );
+    expect(duration).toBeDefined();
+    expect(duration?.labels).toMatchObject({
+      handler: "abort:stream:tail",
+      success: "true",
+      outcome: "aborted",
+    });
+  });
 });
 
 // --- postQuery hooks on standalone (entity-less) queries ---

package/src/pipeline/__tests__/event-consumer-state.integration.test.ts

@@ -0,0 +1,31 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { createTestDb, type TestDb } from "../../stack";
+import { createEventConsumerStateTable } from "../event-consumer-state";
+
+// #1362: two dispatcher instances booting concurrently against the same DB
+// both see kumiko_event_consumers already present but rearm_count missing,
+// and both ALTER TABLE. Without IF NOT EXISTS the loser crashes on boot
+// with "column already exists" — a real TOCTOU since this table is
+// explicitly multi-instance.
+describe("createEventConsumerStateTable — concurrent boot", () => {
+  let testDb: TestDb;
+
+  beforeAll(async () => {
+    testDb = await createTestDb();
+  });
+
+  afterAll(async () => {
+    await testDb.cleanup();
+  });
+
+  test("running it twice concurrently does not throw", async () => {
+    await createEventConsumerStateTable(testDb.db);
+
+    await expect(
+      Promise.all([
+        createEventConsumerStateTable(testDb.db),
+        createEventConsumerStateTable(testDb.db),
+      ]),
+    ).resolves.toBeDefined();
+  });
+});

package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts

@@ -11,6 +11,13 @@ import { createEventStoreExecutor } from "../../db/event-store-executor";
 import { asRawClient } from "../../db/query";
 import { createTenantDb, type TenantDb } from "../../db/tenant-db";
 import { defineFeature } from "../../engine";
+import {
+  DEFAULT_SENSITIVE_CONFIG,
+  type MetricEvent,
+  type ObservabilityProvider,
+  RecordingMeter,
+  RecordingTracer,
+} from "../../observability";
 import { getConsumerState } from "../../pipeline";
 import {
   resetEventStore,
@@ -20,6 +27,7 @@ import {
   unsafeCreateEntityTable,
 } from "../../stack";
 import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
+import { withoutAmbientTemporal } from "../../testing/without-ambient-temporal";
 import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
 
 const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
@@ -151,6 +159,23 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(deadAgain?.rearmCount).toBe(0);
   });
 
+  test("kumiko-framework#1525: cooldown-elapsed re-arm check works without relying on globalThis.Temporal", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+    await backdateUpdatedAt(10);
+
+    // ensureTemporalPolyfill re-installs if the ambient global was torn down,
+    // so runOnce stays safe if a nested path calls it during this window.
+    await withoutAmbientTemporal(async () => {
+      await stack.eventDispatcher?.runOnce();
+    });
+
+    const revived = await getConsumerState(stack.db, qn);
+    expect(revived?.status).toBe("idle");
+    expect(revived?.rearmCount).toBe(1);
+  });
+
   test("permanently dead after maxRearmCount cycles, even once cooldown elapses again", async () => {
     poisonNames.add("poison");
     await appendWidget("poison");
@@ -177,4 +202,62 @@ describe("issue #1350 — bounded dead-consumer re-arm", () => {
     expect(stillExhausted?.status).toBe("dead");
     expect(stillExhausted?.rearmCount).toBe(3);
   });
+
+  test("kumiko_event_consumer_rearm_exhausted_total fires once, not per still-dead pass (#1362)", async () => {
+    // Dedicated stack with a RecordingMeter — module-level `stack` above is
+    // shared across every test in this file and has no metric capture.
+    const metricEvents: MetricEvent[] = [];
+    const meter = new RecordingMeter((e) => metricEvents.push(e));
+    const tracer = new RecordingTracer({
+      sensitiveConfig: DEFAULT_SENSITIVE_CONFIG,
+      onSpanEnd: () => {},
+    });
+    const recordingProvider: ObservabilityProvider = {
+      name: "recording",
+      meter,
+      tracer,
+      shutdown: async () => {},
+    };
+
+    const recStack = await setupTestStack({
+      features: [rearmFeature],
+      systemHooks: [],
+      observability: recordingProvider,
+    });
+    try {
+      await unsafeCreateEntityTable(recStack.db, sharedWidgetEntity, "widget");
+      const recTdb = createTenantDb(recStack.db, admin.tenantId);
+
+      poisonNames.add("poison-metric");
+      await executor.create({ name: "poison-metric" }, admin, recTdb);
+      for (let i = 0; i < 10; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      // The 10th pass is the one whose failed delivery flips status to
+      // "dead" (persisted for the NEXT pass to see) — acquireConsumerState's
+      // dead-branch (where the exhausted-metric fires) only runs starting
+      // from the pass AFTER that transition, same as the "stays dead within
+      // cooldown window" test above needing one extra runOnce().
+      await recStack.eventDispatcher?.runOnce();
+
+      const exhaustedEvents = () =>
+        metricEvents.filter(
+          (e) =>
+            e.type === "counter.inc" && e.name === "kumiko_event_consumer_rearm_exhausted_total",
+        );
+      expect(exhaustedEvents().length).toBe(1);
+
+      // Several more passes while it's still dead — must stay at 1, not
+      // re-fire on every pass that observes "still dead" (log/metric spam
+      // the finding flagged).
+      for (let i = 0; i < 5; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      expect(exhaustedEvents().length).toBe(1);
+      expect(exhaustedEvents()[0]?.labels?.["consumer"]).toBe(qn);
+    } finally {
+      await resetEventStore(recStack, ["read_widgets"]);
+      await recStack.cleanup();
+    }
+  });
 });