@cosmicdrift/kumiko-framework 1.0.0 → 2.0.0

package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts

@@ -0,0 +1,263 @@
+// Issue #1350 — bounded auto-revival of a dead consumer.
+//
+// acquireConsumerState no longer parks a dead consumer forever: once its
+// last write is older than rearmCooldownMs it gets reset to idle and
+// retried, up to maxRearmCount times. A poison event (permanent failure)
+// still ends up permanently dead once the budget is exhausted — the
+// mechanism is a bounded number of extra chances, not unlimited retry.
+
+import { afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { createEventStoreExecutor } from "../../db/event-store-executor";
+import { asRawClient } from "../../db/query";
+import { createTenantDb, type TenantDb } from "../../db/tenant-db";
+import { defineFeature } from "../../engine";
+import {
+  DEFAULT_SENSITIVE_CONFIG,
+  type MetricEvent,
+  type ObservabilityProvider,
+  RecordingMeter,
+  RecordingTracer,
+} from "../../observability";
+import { getConsumerState } from "../../pipeline";
+import {
+  resetEventStore,
+  setupTestStack,
+  type TestStack,
+  TestUsers,
+  unsafeCreateEntityTable,
+} from "../../stack";
+import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
+import { withoutAmbientTemporal } from "../../testing/without-ambient-temporal";
+import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
+
+const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
+  entityName: "widget",
+});
+
+let poisonNames = new Set<string>();
+let observed: Array<{ name: string }> = [];
+
+const rearmFeature = defineFeature("rearmtest", (r) => {
+  r.entity("widget", sharedWidgetEntity);
+
+  r.multiStreamProjection({
+    name: "observer",
+    apply: {
+      "widget.created": async (event) => {
+        const name = event.payload["name"] as string;
+        if (poisonNames.has(name)) {
+          throw new Error(`poisoned: ${name}`);
+        }
+        observed.push({ name });
+      },
+    },
+  });
+});
+
+const admin = TestUsers.admin;
+const qn = "rearmtest:projection:observer";
+let stack: TestStack;
+let tdb: TenantDb;
+
+beforeAll(async () => {
+  stack = await setupTestStack({
+    features: [rearmFeature],
+    systemHooks: [],
+  });
+  await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
+  tdb = createTenantDb(stack.db, admin.tenantId);
+});
+
+afterEach(async () => {
+  poisonNames = new Set();
+  observed = [];
+  await resetEventStore(stack, ["read_widgets"]);
+});
+
+async function appendWidget(name: string): Promise<void> {
+  await executor.create({ name }, admin, tdb);
+}
+
+async function driveUntilDead(): Promise<void> {
+  for (let i = 0; i < 10; i++) {
+    await stack.eventDispatcher?.runOnce();
+  }
+}
+
+// Simulates cooldown elapsed without waiting real wall-clock time.
+async function backdateUpdatedAt(minutesAgo: number): Promise<void> {
+  await asRawClient(stack.db).unsafe(
+    `UPDATE "kumiko_event_consumers" SET "updated_at" = now() - make_interval(mins => $1)
+     WHERE "name" = $2 AND "instance_id" = $3`,
+    [minutesAgo, qn, SHARED_INSTANCE_SENTINEL],
+  );
+}
+
+describe("issue #1350 — bounded dead-consumer re-arm", () => {
+  test("stays dead within the cooldown window", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+
+    const dead = await getConsumerState(stack.db, qn);
+    expect(dead?.status).toBe("dead");
+    expect(dead?.rearmCount).toBe(0);
+
+    // No backdate — cooldown (5min default) hasn't elapsed.
+    await stack.eventDispatcher?.runOnce();
+    const stillDead = await getConsumerState(stack.db, qn);
+    expect(stillDead?.status).toBe("dead");
+    expect(stillDead?.rearmCount).toBe(0);
+  });
+
+  test("auto-revives once cooldown elapses, and re-dies on the same poison event", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+
+    await backdateUpdatedAt(10);
+    await stack.eventDispatcher?.runOnce();
+
+    const revived = await getConsumerState(stack.db, qn);
+    expect(revived?.status).toBe("idle");
+    expect(revived?.rearmCount).toBe(1);
+    // Same pass that revives it also retries the pending poisoned event —
+    // acquireConsumerState's reset (attempts=0) is followed immediately by
+    // one failed delivery attempt within this runOnce() call.
+    expect(revived?.attempts).toBe(1);
+
+    // Handler still throws on "poison" — climbs back to dead.
+    await driveUntilDead();
+    const deadAgain = await getConsumerState(stack.db, qn);
+    expect(deadAgain?.status).toBe("dead");
+    expect(deadAgain?.rearmCount).toBe(1);
+  });
+
+  test("a re-arm that leads to a real delivery resets rearmCount to 0", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+
+    // Cause is gone by the time the cooldown elapses — the retried event
+    // now succeeds instead of re-poisoning the consumer.
+    poisonNames.delete("poison");
+    await backdateUpdatedAt(10);
+    await stack.eventDispatcher?.runOnce();
+
+    const recovered = await getConsumerState(stack.db, qn);
+    expect(recovered?.status).toBe("idle");
+    expect(recovered?.rearmCount).toBe(0);
+    expect(observed.map((o) => o.name)).toEqual(["poison"]);
+
+    // A later, unrelated poison event gets its own fresh maxRearmCount
+    // budget instead of inheriting the spent counter from the resolved one.
+    poisonNames.add("later-poison");
+    await appendWidget("later-poison");
+    await driveUntilDead();
+    const deadAgain = await getConsumerState(stack.db, qn);
+    expect(deadAgain?.status).toBe("dead");
+    expect(deadAgain?.rearmCount).toBe(0);
+  });
+
+  test("kumiko-framework#1525: cooldown-elapsed re-arm check works without relying on globalThis.Temporal", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+    await backdateUpdatedAt(10);
+
+    // ensureTemporalPolyfill re-installs if the ambient global was torn down,
+    // so runOnce stays safe if a nested path calls it during this window.
+    await withoutAmbientTemporal(async () => {
+      await stack.eventDispatcher?.runOnce();
+    });
+
+    const revived = await getConsumerState(stack.db, qn);
+    expect(revived?.status).toBe("idle");
+    expect(revived?.rearmCount).toBe(1);
+  });
+
+  test("permanently dead after maxRearmCount cycles, even once cooldown elapses again", async () => {
+    poisonNames.add("poison");
+    await appendWidget("poison");
+    await driveUntilDead();
+
+    // Cycles 1..3: each backdate+drive re-arms once, then re-dies on the
+    // same poison event. Default maxRearmCount is 3.
+    for (let cycle = 1; cycle <= 3; cycle++) {
+      await backdateUpdatedAt(10);
+      await stack.eventDispatcher?.runOnce();
+      const revived = await getConsumerState(stack.db, qn);
+      expect(revived?.rearmCount).toBe(cycle);
+      await driveUntilDead();
+    }
+
+    const exhausted = await getConsumerState(stack.db, qn);
+    expect(exhausted?.status).toBe("dead");
+    expect(exhausted?.rearmCount).toBe(3);
+
+    // Cooldown elapsed again, but the budget is spent — stays dead.
+    await backdateUpdatedAt(10);
+    await stack.eventDispatcher?.runOnce();
+    const stillExhausted = await getConsumerState(stack.db, qn);
+    expect(stillExhausted?.status).toBe("dead");
+    expect(stillExhausted?.rearmCount).toBe(3);
+  });
+
+  test("kumiko_event_consumer_rearm_exhausted_total fires once, not per still-dead pass (#1362)", async () => {
+    // Dedicated stack with a RecordingMeter — module-level `stack` above is
+    // shared across every test in this file and has no metric capture.
+    const metricEvents: MetricEvent[] = [];
+    const meter = new RecordingMeter((e) => metricEvents.push(e));
+    const tracer = new RecordingTracer({
+      sensitiveConfig: DEFAULT_SENSITIVE_CONFIG,
+      onSpanEnd: () => {},
+    });
+    const recordingProvider: ObservabilityProvider = {
+      name: "recording",
+      meter,
+      tracer,
+      shutdown: async () => {},
+    };
+
+    const recStack = await setupTestStack({
+      features: [rearmFeature],
+      systemHooks: [],
+      observability: recordingProvider,
+    });
+    try {
+      await unsafeCreateEntityTable(recStack.db, sharedWidgetEntity, "widget");
+      const recTdb = createTenantDb(recStack.db, admin.tenantId);
+
+      poisonNames.add("poison-metric");
+      await executor.create({ name: "poison-metric" }, admin, recTdb);
+      for (let i = 0; i < 10; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      // The 10th pass is the one whose failed delivery flips status to
+      // "dead" (persisted for the NEXT pass to see) — acquireConsumerState's
+      // dead-branch (where the exhausted-metric fires) only runs starting
+      // from the pass AFTER that transition, same as the "stays dead within
+      // cooldown window" test above needing one extra runOnce().
+      await recStack.eventDispatcher?.runOnce();
+
+      const exhaustedEvents = () =>
+        metricEvents.filter(
+          (e) =>
+            e.type === "counter.inc" && e.name === "kumiko_event_consumer_rearm_exhausted_total",
+        );
+      expect(exhaustedEvents().length).toBe(1);
+
+      // Several more passes while it's still dead — must stay at 1, not
+      // re-fire on every pass that observes "still dead" (log/metric spam
+      // the finding flagged).
+      for (let i = 0; i < 5; i++) {
+        await recStack.eventDispatcher?.runOnce();
+      }
+      expect(exhaustedEvents().length).toBe(1);
+      expect(exhaustedEvents()[0]?.labels?.["consumer"]).toBe(qn);
+    } finally {
+      await resetEventStore(recStack, ["read_widgets"]);
+      await recStack.cleanup();
+    }
+  });
+});

package/src/pipeline/__tests__/event-dispatcher.integration.test.ts

@@ -159,10 +159,20 @@ describe("event-dispatcher — happy path", () => {
     await stack.eventDispatcher?.runOnce();
     expect(captureA).toHaveLength(1);
 
+    const stateBefore = await getConsumerState(stack.db, qnA);
+
     const second = await stack.eventDispatcher?.runOnce();
     expect(second?.byConsumer[qnA]).toEqual({ processed: 0, failed: 0 });
     // Still only one — consumer correctly saw "nothing new past my cursor".
     expect(captureA).toHaveLength(1);
+
+    // No pending events means no write at all — an idle pass must not touch
+    // updated_at/status, otherwise every empty poll tick burns a WAL record.
+    const stateAfter = await getConsumerState(stack.db, qnA);
+    expect(stateAfter?.updatedAt.epochMilliseconds).toEqual(
+      stateBefore?.updatedAt.epochMilliseconds,
+    );
+    expect(stateAfter?.status).toBe(stateBefore?.status);
   });
 
   test("cursor advances only past successfully-consumed events", async () => {

package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts

@@ -0,0 +1,106 @@
+// createJobTriggerEventConsumer — proves r.job's trigger.on can fire on an
+// r.defineEvent QN appended by a multiStreamProjection's unsafeAppendEvent
+// (kumiko-framework#1505). Mirrors document-ingest-foundation's actual
+// request-ingest MSP (upload → fileRef.created → an owned defineEvent),
+// the motivating case for this fix — fileRef.created itself never reaches
+// jobRunner.handleEvent because the upload route appends it via the raw
+// event-store executor, not a write-handler dispatch (see #1505).
+//
+// Not covered here: a job triggered on a write/query-handler QN still
+// firing exactly once (unaffected by the new consumer). The full suite
+// stays green (e.g. the lane-routing sample), but that's not a positive
+// test of the partition guard — no stored event's `type` is ever a
+// handler QN in practice (entity events are "entity.verb"; custom
+// write-handlers like lane-routing's don't append to the store at all),
+// so `getWriteHandler`/`getQueryHandler` in the new consumer's handler is
+// defense-in-depth for an input shape the framework doesn't currently
+// produce, not something exercised end-to-end by any test today.
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { z } from "zod";
+import { entityEventName } from "../../db";
+import { defineFeature } from "../../engine";
+import { createInMemoryFileProvider, type InMemoryFileProvider } from "../../files";
+import { setupTestStack, type TestStack, TestUsers } from "../../stack";
+import { waitFor } from "../../testing";
+
+const ITEM_REQUESTED_EVENT_QN = "job-trigger-fixture:event:item-requested";
+const FILE_REF_CREATED = entityEventName("fileRef", "created");
+
+const processedItems: Array<{ readonly fileRefId: string }> = [];
+
+const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
+  r.defineEvent("item-requested", z.object({ fileRefId: z.string().min(1) }));
+
+  // Mirrors document-ingest-foundation's request-ingest MSP exactly: reacts
+  // to fileRef.created, appends a NEW event via unsafeAppendEvent — no
+  // write-handler behind the appended event itself.
+  r.multiStreamProjection({
+    name: "request-item",
+    apply: {
+      [FILE_REF_CREATED]: async (event, _tx, ctx) => {
+        await ctx.unsafeAppendEvent({
+          aggregateId: event.aggregateId,
+          aggregateType: "job-trigger-fixture-request",
+          type: ITEM_REQUESTED_EVENT_QN,
+          payload: { fileRefId: event.aggregateId },
+        });
+      },
+    },
+  });
+
+  // Under test: only reachable via createJobTriggerEventConsumer, since
+  // ITEM_REQUESTED_EVENT_QN is an r.defineEvent QN, not a handler QN.
+  r.job(
+    "process-item",
+    { trigger: { on: ITEM_REQUESTED_EVENT_QN }, runIn: "worker" },
+    async (payload) => {
+      processedItems.push({ fileRefId: payload["fileRefId"] as string });
+    },
+  );
+});
+
+let stack: TestStack;
+let provider: InMemoryFileProvider;
+
+beforeAll(async () => {
+  provider = createInMemoryFileProvider();
+  stack = await setupTestStack({
+    features: [jobTriggerFixtureFeature],
+    files: { storageProvider: provider },
+    jobs: { consumerLane: "worker" },
+  });
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+beforeEach(() => {
+  processedItems.length = 0;
+  provider.clear();
+});
+
+describe("job-trigger event consumer", () => {
+  test("a job triggers on an r.defineEvent QN appended by an MSP's unsafeAppendEvent", async () => {
+    const token = await stack.jwt.sign(TestUsers.admin);
+    const formData = new FormData();
+    formData.append("file", new File([Buffer.from("hello")], "note.txt", { type: "text/plain" }));
+    const res = await stack.app.request("/api/files", {
+      method: "POST",
+      headers: { Authorization: `Bearer ${token}` },
+      body: formData,
+    });
+    expect(res.status).toBe(201);
+
+    await waitFor(async () => {
+      // Drives both the MSP (appends item-requested off fileRef.created)
+      // and the new job-trigger consumer (reacts to it) — may need more
+      // than one pass since the MSP's append happens mid-drain.
+      await stack.eventDispatcher?.runOnce();
+      expect(processedItems).toHaveLength(1);
+    });
+
+    expect(processedItems[0]?.fileRefId).toBeTruthy();
+  });
+});