@cosmicdrift/kumiko-framework 0.311.0 → 0.313.0

package/src/pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts

@@ -195,3 +195,30 @@ describe("E: dispatcher survives a DB outage and logs the recovery", () => {
     }
   });
 });
+
+describe("F: stop() waits for a pass still mid-flight", () => {
+  test("stop() waits for a pass still in its idle pre-check", async () => {
+    const consumer: EventConsumer = {
+      name: "dboutage:consumer-stop-race",
+      handler: async () => {},
+    };
+    const dispatcher: EventDispatcher = createEventDispatcher({
+      db: stack.db,
+      consumers: [consumer],
+      context: { db: stack.db },
+      pollIntervalMs: 60_000,
+    });
+
+    await dispatcher.start();
+    let passSettled = false;
+    // A pass woken by runOnce() has not registered any turn yet while it is
+    // still awaiting the idle pre-check (selectIdleConsumerKeys) — stop()
+    // must wait for it too, or it queries the pool stop() is closing.
+    const pass = dispatcher.runOnce().finally(() => {
+      passSettled = true;
+    });
+    await dispatcher.stop();
+    expect(passSettled).toBe(true);
+    await pass;
+  });
+});

package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts

@@ -19,15 +19,22 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
 import * as z from "zod";
 import { entityEventName } from "../../db";
-import { defineFeature } from "../../engine";
+import { defineFeature, type JobContext } from "../../engine";
+import { loadAggregate } from "../../event-store";
 import { createInMemoryFileProvider, type InMemoryFileProvider } from "../../files";
 import { setupTestStack, type TestStack, TestUsers } from "../../stack";
 import { waitFor } from "../../testing";
+import { generateId } from "../../utils";
 
 const ITEM_REQUESTED_EVENT_QN = "job-trigger-fixture:event:item-requested";
 const FILE_REF_CREATED = entityEventName("fileRef", "created");
+const PROVIDER_EVENT_ID_HEADER = "providerEventId";
 
-const processedItems: Array<{ readonly fileRefId: string }> = [];
+const processedItems: Array<{
+  readonly fileRefId: string;
+  readonly triggerEvent: JobContext["triggerEvent"];
+  readonly rawDataKeys: readonly string[];
+}> = [];
 
 const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
   r.defineEvent("item-requested", z.object({ fileRefId: z.string().min(1) }), {
@@ -36,7 +43,8 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
 
   // Mirrors document-ingest-foundation's request-ingest MSP exactly: reacts
   // to fileRef.created, appends a NEW event via unsafeAppendEvent — no
-  // write-handler behind the appended event itself.
+  // write-handler behind the appended event itself. headers carries a
+  // provider-style idempotency key so the job side can assert it round-trips.
   r.multiStreamProjection({
     name: "request-item",
     apply: {
@@ -46,6 +54,7 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
           aggregateType: "job-trigger-fixture-request",
           type: ITEM_REQUESTED_EVENT_QN,
           payload: { fileRefId: event.aggregateId },
+          headers: { [PROVIDER_EVENT_ID_HEADER]: "provider-evt-42" },
         });
       },
     },
@@ -56,31 +65,100 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
   r.job(
     "process-item",
     { trigger: { on: ITEM_REQUESTED_EVENT_QN }, runIn: "worker" },
+    async (payload, ctx) => {
+      processedItems.push({
+        fileRefId: payload["fileRefId"] as string,
+        triggerEvent: ctx.triggerEvent,
+        rawDataKeys: Object.keys(payload),
+      });
+    },
+  );
+});
+
+// trigger.where partition fixture — two jobs share one broad QN, each
+// filtered to a disjoint payload.kind, mirroring document-ingest-foundation's
+// N-providers-one-QN shape without pulling in the whole feature.
+const WHERE_ITEM_REQUESTED_EVENT_QN = "job-trigger-where-fixture:event:item-requested";
+
+const alphaProcessed: Array<{ readonly fileRefId: string }> = [];
+const betaProcessed: Array<{ readonly fileRefId: string }> = [];
+
+const jobTriggerWhereFixtureFeature = defineFeature("job-trigger-where-fixture", (r) => {
+  r.defineEvent(
+    "item-requested",
+    z.object({ fileRefId: z.string().min(1), kind: z.enum(["alpha", "beta"]) }),
+    { piiFields: "none" },
+  );
+
+  r.multiStreamProjection({
+    name: "request-item",
+    apply: {
+      [FILE_REF_CREATED]: async (event, _tx, ctx) => {
+        const fileName = event.payload["fileName"];
+        const kind =
+          typeof fileName === "string" && fileName.startsWith("alpha-") ? "alpha" : "beta";
+        await ctx.unsafeAppendEvent({
+          aggregateId: event.aggregateId,
+          aggregateType: "job-trigger-where-fixture-request",
+          type: WHERE_ITEM_REQUESTED_EVENT_QN,
+          payload: { fileRefId: event.aggregateId, kind },
+        });
+      },
+    },
+  });
+
+  // Under test: both jobs trigger on the SAME QN; only the one whose
+  // `where` matches the appended payload's `kind` may run.
+  r.job(
+    "process-alpha",
+    { trigger: { on: WHERE_ITEM_REQUESTED_EVENT_QN, where: { kind: "alpha" } }, runIn: "worker" },
     async (payload) => {
-      processedItems.push({ fileRefId: payload["fileRefId"] as string });
+      alphaProcessed.push({ fileRefId: payload["fileRefId"] as string });
+    },
+  );
+  r.job(
+    "process-beta",
+    { trigger: { on: WHERE_ITEM_REQUESTED_EVENT_QN, where: { kind: "beta" } }, runIn: "worker" },
+    async (payload) => {
+      betaProcessed.push({ fileRefId: payload["fileRefId"] as string });
     },
   );
 });
 
 let stack: TestStack;
 let provider: InMemoryFileProvider;
+let whereStack: TestStack;
+let whereProvider: InMemoryFileProvider;
 
 beforeAll(async () => {
   provider = createInMemoryFileProvider();
   stack = await setupTestStack({
     features: [jobTriggerFixtureFeature],
     files: { storageProvider: provider },
-    jobs: { consumerLane: "worker" },
+    jobs: { consumerLane: "worker", queueNamePrefix: `job-trigger-fixture-${generateId()}` },
+  });
+  whereProvider = createInMemoryFileProvider();
+  whereStack = await setupTestStack({
+    features: [jobTriggerWhereFixtureFeature],
+    files: { storageProvider: whereProvider },
+    // Distinct prefix from `stack` above — createTestRedis's keyPrefix
+    // isolation doesn't cover BullMQ's own connections, so without this both
+    // JobRunners would race each other on the same default queue name.
+    jobs: { consumerLane: "worker", queueNamePrefix: `job-trigger-where-fixture-${generateId()}` },
   });
 });
 
 afterAll(async () => {
   await stack.cleanup();
+  await whereStack.cleanup();
 });
 
 beforeEach(() => {
   processedItems.length = 0;
   provider.clear();
+  alphaProcessed.length = 0;
+  betaProcessed.length = 0;
+  whereProvider.clear();
 });
 
 describe("job-trigger event consumer", () => {
@@ -103,6 +181,55 @@ describe("job-trigger event consumer", () => {
       expect(processedItems).toHaveLength(1);
     });
 
-    expect(processedItems[0]?.fileRefId).toBeTruthy();
+    const processed = processedItems[0];
+    expect(processed?.fileRefId).toBeTruthy();
+
+    // aggregateId of the appended item-requested event is the fileRef's id
+    // (the MSP apply carries event.aggregateId straight through).
+    const fileRefId = processed?.fileRefId as string;
+    const storedEvents = await loadAggregate(stack.db, fileRefId, TestUsers.admin.tenantId);
+    const itemRequested = storedEvents.find((e) => e.type === ITEM_REQUESTED_EVENT_QN);
+    expect(itemRequested).toBeDefined();
+
+    // triggerEvent carries the stored event's own id and headers through to
+    // the job — the idempotency-key surface at-least-once delivery needs.
+    expect(processed?.triggerEvent?.id).toBe(itemRequested?.id);
+    expect(processed?.triggerEvent?.headers[PROVIDER_EVENT_ID_HEADER]).toBe("provider-evt-42");
+
+    // Reserved keys never leak into the payload the handler receives.
+    expect(processed?.rawDataKeys).not.toContain("_triggerEvent");
+  });
+});
+
+describe("job-trigger event consumer — trigger.where partitions one QN across jobs", () => {
+  test("only the job whose where matches the payload runs — both partitions checked, not just the absence of the other", async () => {
+    const token = await whereStack.jwt.sign(TestUsers.admin);
+    async function uploadKind(fileName: string): Promise<string> {
+      const formData = new FormData();
+      formData.append("file", new File([Buffer.from("hello")], fileName, { type: "text/plain" }));
+      const res = await whereStack.app.request("/api/files", {
+        method: "POST",
+        headers: { Authorization: `Bearer ${token}` },
+        body: formData,
+      });
+      expect(res.status).toBe(201);
+      const body = (await res.json()) as { id: string };
+      return body.id;
+    }
+
+    const alphaFileRefId = await uploadKind("alpha-note.txt");
+    const betaFileRefId = await uploadKind("beta-note.txt");
+
+    await waitFor(async () => {
+      // Drives both jobs' consumers in the same pass — asserting both lists
+      // are non-empty before checking membership rules out "beta simply
+      // hasn't run yet" as an explanation for an empty betaProcessed.
+      await whereStack.eventDispatcher?.runOnce();
+      expect(alphaProcessed).toHaveLength(1);
+      expect(betaProcessed).toHaveLength(1);
+    });
+
+    expect(alphaProcessed).toEqual([{ fileRefId: alphaFileRefId }]);
+    expect(betaProcessed).toEqual([{ fileRefId: betaFileRefId }]);
   });
 });

package/src/pipeline/event-dispatcher-delivery.ts

@@ -3,12 +3,13 @@
 // resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
 import { Temporal as TemporalPolyfill } from "temporal-polyfill";
 import { type RequestContextData, requestContext } from "../api/request-context";
-import type { DbConnection, DbTx } from "../db/connection";
+import type { DbConnection, DbRunner, DbTx } from "../db/connection";
 import {
   insertConsumerIfAbsent,
   markConsumerProcessing,
   rearmDeadConsumer,
   recordConsumerPassFailure,
+  selectConsumerForUpdate,
   selectConsumerForUpdateSkipLocked,
   selectProvablyIdleConsumerPairs,
   updateConsumerDeliveryOutcome,
@@ -160,6 +161,26 @@ export async function acquireConsumerState(
   return { state, skip: null };
 }
 
+export type ConsumerCursor = {
+  readonly lastProcessedEventId: bigint;
+  readonly pendingGaps: readonly PendingGapEntry[];
+};
+
+// Blocking FOR UPDATE (not SKIP LOCKED): callers of this one need to actually
+// wait for the dispatcher's own turn to finish, not skip past it, so their
+// read of the already-applied cursor is guaranteed to be up to date rather
+// than possibly stale.
+export async function selectConsumerCursorForUpdate(
+  db: DbRunner,
+  name: string,
+  instanceId: string,
+): Promise<ConsumerCursor | undefined> {
+  const rawState = await selectConsumerForUpdate(db, name, instanceId);
+  if (!rawState) return undefined;
+  const state = coerceRow(rawState, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow;
+  return { lastProcessedEventId: state.lastProcessedEventId, pendingGaps: state.pendingGaps };
+}
+
 // Read-only pre-check run once per doPass, before any consumer's turn opens
 // a transaction (event-dispatcher.ts's doPass/runConsumerTurn). Keyed like
 // consumerBackoff/inFlightTurns: `${name}:${instanceId}`. Consumers not

package/src/pipeline/event-dispatcher.ts

@@ -385,13 +385,34 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
     );
   }
 
+  // A pass still in doPass's idle pre-check (selectIdleConsumerKeys) hasn't
+  // registered any consumer turn in inFlightTurns yet — stop() must wait for
+  // it too, or it keeps querying the pool stop() is about to close.
+  const inFlightPasses = new Set<Promise<DispatcherPassResult>>();
+
+  async function drainInFlightPasses(): Promise<void> {
+    await Promise.all(
+      [...inFlightPasses].map((pass) =>
+        pass.catch(() => {
+          // skip: errors already recorded per-consumer inside the pass
+        }),
+      ),
+    );
+  }
+
   async function runOnce(): Promise<DispatcherPassResult> {
     if (!preRegistered) {
       throw new Error(
         "EventDispatcher.runOnce() called before start() — consumer state rows are not registered. Call start() first (production) or ensureRegistered() (tests after truncating kumiko_event_consumers).",
       );
     }
-    return doPass();
+    const pass = doPass();
+    inFlightPasses.add(pass);
+    try {
+      return await pass;
+    } finally {
+      inFlightPasses.delete(pass);
+    }
   }
 
   async function doPass(): Promise<DispatcherPassResult> {
@@ -764,7 +785,9 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
         emitEventDispatcherListenConnected(meter, false);
       }
 
-      // Drain any in-flight pass so shutdown observes consistent state.
+      // Drain any in-flight pass so shutdown observes consistent state —
+      // including one still stuck in the idle pre-check above inFlightTurns.
+      await drainInFlightPasses();
       await drainInFlightTurns();
       // preRegistered stays true — the rows survive stop(). runOnce()
       // after a stop() still works (tests stop the timer and then drain

package/src/pipeline/index.ts

@@ -15,11 +15,12 @@ export type { DistributedLock } from "./distributed-lock";
 export { createDistributedLock } from "./distributed-lock";
 export type { EntityCache, EntityCacheOptions } from "./entity-cache";
 export { createEntityCache } from "./entity-cache";
-export type { ConsumerStatus } from "./event-consumer-state";
+export type { ConsumerStatus, PendingGapEntry } from "./event-consumer-state";
 export {
   ConsumerStatuses,
   createEventConsumerStateTable,
   eventConsumerStateTable,
+  SHARED_INSTANCE_SENTINEL,
 } from "./event-consumer-state";
 export type { EventDedup } from "./event-dedup";
 export { createEventDedup } from "./event-dedup";
@@ -43,6 +44,8 @@ export {
   restartConsumer,
   skipPoisonEvent,
 } from "./event-dispatcher";
+export type { ConsumerCursor } from "./event-dispatcher-delivery";
+export { selectConsumerCursorForUpdate } from "./event-dispatcher-delivery";
 export type { PruneEventsOptions, PruneEventsResult } from "./event-retention";
 export { ConsumerLagError, pruneEvents } from "./event-retention";
 export type { IdempotencyGuard } from "./idempotency";

package/src/pipeline/msp-rebuild.ts

@@ -115,6 +115,7 @@ function createRebuildCtx(
     });
   };
   return {
+    registry,
     appendEvent: refuseAppend as MultiStreamApplyContext["appendEvent"], // @cast-boundary engine-bridge
     unsafeAppendEvent: refuseAppend,
     loadAggregate: async (aggregateId, options) => {

package/src/pipeline/multi-stream-apply-context.ts

@@ -38,6 +38,7 @@ export function createMultiStreamApplyContext(
   deps: MultiStreamApplyContextDeps,
 ): MultiStreamApplyContext {
   return {
+    registry: deps.registry,
     ...(deps.files ? { files: deps.files } : {}),
     ...(deps.derivatives ? { derivatives: deps.derivatives } : {}),
     appendEvent: (async (args: AppendEventArgs) => {

package/src/pipeline/system-hooks.ts

@@ -542,7 +542,10 @@ export function createJobTriggerEventConsumer(
         tenantId: event.tenantId,
         roles: [],
       };
-      await jobRunner.handleEvent(event.type, event.payload, user);
+      await jobRunner.handleEvent(event.type, event.payload, user, {
+        id: event.id,
+        headers: event.metadata.headers ?? {},
+      });
     },
   };
 }