@cosmicdrift/kumiko-framework 0.312.0 → 0.314.0

package/src/pipeline/system-hooks.ts

@@ -468,7 +468,8 @@ export async function buildSearchDocument(
 //    die auf optimistic-update setzen merken das nicht; strictly-waiting
 //    Clients müssten poll-after-write.
 //
-// Tests drain deterministisch via `await stack.eventDispatcher.runOnce()`.
+// Tests drain deterministically via `await stack.eventDispatcher.runOnce()`,
+// or via `drainEventConsumers` for this consumer name specifically.
 export const SSE_BROADCAST_CONSUMER_NAME = "system:consumer:sse-broadcast";
 
 export function createSseBroadcastEventConsumer(sseBroker: SseBroker): EventConsumer {
@@ -542,7 +543,10 @@ export function createJobTriggerEventConsumer(
         tenantId: event.tenantId,
         roles: [],
       };
-      await jobRunner.handleEvent(event.type, event.payload, user);
+      await jobRunner.handleEvent(event.type, event.payload, user, {
+        id: event.id,
+        headers: event.metadata.headers ?? {},
+      });
     },
   };
 }

package/src/stack/__tests__/drain-event-consumers.integration.test.ts

@@ -0,0 +1,166 @@
+// drainEventConsumers — pass-budgeted alternative to
+// `while ((await eventDispatcher.runOnce())?.processed ?? 0) > 0) {}`, which
+// under-drains a backlog bigger than one dispatcher batch and gives no
+// diagnostics when a consumer is stuck.
+
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { insertOne } from "../../db";
+import { defineFeature } from "../../engine";
+import { eventsTable } from "../../event-store";
+import { drainEventConsumers } from "../drain-event-consumers";
+import { resetEventStore } from "../table-helpers";
+import { setupTestStack, type TestStack } from "../test-stack";
+import { TestUsers } from "../test-users";
+
+const admin = TestUsers.admin;
+const TRACKER_QN = "drain-test:projection:tracker";
+const POISON_QN = "drain-test:projection:poison";
+
+// Distinct from POISON_TARGET_NAME (test 3's own poison consumer) so the two
+// scenarios can't accidentally cross-trigger each other's throw branch.
+const CASCADE_TRIGGER_NAME = "cascade-trigger";
+const CASCADE_POISON_NAME = "cascade-poison-must-not-be-processed";
+const POISON_TARGET_NAME = "always-poison";
+
+let observed: string[] = [];
+
+const drainTestFeature = defineFeature("drain-test", (r) => {
+  r.multiStreamProjection({
+    name: "tracker",
+    apply: {
+      "drain-test.tick": async (event, tx) => {
+        const name = (event.payload as { name: string }).name;
+        if (name === CASCADE_POISON_NAME) {
+          throw new Error(`tracker must not process the post-snapshot cascade: ${name}`);
+        }
+        observed.push(name);
+        // Simulates a consumer whose own handler writes a follow-up event —
+        // drainEventConsumers snapshots the target once, so this write must
+        // never be waited on.
+        if (name === CASCADE_TRIGGER_NAME) {
+          await insertOne(tx, eventsTable, {
+            aggregateId: crypto.randomUUID(),
+            aggregateType: "drain-test-source",
+            tenantId: admin.tenantId,
+            version: 1,
+            type: "drain-test.tick",
+            eventVersion: 1,
+            payload: { name: CASCADE_POISON_NAME },
+            metadata: { userId: admin.id },
+            createdBy: admin.id,
+          });
+        }
+      },
+    },
+  });
+  r.multiStreamProjection({
+    name: "poison",
+    apply: {
+      "drain-test.tick": async (event) => {
+        const name = (event.payload as { name: string }).name;
+        if (name === POISON_TARGET_NAME) {
+          throw new Error(`boom: ${name}`);
+        }
+      },
+    },
+  });
+});
+
+async function appendTick(name: string): Promise<void> {
+  await insertOne(stack.db, eventsTable, {
+    aggregateId: crypto.randomUUID(),
+    aggregateType: "drain-test-source",
+    tenantId: admin.tenantId,
+    version: 1,
+    type: "drain-test.tick",
+    eventVersion: 1,
+    payload: { name },
+    metadata: { userId: admin.id },
+    createdBy: admin.id,
+  });
+}
+
+let stack: TestStack;
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [drainTestFeature], systemHooks: [] });
+});
+
+afterEach(async () => {
+  observed = [];
+  await resetEventStore(stack);
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+describe("drainEventConsumers", () => {
+  test("backlog larger than the dispatcher batch size gets fully processed", async () => {
+    const names = Array.from({ length: 220 }, (_, i) => `evt-${i}`);
+    for (const name of names) await appendTick(name);
+
+    await drainEventConsumers({ db: stack.db, eventDispatcher: stack.eventDispatcher }, [
+      TRACKER_QN,
+    ]);
+
+    expect(observed.sort()).toEqual([...names].sort());
+  });
+
+  test("an event written after the HWM snapshot does not block", async () => {
+    await appendTick(CASCADE_TRIGGER_NAME);
+
+    await drainEventConsumers({ db: stack.db, eventDispatcher: stack.eventDispatcher }, [
+      TRACKER_QN,
+    ]);
+
+    // The cascade event was written by the handler DURING the drain, after
+    // the target snapshot — proven never processed (would throw otherwise).
+    expect(observed).toEqual([CASCADE_TRIGGER_NAME]);
+  });
+
+  test("a throwing consumer exhausts the budget and the error contains lastError", async () => {
+    await appendTick(POISON_TARGET_NAME);
+
+    let caughtMessage: string | undefined;
+    try {
+      await drainEventConsumers(
+        { db: stack.db, eventDispatcher: stack.eventDispatcher },
+        [POISON_QN],
+        { maxPasses: 2 },
+      );
+    } catch (e) {
+      caughtMessage = e instanceof Error ? e.message : String(e);
+    }
+
+    expect(caughtMessage).toBeDefined();
+    expect(caughtMessage).toContain(POISON_QN);
+    expect(caughtMessage).toContain(`lastError=boom: ${POISON_TARGET_NAME}`);
+  });
+
+  test("an unknown consumer name throws", async () => {
+    let caughtMessage: string | undefined;
+    try {
+      await drainEventConsumers({ db: stack.db, eventDispatcher: stack.eventDispatcher }, [
+        "drain-test:projection:does-not-exist",
+      ]);
+    } catch (e) {
+      caughtMessage = e instanceof Error ? e.message : String(e);
+    }
+
+    expect(caughtMessage).toContain(
+      'consumer "drain-test:projection:does-not-exist" not registered or per-instance',
+    );
+  });
+
+  test("missing eventDispatcher raises a clear error", async () => {
+    let caughtMessage: string | undefined;
+    try {
+      await drainEventConsumers({ db: stack.db, eventDispatcher: undefined }, [TRACKER_QN]);
+    } catch (e) {
+      caughtMessage = e instanceof Error ? e.message : String(e);
+    }
+
+    expect(caughtMessage).toContain("eventDispatcher is undefined");
+  });
+});

package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts

@@ -8,6 +8,7 @@ import * as z from "zod";
 import { defineFeature } from "../../engine";
 import { InternalError, writeFailure } from "../../errors";
 import { waitFor } from "../../testing";
+import { generateId } from "../../utils";
 import { setupTestStack, type TestStack } from "../test-stack";
 import { TestUsers } from "../test-users";
 
@@ -57,11 +58,22 @@ const jobContextFeature = defineFeature("jobcontextcheck", (r) => {
   });
 });
 
+const laneIsolationRuns: Array<{ from: unknown }> = [];
+
+const laneIsolationFeature = defineFeature("laneisolation", (r) => {
+  r.job("record", { trigger: { manual: true }, runIn: "worker" }, async (payload) => {
+    laneIsolationRuns.push({ from: payload["from"] });
+  });
+});
+
 let stack: TestStack | undefined;
+let secondStack: TestStack | undefined;
 
 afterEach(async () => {
   if (stack) await stack.cleanup();
+  if (secondStack) await secondStack.cleanup();
   stack = undefined;
+  secondStack = undefined;
 });
 
 describe("setupTestStack({ jobs }) wires ctx.jobRunner for manual dispatch", () => {
@@ -98,6 +110,54 @@ describe("setupTestStack({ jobs }) wires ctx.jobRunner for manual dispatch", ()
   });
 });
 
+// Stack "api" consumes only the api lane, so its worker-lane job can only run
+// if the "worker" stack's consumer reads the same queue.
+describe("setupTestStack({ jobs }) queue isolation between parallel stacks", () => {
+  async function dispatchFromApiStackThenRunOwnJob(): Promise<void> {
+    await stack?.jobRunner?.dispatch("laneisolation:job:record", { from: "api-stack" });
+    await secondStack?.jobRunner?.dispatch("laneisolation:job:record", { from: "worker-stack" });
+    await waitFor(() => {
+      expect(laneIsolationRuns.some((run) => run.from === "worker-stack")).toBe(true);
+    });
+  }
+
+  test("without an explicit queueNamePrefix, one stack's consumer never runs another stack's job", async () => {
+    laneIsolationRuns.length = 0;
+    [stack, secondStack] = await Promise.all([
+      setupTestStack({ features: [laneIsolationFeature], jobs: { consumerLane: "api" } }),
+      setupTestStack({ features: [laneIsolationFeature], jobs: { consumerLane: "worker" } }),
+    ]);
+
+    await dispatchFromApiStackThenRunOwnJob();
+
+    expect(laneIsolationRuns.map((run) => run.from)).toEqual(["worker-stack"]);
+  });
+
+  test("an explicit shared queueNamePrefix wins, so both stacks read one queue", async () => {
+    laneIsolationRuns.length = 0;
+    const queueNamePrefix = `laneisolation-shared-${generateId()}`;
+    [stack, secondStack] = await Promise.all([
+      setupTestStack({
+        features: [laneIsolationFeature],
+        jobs: { consumerLane: "api", queueNamePrefix },
+      }),
+      setupTestStack({
+        features: [laneIsolationFeature],
+        jobs: { consumerLane: "worker", queueNamePrefix },
+      }),
+    ]);
+
+    await dispatchFromApiStackThenRunOwnJob();
+
+    await waitFor(() => {
+      expect(laneIsolationRuns.map((run) => run.from).sort()).toEqual([
+        "api-stack",
+        "worker-stack",
+      ]);
+    });
+  });
+});
+
 describe("setupTestStack({ jobs }) job context matches the request-path context", () => {
   test("job handler's context carries tracer + searchAdapter, mirrors prod's effectiveFeatures-less job context", async () => {
     jobContextFields.length = 0;

package/src/stack/drain-event-consumers.ts

@@ -0,0 +1,74 @@
+import type { DbConnection } from "../db";
+import { getEventsHighWaterMark } from "../event-store";
+import { type ConsumerRecoveryState, getConsumerState } from "../pipeline";
+import type { TestStack } from "./test-stack";
+
+const DEFAULT_MAX_PASSES = 25;
+
+// Drives eventDispatcher.runOnce() until every named consumer has caught up
+// to the events high-water mark captured at call time. No "no progress ⇒
+// throw" shortcut: a pass can legitimately advance zero cursors even though
+// work remains — runOnce's FOR UPDATE SKIP LOCKED lets a concurrent pass
+// (the test stack's own 50ms poll timer) hold a consumer's row, and a
+// consumer coming off a thrown-pass backoff sits idle until its retryAtMs
+// elapses. A bounded pass budget tolerates both without pretending to detect
+// "stuck" from a single empty pass. The target is a snapshot: events a
+// consumer's own handler writes afterward (cascades) are not waited on.
+export async function drainEventConsumers(
+  stack: Pick<TestStack, "db" | "eventDispatcher">,
+  consumerNames: readonly [string, ...string[]],
+  options?: { readonly maxPasses?: number },
+): Promise<void> {
+  const { db, eventDispatcher } = stack;
+  if (!eventDispatcher) {
+    throw new Error(
+      "drainEventConsumers: stack.eventDispatcher is undefined — no system consumer or " +
+        "multiStreamProjection is wired for this stack.",
+    );
+  }
+  const maxPasses = options?.maxPasses ?? DEFAULT_MAX_PASSES;
+  const target = await getEventsHighWaterMark(db);
+
+  for (let pass = 0; pass < maxPasses; pass++) {
+    await eventDispatcher.runOnce();
+    const states = await loadConsumerStates(db, consumerNames);
+    // skip: every named consumer has already caught up to the target snapshot — the
+    // remaining budgeted passes are unnecessary, not a swallowed error.
+    if (states.every((state) => state.lastProcessedEventId >= target)) return;
+    if (pass === maxPasses - 1) {
+      throw budgetExhaustedError(maxPasses, target, states);
+    }
+  }
+}
+
+async function loadConsumerStates(
+  db: DbConnection,
+  consumerNames: readonly [string, ...string[]],
+): Promise<readonly ConsumerRecoveryState[]> {
+  return Promise.all(
+    consumerNames.map(async (name) => {
+      const state = await getConsumerState(db, name);
+      if (!state) {
+        throw new Error(`drainEventConsumers: consumer "${name}" not registered or per-instance`);
+      }
+      return state;
+    }),
+  );
+}
+
+function budgetExhaustedError(
+  maxPasses: number,
+  target: bigint,
+  states: readonly ConsumerRecoveryState[],
+): Error {
+  const unfinished = states.filter((state) => state.lastProcessedEventId < target);
+  const lines = unfinished.map(
+    (state) =>
+      `  ${state.name}: lastProcessedEventId=${state.lastProcessedEventId} target=${target} ` +
+      `status=${state.status} attempts=${state.attempts} lastError=${state.lastError ?? "null"}`,
+  );
+  return new Error(
+    `drainEventConsumers: budget of ${maxPasses} passes exhausted, consumers still behind target:\n` +
+      lines.join("\n"),
+  );
+}

package/src/stack/index.ts

@@ -14,6 +14,7 @@ export {
   createTestDb,
   type TestDb,
 } from "./db";
+export { drainEventConsumers } from "./drain-event-consumers";
 export { createEventCollector, type EventCollector } from "./event-collector";
 export { pushEntityProjectionTables } from "./push-entity-projection-tables";
 export { createTestRedis, type TestRedis } from "./redis";

package/src/stack/redis.ts

@@ -10,6 +10,8 @@ export type TestRedis = {
   // kumiko-framework #1036/2) — callers needing a fresh connection should use
   // this raw string, not rebuild one from parsed options.
   redisUrl: string;
+  /** The per-test `keyPrefix` on `redis`, e.g. `kt:1a2b3c4d:`. */
+  keyPrefix: string;
   /** Delete every key this test created (prefix-scoped). Replaces the old
    *  `redis.flushdb()` — that wiped other parallel tests' BullMQ state. */
   flushNamespace: () => Promise<void>;
@@ -43,6 +45,7 @@ export async function createTestRedis(): Promise<TestRedis> {
   return {
     redis,
     redisUrl,
+    keyPrefix: prefix,
     flushNamespace,
     cleanup: async () => {
       await flushNamespace();

package/src/stack/test-stack.ts

@@ -52,7 +52,8 @@ export type TestStack = {
   // event-triggered job reaching for ctx.files die where the request path works.
   context: AppContext;
   // Present whenever a system consumer (SSE, Search) or
-  // r.multiStreamProjection is wired. Tests drain it via runOnce() for
+  // r.multiStreamProjection is wired. Tests drain it via runOnce() (or the
+  // drainEventConsumers helper for a named-consumer backlog) for
   // deterministic assertion — no timer-induced flakiness.
   eventDispatcher?: EventDispatcher;
   // Only set when the caller passed `lifecycle` via options. Tests that
@@ -167,6 +168,8 @@ export type TestStackOptions = {
    *  `runOnBoot`/cron jobs against a caller-owned consumer). */
   jobs?: {
     consumerLane?: JobRunIn;
+    /** Default: unique per stack, derived from its test Redis keyPrefix. Pass
+     *  the same value to every runner that has to share this stack's queues. */
     queueNamePrefix?: string;
     /** Source of active tenant ids for `perTenant: true` jobs. Omit when the
      *  tenant feature is among `options.features` — the framework then
@@ -188,6 +191,13 @@ export type TestStackOptions = {
 
 const DEFAULT_JWT_SECRET = "test-stack-secret-minimum-32-characters!!";
 
+// BullMQ queues live on the raw redisUrl, outside the test Redis keyPrefix, so
+// parallel stacks sharing the prod default queue name would consume each
+// other's jobs. BullMQ rejects `:` in queue names.
+function queueNamePrefixForTestRedis(keyPrefix: string): string {
+  return keyPrefix.split(":").filter(Boolean).join("-");
+}
+
 export async function setupTestStack(options: TestStackOptions): Promise<TestStack> {
   const jwtSecret = options.jwtSecret ?? DEFAULT_JWT_SECRET;
   const enabledHooks = options.systemHooks ?? ["sse", "search"];
@@ -355,9 +365,8 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
       // password/username/tls/path (pr-review kumiko-framework #1036/2).
       redisUrl: testRedis.redisUrl,
       ...(options.jobs.consumerLane !== undefined && { consumerLane: options.jobs.consumerLane }),
-      ...(options.jobs.queueNamePrefix !== undefined && {
-        queueNamePrefix: options.jobs.queueNamePrefix,
-      }),
+      queueNamePrefix:
+        options.jobs.queueNamePrefix ?? queueNamePrefixForTestRedis(testRedis.keyPrefix),
       ...(options.jobs.getActiveTenantIds !== undefined && {
         getActiveTenantIds: options.jobs.getActiveTenantIds,
       }),

package/src/upgrade-cli.ts

@@ -304,9 +304,6 @@ function readMarkerVersion(targetDir: string): string | null {
   }
 }
 
-/** Highest pending version strictly below the earliest open manual breaking
- *  entry (no codemod). Falls back to highest non-breaking pending when every
- *  pending version is at/after that manual — never advances onto the manual. */
 /** Highest pending version strictly below the earliest open manual breaking
  *  entry (no codemod). Falls back to highest non-breaking pending when every
  *  pending version is at/after that manual. If only manuals remain, returns
@@ -340,6 +337,21 @@ function markerVersionForPending(
   return fallbackInstalled;
 }
 
+function logManualMarkerOutcome(
+  out: UpgradeCliOut,
+  manualEntries: readonly ChangelogEntry[],
+  markerVersion: string,
+): void {
+  if (manualEntries.length > 0) {
+    const markerHeldBack = manualEntries.some((e) => compareVersions(e.version, markerVersion) > 0);
+    out.log(
+      markerHeldBack
+        ? `  ⚠ Marker stays at ${markerVersion}, so the upgrade guard keeps listing the manual change(s) above. Migrate them by hand, then run --apply again to acknowledge them.`
+        : `  ⚠ Marker moved past the manual change(s) above to ${markerVersion}; the upgrade guard no longer lists them, so make sure they are migrated.`,
+    );
+  }
+}
+
 // Runs every pending breaking entry's codemod, oldest version first (so a
 // later codemod can assume an earlier one already ran). Stops on the first
 // failure — no partial marker. Writes the marker whenever dryRun is false —
@@ -394,6 +406,7 @@ async function applyCodemods(
         codemods: [],
       });
       out.log(`  ✓ Applied 0 codemod(s). Wrote ${join(targetDir, ".kumiko/upgrade-state.json")}`);
+      logManualMarkerOutcome(out, manualEntries, markerVer);
     }
     return 0;
   }
@@ -442,6 +455,7 @@ async function applyCodemods(
   if (pendingManual.length > 0) {
     out.log(`  ⚠ ${pendingManual.length} breaking change(s) still need manual migration.`);
   }
+  logManualMarkerOutcome(out, manualEntries, latestVersion);
   return 0;
 }