@cosmicdrift/kumiko-framework 0.322.0 → 0.324.0

package/src/stack/__tests__/drain-jobs.integration.test.ts

@@ -0,0 +1,159 @@
+// fw#3360: stack.drainJobs() waits deterministically for every in-flight
+// job AND its follow-up event-consumer / job-trigger cascade — replaces
+// `waitFor` polling on a specific job's side effect.
+
+import { afterEach, describe, expect, test } from "bun:test";
+import * as z from "zod";
+import { defineFeature } from "../../engine";
+import { SYSTEM_ROLE } from "../../engine/system-user";
+import { setupTestStack, type TestStack } from "../test-stack";
+import { TestUsers } from "../test-users";
+
+const chainResults: Array<{ value: string }> = [];
+
+// Job A → write handler that appends an r.defineEvent event → the async
+// job-trigger event-consumer → Job B. Exercises exactly the gap drainJobs
+// exists for: a job-completion that only cascades into a follow-up job via
+// the event-dispatcher, not the synchronous write-handler dispatch path.
+const chainFeature = defineFeature("draintestchain", (r) => {
+  const relayed = r.defineEvent("relayed", z.object({ value: z.string() }), {
+    piiFields: "none",
+  });
+
+  r.writeHandler(
+    "start",
+    z.object({ value: z.string() }),
+    async () => ({ isSuccess: true as const, data: {} }),
+    { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
+  );
+
+  r.writeHandler(
+    "emit-relayed",
+    z.object({ value: z.string() }),
+    async (event, ctx) => {
+      await ctx.unsafeAppendEvent({
+        aggregateId: crypto.randomUUID(),
+        aggregateType: "draintestchain",
+        type: relayed.name,
+        payload: { value: event.payload.value },
+      });
+      return { isSuccess: true as const, data: {} };
+    },
+    { access: { roles: [SYSTEM_ROLE] } },
+  );
+
+  r.job(
+    "job-a",
+    { trigger: { on: "draintestchain:write:start" }, retries: 0 },
+    async (payload, ctx) => {
+      await ctx.write("draintestchain:write:emit-relayed", { value: payload["value"] });
+    },
+  );
+
+  r.job("job-b", { trigger: { on: relayed.name }, retries: 0 }, async (payload) => {
+    chainResults.push({ value: payload["value"] as string }); // @cast-boundary test-fixture
+  });
+});
+
+const alwaysFailsRuns: string[] = [];
+const retrySucceedsAttempts: number[] = [];
+
+const failureFeature = defineFeature("draintestfailure", (r) => {
+  r.job("always-fails", { trigger: { manual: true }, retries: 0 }, async () => {
+    alwaysFailsRuns.push("ran");
+    throw new Error("job-c always fails");
+  });
+
+  r.job(
+    "retry-succeeds",
+    { trigger: { manual: true }, retries: 1, backoff: { type: "fixed", delayMs: 10 } },
+    async () => {
+      retrySucceedsAttempts.push(Date.now());
+      if (retrySucceedsAttempts.length === 1) throw new Error("first attempt fails on purpose");
+    },
+  );
+});
+
+let stack: TestStack | undefined;
+
+afterEach(async () => {
+  if (stack) await stack.cleanup();
+  stack = undefined;
+});
+
+describe("stack.drainJobs() waits for a job's async event-trigger cascade", () => {
+  test("job A → event → job B settles before drainJobs() resolves", async () => {
+    chainResults.length = 0;
+    stack = await setupTestStack({
+      features: [chainFeature],
+      jobs: { consumerLane: "worker" },
+    });
+
+    await stack.http.writeOk("draintestchain:write:start", { value: "hi" }, TestUsers.admin);
+    await stack.drainJobs();
+
+    expect(chainResults).toEqual([{ value: "hi" }]);
+  });
+
+  test("a huge event-dispatcher poll interval doesn't stop drainJobs() from driving the cascade itself", async () => {
+    chainResults.length = 0;
+    stack = await setupTestStack({
+      features: [chainFeature],
+      jobs: { consumerLane: "worker" },
+    });
+    // stop() tears down both the poll timer AND the LISTEN/NOTIFY
+    // subscription (idempotent — cleanup()'s own stop() later is a no-op).
+    // runOnce() still works afterward (event-dispatcher.ts's own contract),
+    // so this leaves drainJobs() as the ONLY thing that can advance the
+    // job-trigger consumer past job A's emitted event — without this, the
+    // always-on pgClient wake-up races the assertion regardless of
+    // eventDispatcherPollIntervalMs.
+    await stack.eventDispatcher?.stop();
+
+    await stack.http.writeOk("draintestchain:write:start", { value: "race" }, TestUsers.admin);
+    await stack.drainJobs();
+
+    expect(chainResults).toEqual([{ value: "race" }]);
+  });
+});
+
+describe("stack.drainJobs() surfaces final job failures", () => {
+  test("a job that always throws rejects drainJobs() with its name and error", async () => {
+    alwaysFailsRuns.length = 0;
+    stack = await setupTestStack({
+      features: [failureFeature],
+      jobs: { consumerLane: "worker" },
+    });
+
+    await stack.jobRunner?.dispatch("draintestfailure:job:always-fails");
+
+    await expect(stack.drainJobs()).rejects.toThrow(
+      /draintestfailure:job:always-fails.*job-c always fails/s,
+    );
+
+    // The tracker is cleared after a rejection — a later drainJobs() call
+    // (e.g. the next test's setup) must not still see this failure.
+    await expect(stack.drainJobs()).resolves.toBeUndefined();
+  });
+
+  test("a job with retries:1 that fails once then succeeds resolves drainJobs()", async () => {
+    retrySucceedsAttempts.length = 0;
+    stack = await setupTestStack({
+      features: [failureFeature],
+      jobs: { consumerLane: "worker" },
+    });
+
+    await stack.jobRunner?.dispatch("draintestfailure:job:retry-succeeds");
+
+    await expect(stack.drainJobs()).resolves.toBeUndefined();
+    expect(retrySucceedsAttempts).toHaveLength(2);
+  });
+});
+
+describe("stack.drainJobs() without a jobRunner", () => {
+  test("rejects with a clear message when the stack has no `jobs` opt-in", async () => {
+    stack = await setupTestStack({ features: [chainFeature] });
+
+    await expect(stack.drainJobs()).rejects.toThrow(/jobRunner is undefined/);
+  });
+});

package/src/stack/drain-jobs.ts

@@ -0,0 +1,129 @@
+import type { DbConnection } from "../db";
+import { getEventsHighWaterMark } from "../event-store";
+import type { JobRunner } from "../jobs";
+import { type EventDispatcher, getConsumerState } from "../pipeline";
+
+// Gap between idle-checks while waiting for a job hook to fire. Short enough
+// that a test never feels it, long enough that consecutive idle passes don't
+// hammer Postgres/Redis with pointless polling between hook-driven wakeups.
+const DRAIN_IDLE_POLL_MS = 25;
+// Two consecutive idle passes (not one) before declaring the stack drained —
+// a single "pending===0 and consumers caught up" pass can be a race: a job
+// hook fires and enqueues a follow-up between the countPendingJobs() read
+// and the consumer check of the SAME pass. A DRAIN_IDLE_POLL_MS wait sits
+// between every pass, including the first and second idle one, so an
+// enqueue that lands just after the first idle read still gets picked up
+// by the second pass instead of racing straight past it.
+const REQUIRED_IDLE_PASSES = 2;
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export type JobFailureTracker = {
+  readonly onJobComplete: (jobName: string, jobId: string) => void;
+  readonly onJobFailed: (jobName: string, jobId: string, error: string) => void;
+  waitForActivity(): Promise<void>;
+  readonly failures: ReadonlyMap<string, { readonly jobName: string; readonly error: string }>;
+  clear(): void;
+};
+
+export function createJobFailureTracker(): JobFailureTracker {
+  const failures = new Map<string, { jobName: string; error: string }>();
+  let wake: (() => void) | undefined;
+  let armed: Promise<void> | undefined;
+
+  function fire(): void {
+    wake?.();
+    armed = undefined;
+    wake = undefined;
+  }
+
+  return {
+    onJobComplete(_jobName, jobId) {
+      // A retry that eventually succeeds must not leave a stale failure
+      // behind from its earlier attempt(s).
+      failures.delete(jobId);
+      fire();
+    },
+    onJobFailed(jobName, jobId, error) {
+      failures.set(jobId, { jobName, error });
+      fire();
+    },
+    waitForActivity() {
+      armed ??= new Promise<void>((resolve) => {
+        wake = resolve;
+      });
+      return armed;
+    },
+    failures,
+    clear() {
+      failures.clear();
+    },
+  };
+}
+
+export type DrainJobsStack = {
+  readonly db: DbConnection;
+  readonly eventDispatcher?: EventDispatcher;
+  readonly jobRunner?: JobRunner;
+};
+
+async function sharedConsumersCaughtUp(
+  db: DbConnection,
+  consumerNames: readonly string[],
+): Promise<boolean> {
+  // skip: no shared consumers wired — nothing to catch up on.
+  if (consumerNames.length === 0) return true;
+  const target = await getEventsHighWaterMark(db);
+  let allCaughtUp = true;
+  for (const name of consumerNames) {
+    const state = await getConsumerState(db, name);
+    if (!state) {
+      throw new Error(`drainJobs: consumer "${name}" not registered or per-instance`);
+    }
+    if (state.status === "dead") {
+      throw new Error(
+        `drainJobs: consumer "${state.name}" is dead — lastError: ${state.lastError ?? "unknown"}`,
+      );
+    }
+    if (state.lastProcessedEventId < target) allCaughtUp = false;
+  }
+  return allCaughtUp;
+}
+
+export async function drainJobs(stack: DrainJobsStack, tracker: JobFailureTracker): Promise<void> {
+  const { db, eventDispatcher, jobRunner } = stack;
+  if (!jobRunner) {
+    throw new Error(
+      "drainJobs: stack.jobRunner is undefined — pass `jobs: {...}` to setupTestStack and " +
+        "register at least one r.job(...) so a runner is built.",
+    );
+  }
+
+  const sharedConsumerNames = (eventDispatcher?.consumers ?? [])
+    .filter((consumer) => (consumer.delivery ?? "shared") === "shared")
+    .map((consumer) => consumer.name);
+
+  try {
+    let consecutiveIdlePasses = 0;
+    while (consecutiveIdlePasses < REQUIRED_IDLE_PASSES) {
+      await eventDispatcher?.runOnce();
+      const pending = await jobRunner.countPendingJobs();
+      const caughtUp = await sharedConsumersCaughtUp(db, sharedConsumerNames);
+      consecutiveIdlePasses = pending === 0 && caughtUp ? consecutiveIdlePasses + 1 : 0;
+      if (consecutiveIdlePasses < REQUIRED_IDLE_PASSES) {
+        await Promise.race([tracker.waitForActivity(), sleep(DRAIN_IDLE_POLL_MS)]);
+      }
+    }
+
+    if (tracker.failures.size > 0) {
+      const details = [...tracker.failures.values()]
+        .map((failure) => `${failure.jobName}: ${failure.error}`)
+        .join("; ");
+      throw new Error(`drainJobs: job(s) failed: ${details}`);
+    }
+  } finally {
+    tracker.clear();
+  }
+}

package/src/stack/test-stack.ts

@@ -22,6 +22,7 @@ import { createRateLimitResolver } from "../rate-limit";
 import { createInMemorySearchAdapter } from "../search";
 import type { SearchAdapter } from "../search/types";
 import { createTestDb } from "./db";
+import { createJobFailureTracker, drainJobs } from "./drain-jobs";
 import { createEventCollector, type EventCollector } from "./event-collector";
 import { createTestRedis, type TestRedis } from "./redis";
 import { createRequestHelper, type RequestHelper } from "./request-helper";
@@ -64,6 +65,16 @@ export type TestStack = {
   // registered in the mounted features. Lets integration tests call
   // `stack.jobRunner.dispatch(...)` directly, mirroring `ctx.jobRunner`.
   jobRunner?: JobRunner;
+  // Waits for every in-flight job and its follow-up event-consumer /
+  // job-trigger cascade to settle — instead of `waitFor` polling on a
+  // specific job's side effect. Rejects with the failing job's name and
+  // error once a job exhausts its retries, or with a clear error when this
+  // stack has no jobRunner at all. Tracked failures accumulate since the
+  // PREVIOUS drainJobs() call (or setup, if none yet) — on a shared stack
+  // (e.g. beforeAll), a test that deliberately fails a job and never calls
+  // drainJobs() itself leaves that failure for the next test's drainJobs()
+  // call to reject on.
+  drainJobs: () => Promise<void>;
   cleanup: () => Promise<void>;
 };
 
@@ -361,6 +372,11 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
   // context = appContext + tracer/meter, mirroring the prod entrypoint's
   // `contextWithObservability(options.context, observability)`.
   let jobRunner: JobRunner | undefined;
+  // Fed to createJobRunner's onJobComplete/onJobFailed below so drainJobs()
+  // can tell a job's final outcome apart from an attempt that will retry —
+  // built before createJobRunner regardless of whether a runner ends up
+  // existing, so the returned drainJobs() closure always has one to read.
+  const jobFailureTracker = createJobFailureTracker();
   if (options.jobs && registry.getAllJobs().size > 0) {
     jobRunner = createJobRunner({
       registry,
@@ -374,6 +390,8 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
       ...(options.jobs.getActiveTenantIds !== undefined && {
         getActiveTenantIds: options.jobs.getActiveTenantIds,
       }),
+      onJobComplete: (jobName, jobId) => jobFailureTracker.onJobComplete(jobName, jobId),
+      onJobFailed: (jobName, jobId, error) => jobFailureTracker.onJobFailed(jobName, jobId, error),
     });
   }
 
@@ -540,6 +558,7 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
       ...(eventDispatcher ? { eventDispatcher } : {}),
       ...(server.lifecycle ? { lifecycle: server.lifecycle } : {}),
       ...(jobRunner ? { jobRunner } : {}),
+      drainJobs: () => drainJobs({ db: testDb.db, eventDispatcher, jobRunner }, jobFailureTracker),
       cleanup: async () => {
         if (jobRunner) await jobRunner.stop();
         if (eventDispatcher) await eventDispatcher.stop();