@cosmicdrift/kumiko-framework 0.304.0 → 0.306.0

package/src/errors/write-error-info.ts

@@ -58,6 +58,12 @@ export function failTransition(from: string, to: string, allowed: readonly strin
   );
 }
 
+// The cause travels out-of-band, keyed on the info object's identity: a field
+// on WriteErrorInfo would leak the underlying Error into the /batch failure
+// body and the idempotency cache, both of which serialize the info to JSON.
+// A cache replay yields a fresh object without an entry, so no cause there.
+const causeByWriteErrorInfo = new WeakMap<WriteErrorInfo, Error>();
+
 export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
   // In dev/test surface the cause-snapshot through `details` so the
   // HTTP response carries something useful. Without this, internal_error
@@ -79,7 +85,7 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
         : undefined
       : undefined;
   const effectiveDetails = err.details ?? causeDetails;
-  return {
+  const info: WriteErrorInfo = {
     code: err.code,
     httpStatus: err.httpStatus,
     i18nKey: err.i18nKey,
@@ -87,6 +93,8 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
     ...(err.i18nParams && { i18nParams: err.i18nParams }),
     ...(effectiveDetails !== undefined && { details: effectiveDetails }),
   };
+  if (err.cause instanceof Error) causeByWriteErrorInfo.set(info, err.cause);
+  return info;
 }
 
 // Reconstitutes an error from WriteErrorInfo so command() (throw-based) can
@@ -95,19 +103,20 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
 // httpStatus / details but `instanceof NotFoundError` won't work. That's OK:
 // the HTTP layer keys off code + httpStatus, not class identity.
 export function reraiseAsKumikoError(info: WriteErrorInfo): KumikoError {
-  return new ReraisedError(info);
+  return new ReraisedError(info, causeByWriteErrorInfo.get(info));
 }
 
 class ReraisedError extends KumikoError {
   readonly code: string;
   readonly httpStatus: number;
 
-  constructor(info: WriteErrorInfo) {
+  constructor(info: WriteErrorInfo, cause?: Error) {
     super({
       message: info.message,
       i18nKey: info.i18nKey,
       ...(info.i18nParams && { i18nParams: info.i18nParams }),
       ...(info.details !== undefined && { details: info.details }),
+      ...(cause && { cause }),
     });
     this.code = info.code;
     this.httpStatus = info.httpStatus;

package/src/jobs/__tests__/job-backoff.integration.test.ts

@@ -0,0 +1,155 @@
+// fw#3167: buildRetryBullOpts only passed `{ type }` to BullMQ, never
+// `delay` — BullMQ's fixed/exponential backoff strategies compute
+// NaN/undefined without it (falsy), so a job with `backoff` set retried
+// immediately instead of waiting. These tests assert real inter-attempt
+// gaps via a real HTTP write that triggers the job through `trigger.on`,
+// same delivery path as dispatch-write.ts's afterCommitHooks.
+//
+// Only lower bounds are asserted (never upper) — BullMQ schedules the next
+// attempt at failureTime + delay using the same wall clock this test reads,
+// so a >= assertion is flake-free without inflating waitFor's budget.
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { z } from "zod";
+import { defineFeature } from "../../engine";
+import { setupTestStack, type TestStack, TestUsers } from "../../stack";
+import { waitFor } from "../../testing";
+import { generateId } from "../../utils";
+
+const exponentialDefaultStarts: number[] = [];
+const exponentialCustomStarts: number[] = [];
+const fixedCustomStarts: number[] = [];
+
+function gapsBetween(starts: readonly number[]): number[] {
+  return starts.slice(1).map((start, index) => start - (starts[index] ?? start));
+}
+
+const backoffFixtureFeature = defineFeature("backofffixture", (r) => {
+  r.writeHandler(
+    "trigger-exponential-default",
+    z.object({}),
+    async () => ({ isSuccess: true as const, data: {} }),
+    { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
+  );
+  // AC: retries: 3, backoff: "exponential" — fails once, succeeds on retry 2.
+  // Default base delay is 1000ms, so the single retry gap must be >= 1000ms.
+  r.job(
+    "exponential-default",
+    {
+      trigger: { on: "backofffixture:write:trigger-exponential-default" },
+      retries: 3,
+      backoff: "exponential",
+    },
+    async () => {
+      exponentialDefaultStarts.push(Date.now());
+      if (exponentialDefaultStarts.length === 1) throw new Error("fails on attempt 1");
+    },
+  );
+
+  r.writeHandler(
+    "trigger-exponential-custom",
+    z.object({}),
+    async () => ({ isSuccess: true as const, data: {} }),
+    { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
+  );
+  // Per-job configurable + growing: delayMs 50 → gaps 50, 100, 200ms.
+  r.job(
+    "exponential-custom",
+    {
+      trigger: { on: "backofffixture:write:trigger-exponential-custom" },
+      retries: 3,
+      backoff: { type: "exponential", delayMs: 50 },
+    },
+    async () => {
+      exponentialCustomStarts.push(Date.now());
+      if (exponentialCustomStarts.length < 4)
+        throw new Error(`fails on attempt ${exponentialCustomStarts.length}`);
+    },
+  );
+
+  r.writeHandler(
+    "trigger-fixed-custom",
+    z.object({}),
+    async () => ({ isSuccess: true as const, data: {} }),
+    { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
+  );
+  r.job(
+    "fixed-custom",
+    {
+      trigger: { on: "backofffixture:write:trigger-fixed-custom" },
+      retries: 1,
+      backoff: { type: "fixed", delayMs: 300 },
+    },
+    async () => {
+      fixedCustomStarts.push(Date.now());
+      if (fixedCustomStarts.length === 1) throw new Error("fails on attempt 1");
+    },
+  );
+});
+
+let stack: TestStack;
+
+beforeAll(async () => {
+  stack = await setupTestStack({
+    features: [backoffFixtureFeature],
+    // Own queue namespace: the default "kumiko-jobs" queue on the shared test
+    // Redis is consumed by every parallel test process, and a foreign worker
+    // grabbing a promoted delayed retry fails it as "Unknown job".
+    jobs: { consumerLane: "worker", queueNamePrefix: `backoff-test-${generateId()}` },
+  });
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+beforeEach(() => {
+  exponentialDefaultStarts.length = 0;
+  exponentialCustomStarts.length = 0;
+  fixedCustomStarts.length = 0;
+});
+
+describe("job backoff waits between retries (fw#3167)", () => {
+  test('backoff: "exponential" without delayMs waits the default 1000ms base delay', async () => {
+    await stack.http.writeOk(
+      "backofffixture:write:trigger-exponential-default",
+      {},
+      TestUsers.admin,
+    );
+
+    await waitFor(() => {
+      expect(exponentialDefaultStarts).toHaveLength(2);
+    });
+
+    const [gap = 0] = gapsBetween(exponentialDefaultStarts);
+    expect(gap).toBeGreaterThanOrEqual(1000);
+  });
+
+  test("object-form backoff.delayMs is configurable per job and grows exponentially", async () => {
+    await stack.http.writeOk(
+      "backofffixture:write:trigger-exponential-custom",
+      {},
+      TestUsers.admin,
+    );
+
+    await waitFor(() => {
+      expect(exponentialCustomStarts).toHaveLength(4);
+    });
+
+    const [gap1 = 0, gap2 = 0, gap3 = 0] = gapsBetween(exponentialCustomStarts);
+    expect(gap1).toBeGreaterThanOrEqual(50);
+    expect(gap2).toBeGreaterThanOrEqual(100);
+    expect(gap3).toBeGreaterThanOrEqual(200);
+  });
+
+  test('backoff: { type: "fixed", delayMs } waits the configured constant delay', async () => {
+    await stack.http.writeOk("backofffixture:write:trigger-fixed-custom", {}, TestUsers.admin);
+
+    await waitFor(() => {
+      expect(fixedCustomStarts).toHaveLength(2);
+    });
+
+    const [gap = 0] = gapsBetween(fixedCustomStarts);
+    expect(gap).toBeGreaterThanOrEqual(300);
+  });
+});

package/src/jobs/__tests__/job-retention.integration.test.ts

@@ -0,0 +1,316 @@
+// BullMQ's moveToFinished lua sweeps a queue's WHOLE
+// completed/failed zset whenever any job finishes with keepJobs set, not
+// just the finishing job, and only lazily (on a later finish into the same
+// set). These tests exercise the real sweep against a real Redis, on every
+// enqueue path, plus the runOnBoot marker and the perTenant invariant that
+// keeps the sweep from breaking wrapper-retry child dedup.
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { Queue } from "bullmq";
+import { z } from "zod";
+import { createRegistry, defineFeature } from "../../engine";
+import type { AppContext, TenantId } from "../../engine/types";
+import { createTestRedis, type TestRedis } from "../../stack";
+import { sleep, waitFor } from "../../testing";
+import { bootJobIdForJobName, createJobRunner } from "../job-runner";
+
+let testRedis: TestRedis;
+let redisUrl: string;
+
+beforeAll(async () => {
+  testRedis = await createTestRedis();
+  redisUrl = `redis://${testRedis.redis.options.host}:${testRedis.redis.options.port}/${testRedis.redis.options.db}`;
+});
+
+afterAll(async () => {
+  await testRedis.cleanup();
+});
+
+function uniquePrefix(tag: string): string {
+  return `kumiko-test-retention-${tag}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+async function purgeQueueKeys(queueNamePrefix: string): Promise<void> {
+  const workerKeys = await testRedis.redis.keys(`bull:${queueNamePrefix}-worker:*`);
+  if (workerKeys.length > 0) await testRedis.redis.del(...workerKeys);
+  const apiKeys = await testRedis.redis.keys(`bull:${queueNamePrefix}-api:*`);
+  if (apiKeys.length > 0) await testRedis.redis.del(...apiKeys);
+}
+
+function rawWorkerQueue(queueNamePrefix: string): Queue {
+  const queue = new Queue(`${queueNamePrefix}-worker`, {
+    connection: { host: testRedis.redis.options.host, port: testRedis.redis.options.port },
+  });
+  // A post-close 'error' here is otherwise unhandled and bun:test attributes
+  // it to whichever test runs next (fw#1805).
+  queue.on("error", () => {});
+  return queue;
+}
+
+describe("bounded job retention (fw#3199)", () => {
+  test("completed jobs from every enqueue path are swept after the age once a later job completes", async () => {
+    const queueNamePrefix = uniquePrefix("completed");
+    const tenants = ["ret-a", "ret-b"] as TenantId[];
+    const completed: Array<{ name: string; jobId: string }> = [];
+
+    const feature = defineFeature("retention", (r) => {
+      const someEvent = r.defineEvent("some-event", z.object({}), { piiFields: "none" });
+      r.job("tick", { trigger: { manual: true } }, async () => {});
+      r.job("onEvent", { trigger: { on: someEvent.name } }, async () => {});
+      r.job("fanout", { trigger: { manual: true }, perTenant: true }, async () => {});
+      r.job("cronJob", { trigger: { cron: "* * * * * *" } }, async () => {});
+      r.job("sweeper", { trigger: { manual: true } }, async () => {});
+    });
+
+    const registry = createRegistry([feature]);
+    const context: AppContext = {};
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds: async () => tenants,
+      jobRetention: { completedAgeSec: 1, failedAgeSec: 1 },
+      onJobComplete: (name, jobId) => {
+        completed.push({ name, jobId });
+      },
+    });
+
+    const rawQueue = rawWorkerQueue(queueNamePrefix);
+    try {
+      await runner.start();
+      const tickId = await runner.dispatch("retention:job:tick");
+      await runner.handleEvent("retention:event:some-event", {});
+      const wrapperId = await runner.dispatch("retention:job:fanout");
+
+      await waitFor(() => {
+        expect(completed.some((c) => c.jobId === tickId)).toBe(true);
+        expect(completed.some((c) => c.name === "retention:job:on-event")).toBe(true);
+        expect(completed.filter((c) => c.name === "retention:job:fanout").length).toBe(2);
+        expect(completed.some((c) => c.name === "retention:job:cron-job")).toBe(true);
+      });
+
+      const cronId = completed.find((c) => c.name === "retention:job:cron-job")?.jobId;
+      const eventId = completed.find((c) => c.name === "retention:job:on-event")?.jobId;
+      const childIds = completed
+        .filter((c) => c.name === "retention:job:fanout")
+        .map((c) => c.jobId);
+
+      await sleep(1200);
+
+      await runner.dispatch("retention:job:sweeper");
+      await waitFor(() => {
+        expect(completed.some((c) => c.name === "retention:job:sweeper")).toBe(true);
+      });
+
+      const idsToCheck = [tickId, wrapperId, eventId, cronId, ...childIds].filter(
+        (id): id is string => id !== undefined,
+      );
+      for (const id of idsToCheck) {
+        await waitFor(async () => {
+          expect(await rawQueue.getJob(id)).toBeUndefined();
+        });
+      }
+    } finally {
+      await rawQueue.close();
+      await runner.stop();
+      await purgeQueueKeys(queueNamePrefix);
+    }
+  });
+
+  test("failed jobs are swept after the age once a later failure completes", async () => {
+    const queueNamePrefix = uniquePrefix("failed");
+    const failed: Array<{ name: string; jobId: string }> = [];
+
+    const feature = defineFeature("retentionfail", (r) => {
+      r.job("boom", { trigger: { manual: true } }, async () => {
+        throw new Error("always fails");
+      });
+    });
+
+    const registry = createRegistry([feature]);
+    const context: AppContext = {};
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      jobRetention: { completedAgeSec: 1, failedAgeSec: 1 },
+      onJobFailed: (name, jobId) => {
+        failed.push({ name, jobId });
+      },
+    });
+
+    const rawQueue = rawWorkerQueue(queueNamePrefix);
+    try {
+      await runner.start();
+      const firstId = await runner.dispatch("retentionfail:job:boom");
+      await waitFor(() => {
+        expect(failed.some((f) => f.jobId === firstId)).toBe(true);
+      });
+
+      await sleep(1200);
+
+      const secondId = await runner.dispatch("retentionfail:job:boom");
+      await waitFor(() => {
+        expect(failed.some((f) => f.jobId === secondId)).toBe(true);
+      });
+
+      await waitFor(async () => {
+        expect(await rawQueue.getJob(firstId)).toBeUndefined();
+      });
+    } finally {
+      await rawQueue.close();
+      await runner.stop();
+      await purgeQueueKeys(queueNamePrefix);
+    }
+  });
+
+  test("runOnBoot marker survives the retention sweep and still dedupes across restarts", async () => {
+    const queueNamePrefix = uniquePrefix("boot");
+    let bootRuns = 0;
+    let tickRuns = 0;
+
+    const feature = defineFeature("retentionboot", (r) => {
+      r.job("boot", { trigger: { manual: true }, runOnBoot: true }, async () => {
+        bootRuns += 1;
+      });
+      r.job("tick", { trigger: { manual: true } }, async () => {
+        tickRuns += 1;
+      });
+    });
+
+    const registry1 = createRegistry([feature]);
+    const context: AppContext = {};
+    const runner1 = createJobRunner({
+      registry: registry1,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      jobRetention: { completedAgeSec: 1, failedAgeSec: 1 },
+    });
+
+    const rawQueue = rawWorkerQueue(queueNamePrefix);
+    const bootJobId = bootJobIdForJobName("retentionboot:job:boot");
+    try {
+      await runner1.start();
+      await waitFor(() => {
+        expect(bootRuns).toBe(1);
+      });
+
+      await sleep(1200);
+      await runner1.dispatch("retentionboot:job:tick");
+      await waitFor(() => {
+        expect(tickRuns).toBe(1);
+      });
+
+      // Proves the sweep actually ran — without it this assertion (and the
+      // regression it guards) would pass on the old fixed-job-id dedup too.
+      await waitFor(async () => {
+        expect(await rawQueue.getJob(bootJobId)).toBeUndefined();
+      });
+
+      await runner1.stop();
+
+      const registry2 = createRegistry([feature]);
+      const runner2 = createJobRunner({
+        registry: registry2,
+        context,
+        redisUrl,
+        consumerLane: "worker",
+        queueNamePrefix,
+        jobRetention: { completedAgeSec: 1, failedAgeSec: 1 },
+      });
+      try {
+        await runner2.start();
+        await sleep(300);
+        expect(bootRuns).toBe(1);
+      } finally {
+        await runner2.stop();
+      }
+    } finally {
+      await rawQueue.close();
+      await purgeQueueKeys(queueNamePrefix);
+    }
+  });
+
+  test("createJobRunner throws for a perTenant job whose retry window reaches the completed retention", () => {
+    const feature = defineFeature("retentioninvariant", (r) => {
+      r.job(
+        "risky",
+        {
+          trigger: { manual: true },
+          perTenant: true,
+          retries: 2,
+          backoff: { type: "fixed", delayMs: 1000 },
+        },
+        async () => {},
+      );
+    });
+    const registry = createRegistry([feature]);
+    expect(() =>
+      createJobRunner({
+        registry,
+        context: {},
+        redisUrl,
+        consumerLane: "worker",
+        queueNamePrefix: uniquePrefix("invariant"),
+        jobRetention: { completedAgeSec: 1, failedAgeSec: 1 },
+      }),
+    ).toThrow(/retry window/);
+  });
+
+  test("createJobRunner does not throw for a non-perTenant job with the same retry config", async () => {
+    const feature = defineFeature("retentionnonpertenant", (r) => {
+      r.job(
+        "risky",
+        { trigger: { manual: true }, retries: 2, backoff: { type: "fixed", delayMs: 1000 } },
+        async () => {},
+      );
+    });
+    const registry = createRegistry([feature]);
+    const queueNamePrefix = uniquePrefix("nonpertenant");
+    let ran = false;
+    const runner = createJobRunner({
+      registry,
+      context: {},
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      jobRetention: { completedAgeSec: 1, failedAgeSec: 1 },
+      onJobComplete: () => {
+        ran = true;
+      },
+    });
+    try {
+      await runner.start();
+      await runner.dispatch("retentionnonpertenant:job:risky");
+      await waitFor(() => {
+        expect(ran).toBe(true);
+      });
+    } finally {
+      await runner.stop();
+      await purgeQueueKeys(queueNamePrefix);
+    }
+  });
+
+  test.each([0, 1.5])("createJobRunner throws for jobRetention.completedAgeSec = %p", (value) => {
+    const feature = defineFeature("retentionbadvalue", (r) => {
+      r.job("noop", { trigger: { manual: true } }, async () => {});
+    });
+    const registry = createRegistry([feature]);
+    expect(() =>
+      createJobRunner({
+        registry,
+        context: {},
+        redisUrl,
+        consumerLane: "worker",
+        queueNamePrefix: uniquePrefix("badvalue"),
+        jobRetention: { completedAgeSec: value },
+      }),
+    ).toThrow();
+  });
+});