@cosmicdrift/kumiko-framework 0.304.0 → 0.306.0

package/src/jobs/__tests__/job-retry-enqueue-paths.integration.test.ts

@@ -0,0 +1,309 @@
+// fw#3184: buildRetryBullOpts (attempts/backoff derived from a job's
+// `retries`/`backoff`) was only wired into dispatch() and handleEvent().
+// Cron, runOnBoot, the perTenant wrapper, its fanned-out children, and the
+// sequential re-enqueue path all called queue.add() without it, so a job
+// with `retries` set still failed for good on its very first error when
+// reached through any of those paths. Where a fresh job could also rerun the
+// handler (a new cron tick, a sequential re-enqueue), the test asserts a
+// SECOND attempt on the SAME BullMQ job id — "the handler ran again
+// eventually" would pass without the fix there.
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { Queue } from "bullmq";
+import { createRegistry, defineFeature } from "../../engine";
+import type { AppContext, TenantId } from "../../engine/types";
+import { createTestRedis, type TestRedis } from "../../stack";
+import { sleep, waitFor } from "../../testing";
+import { bootJobIdForJobName, createJobRunner, type JobMeta } from "../job-runner";
+
+let testRedis: TestRedis;
+let redisUrl: string;
+
+beforeAll(async () => {
+  testRedis = await createTestRedis();
+  redisUrl = `redis://${testRedis.redis.options.host}:${testRedis.redis.options.port}/${testRedis.redis.options.db}`;
+});
+
+afterAll(async () => {
+  await testRedis.cleanup();
+});
+
+function uniquePrefix(tag: string): string {
+  return `kumiko-test-retry-${tag}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+async function purgeWorkerKeys(queueNamePrefix: string): Promise<void> {
+  // Cron leaves its repeatable scheduler firing into a stopped worker if not
+  // purged — the same hazard the jobs.integration.test.ts helper guards
+  // against.
+  const keys = await testRedis.redis.keys(`bull:${queueNamePrefix}-worker:*`);
+  if (keys.length > 0) await testRedis.redis.del(...keys);
+}
+
+describe("job retries on every enqueue path (fw#3184)", () => {
+  test("cron: a job with retries recovers via a second attempt on the same BullMQ job id", async () => {
+    let callCount = 0;
+    const starts: Array<{ jobId: string; attempt: number | undefined }> = [];
+    const cronFeature = defineFeature("retrycron", (r) => {
+      r.job("flaky", { trigger: { cron: "* * * * * *" }, retries: 1 }, async () => {
+        callCount += 1;
+        if (callCount === 1) throw new Error("fails on first cron tick");
+      });
+    });
+
+    const registry = createRegistry([cronFeature]);
+    const context: AppContext = {};
+    const queueNamePrefix = uniquePrefix("cron");
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      onJobStart: (_name, jobId, meta: JobMeta) => {
+        starts.push({ jobId, attempt: meta.attempt });
+      },
+    });
+
+    try {
+      await runner.start();
+      await waitFor(
+        () => {
+          const byJob = new Map<string, number>();
+          for (const s of starts)
+            byJob.set(s.jobId, Math.max(byJob.get(s.jobId) ?? 0, s.attempt ?? 0));
+          const reachedRetry = [...byJob.values()].some((attempt) => attempt >= 2);
+          expect(reachedRetry).toBe(true);
+        },
+        { delays: [500, 1000, 2000, 3000] },
+      );
+    } finally {
+      await runner.stop();
+      await purgeWorkerKeys(queueNamePrefix);
+    }
+  });
+
+  test("runOnBoot: a job with retries recovers via a second attempt on the boot job id", async () => {
+    const starts: Array<{ jobId: string; attempt: number | undefined }> = [];
+    let callCount = 0;
+    const bootFeature = defineFeature("retryboot", (r) => {
+      r.job("flaky", { trigger: { manual: true }, runOnBoot: true, retries: 1 }, async () => {
+        callCount += 1;
+        if (callCount === 1) throw new Error("fails on first boot run");
+      });
+    });
+
+    const registry = createRegistry([bootFeature]);
+    const context: AppContext = {};
+    const queueNamePrefix = uniquePrefix("boot");
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      onJobStart: (_name, jobId, meta: JobMeta) => {
+        starts.push({ jobId, attempt: meta.attempt });
+      },
+    });
+
+    const expectedJobId = bootJobIdForJobName("retryboot:job:flaky");
+    try {
+      await runner.start();
+      await waitFor(() => {
+        const attempts = starts.filter((s) => s.jobId === expectedJobId).map((s) => s.attempt);
+        expect(attempts).toContain(2);
+      });
+    } finally {
+      await runner.stop();
+      await purgeWorkerKeys(queueNamePrefix);
+    }
+  });
+
+  test("perTenant child: a job with retries recovers per tenant via a second attempt", async () => {
+    const starts: Array<{ jobId: string; attempt: number | undefined }> = [];
+    const callCountByTenant = new Map<string, number>();
+    const tenants = ["retry-child-a", "retry-child-b"] as TenantId[];
+    const perTenantFeature = defineFeature("retrychild", (r) => {
+      r.job(
+        "flaky",
+        { trigger: { manual: true }, perTenant: true, retries: 1 },
+        async (_payload, ctx) => {
+          const tenantId = String(ctx.systemUser.tenantId);
+          const count = (callCountByTenant.get(tenantId) ?? 0) + 1;
+          callCountByTenant.set(tenantId, count);
+          if (count === 1) throw new Error(`fails on first attempt for ${tenantId}`);
+        },
+      );
+    });
+
+    const registry = createRegistry([perTenantFeature]);
+    const context: AppContext = {};
+    const queueNamePrefix = uniquePrefix("child");
+    const getActiveTenantIds = async () => tenants;
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+      onJobStart: (name, jobId, meta: JobMeta) => {
+        if (name === "retrychild:job:flaky") starts.push({ jobId, attempt: meta.attempt });
+      },
+    });
+
+    try {
+      await runner.start();
+      await runner.dispatch("retrychild:job:flaky");
+      await waitFor(() => {
+        for (const tenantId of tenants) {
+          expect(callCountByTenant.get(tenantId)).toBe(2);
+        }
+      });
+      const byJob = new Map<string, number>();
+      for (const s of starts) byJob.set(s.jobId, Math.max(byJob.get(s.jobId) ?? 0, s.attempt ?? 0));
+      expect([...byJob.values()].some((attempt) => attempt >= 2)).toBe(true);
+    } finally {
+      await runner.stop();
+      await purgeWorkerKeys(queueNamePrefix);
+    }
+  });
+
+  test("perTenant wrapper: retries on getActiveTenantIds failure without double-fanning-out", async () => {
+    const tenants = ["retry-wrapper-a", "retry-wrapper-b"] as TenantId[];
+    const handlerCalls = new Map<string, number>();
+    let getActiveTenantIdsCalls = 0;
+    const wrapperFeature = defineFeature("retrywrapper", (r) => {
+      r.job(
+        "fanout",
+        { trigger: { manual: true }, perTenant: true, retries: 1 },
+        async (_payload, ctx) => {
+          const tenantId = String(ctx.systemUser.tenantId);
+          handlerCalls.set(tenantId, (handlerCalls.get(tenantId) ?? 0) + 1);
+        },
+      );
+    });
+
+    const registry = createRegistry([wrapperFeature]);
+    const context: AppContext = {};
+    const queueNamePrefix = uniquePrefix("wrapper");
+    const getActiveTenantIds = async () => {
+      getActiveTenantIdsCalls += 1;
+      if (getActiveTenantIdsCalls === 1)
+        throw new Error("fails resolving tenants on first attempt");
+      return tenants;
+    };
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+
+    try {
+      await runner.start();
+      await runner.dispatch("retrywrapper:job:fanout");
+      await waitFor(() => {
+        for (const tenantId of tenants) {
+          expect(handlerCalls.get(tenantId)).toBe(1);
+        }
+      });
+      // A wrapper retry re-derives the same child ids from its own stable
+      // BullMQ job id, so a late second fan-out would be a no-op on
+      // existing-jobId add() — but nothing should even attempt one here.
+      await sleep(400);
+      for (const tenantId of tenants) {
+        expect(handlerCalls.get(tenantId)).toBe(1);
+      }
+      expect(getActiveTenantIdsCalls).toBe(2);
+    } finally {
+      await runner.stop();
+      await purgeWorkerKeys(queueNamePrefix);
+    }
+  });
+
+  test("sequential re-enqueue: a job with retries recovers via a second attempt on the re-enqueued job id", async () => {
+    const starts: Array<{ jobId: string; attempt: number | undefined }> = [];
+    let callCount = 0;
+    let releaseGate: (() => void) | undefined;
+    const gate = new Promise<void>((resolve) => {
+      releaseGate = resolve;
+    });
+    const sequentialFeature = defineFeature("retryseq", (r) => {
+      r.job(
+        "flaky",
+        { trigger: { manual: true }, concurrency: "sequential", retries: 1 },
+        async () => {
+          callCount += 1;
+          const attemptNumber = callCount;
+          if (attemptNumber === 1) {
+            await gate;
+            return;
+          }
+          if (attemptNumber === 2) throw new Error("fails on second call (first retry slot)");
+        },
+      );
+    });
+
+    const registry = createRegistry([sequentialFeature]);
+    const context: AppContext = {};
+    const queueNamePrefix = uniquePrefix("seq");
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      onJobStart: (_name, jobId, meta: JobMeta) => {
+        starts.push({ jobId, attempt: meta.attempt });
+      },
+    });
+
+    const rawQueue = new Queue(`${queueNamePrefix}-worker`, {
+      connection: { host: testRedis.redis.options.host, port: testRedis.redis.options.port },
+    });
+    // A post-close 'error' here is otherwise unhandled and bun:test
+    // attributes it to whichever test runs next (fw#1805).
+    rawQueue.on("error", () => {});
+
+    try {
+      await runner.start();
+      const firstJobId = await runner.dispatch("retryseq:job:flaky");
+      const secondJobId = await runner.dispatch("retryseq:job:flaky");
+
+      await waitFor(() => {
+        expect(callCount).toBeGreaterThanOrEqual(1);
+      });
+
+      // Wait until the second dispatch's held-lock re-enqueue has actually
+      // landed as a delayed job before releasing the gate — otherwise the
+      // gate could resolve before there is anything left to retry.
+      await waitFor(
+        async () => {
+          const delayed = await rawQueue.getDelayed();
+          expect(delayed.some((j) => j.name === "retryseq:job:flaky")).toBe(true);
+        },
+        { delays: [100, 200, 400, 800] },
+      );
+
+      releaseGate?.();
+
+      await waitFor(() => {
+        const byJob = new Map<string, number>();
+        for (const s of starts)
+          byJob.set(s.jobId, Math.max(byJob.get(s.jobId) ?? 0, s.attempt ?? 0));
+        const retriedJobId = [...byJob.entries()].find(
+          ([jobId, attempt]) => attempt >= 2 && jobId !== firstJobId && jobId !== secondJobId,
+        );
+        expect(retriedJobId).toBeDefined();
+      });
+    } finally {
+      await rawQueue.close();
+      await runner.stop();
+      await purgeWorkerKeys(queueNamePrefix);
+    }
+  });
+});

package/src/jobs/__tests__/jobs.integration.test.ts

@@ -1361,9 +1361,9 @@ describe("boot gates", () => {
       await second.start();
       await sleep(300);
       expect(gateLog.filter((e) => e === "gate").length).toBe(2);
-      // Same fixed boot job id, still present in Redis from the first start,
-      // so BullMQ drops the second enqueue — that dedup is why runOnBoot
-      // cannot gate a deploy.
+      // The persistent boot-enqueued marker set from the first start is
+      // still present in Redis, so the second start skips re-enqueueing —
+      // that dedup is why runOnBoot cannot gate a deploy.
       expect(gateLog.filter((e) => e === "boot").length).toBe(1);
     } finally {
       await second.stop();
@@ -1390,4 +1390,39 @@ describe("boot gates", () => {
     });
     expect(() => createRegistry([sequentialGate])).toThrow(/bootGate with concurrency/);
   });
+
+  test("object-form backoff.delayMs must be a positive integer", () => {
+    for (const badDelayMs of [0, -5, 1.5]) {
+      const feature = defineFeature("badbackoff", (r) => {
+        r.job(
+          "check",
+          { trigger: { manual: true }, backoff: { type: "fixed", delayMs: badDelayMs } },
+          async () => {
+            gateLog.push("never");
+          },
+        );
+      });
+      expect(() => createRegistry([feature])).toThrow(
+        /backoff\.delayMs must be a positive integer/,
+      );
+    }
+
+    const validObjectForm = defineFeature("goodbackoffobject", (r) => {
+      r.job(
+        "check",
+        { trigger: { manual: true }, backoff: { type: "exponential", delayMs: 100 } },
+        async () => {
+          gateLog.push("never");
+        },
+      );
+    });
+    expect(() => createRegistry([validObjectForm])).not.toThrow();
+
+    const validStringForm = defineFeature("goodbackoffstring", (r) => {
+      r.job("check", { trigger: { manual: true }, backoff: "fixed" }, async () => {
+        gateLog.push("never");
+      });
+    });
+    expect(() => createRegistry([validStringForm])).not.toThrow();
+  });
 });

package/src/jobs/job-runner.ts

@@ -1,4 +1,4 @@
-import { Queue, Worker } from "bullmq";
+import { type JobsOptions, Queue, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context";
 import type { DbConnection, DbRow } from "../db/connection";
@@ -85,8 +85,10 @@ function legacySchedulerIdForJobName(jobName: string): string {
 // for the same trigger (retry after failure, a Redis drop, an instance
 // restarting mid-run) re-derives the *same* child ids, and BullMQ's
 // existing-jobId add() no-ops the second batch instead of creating
-// duplicates — the same dedup-on-id mechanism bootJobIdForJobName already
-// relies on above. Same colon-in-BullMQ-id hazard as schedulerIdForJobName
+// duplicates — safe only as long as the child job hash outlives the
+// wrapper's retry window, enforced at construction against
+// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
+// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
 // (fw#1603/#1604) — the wrapper id already contains ":", so strip
 // separators from both halves before joining instead of interpolating raw.
 function perTenantChildJobId(wrapperJobId: string, tenantId: string): string {
@@ -225,6 +227,13 @@ export type JobRunnerOptions = {
   // before failing boot. Defaults to BOOT_REDIS_TIMEOUT_MS; tests shrink it
   // to keep an unreachable-Redis assertion fast.
   bootRedisTimeoutMs?: number | undefined;
+  // Override how long completed/failed jobs stay in Redis before BullMQ's
+  // lazy queue-wide sweep evicts them. Defaults to
+  // COMPLETED_JOB_RETENTION_AGE_SEC / FAILED_JOB_RETENTION_AGE_SEC; tests
+  // shrink both to exercise the sweep without a real 24h/7d wait.
+  jobRetention?:
+    | { completedAgeSec?: number | undefined; failedAgeSec?: number | undefined }
+    | undefined;
   getActiveTenantIds?: () => Promise<TenantId[]>;
   onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
   onJobComplete?: (
@@ -317,16 +326,59 @@ function timeoutReject(
   };
 }
 
-// Shared by dispatch() and handleEvent() — an event-triggered job must retry
-// on failure the same way a directly-dispatched one does; a duplicated
-// inline computation in handleEvent previously dropped both options.
-function buildRetryBullOpts(jobDef: JobDefinition): Record<string, unknown> {
-  const opts: Record<string, unknown> = {};
-  if (jobDef.retries !== undefined) opts["attempts"] = jobDef.retries + 1;
-  if (jobDef.backoff) opts["backoff"] = { type: jobDef.backoff };
+// Default base delay when a job opts into backoff without an explicit
+// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
+// NaN/undefined, which is falsy — the job retries immediately instead of
+// waiting.
+const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
+
+// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
+// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
+// set must retry the same way regardless of how it got enqueued, or it
+// fails for good on the very first error on whichever path skips this.
+function buildRetryBullOpts(jobDef: JobDefinition): Pick<JobsOptions, "attempts" | "backoff"> {
+  const opts: Pick<JobsOptions, "attempts" | "backoff"> = {};
+  if (jobDef.retries !== undefined) opts.attempts = jobDef.retries + 1;
+  if (jobDef.backoff) {
+    opts.backoff =
+      typeof jobDef.backoff === "string"
+        ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
+        : {
+            type: jobDef.backoff.type,
+            delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
+          };
+  }
   return opts;
 }
 
+// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
+// OTHER jobs in the same completed/failed zset (moveToFinished lua,
+// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
+// the sweep only runs lazily on a later finish into that set. Per-job
+// retention is therefore meaningless on a shared queue — only queue-wide,
+// age-only retention (no count) is safe: a count would evict perTenant
+// children still inside their wrapper's retry window, and boot jobs. The
+// completed age must stay above every perTenant wrapper's own worst-case
+// retry window (see maxRetryWindowMs and the invariant check in
+// createJobRunner below), because child dedup (perTenantChildJobId) relies
+// on the children still existing in Redis when a wrapper retry re-derives
+// their ids. The margin here is generous — queue wait time isn't bounded by
+// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
+// not BullMQ (fw#3199).
+const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
+const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
+
+// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
+// retries, consistent with buildRetryBullOpts's own backoff computation.
+function maxRetryWindowMs(jobDef: JobDefinition): number {
+  if (!jobDef.backoff || jobDef.retries === undefined) return 0;
+  const delayMs =
+    (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
+    DEFAULT_JOB_BACKOFF_DELAY_MS;
+  const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
+  return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
+}
+
 export function createJobRunner(options: JobRunnerOptions): JobRunner {
   const { registry, context, redisUrl, consumerLane } = options;
   const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
@@ -383,6 +435,49 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
 
   const allJobs = registry.getAllJobs();
 
+  function positiveIntRetentionSec(
+    value: number | undefined,
+    fallback: number,
+    label: string,
+  ): number {
+    const resolved = value ?? fallback;
+    if (!Number.isInteger(resolved) || resolved <= 0) {
+      throw new Error(
+        `job-runner: jobRetention.${label} must be a positive integer, got ${resolved}`,
+      );
+    }
+    return resolved;
+  }
+  const completedAgeSec = positiveIntRetentionSec(
+    options.jobRetention?.completedAgeSec,
+    COMPLETED_JOB_RETENTION_AGE_SEC,
+    "completedAgeSec",
+  );
+  const failedAgeSec = positiveIntRetentionSec(
+    options.jobRetention?.failedAgeSec,
+    FAILED_JOB_RETENTION_AGE_SEC,
+    "failedAgeSec",
+  );
+
+  // perTenant child dedup (perTenantChildJobId) only holds as long as the
+  // children are still in Redis when a wrapper retry re-derives their ids,
+  // which requires the completed-job retention to outlast the wrapper's own
+  // worst-case retry window. Enforced here, before any Redis connection
+  // opens below, so a misconfigured job fails boot loudly instead of
+  // silently losing dedup in prod.
+  for (const [name, jobDef] of allJobs) {
+    if (!jobDef.perTenant) continue;
+    const windowMs = maxRetryWindowMs(jobDef);
+    if (windowMs >= completedAgeSec * 1000) {
+      throw new Error(
+        `job-runner: perTenant job "${name}" has a retry window of ${windowMs}ms, which is >= ` +
+          `the completed-job retention (${completedAgeSec}s). Child dedup relies on children ` +
+          "staying in Redis for the whole retry window — lower retries/backoff or raise " +
+          "jobRetention.completedAgeSec.",
+      );
+    }
+  }
+
   // Resolve the lane for a job — "worker" is the default because that's the
   // sensible prod lane (heavy async off the request path). Jobs that opted
   // into "api" must have been validated at registry boot already.
@@ -426,9 +521,22 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
   // queue matching the target job's runIn. Client-creation is cheap (shared
   // ioredis connection via bullmq), so this doesn't scale with number of
   // processes.
+  // Queue-wide, age-only retention (see COMPLETED_JOB_RETENTION_AGE_SEC
+  // above for why) — merged into every add()/addBulk()/upsertJobScheduler()
+  // template on both lanes.
+  const jobRetentionOpts: Pick<JobsOptions, "removeOnComplete" | "removeOnFail"> = {
+    removeOnComplete: { age: completedAgeSec },
+    removeOnFail: { age: failedAgeSec },
+  };
   const queues: Readonly<Record<JobRunIn, Queue>> = {
-    api: new Queue(queueNameFor(queueNamePrefix, "api"), { connection: redisOpts }),
-    worker: new Queue(queueNameFor(queueNamePrefix, "worker"), { connection: redisOpts }),
+    api: new Queue(queueNameFor(queueNamePrefix, "api"), {
+      connection: redisOpts,
+      defaultJobOptions: jobRetentionOpts,
+    }),
+    worker: new Queue(queueNameFor(queueNamePrefix, "worker"), {
+      connection: redisOpts,
+      defaultJobOptions: jobRetentionOpts,
+    }),
   };
   // Same unhandled-'error'-crash hazard as lockRedis above, just via
   // BullMQ's internal ioredis client (fw#1805).
@@ -522,9 +630,15 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
         await targetQueue.add(
           actualName,
           { ...bullJob.data, _tenantId: tenantId },
-          wrapperJobId !== undefined
-            ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
-            : undefined,
+          {
+            ...buildRetryBullOpts(actualDef),
+            // Dedup over wrapper retries only holds as long as the children
+            // stay in Redis for the wrapper's whole retry window — see the
+            // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
+            ...(wrapperJobId !== undefined
+              ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
+              : {}),
+          },
         );
       }
       // skip: fan-out dispatcher job, per-tenant children enqueued
@@ -553,8 +667,13 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
         // same queue the worker just picked from (since only the consuming
         // lane runs handleJob at all), but route explicitly — no implicit
         // coupling to "whichever queue the caller happened to be on".
+        // The re-enqueued job starts with a full retry budget; a remaining
+        // budget is deliberately not carried over, since finalAttempt/
+        // tenantVisibleFailure are computed from jobDef.retries against
+        // attemptsMade, not against some inherited remainder.
         await queues[laneForJob(jobDef)].add(jobName, bullJob.data, {
           delay: SEQUENTIAL_RETRY_DELAY_MS,
+          ...buildRetryBullOpts(jobDef),
         });
         // skip: lock taken, work re-enqueued with delay, current invocation done
         return;
@@ -900,20 +1019,48 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
             {
               name: jobDef.perTenant ? `_perTenant:${name}` : name,
               data: {},
+              // Queue-level defaultJobOptions (jobRetentionOpts) covers
+              // retention; a count here would sweep queue-wide again and
+              // evict perTenant children and boot jobs.
               opts: {
-                removeOnComplete: { count: 100 },
-                removeOnFail: { count: 50 },
+                ...buildRetryBullOpts(jobDef),
               },
             },
           );
         }
       }
 
+      // Persistent marker outside the swept completed/failed sets: a plain
+      // Redis hash at one key per consumer queue, fields = boot job ids
+      // already enqueued (BullMQ's IRedisClient has no set commands, only
+      // hash/hexists — a hash with dummy values does the same job). The job
+      // hash itself (removeOnComplete/-Fail: age-bound) is not a safe dedup
+      // target any more — retention now evicts it eventually, which would
+      // otherwise re-run a boot job "once per dataset" every time it ages
+      // out. Order matters: HEXISTS check, then add(), then HSET.
+      // Concurrent starts still dedupe on the existing job hash from add()'s
+      // own jobId no-op; a crash between add() and HSET dedupes again on the
+      // *next* boot (the job hash is still there) instead of losing the
+      // boot job forever, which HSET-first would risk if the process died
+      // before add() ran.
+      const bootEnqueuedKey = consumerQueue.toKey("kumiko-boot-enqueued");
       for (const [name, jobDef] of allJobs) {
         if (laneForJob(jobDef) !== consumerLane) continue;
         if (jobDef.runOnBoot) {
           const bootName = jobDef.perTenant ? `_perTenant:${name}` : name;
-          await consumerQueue.add(bootName, {}, { jobId: bootJobIdForJobName(name) });
+          const bootJobId = bootJobIdForJobName(name);
+          const client = await consumerQueue.client;
+          const alreadyEnqueued = await client.hexists(bootEnqueuedKey, bootJobId);
+          if (alreadyEnqueued) continue;
+          await consumerQueue.add(
+            bootName,
+            {},
+            {
+              jobId: bootJobId,
+              ...buildRetryBullOpts(jobDef),
+            },
+          );
+          await client.hset(bootEnqueuedKey, { [bootJobId]: 1 });
         }
       }
 
@@ -960,7 +1107,11 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
 
       // perTenant: dispatch the fan-out wrapper instead
       if (jobDef.perTenant) {
-        const job = await targetQueue.add(`_perTenant:${jobName}`, payload ?? {});
+        const job = await targetQueue.add(
+          `_perTenant:${jobName}`,
+          payload ?? {},
+          buildRetryBullOpts(jobDef),
+        );
         return job.id ?? "unknown";
       }