@johpaz/hive-sdk 0.1.4 → 0.1.5

package/test/job-retry-backoff.test.ts

@@ -0,0 +1,184 @@
+/**
+ * Tests for gateway/job-store.ts — failJobOrRetry: logical-failure retries
+ * with exponential backoff + jitter, separate from `attempts` (crash/lease
+ * only). Plus DurableLaneQueue wiring: chat_turn never retries, other types
+ * retry via the queue when the executor returns retryable:false-or-absent.
+ */
+
+process.env.HIVE_DB_PATH = ":memory:";
+
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { closeHiveDb } from "../packages/core/src/storage/hivedb";
+import { ensureHiveDb } from "../packages/core/src/storage/bootstrap";
+import { resetBootId, getBootId } from "../packages/core/src/storage/boot-id";
+import {
+  createJob,
+  claimJob,
+  failJobOrRetry,
+  getJob,
+  computeBackoffDelay,
+  type JobRetryPolicy,
+} from "../packages/core/src/gateway/job-store";
+import { DurableLaneQueue, registerExecutor } from "../packages/core/src/gateway/durable-queue";
+import { col } from "../packages/core/src/storage/hive";
+import type { JobDoc } from "../packages/core/src/storage/collections";
+
+beforeEach(async () => {
+  closeHiveDb();
+  resetBootId();
+  await ensureHiveDb();
+});
+
+afterEach(() => {
+  closeHiveDb();
+});
+
+const FAST_POLICY: JobRetryPolicy = {
+  maxRetries: 2,
+  initialDelayMs: 10,
+  backoffMultiplier: 2,
+  maxDelayMs: 1000,
+  jitter: 0,
+};
+
+describe("job-store: computeBackoffDelay", () => {
+  test("grows exponentially and caps at maxDelayMs", () => {
+    const policy: JobRetryPolicy = { maxRetries: 10, initialDelayMs: 100, backoffMultiplier: 2, maxDelayMs: 500, jitter: 0 };
+    expect(computeBackoffDelay(0, policy)).toBe(100);
+    expect(computeBackoffDelay(1, policy)).toBe(200);
+    expect(computeBackoffDelay(2, policy)).toBe(400);
+    expect(computeBackoffDelay(3, policy)).toBe(500); // capped
+    expect(computeBackoffDelay(10, policy)).toBe(500); // still capped
+  });
+
+  test("jitter adds a bounded random amount on top of the base delay", () => {
+    const policy: JobRetryPolicy = { maxRetries: 10, initialDelayMs: 100, backoffMultiplier: 2, maxDelayMs: 10_000, jitter: 0.5 };
+    for (let i = 0; i < 20; i++) {
+      const delay = computeBackoffDelay(0, policy);
+      expect(delay).toBeGreaterThanOrEqual(100);
+      expect(delay).toBeLessThanOrEqual(150);
+    }
+  });
+});
+
+describe("job-store: failJobOrRetry", () => {
+  test("schedules a retry: pending with a future not_before, retry_count incremented", async () => {
+    const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
+    await claimJob(job.id);
+
+    const before = Date.now();
+    const result = await failJobOrRetry(job.id, "transient error", getBootId(), FAST_POLICY);
+
+    expect(result).not.toBeNull();
+    expect(result!.status).toBe("pending");
+    expect(result!.retry_count).toBe(1);
+    expect(result!.last_error).toBe("transient error");
+    expect(result!.not_before).toBeGreaterThan(before);
+    expect(result!.boot_id).toBeNull();
+    expect(result!.lease_expires_at).toBeNull();
+
+    const fromDb = await getJob(job.id);
+    expect(fromDb!.status).toBe("pending");
+  });
+
+  test("does not touch `attempts` (crash counter) — only `retry_count`", async () => {
+    const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
+    const claimed = await claimJob(job.id);
+    expect(claimed!.attempts).toBe(1);
+
+    const result = await failJobOrRetry(job.id, "boom", getBootId(), FAST_POLICY);
+    expect(result!.attempts).toBe(1); // unchanged
+    expect(result!.retry_count).toBe(1);
+  });
+
+  test("fails terminally once retry_count reaches policy.maxRetries", async () => {
+    const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
+    const c = await col<JobDoc>("jobQueue");
+
+    // Exhaust retries one at a time (each retry re-pends the job; re-claim before the next failure).
+    // Force not_before into the past after each retry so claimJob doesn't block on the backoff delay.
+    for (let i = 0; i < FAST_POLICY.maxRetries; i++) {
+      await claimJob(job.id);
+      const r = await failJobOrRetry(job.id, `attempt ${i}`, getBootId(), FAST_POLICY);
+      expect(r!.status).toBe("pending");
+      const entry = await c.get(job.id);
+      await c.put(job.id, { ...entry!.doc, not_before: Date.now() - 1 }, { expectedVersion: entry!.version });
+    }
+
+    await claimJob(job.id);
+    const final = await failJobOrRetry(job.id, "final failure", getBootId(), FAST_POLICY);
+    expect(final!.status).toBe("failed");
+    expect(final!.error).toBe("final failure");
+    expect(final!.last_error).toBe("final failure");
+  });
+
+  test("ignores a job owned by a different boot_id", async () => {
+    const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
+    await claimJob(job.id, "boot-A");
+
+    const result = await failJobOrRetry(job.id, "err", "boot-B", FAST_POLICY);
+    expect(result).toBeNull();
+
+    const fromDb = await getJob(job.id);
+    expect(fromDb!.status).toBe("running");
+  });
+});
+
+async function waitFor(predicate: () => Promise<boolean>, timeoutMs = 3000): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    if (await predicate()) return;
+    await new Promise((r) => setTimeout(r, 10));
+  }
+  throw new Error("waitFor timed out");
+}
+
+describe("durable-queue: retry-on-logical-failure wiring", () => {
+  let queue: DurableLaneQueue | null = null;
+
+  afterEach(() => {
+    queue?.stop();
+    queue = null;
+  });
+
+  test("a retryable logical failure eventually completes after retrying", async () => {
+    let calls = 0;
+    registerExecutor("worker_task", async () => {
+      calls++;
+      if (calls < 2) return { ok: false, error: "flaky", retryable: true };
+      return { ok: true, result: "recovered" };
+    });
+
+    queue = new DurableLaneQueue({ maxGlobalConcurrency: 2, jobRetryPolicy: FAST_POLICY });
+    const job = await queue.enqueue({ lane: "task:retry-1", type: "worker_task", run_id: "r1", payload: {} });
+
+    // Wait past the (10ms) backoff delay, then use the public start() — it
+    // re-dispatches any due-pending lane — instead of the real 10s timer.
+    // The extra buffer avoids racing the exact not_before boundary.
+    await waitFor(async () => (await getJob(job.id))?.status === "pending" && (await getJob(job.id))!.retry_count === 1);
+    await new Promise((r) => setTimeout(r, 30));
+    queue.start();
+
+    await waitFor(async () => (await getJob(job.id))?.status === "completed");
+    const final = await getJob(job.id);
+    expect(calls).toBe(2);
+    expect(final!.status).toBe("completed");
+    expect(JSON.parse(final!.result_json!)).toBe("recovered");
+  });
+
+  test("chat_turn jobs never auto-retry a logical failure", async () => {
+    let calls = 0;
+    registerExecutor("chat_turn", async () => {
+      calls++;
+      return { ok: false, error: "user-facing failure" };
+    });
+
+    queue = new DurableLaneQueue({ maxGlobalConcurrency: 2, jobRetryPolicy: FAST_POLICY });
+    const job = await queue.enqueue({ lane: "session-1", type: "chat_turn", run_id: "r1", payload: {} });
+
+    await waitFor(async () => (await getJob(job.id))?.status === "failed");
+    expect(calls).toBe(1);
+    const final = await getJob(job.id);
+    expect(final!.retry_count).toBe(0);
+  });
+});

package/test/job-store.test.ts

@@ -0,0 +1,381 @@
+/**
+ * Tests for gateway/job-store.ts — JobDoc lifecycle, claim OCC, lease renewal,
+ * reclaim after expiry, cancel.
+ * Uses HIVE_DB_PATH=":memory:" so no state persists between runs.
+ */
+
+process.env.HIVE_DB_PATH = ":memory:";
+
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { closeHiveDb } from "../packages/core/src/storage/hivedb";
+import { ensureHiveDb } from "../packages/core/src/storage/bootstrap";
+import { col } from "../packages/core/src/storage/hive";
+import type { JobDoc } from "../packages/core/src/storage/collections";
+import { getBootId, resetBootId } from "../packages/core/src/storage/boot-id";
+import {
+  createJob,
+  claimJob,
+  renewLease,
+  completeJob,
+  failJob,
+  reclaimOrInterrupt,
+  cancelJob,
+  findPendingJobsByLane,
+  findExpiredLeases,
+  getJob,
+} from "../packages/core/src/gateway/job-store";
+
+beforeEach(async () => {
+  closeHiveDb();
+  resetBootId();
+  await ensureHiveDb();
+});
+
+afterEach(() => {
+  closeHiveDb();
+});
+
+describe("job-store: createJob", () => {
+  test("creates a pending job with correct defaults", async () => {
+    const job = await createJob({
+      lane: "session-1",
+      type: "chat_turn",
+      payload: { threadId: "t1", message: "hello" },
+      run_id: "run1",
+    });
+
+    expect(job.status).toBe("pending");
+    expect(job.lane).toBe("session-1");
+    expect(job.type).toBe("chat_turn");
+    expect(job.attempts).toBe(0);
+    expect(job.max_attempts).toBe(2);
+    expect(job.priority).toBe(0);
+    expect(job.boot_id).toBeNull();
+    expect(job.lease_expires_at).toBeNull();
+    expect(job.result_json).toBeNull();
+    expect(job.error).toBeNull();
+
+    const payload = JSON.parse(job.payload_json);
+    expect(payload.threadId).toBe("t1");
+    expect(payload.message).toBe("hello");
+  });
+
+  test("creates with priority and max_attempts overrides", async () => {
+    const job = await createJob({
+      lane: "task:1",
+      type: "worker_task",
+      payload: { task: "build" },
+      run_id: "run2",
+      priority: 10,
+      max_attempts: 5,
+    });
+
+    expect(job.priority).toBe(10);
+    expect(job.max_attempts).toBe(5);
+  });
+});
+
+describe("job-store: claimJob OCC", () => {
+  test("claims a pending job → running with boot_id + lease", async () => {
+    const job = await createJob({
+      lane: "session-1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "run1",
+    });
+
+    const claimed = await claimJob(job.id);
+    expect(claimed).not.toBeNull();
+    expect(claimed!.status).toBe("running");
+    expect(claimed!.attempts).toBe(1);
+    expect(claimed!.boot_id).toBe(getBootId());
+    expect(claimed!.lease_expires_at).toBeGreaterThan(Date.now());
+
+    const fromDb = await getJob(job.id);
+    expect(fromDb!.status).toBe("running");
+  });
+
+  test("claiming a non-pending job returns null", async () => {
+    const job = await createJob({
+      lane: "session-1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "run1",
+    });
+
+    await claimJob(job.id);
+    const secondAttempt = await claimJob(job.id);
+    expect(secondAttempt).toBeNull();
+  });
+
+  test("claiming with not_before in future returns null", async () => {
+    const job = await createJob({
+      lane: "session-1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "run1",
+      not_before: Date.now() + 60_000,
+    });
+
+    const claimed = await claimJob(job.id);
+    expect(claimed).toBeNull();
+  });
+
+  test("two bootIds race — only one wins (OCC guarantee)", async () => {
+    const job = await createJob({
+      lane: "session-1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "run1",
+    });
+
+    // Simulate two different boots attempting claim simultaneously
+    const [claim1, claim2] = await Promise.all([
+      claimJob(job.id, "boot-A"),
+      claimJob(job.id, "boot-B"),
+    ]);
+
+    // At least one should win, at most one should win
+    const winners = [claim1, claim2].filter((c) => c !== null);
+    expect(winners.length).toBe(1);
+    const winningBoot = winners[0]!.boot_id;
+    expect(["boot-A", "boot-B"]).toContain(winningBoot);
+  });
+});
+
+describe("job-store: lease renewal", () => {
+  test("renewLease extends lease_expires_at", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id);
+    const before = await getJob(job.id);
+    const oldLease = before!.lease_expires_at!;
+
+    // Wait a bit so the new lease is strictly greater
+    await new Promise((r) => setTimeout(r, 10));
+    const ok = await renewLease(job.id);
+    expect(ok).toBe(true);
+
+    const after = await getJob(job.id);
+    expect(after!.lease_expires_at!).toBeGreaterThan(oldLease);
+  });
+
+  test("renewLease fails for wrong boot_id", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id, "boot-A");
+
+    const ok = await renewLease(job.id, "boot-B");
+    expect(ok).toBe(false);
+  });
+});
+
+describe("job-store: completeJob / failJob", () => {
+  test("completeJob saves result_json and clears lease", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id);
+
+    await completeJob(job.id, { response: "done" });
+
+    const saved = await getJob(job.id);
+    expect(saved!.status).toBe("completed");
+    expect(saved!.result_json).not.toBeNull();
+    const result = JSON.parse(saved!.result_json!);
+    expect(result.response).toBe("done");
+    expect(saved!.boot_id).toBeNull();
+    expect(saved!.lease_expires_at).toBeNull();
+    expect(saved!.finished_at).not.toBeNull();
+  });
+
+  test("failJob saves error and clears lease", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "worker_task",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id);
+
+    await failJob(job.id, "Worker crashed");
+
+    const saved = await getJob(job.id);
+    expect(saved!.status).toBe("failed");
+    expect(saved!.error).toBe("Worker crashed");
+    expect(saved!.boot_id).toBeNull();
+  });
+
+  test("completeJob refuses to overwrite a job owned by different boot", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id, "boot-A");
+
+    // boot-B tries to complete — should be ignored
+    await completeJob(job.id, { response: "bad" }, "boot-B");
+
+    const saved = await getJob(job.id);
+    expect(saved!.status).toBe("running");
+    expect(saved!.result_json).toBeNull();
+  });
+});
+
+describe("job-store: reclaimOrInterrupt", () => {
+  test("reclaims job back to pending when lease expired and attempts < max", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+      max_attempts: 3,
+    });
+    await claimJob(job.id, "boot-A");
+
+    // Expire the lease
+    const c = await col<JobDoc>("jobQueue");
+    const entry = await c.get(job.id);
+    await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
+
+    const result = await reclaimOrInterrupt(job.id);
+    expect(result).not.toBeNull();
+    expect(result!.status).toBe("pending");
+    expect(result!.attempts).toBe(1);
+    expect(result!.boot_id).toBeNull();
+  });
+
+  test("interrupts job when attempts exhausted", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "worker_task",
+      payload: {},
+      run_id: "r1",
+      max_attempts: 2,
+    });
+    // Claim twice to exhaust attempts
+    await claimJob(job.id, "boot-A");
+    const c = await col<JobDoc>("jobQueue");
+    let entry = await c.get(job.id);
+    await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
+    await reclaimOrInterrupt(job.id); // attempt 1 → pending
+
+    entry = await c.get(job.id);
+    expect(entry!.doc.status).toBe("pending");
+
+    await claimJob(job.id, "boot-B"); // attempt 2 → running
+    entry = await c.get(job.id);
+    await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
+    const result = await reclaimOrInterrupt(job.id); // attempt 2 = max → interrupted
+
+    expect(result!.status).toBe("interrupted");
+    expect(result!.error).toContain("Max attempts");
+  });
+});
+
+describe("job-store: cancelJob", () => {
+  test("cancels a pending job", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+
+    const ok = await cancelJob(job.id);
+    expect(ok).toBe(true);
+
+    const saved = await getJob(job.id);
+    expect(saved!.status).toBe("cancelled");
+  });
+
+  test("cancels a running job", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id);
+
+    const ok = await cancelJob(job.id);
+    expect(ok).toBe(true);
+
+    const saved = await getJob(job.id);
+    expect(saved!.status).toBe("cancelled");
+  });
+
+  test("refuses to cancel an already completed job", async () => {
+    const job = await createJob({
+      lane: "s1",
+      type: "chat_turn",
+      payload: {},
+      run_id: "r1",
+    });
+    await claimJob(job.id);
+    await completeJob(job.id, { response: "done" });
+
+    const ok = await cancelJob(job.id);
+    expect(ok).toBe(false);
+  });
+});
+
+describe("job-store: findPendingJobsByLane", () => {
+  test("returns pending jobs ordered by priority desc then FIFO by id", async () => {
+    const jobLow = await createJob({ lane: "sports", type: "chat_turn", payload: {}, run_id: "r1", priority: 0 });
+    const jobHigh = await createJob({ lane: "sports", type: "chat_turn", payload: {}, run_id: "r2", priority: 10 });
+    const jobMid = await createJob({ lane: "sports", type: "chat_turn", payload: {}, run_id: "r3", priority: 5 });
+    const other = await createJob({ lane: "news", type: "chat_turn", payload: {}, run_id: "r4", priority: 100 });
+
+    const pending = await findPendingJobsByLane("sports");
+    expect(pending.length).toBe(3);
+    expect(pending[0].id).toBe(jobHigh.id);   // priority 10
+    expect(pending[1].id).toBe(jobMid.id);    // priority 5
+    expect(pending[2].id).toBe(jobLow.id);    // priority 0
+  });
+
+  test("excludes pending jobs with not_before in future", async () => {
+    await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r1" });
+    await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r2", not_before: Date.now() + 60_000 });
+
+    const pending = await findPendingJobsByLane("s1");
+    expect(pending.length).toBe(1);
+  });
+});
+
+describe("job-store: findExpiredLeases", () => {
+  test("finds running jobs with expired leases", async () => {
+    const job = await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r1" });
+    await claimJob(job.id, "boot-A");
+
+    // Expire the lease
+    const c = await col<JobDoc>("jobQueue");
+    const entry = await c.get(job.id);
+    await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
+
+    const expired = await findExpiredLeases();
+    expect(expired.length).toBe(1);
+    expect(expired[0].id).toBe(job.id);
+  });
+
+  test("excludes jobs with active leases", async () => {
+    const job = await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r1" });
+    await claimJob(job.id);
+
+    const expired = await findExpiredLeases();
+    expect(expired.length).toBe(0);
+  });
+});

package/test/llm-retry.test.ts

@@ -0,0 +1,97 @@
+/**
+ * Tests for resilience/retry.ts — isRetryableError classification and
+ * withRetry's backoff/retry-after behavior, plus llm-client.ts's callLLM
+ * wiring (retries a 429 then succeeds; never retries after abort).
+ */
+
+import { describe, test, expect } from "bun:test";
+import { isRetryableError, withRetry, computeRetryDelay, type RetryPolicy } from "../packages/core/src/resilience/retry";
+
+describe("resilience/retry: isRetryableError", () => {
+  test("classifies HTTP status codes", () => {
+    expect(isRetryableError({ status: 429 })).toBe(true);
+    expect(isRetryableError({ status: 408 })).toBe(true);
+    expect(isRetryableError({ status: 500 })).toBe(true);
+    expect(isRetryableError({ status: 503 })).toBe(true);
+    expect(isRetryableError({ status: 400 })).toBe(false);
+    expect(isRetryableError({ status: 401 })).toBe(false);
+    expect(isRetryableError({ status: 404 })).toBe(false);
+  });
+
+  test("reads status from a nested response object", () => {
+    expect(isRetryableError({ response: { status: 429 } })).toBe(true);
+    expect(isRetryableError({ response: { status: 403 } })).toBe(false);
+  });
+
+  test("classifies network/timeout errors by name/message when no status is present", () => {
+    expect(isRetryableError(new Error("fetch failed"))).toBe(true);
+    expect(isRetryableError({ name: "TimeoutError", message: "Request timed out" })).toBe(true);
+    expect(isRetryableError({ message: "ECONNRESET" })).toBe(true);
+    expect(isRetryableError({ name: "AbortError", message: "The operation was aborted" })).toBe(false);
+  });
+
+  test("unclassifiable errors default to non-retryable", () => {
+    expect(isRetryableError(new Error("invalid API key"))).toBe(false);
+  });
+});
+
+describe("resilience/retry: computeRetryDelay", () => {
+  test("honors an explicit Retry-After over the computed backoff", () => {
+    const policy: RetryPolicy = { maxAttempts: 3, initialDelayMs: 1000, backoffMultiplier: 2, maxDelayMs: 30_000 };
+    expect(computeRetryDelay(0, policy, 5000)).toBe(5000);
+    expect(computeRetryDelay(0, policy, 60_000)).toBe(30_000); // still capped
+  });
+});
+
+describe("resilience/retry: withRetry", () => {
+  const FAST_POLICY: RetryPolicy = { maxAttempts: 3, initialDelayMs: 1, backoffMultiplier: 1, maxDelayMs: 5 };
+
+  test("retries a transient failure then returns the eventual success", async () => {
+    let calls = 0;
+    const result = await withRetry(async () => {
+      calls++;
+      if (calls < 2) throw { status: 429, message: "rate limited" };
+      return "ok";
+    }, FAST_POLICY);
+
+    expect(result).toBe("ok");
+    expect(calls).toBe(2);
+  });
+
+  test("stops retrying once maxAttempts is reached and throws the last error", async () => {
+    let calls = 0;
+    await expect(
+      withRetry(async () => {
+        calls++;
+        throw { status: 500, message: "always fails" };
+      }, FAST_POLICY)
+    ).rejects.toThrow("always fails");
+    expect(calls).toBe(FAST_POLICY.maxAttempts);
+  });
+
+  test("does not retry a non-retryable (terminal) error", async () => {
+    let calls = 0;
+    await expect(
+      withRetry(async () => {
+        calls++;
+        throw { status: 401, message: "unauthorized" };
+      }, FAST_POLICY)
+    ).rejects.toThrow("unauthorized");
+    expect(calls).toBe(1);
+  });
+
+  test("a custom isRetryable classifier overrides the default", async () => {
+    let calls = 0;
+    const result = await withRetry(
+      async () => {
+        calls++;
+        if (calls < 2) throw new Error("domain-specific transient");
+        return "recovered";
+      },
+      FAST_POLICY,
+      (err) => (err as Error).message === "domain-specific transient"
+    );
+    expect(result).toBe("recovered");
+    expect(calls).toBe(2);
+  });
+});