cairnq 0.6.0 → 0.8.0

package/src/errors.ts

@@ -77,8 +77,12 @@ export class QueueFull extends CairnQError {
  * observed. No worker running, no handler for the name, wrong queue, and two
  * processes on different database files all look identical from the API side —
  * queued, never claimed — so that case names the likely causes. */
-function timeoutDetail(task: Task | null): string {
-  if (!task) return "task not found — wrong database file, or already purged?";
+function timeoutDetail(task: Task | null, key: string | null): string {
+  if (!task) {
+    return key === null
+      ? "task not found — wrong database file, or already purged?"
+      : "no task under this key — never submitted, or already purged?";
+  }
   if (isQueued(task)) {
     const delayMs = task.run_at_ms - nowMs();
     if (task.attempt === 0 && delayMs <= 0) {
@@ -94,23 +98,33 @@ function timeoutDetail(task: Task | null): string {
   return `still running (attempt ${task.attempt}/${task.max_attempts})`;
 }
 
-/** wait/call did not reach a terminal status in time. The task keeps running.
- * `task` is the last snapshot wait() observed (null if get() found nothing), and
- * the message says what state it was stuck in — a queued-never-claimed task is
- * the classic first-run failure (no worker, no handler, wrong queue or file). */
+/** wait/call did not reach a terminal status in time. The task keeps running, so
+ * `taskId` is the handle for picking the wait back up — `wait(err.taskId)`
+ * re-attaches to the same task from anywhere that can reach the store. `task` is
+ * the last snapshot wait() observed (null if the lookup found nothing), and the
+ * message says what state it was stuck in — a queued-never-claimed task is the
+ * classic first-run failure (no worker, no handler, wrong queue or file).
+ *
+ * `key` is set when the wait watched a key rather than an id; `taskId` is then
+ * the task the key pointed at, or the key itself when it pointed at nothing —
+ * there was no id to report. */
 export class TaskTimeout extends CairnQError {
   readonly task: Task | null;
+  readonly key: string | null;
   constructor(
     public taskId: string,
-    opts: { timeoutMs?: number; task?: Task | null } = {},
+    opts: { timeoutMs?: number; task?: Task | null; key?: string | null } = {},
   ) {
+    const key = opts.key ?? null;
+    const subject = key === null ? `task ${taskId}` : `key ${key}`;
     super(
       opts.timeoutMs == null
-        ? `task ${taskId} did not finish in time`
-        : `task ${taskId} did not finish within ${opts.timeoutMs}ms: ${timeoutDetail(opts.task ?? null)}`,
+        ? `${subject} did not finish in time`
+        : `${subject} did not finish within ${opts.timeoutMs}ms: ${timeoutDetail(opts.task ?? null, key)}`,
     );
     this.name = "TaskTimeout";
     this.task = opts.task ?? null;
+    this.key = key;
   }
 }
 
@@ -146,6 +160,39 @@ export class TaskCanceled extends CairnQError {
   }
 }
 
+/**
+ * A heartbeat beat came back later than its own interval allowed.
+ *
+ * The heartbeat shares the event loop with the handlers whose leases it renews,
+ * so a handler that blocks the loop stops the renewal with it: the lease expires,
+ * the task is recovered and redelivered, and a second worker starts computing
+ * what the first is still computing — one task, billed twice, with no error
+ * anywhere. Nothing inside the blocked handler can observe that, which is why it
+ * is reported through `onError` alongside the other things the run loop survived.
+ *
+ * The usual cause is synchronous work in a handler: a tight loop, a large
+ * JSON.parse, a `*Sync` filesystem or crypto call. Node has one loop and no way
+ * to preempt it — move the work to a worker thread, a child process, or an async
+ * API that yields. The other cause is a worker simply oversubscribed for its
+ * `leaseMs` — nothing is blocking, there is just more work than turns — which the
+ * same report covers, because the lease is at equal risk either way.
+ */
+export class EventLoopBlocked extends CairnQError {
+  constructor(
+    readonly lateMs: number,
+    readonly intervalMs: number,
+    readonly leaseMs: number,
+  ) {
+    super(
+      `heartbeat beat was ${lateMs}ms late (interval ${intervalMs}ms, lease ${leaseMs}ms): ` +
+        `the event loop was blocked long enough to miss a beat, so this worker's leases ` +
+        `are at risk. Usually synchronous work in a handler (move it off the loop); ` +
+        `otherwise the worker is oversubscribed for its leaseMs.`,
+    );
+    this.name = "EventLoopBlocked";
+  }
+}
+
 /** A worker write affected 0 rows: the lease expired and was reclaimed. */
 export class LostLease extends CairnQError {
   constructor(public taskId: string) {

package/src/index.ts

@@ -1,7 +1,9 @@
 export { CairnQ } from "./client.js";
-export type { CallOptions, ClientOptions, SubmitOptions } from "./client.js";
+export type { CallOptions, ClientOptions, SubmitOptions, WaitOptions } from "./client.js";
 export { QueueDepthGate } from "./backpressure.js";
 export type { BackpressureOptions, QueueDepthLimit } from "./backpressure.js";
+export { RetentionSweeper } from "./retention.js";
+export type { RetentionCutoffs, RetentionOptions } from "./retention.js";
 export { Worker } from "./worker.js";
 export type { BatchHandler, Handler, TypedHandler, WorkerOptions } from "./worker.js";
 export { TaskContext } from "./context.js";
@@ -12,10 +14,11 @@ export { SQLiteStore } from "./store/sqlite.js";
 export { PostgresStore } from "./store/postgres.js";
 export { TaskStore } from "./store/base.js";
 export type { ListInput, PurgeInput, SubmitInput, Conflict } from "./store/base.js";
-export type { Task, TaskStatus } from "./models.js";
+export type { Task, TaskRef, TaskStatus, TerminalStatus } from "./models.js";
 export {
   STATUSES,
   isTerminal,
+  isTerminalStatus,
   cancelRequested,
   isQueued,
   isRunning,
@@ -32,6 +35,7 @@ export {
   TaskCanceled,
   TaskError,
   LostLease,
+  EventLoopBlocked,
   ProtocolVersionMismatch,
   SerializationError,
 } from "./errors.js";

package/src/models.ts

@@ -31,8 +31,28 @@ export interface Task {
   completed_at_ms: number | null;
 }
 
+/** The id + status pair the wait loop polls on (see get_status.sql) — a probe,
+ * not a snapshot: everything else about the task is deliberately not read. */
+export interface TaskRef {
+  id: string;
+  status: TaskStatus;
+}
+
 const JSON_COLUMNS = ["payload", "result", "error", "metadata"] as const;
-export const TERMINAL: TaskStatus[] = ["succeeded", "failed", "canceled"];
+// As a const tuple so TerminalStatus derives from it — the same declare-once
+// pattern as STATUSES/TaskStatus above.
+export const TERMINAL = ["succeeded", "failed", "canceled"] as const;
+export type TerminalStatus = (typeof TERMINAL)[number];
+
+export function isTerminalStatus(status: TaskStatus): status is TerminalStatus {
+  return (TERMINAL as readonly TaskStatus[]).includes(status);
+}
+
+/** Map a probe row (see get_status.sql) to a TaskRef — the ref twin of
+ * rowToTask, so the row shape stays models' knowledge alone. */
+export function rowToRef(row: Record<string, unknown>): TaskRef {
+  return { id: row.id as string, status: row.status as TaskStatus };
+}
 
 export function rowToTask(row: Record<string, unknown>): Task {
   const t: Record<string, unknown> = { ...row };
@@ -46,8 +66,9 @@ export function rowToTask(row: Record<string, unknown>): Task {
   return t as unknown as Task;
 }
 
-export function isTerminal(task: Task): boolean {
-  return TERMINAL.includes(task.status);
+/** Accepts anything carrying a status — a Task or a TaskRef probe. */
+export function isTerminal(task: Pick<Task, "status">): boolean {
+  return isTerminalStatus(task.status);
 }
 
 export function cancelRequested(task: Task): boolean {

package/src/retention.ts

@@ -0,0 +1,166 @@
+import type { TaskStatus, TerminalStatus } from "./models.js";
+import { validatePurgeInput, type PurgeInput, type TaskStore } from "./store/base.js";
+
+/** Sweep every hour unless asked otherwise — often enough that a queue with a
+ * day of retention never carries more than an hour of extra rows, rare enough
+ * that the sweep is invisible next to the task traffic. */
+const DEFAULT_INTERVAL_MS = 3_600_000;
+/** Rows per purge statement. The same bound `purge` defaults to: big enough that
+ * a backlog drains in few statements, small enough that each is a short write. */
+const DEFAULT_LIMIT = 1_000;
+
+/** Per-status cutoffs. A status left out is never swept — granular retention is
+ * an explicit statement of what may go, not a default for what wasn't named. */
+export type RetentionCutoffs = Partial<Record<TerminalStatus, number>>;
+
+export interface RetentionOptions {
+  /**
+   * How long a terminal task is kept after it finished. Required: there is no
+   * safe default for how long someone else's results stay readable.
+   *
+   * A number keeps every terminal status the same time. Retention needs are
+   * often tiered — a succeeded row is spent once its result is consumed, while
+   * a failed one is worth keeping for diagnosis — so a per-status map sets a
+   * cutoff per status instead: `{ succeeded: 300_000, failed: 86_400_000 }`.
+   */
+  olderThanMs: number | RetentionCutoffs;
+  /** Time between sweeps. Default 3_600_000 (one hour). */
+  intervalMs?: number;
+  /** Rows deleted per statement while draining. Default 1_000. */
+  limit?: number;
+  /**
+   * Called for a sweep that threw. The next sweep runs on schedule regardless —
+   * a purge that failed because the database was busy is not a reason to stop
+   * retaining — so without this a store quietly stops being swept. Must not throw.
+   */
+  onError?: (err: unknown) => void;
+}
+
+/**
+ * Deletes terminal tasks on a schedule, for as long as the handle is open.
+ *
+ * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
+ * payloads carry real data — an image, a document, a batch of embeddings — turns
+ * that into a disk leak measured in gigabytes per backfill. Every deployment
+ * that runs longer than a demo needs the sweep; leaving it to an external
+ * scheduler means the leak is the default and remembering is the opt-in.
+ *
+ * It sweeps in bounded batches with a yield between them, so draining a backlog
+ * that accumulated while nothing was sweeping stays a sequence of short writes
+ * rather than one long one — on SQLite that matters, since a long write holds
+ * the single write lock against every producer and worker on the file.
+ */
+export class RetentionSweeper {
+  /** Whether the scheduled loop is running. */
+  private active = false;
+  /** Set by stop(), so a drain in progress can cut itself short too. */
+  private stopping = false;
+  /** Resolves the current sleep early, so stop() need not wait out an interval. */
+  private wake: (() => void) | null = null;
+  /** The loop itself, awaited by stop() so no purge outlives the store. */
+  private loop: Promise<void> | null = null;
+  private readonly intervalMs: number;
+  /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
+  private readonly limit: number;
+  /** One purge per cutoff: a lone entry for a number, one per status for a map. */
+  private readonly purgeInputs: PurgeInput[];
+
+  constructor(
+    private readonly store: TaskStore,
+    private readonly opts: RetentionOptions,
+  ) {
+    this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
+    if (!Number.isFinite(this.intervalMs) || this.intervalMs < 1) {
+      throw new Error(`retention.intervalMs must be >= 1, got ${this.intervalMs}`);
+    }
+    this.limit = opts.limit ?? DEFAULT_LIMIT;
+    const cutoffs: [TaskStatus | undefined, number][] =
+      typeof opts.olderThanMs === "number"
+        ? [[undefined, opts.olderThanMs]]
+        : (Object.entries(opts.olderThanMs) as [TaskStatus, number][]);
+    // An empty map retains nothing and sweeps nothing — almost certainly a bug
+    // upstream of this call, so refuse it rather than silently never purging.
+    if (!cutoffs.length) {
+      throw new Error("retention.olderThanMs must name at least one status");
+    }
+    this.purgeInputs = cutoffs.map(([status, ms]) => ({
+      olderThanMs: ms,
+      status,
+      limit: this.limit,
+    }));
+    // Fail fast on the store's own purge rules (terminal status, cutoff >= 0):
+    // the sweep runs an hour from now, and its errors only surface via onError.
+    for (const input of this.purgeInputs) validatePurgeInput(input);
+  }
+
+  start(): void {
+    if (this.active) return;
+    this.active = true;
+    this.stopping = false;
+    this.loop = this.run();
+  }
+
+  /** Stop sweeping and wait for the sweep in flight, if any. */
+  async stop(): Promise<void> {
+    this.stopping = true;
+    this.active = false;
+    this.wake?.();
+    await this.loop;
+    this.loop = null;
+  }
+
+  private async run(): Promise<void> {
+    // Sleep first: a process that restarts often would otherwise purge on every
+    // boot, which is a write burst exactly when the store is busiest.
+    while (!this.stopping) {
+      await this.sleep(this.intervalMs);
+      if (this.stopping) return;
+      try {
+        await this.sweep();
+      } catch (err) {
+        try {
+          this.opts.onError?.(err);
+        } catch {
+          // A reporting hook must never take the sweep down with it — the same
+          // rule the worker's onError follows.
+        }
+      }
+    }
+  }
+
+  /**
+   * Delete everything past the cutoff now, in bounded batches, and return how
+   * many rows went. The scheduled loop calls this; call it directly to drain on
+   * demand — after a backfill, or from a maintenance command.
+   */
+  async sweep(): Promise<number> {
+    let deleted = 0;
+    for (const input of this.purgeInputs) {
+      for (;;) {
+        const ids = await this.store.purge(input);
+        deleted += ids.length;
+        if (this.stopping) return deleted;
+        if (ids.length < this.limit) break;
+        // Hand the loop back between batches: a large drain must not starve the
+        // submits and claims sharing this process.
+        await this.sleep(0);
+      }
+    }
+    return deleted;
+  }
+
+  /** Sleep, interruptible by stop(). Unref'd: retention is housekeeping, and a
+   * pending sweep must never be the reason a process refuses to exit. */
+  private sleep(ms: number): Promise<void> {
+    return new Promise<void>((resolve) => {
+      const timer = setTimeout(resolve, ms);
+      timer.unref?.();
+      this.wake = () => {
+        clearTimeout(timer);
+        resolve();
+      };
+    }).finally(() => {
+      this.wake = null;
+    });
+  }
+}

package/src/store/base.ts

@@ -6,7 +6,15 @@ import {
   ProtocolVersionMismatch,
   SerializationError,
 } from "../errors.js";
-import { rowToTask, STATUSES, type Task, type TaskStatus } from "../models.js";
+import {
+  isTerminalStatus,
+  rowToRef,
+  rowToTask,
+  STATUSES,
+  type Task,
+  type TaskRef,
+  type TaskStatus,
+} from "../models.js";
 import { type BackpressureOptions, QueueDepthGate } from "../backpressure.js";
 
 const rejectMangled = function (this: unknown, _key: string, v: unknown): unknown {
@@ -62,9 +70,25 @@ export function checkProtocolVersion(version: number): void {
 // CONFLICTS is the canonical declaration; the type derives from it so the
 // runtime guard in submit() and the type can't drift apart (same pattern as
 // STATUSES/TaskStatus in models.ts).
-const CONFLICTS = ["reuse", "reject", "replace"] as const;
+const CONFLICTS = ["reuse", "reuse-succeeded", "reject", "replace"] as const;
 export type Conflict = (typeof CONFLICTS)[number];
 
+/**
+ * Whether a keyed submit's strategy accepts the task the key already points at.
+ *
+ * Both reuse strategies deduplicate work that is still in play — that is what a
+ * key is for, and the answer cannot depend on the outcome of a task that has no
+ * outcome yet. They differ only on what a *finished* task means: `reuse` treats
+ * the key as free again, while `reuse-succeeded` reads a succeeded task as a
+ * cached result. Neither ever hands back a failed or canceled one, which would
+ * poison the key for every later submit (see PROTOCOL.md "Key conflict").
+ */
+function reusable(conflict: Conflict, status: TaskStatus): boolean {
+  if (conflict === "replace") return false;
+  if (!isTerminalStatus(status)) return true;
+  return conflict === "reuse-succeeded" && status === "succeeded";
+}
+
 /** The queue a submit lands on when it names none. Owned here, where the
  * default is applied, so nothing above has to re-derive it. */
 export const DEFAULT_QUEUE = "default";
@@ -94,8 +118,32 @@ export interface ListInput {
   offset?: number;
 }
 
+/** Validate a purge's inputs. Shared with RetentionSweeper, which fail-fasts at
+ * construction on the same rules an hourly sweep would otherwise only surface
+ * through its onError hook — one statement of the rules, two callers. */
+export function validatePurgeInput(input: PurgeInput): void {
+  if (input.olderThanMs != null && (!Number.isFinite(input.olderThanMs) || input.olderThanMs < 0)) {
+    throw new Error(`olderThanMs must be >= 0, got ${input.olderThanMs}`);
+  }
+  if (input.limit != null && input.limit < 1) {
+    throw new Error(`limit must be >= 1, got ${input.limit}`);
+  }
+  // Terminal only: purge never deletes live work, so accepting `queued` here
+  // would be accepting a filter that silently matches nothing.
+  if (input.status != null && !isTerminalStatus(input.status)) {
+    throw new Error(`status must be terminal, got ${input.status}`);
+  }
+}
+
 export interface PurgeInput {
   olderThanMs?: number;
+  /** Restrict the sweep to one terminal status. Retention needs are tiered —
+   * succeeded rows are spent once their result is consumed, failed ones are
+   * worth keeping for diagnosis — and without this the shortest-lived tier
+   * sets the retention for every row. Absent means all terminal statuses. */
+  status?: TaskStatus;
+  /** Restrict the sweep to one task name. Absent means all names. */
+  name?: string;
   limit?: number;
 }
 
@@ -218,6 +266,10 @@ export abstract class TaskStore {
     return rows.length ? rowToTask(rows[0]) : null;
   }
 
+  private static oneRef(rows: any[]): TaskRef | null {
+    return rows.length ? rowToRef(rows[0]) : null;
+  }
+
   // ------------------------------------------------------------- client side
   /**
    * Bound how deep a queue may get before `submit` blocks. Off unless set.
@@ -283,10 +335,15 @@ export abstract class TaskStore {
         // free after all, whatever the strategy.
         const current = (await fetch("get", { id: existing[0].task_id }))[0];
         if (current) {
-          if (conflict === "reuse") return rowToTask(current);
           if (conflict === "reject") throw new AlreadyExists(key);
-          // "replace": cancel the recorded task, then repoint the key below.
-          await fetch("cancel", { id: existing[0].task_id });
+          if (reusable(conflict, current.status as TaskStatus)) return rowToTask(current);
+          // The strategy declined the recorded task, so the key repoints to the
+          // fresh one inserted below. Cancel only what is still live: a terminal
+          // task has nothing to stop, and cancelling it would rewrite a settled
+          // row (and hand a `canceled` back to whoever is waiting on it).
+          if (!isTerminalStatus(current.status as TaskStatus)) {
+            await fetch("cancel", { id: existing[0].task_id });
+          }
         }
       }
       const row = (await fetch("insert_task", ins))[0];
@@ -303,6 +360,18 @@ export abstract class TaskStore {
     return TaskStore.one(await this.fetch("get_by_key", { key }));
   }
 
+  /** The wait loop's probe: id + status alone, so polling a task with a large
+   * payload does not re-read and re-parse that payload on every beat. */
+  async getStatus(taskId: string): Promise<TaskRef | null> {
+    return TaskStore.oneRef(await this.fetch("get_status", { id: taskId }));
+  }
+
+  /** getStatus, following a key instead of an id — re-resolved per call, so a
+   * `replace` moves the probe onto the new task. */
+  async getStatusByKey(key: string): Promise<TaskRef | null> {
+    return TaskStore.oneRef(await this.fetch("get_status_by_key", { key }));
+  }
+
   async list(input: ListInput = {}): Promise<Task[]> {
     // Validate up front, like submit's conflict guard: a typo'd status otherwise
     // matches nothing and returns [] indistinguishably from "no such tasks".
@@ -364,14 +433,11 @@ export abstract class TaskStore {
    * call it in a loop until it returns fewer than `limit`.
    */
   async purge(input: PurgeInput = {}): Promise<string[]> {
-    if (input.olderThanMs != null && input.olderThanMs < 0) {
-      throw new Error(`olderThanMs must be >= 0, got ${input.olderThanMs}`);
-    }
-    if (input.limit != null && input.limit < 1) {
-      throw new Error(`limit must be >= 1, got ${input.limit}`);
-    }
+    validatePurgeInput(input);
     const rows = await this.fetch("purge", {
       older_than_ms: input.olderThanMs ?? 0,
+      status: input.status ?? null,
+      name: input.name ?? null,
       limit: input.limit ?? 1_000,
     });
     return rows.map((r) => r.id as string);

package/src/wait.ts

@@ -1,12 +1,25 @@
 import { TaskTimeout } from "./errors.js";
 import { nowMs } from "./ids.js";
-import { isTerminal, type Task } from "./models.js";
+import { isTerminal, type Task, type TaskRef } from "./models.js";
 import type { TaskStore } from "./store/base.js";
 
+export const DEFAULT_WAIT_TIMEOUT_MS = 30_000;
 export const DEFAULT_POLL_MS = 100;
 export const MAX_POLL_MS = 500;
 const GROWTH = 1.5;
 
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+export interface PollOptions {
+  timeoutMs: number;
+  /** The first poll interval (default 100). */
+  pollMs?: number;
+  /** Ceiling the poll interval backs off to (default 500). Worth raising for a
+   * task known to take minutes — fewer reads — or lowering when shaving the
+   * average half-interval of completion-detection latency matters. */
+  maxPollMs?: number;
+}
+
 /**
  * Grow the polling interval towards the ceiling.
  *
@@ -19,28 +32,80 @@ export function nextPollMs(current: number, maxMs: number): number {
   return Math.min(maxMs, Math.max(current + 1, Math.floor(current * GROWTH)));
 }
 
-/** Poll get() until terminal or timeout. Returns the terminal Task (any status).
- * Throws TaskTimeout, leaving the task running. `pollMs` is the *first* interval;
- * it backs off towards `maxPollMs`. */
-export async function pollWait(
-  store: TaskStore,
-  taskId: string,
-  {
-    timeoutMs,
-    pollMs = DEFAULT_POLL_MS,
-    maxPollMs = MAX_POLL_MS,
-  }: { timeoutMs: number; pollMs?: number; maxPollMs?: number },
+/**
+ * Poll `probe` until it reports a terminal status, then return the full task
+ * via `read`; or throw once the timeout elapses.
+ *
+ * The loop's repeated read is the status-only `probe` (see get_status.sql): a
+ * waiting caller asks nothing but "is it finished yet", and re-reading the whole
+ * row would drag the payload back — and re-parse it — on every beat for the life
+ * of the wait. The full row is read once, when the probe turns terminal or, on
+ * the timeout beat, for the error's snapshot. Between the probe and that read
+ * the row can vanish (purge) or the key repoint (`replace`); a read that comes
+ * back empty or non-terminal is simply not finished, and the loop keeps polling.
+ *
+ * `wake` is what the loop sleeps on between reads: a store with a push channel
+ * (Postgres) cuts it short when the task goes terminal, but the re-probe is the
+ * source of truth either way, so a plain sleep is always a correct answer.
+ */
+async function poll(
+  probe: () => Promise<TaskRef | null>,
+  read: () => Promise<Task | null>,
+  wake: (ref: TaskRef | null, ms: number) => Promise<void>,
+  subject: string,
+  key: string | null,
+  { timeoutMs, pollMs = DEFAULT_POLL_MS, maxPollMs = MAX_POLL_MS }: PollOptions,
 ): Promise<Task> {
   const deadline = nowMs() + timeoutMs;
   let interval = pollMs;
   for (;;) {
-    const task = await store.get(taskId);
-    if (task && isTerminal(task)) return task;
+    const ref = await probe();
     const remaining = deadline - nowMs();
-    if (remaining <= 0) throw new TaskTimeout(taskId, { timeoutMs, task });
-    // A store with a push channel (Postgres) cuts the sleep short when the task
-    // goes terminal; the re-get above stays the source of truth either way.
-    await store.taskDoneWake(taskId, Math.min(interval, remaining));
+    // The one full-read site: when the probe says finished, or on the timeout
+    // beat for the error's stuck-in-what-state snapshot. No ref means no row,
+    // so there is nothing for a read to add to either case.
+    const task = ref && (isTerminal(ref) || remaining <= 0) ? await read() : null;
+    if (task && isTerminal(task)) return task;
+    if (remaining <= 0) throw new TaskTimeout(ref?.id ?? subject, { timeoutMs, task, key });
+    await wake(ref, Math.min(interval, remaining));
     interval = nextPollMs(interval, maxPollMs);
   }
 }
+
+/** Poll the task's status until terminal or timeout. Returns the terminal Task
+ * (any status). Throws TaskTimeout, leaving the task running. `pollMs` is the
+ * *first* interval; it backs off towards `maxPollMs`. */
+export function pollWait(store: TaskStore, taskId: string, opts: PollOptions): Promise<Task> {
+  return poll(
+    () => store.getStatus(taskId),
+    () => store.get(taskId),
+    (_ref, ms) => store.taskDoneWake(taskId, ms),
+    taskId,
+    null,
+    opts,
+  );
+}
+
+/**
+ * The same wait, following a key instead of an id.
+ *
+ * The key is re-resolved on every probe, because that is what a key means: a
+ * pointer to the task that is *current* under it. A `replace` landing mid-wait
+ * moves the wait onto the new task rather than reporting the cancellation of the
+ * old one, and a key that points at nothing yet is simply not finished — it
+ * polls until something appears, the same way waiting on an id that does not
+ * exist yet does.
+ *
+ * There is nothing to subscribe to before the key resolves, so those naps are
+ * plain sleeps; once it resolves, the store's push channel applies as usual.
+ */
+export function pollWaitByKey(store: TaskStore, key: string, opts: PollOptions): Promise<Task> {
+  return poll(
+    () => store.getStatusByKey(key),
+    () => store.getByKey(key),
+    (ref, ms) => (ref ? store.taskDoneWake(ref.id, ms) : sleep(ms)),
+    key,
+    key,
+    opts,
+  );
+}

package/src/worker.ts

@@ -4,6 +4,7 @@ import { TaskContext } from "./context.js";
 import {
   asEnvelope,
   errorEnvelope,
+  EventLoopBlocked,
   exceptionEnvelope,
   type FailReason,
   LostLease,
@@ -923,6 +924,28 @@ export class Worker {
     let wake: (() => void) | null = null;
     // lease/3 gives two beats of slack; the floor only matters for sub-150ms leases.
     const interval = this.opts.heartbeatIntervalMs ?? Math.max(50, Math.floor(leaseMs / 3));
+
+    // When this heartbeat last ran. A loop cannot report its own absence — a
+    // handler that blocks for its whole attempt never lets the timer fire at all
+    // — so the check lives outside the loop and cancel() runs it too.
+    let lastBeatAt = Date.now();
+    /** Report a heartbeat that has not run for more than two intervals: a whole
+     * beat missed, which at lease/3 means the next such block loses the lease
+     * outright. Fires while the lease still holds — after it expires the only
+     * evidence is a task that ran twice, in two workers' logs, with no error in
+     * either. */
+    const checkBeat = (): void => {
+      const now = Date.now();
+      const lateMs = now - lastBeatAt - interval;
+      if (lateMs > interval) {
+        this.report(new EventLoopBlocked(lateMs, interval, leaseMs), {
+          phase: "execute",
+          taskId: ctxs[0].taskId,
+        });
+      }
+      lastBeatAt = now;
+    };
+
     const done = (async () => {
       while (active) {
         // Cancellable sleep: cancel() resolves this immediately and clears the
@@ -936,6 +959,7 @@ export class Worker {
         });
         wake = null;
         if (!active) break;
+        checkBeat();
         const live = ctxs.filter((c) => !c.settled && !c.lostLease);
         if (!live.length) break;
         try {
@@ -960,6 +984,10 @@ export class Worker {
       cancel: () => {
         active = false;
         if (wake) wake();
+        // The attempt is over, so this is the last chance to notice that the
+        // heartbeat never got one — the whole-attempt block, which is also the
+        // one where the lease is already gone.
+        checkBeat();
       },
       done,
     };