cairnq 0.14.0 → 0.15.1

package/README.md

@@ -78,9 +78,9 @@ const worker = Worker.sqlite("tasks.db", {
 
 // Nothing else deletes rows, so give the client a retention cutoff — it sweeps
 // terminal tasks in bounded batches for as long as the handle is open. Tiered
-// retention (per queue, per status) is purge() with filters, from your own
-// scheduler.
-const tasks = CairnQ.sqlite("tasks.db", { retentionMs: 7 * 24 * 3600_000 });
+// retention is the same option in its wider forms: a per-status map, or a list
+// of RetentionRule filtering by anything purge() can (queue, status, name).
+const tasks = CairnQ.sqlite("tasks.db", { retention: 7 * 24 * 3600_000 });
 ```
 
 A handler that does real side effects should bail out when it loses its lease —

package/dist/_protocol/sql/postgres/stats.sql

@@ -0,0 +1,24 @@
+-- Task counts grouped by queue and status. Read-only.
+-- A queue appears only while it has rows — terminal tasks count until purge
+-- removes them. The SDK zero-fills the statuses a queue has no rows in.
+--
+-- :queue is optional (pass NULL for every queue; `::text` pins the param's type,
+-- as in list.sql). Unfiltered, this reads every row in the table, so its cost
+-- grows with everything the installation has ever run — and one store carrying
+-- two workloads is the coordination cairnq recommends, so a caller asking about
+-- its own queue should not pay for the other's backlog. Filtered to one queue it
+-- can be served from cairnq_tasks_claim_idx's (queue, status) prefix instead.
+--
+-- Filtered or not, this still COUNTS: the cost is proportional to the rows being
+-- counted, which is the whole queue, terminal rows included. That is fine for a
+-- dashboard and wrong for a poll loop — queue_depth.sql is the bounded question,
+-- and the one to ask on an interval.
+-- When a queue IS named, the SDK rewrites the optional filter below to a plain
+-- equality before preparing this statement (`specialize`) — see purge.sql for
+-- why the optional form cannot be indexed.
+-- params: queue
+select queue, status, count(*) as count
+from cairnq_tasks
+where (:queue::text is null or queue = :queue)
+group by queue, status
+order by queue asc, status asc;

package/dist/_protocol/sql/sqlite/stats.sql

@@ -0,0 +1,24 @@
+-- Task counts grouped by queue and status. Read-only.
+-- A queue appears only while it has rows — terminal tasks count until purge
+-- removes them. The SDK zero-fills the statuses a queue has no rows in.
+--
+-- :queue is optional (pass NULL for every queue). Unfiltered, this reads every
+-- row in the table, so its cost grows with everything the installation has ever
+-- run — and one store carrying two workloads is the coordination cairnq
+-- recommends, so a caller asking about its own queue should not pay for the
+-- other's backlog. Filtered to one queue it can be served from
+-- cairnq_tasks_claim_idx's (queue, status) prefix instead.
+--
+-- Filtered or not, this still COUNTS: the cost is proportional to the rows being
+-- counted, which is the whole queue, terminal rows included. That is fine for a
+-- dashboard and wrong for a poll loop — queue_depth.sql is the bounded question,
+-- and the one to ask on an interval.
+-- When a queue IS named, the SDK rewrites the optional filter below to a plain
+-- equality before preparing this statement (`specialize`) — see purge.sql for
+-- why the optional form cannot be indexed.
+-- params: queue
+select queue, status, count(*) as count
+from cairnq_tasks
+where (:queue is null or queue = :queue)
+group by queue, status
+order by queue asc, status asc;

package/dist/client.d.ts

@@ -1,4 +1,5 @@
-import { type Task } from "./models.js";
+import { type RetentionOptions } from "./retention.js";
+import { type Task, type TaskRef, type TaskStatus } from "./models.js";
 import type { PgExecutor } from "./store/pg-executor.js";
 import type { Conflict, ListInput, PurgeInput, TaskStore } from "./store/base.js";
 import { type TaskDef } from "./task.js";
@@ -33,18 +34,20 @@ export interface ClientOptions {
     /** How long a blocked submit waits before raising QueueFull. Default 600_000. */
     maxQueueWaitMs?: number;
     /**
-     * Delete terminal tasks this many ms after they finished, on a schedule, for
-     * as long as this handle is open. Off unless set — and off means rows
-     * accumulate forever, because nothing else in CairnQ removes them. Tiered
-     * retention (per queue, per status) is `purge()` with filters, from your own
-     * scheduler.
+     * Delete terminal tasks on a schedule, for as long as this handle is open.
+     * Off unless set — and off means rows accumulate forever, because nothing
+     * else in CairnQ removes them.
+     *
+     * A number keeps every terminal row that many ms. The option form is the
+     * same cutoff in its tiered shapes — per status, or per anything `purge`
+     * can filter on — plus the sweep's own knobs; see RetentionOptions.
      */
-    retentionMs?: number;
+    retention?: number | RetentionOptions;
 }
 /** API-side handle. Thin wrapper over a TaskStore + SDK-orchestrated wait/call. */
 export declare class CairnQ {
     private readonly _store;
-    /** null unless `retentionMs` was configured. */
+    /** null unless `retention` was configured. */
     private readonly sweeper;
     constructor(_store: TaskStore, opts?: ClientOptions);
     static sqlite(path: string, opts?: {
@@ -75,6 +78,11 @@ export declare class CairnQ {
     queueDepth(queue: string, maxDepth: number): Promise<number>;
     get(taskId: string): Promise<Task | null>;
     getByKey(key: string): Promise<Task | null>;
+    /** The status-only probe wait polls on: id + status, no payload. Public for
+     * the same reason it exists — a dashboard or poller that only asks "is it
+     * finished yet" should not drag the payload back per ask. */
+    getStatus(taskId: string): Promise<TaskRef | null>;
+    getStatusByKey(key: string): Promise<TaskRef | null>;
     list(input?: ListInput): Promise<Task[]>;
     cancel(taskId: string): Promise<Task | null>;
     cancelByKey(key: string): Promise<Task | null>;
@@ -86,13 +94,22 @@ export declare class CairnQ {
     }): Promise<Task | null>;
     /** Delete terminal tasks that finished more than `olderThanMs` ago and return
      * their ids. Nothing else in CairnQ removes rows, so a long-lived database
-     * needs this on a schedule — `retentionMs` is this call on a timer. Each call
+     * needs this on a schedule — `retention` is this call on a timer. Each call
      * is bounded by `limit` to keep the write short; loop until it returns fewer
      * than `limit`.
      *
      * `queue` / `status` / `name` narrow the sweep — one installation carrying two
      * workloads needs a retention per workload, not one for the whole database. */
     purge(input?: PurgeInput): Promise<string[]>;
+    /** Task counts per queue, keyed by status and zero-filled across all statuses
+     * — `(await stats()).default.queued` is the backlog of a queue. `queue` narrows
+     * the aggregate to one queue, which is also what keeps a caller from paying for
+     * the other workloads sharing the installation; a named queue is always
+     * present, zero-filled if it has no rows.
+     *
+     * This counts rows, so it costs what it counts — use it for a dashboard, and
+     * poll `queueDepth()` instead, which is bounded. */
+    stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>>;
     /** Wait for a task to finish. Resolves with the terminal Task (any status);
      * throws TaskTimeout without stopping the task, so `wait(err.taskId)` picks the
      * same wait back up — from another process, or after a longer deadline. */

package/dist/client.js

@@ -8,7 +8,7 @@ import { DEFAULT_WAIT_TIMEOUT_MS, pollWait, pollWaitByKey } from "./wait.js";
 /** API-side handle. Thin wrapper over a TaskStore + SDK-orchestrated wait/call. */
 export class CairnQ {
     _store;
-    /** null unless `retentionMs` was configured. */
+    /** null unless `retention` was configured. */
     sweeper;
     constructor(_store, opts = {}) {
         this._store = _store;
@@ -25,7 +25,10 @@ export class CairnQ {
         // Started here rather than in connect(), which is optional — every other
         // path connects lazily, and retention that silently depends on an optional
         // call is retention that silently does not happen.
-        this.sweeper = opts.retentionMs != null ? new RetentionSweeper(_store, opts.retentionMs) : null;
+        this.sweeper =
+            opts.retention != null
+                ? new RetentionSweeper(_store, typeof opts.retention === "number" ? { olderThanMs: opts.retention } : opts.retention)
+                : null;
         this.sweeper?.start();
     }
     static sqlite(path, opts = {}) {
@@ -81,6 +84,15 @@ export class CairnQ {
     getByKey(key) {
         return this._store.getByKey(key);
     }
+    /** The status-only probe wait polls on: id + status, no payload. Public for
+     * the same reason it exists — a dashboard or poller that only asks "is it
+     * finished yet" should not drag the payload back per ask. */
+    getStatus(taskId) {
+        return this._store.getStatus(taskId);
+    }
+    getStatusByKey(key) {
+        return this._store.getStatusByKey(key);
+    }
     list(input) {
         return this._store.list(input);
     }
@@ -98,7 +110,7 @@ export class CairnQ {
     }
     /** Delete terminal tasks that finished more than `olderThanMs` ago and return
      * their ids. Nothing else in CairnQ removes rows, so a long-lived database
-     * needs this on a schedule — `retentionMs` is this call on a timer. Each call
+     * needs this on a schedule — `retention` is this call on a timer. Each call
      * is bounded by `limit` to keep the write short; loop until it returns fewer
      * than `limit`.
      *
@@ -107,6 +119,17 @@ export class CairnQ {
     purge(input) {
         return this._store.purge(input);
     }
+    /** Task counts per queue, keyed by status and zero-filled across all statuses
+     * — `(await stats()).default.queued` is the backlog of a queue. `queue` narrows
+     * the aggregate to one queue, which is also what keeps a caller from paying for
+     * the other workloads sharing the installation; a named queue is always
+     * present, zero-filled if it has no rows.
+     *
+     * This counts rows, so it costs what it counts — use it for a dashboard, and
+     * poll `queueDepth()` instead, which is bounded. */
+    stats(queue) {
+        return this._store.stats(queue);
+    }
     /** Wait for a task to finish. Resolves with the terminal Task (any status);
      * throws TaskTimeout without stopping the task, so `wait(err.taskId)` picks the
      * same wait back up — from another process, or after a longer deadline. */

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { CairnQ } from "./client.js";
 export type { CallOptions, ClientOptions, SubmitOptions, WaitOptions } from "./client.js";
+export type { RetentionCutoffs, RetentionOptions, RetentionRule } from "./retention.js";
 export { Worker } from "./worker.js";
 export type { BatchHandler, Handler, TypedHandler, WorkerOptions } from "./worker.js";
 export { TaskContext } from "./context.js";

package/dist/retention.d.ts

@@ -1,15 +1,67 @@
-import type { TaskStore } from "./store/base.js";
+import type { TerminalStatus } from "./models.js";
+import { type TaskStore } from "./store/base.js";
+/** Per-status cutoffs. A status left out is never swept — granular retention is
+ * an explicit statement of what may go, not a default for what wasn't named. */
+export type RetentionCutoffs = Partial<Record<TerminalStatus, number>>;
 /**
- * Deletes terminal tasks on a schedule, for as long as the handle is open —
- * `purge(olderThanMs)` on a timer, which is the whole mechanism.
+ * One "these rows may go after this long" statement. Each field left out widens
+ * what the rule covers; `olderThanMs` is the only required one.
+ *
+ * A rule is one `purge` call's filters, so the fields are exactly `PurgeInput`'s
+ * — deliberately, since a rule the sweeper can express but the store cannot
+ * enforce would be a lie about what is being deleted.
+ */
+export interface RetentionRule {
+    /** Only this queue. Absent means every queue. */
+    queue?: string;
+    /** Only this terminal status. Absent means all three. */
+    status?: TerminalStatus;
+    /** Only this task name. Absent means every name. */
+    name?: string;
+    /** How long a row matching this rule is kept after it finished. */
+    olderThanMs: number;
+}
+export interface RetentionOptions {
+    /**
+     * How long a terminal task is kept after it finished. Required: there is no
+     * safe default for how long someone else's results stay readable.
+     *
+     * Three forms, widening as the deployment does:
+     *
+     * - A number keeps every terminal row the same time.
+     * - A per-status map tiers by outcome — a succeeded row is spent once its
+     *   result is consumed, a failed one is worth keeping for diagnosis:
+     *   `{ succeeded: 300_000, failed: 86_400_000 }`. A status left out is never
+     *   swept.
+     * - An array of rules tiers by anything `purge` can filter on, which is what a
+     *   store shared by two workloads needs — the recommended way for two
+     *   languages to coordinate is one installation, and an RPC queue read once
+     *   has nothing in common with a durable queue kept for a week:
+     *   `[{ queue: "rpc", olderThanMs: 300_000 },
+     *      { queue: "jobs", status: "failed", olderThanMs: 604_800_000 }]`.
+     *   Rules are independent, each its own sweep — nothing a rule does not match
+     *   is swept, and rules that overlap simply delete the same row once.
+     */
+    olderThanMs: number | RetentionCutoffs | RetentionRule[];
+    /** Time between sweeps. Default 3_600_000 (one hour). */
+    intervalMs?: number;
+    /** Rows deleted per statement while draining. Default 1_000. */
+    limit?: number;
+    /**
+     * Called for a sweep that threw. The next sweep runs on schedule regardless —
+     * a purge that failed because the database was busy is not a reason to stop
+     * retaining — so without this a store quietly stops being swept. Must not throw.
+     */
+    onError?: (err: unknown) => void;
+}
+/**
+ * Deletes terminal tasks on a schedule, for as long as the handle is open.
  *
  * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
  * payloads carry real data — an image, a document, a batch of embeddings — turns
  * that into a disk leak measured in gigabytes per backfill. Every deployment
  * that runs longer than a demo needs the sweep; leaving it to an external
  * scheduler means the leak is the default and remembering is the opt-in.
- * A deployment whose retention is tiered (per queue, per status) calls `purge`
- * with those filters from its own scheduler instead.
  *
  * It sweeps in bounded batches with a yield between them, so draining a backlog
  * that accumulated while nothing was sweeping stays a sequence of short writes
@@ -18,7 +70,7 @@ import type { TaskStore } from "./store/base.js";
  */
 export declare class RetentionSweeper {
     private readonly store;
-    private readonly olderThanMs;
+    private readonly opts;
     /** Whether the scheduled loop is running. */
     private active;
     /** Set by stop(), so a drain in progress can cut itself short too. */
@@ -38,9 +90,12 @@ export declare class RetentionSweeper {
     /** The loop itself, awaited by stop() so no purge outlives the store. */
     private loop;
     private readonly intervalMs;
-    constructor(store: TaskStore, olderThanMs: number, opts?: {
-        intervalMs?: number;
-    });
+    /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
+    private readonly limit;
+    /** One purge per rule: a lone entry for a number, one per status for a map,
+     * one per element for an array. */
+    private readonly purgeInputs;
+    constructor(store: TaskStore, opts: RetentionOptions);
     start(): void;
     /** Mint a fresh stop signal. */
     private arm;

package/dist/retention.js

@@ -1,3 +1,4 @@
+import { validatePurgeInput } from "./store/base.js";
 /** Sweep every hour unless asked otherwise — often enough that a queue with a
  * day of retention never carries more than an hour of extra rows, rare enough
  * that the sweep is invisible next to the task traffic. */
@@ -5,17 +6,27 @@ const DEFAULT_INTERVAL_MS = 3_600_000;
 /** Rows per purge statement. The same bound `purge` defaults to: big enough that
  * a backlog drains in few statements, small enough that each is a short write. */
 const DEFAULT_LIMIT = 1_000;
+/** The three `olderThanMs` forms as the one form the sweep runs on. The number
+ * and the per-status map are the rule array's common cases spelled shorter, so
+ * they are widened here rather than handled separately downstream. */
+function toRules(spec) {
+    if (typeof spec === "number")
+        return [{ olderThanMs: spec }];
+    if (Array.isArray(spec))
+        return spec;
+    return Object.entries(spec).map(([status, olderThanMs]) => ({
+        status,
+        olderThanMs,
+    }));
+}
 /**
- * Deletes terminal tasks on a schedule, for as long as the handle is open —
- * `purge(olderThanMs)` on a timer, which is the whole mechanism.
+ * Deletes terminal tasks on a schedule, for as long as the handle is open.
  *
  * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
  * payloads carry real data — an image, a document, a batch of embeddings — turns
  * that into a disk leak measured in gigabytes per backfill. Every deployment
  * that runs longer than a demo needs the sweep; leaving it to an external
  * scheduler means the leak is the default and remembering is the opt-in.
- * A deployment whose retention is tiered (per queue, per status) calls `purge`
- * with those filters from its own scheduler instead.
  *
  * It sweeps in bounded batches with a yield between them, so draining a backlog
  * that accumulated while nothing was sweeping stays a sequence of short writes
@@ -24,7 +35,7 @@ const DEFAULT_LIMIT = 1_000;
  */
 export class RetentionSweeper {
     store;
-    olderThanMs;
+    opts;
     /** Whether the scheduled loop is running. */
     active = false;
     /** Set by stop(), so a drain in progress can cut itself short too. */
@@ -44,17 +55,32 @@ export class RetentionSweeper {
     /** The loop itself, awaited by stop() so no purge outlives the store. */
     loop = null;
     intervalMs;
-    constructor(store, olderThanMs, opts = {}) {
+    /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
+    limit;
+    /** One purge per rule: a lone entry for a number, one per status for a map,
+     * one per element for an array. */
+    purgeInputs;
+    constructor(store, opts) {
         this.store = store;
-        this.olderThanMs = olderThanMs;
-        if (!Number.isFinite(olderThanMs) || olderThanMs < 0) {
-            throw new Error(`retentionMs must be >= 0, got ${olderThanMs}`);
-        }
+        this.opts = opts;
         this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
         if (!Number.isFinite(this.intervalMs) || this.intervalMs < 1) {
-            throw new Error(`retention intervalMs must be >= 1, got ${this.intervalMs}`);
+            throw new Error(`retention.intervalMs must be >= 1, got ${this.intervalMs}`);
+        }
+        this.limit = opts.limit ?? DEFAULT_LIMIT;
+        const rules = toRules(opts.olderThanMs);
+        // An empty map or array retains nothing and sweeps nothing — almost
+        // certainly a bug upstream of this call, so refuse it rather than silently
+        // never purging.
+        if (!rules.length) {
+            throw new Error("retention.olderThanMs must name at least one rule");
         }
         this.arm();
+        this.purgeInputs = rules.map((rule) => ({ ...rule, limit: this.limit }));
+        // Fail fast on the store's own purge rules (terminal status, cutoff >= 0):
+        // the sweep runs an hour from now, and its errors only surface via onError.
+        for (const input of this.purgeInputs)
+            validatePurgeInput(input);
     }
     start() {
         if (this.active)
@@ -97,9 +123,14 @@ export class RetentionSweeper {
             try {
                 await this.sweep();
             }
-            catch {
-                // A purge that failed because the database was busy is not a reason to
-                // stop retaining — the next sweep runs on schedule regardless.
+            catch (err) {
+                try {
+                    this.opts.onError?.(err);
+                }
+                catch {
+                    // A reporting hook must never take the sweep down with it — the same
+                    // rule the worker's onError follows.
+                }
             }
         }
     }
@@ -110,16 +141,18 @@ export class RetentionSweeper {
      */
     async sweep() {
         let deleted = 0;
-        for (;;) {
-            const ids = await this.store.purge({ olderThanMs: this.olderThanMs, limit: DEFAULT_LIMIT });
-            deleted += ids.length;
-            if (this.stopping)
-                return deleted;
-            if (ids.length < DEFAULT_LIMIT)
-                break;
-            // Hand the loop back between batches: a large drain must not starve the
-            // submits and claims sharing this process. Ref'd — see sleep().
-            await this.sleep(0, true);
+        for (const input of this.purgeInputs) {
+            for (;;) {
+                const ids = await this.store.purge(input);
+                deleted += ids.length;
+                if (this.stopping)
+                    return deleted;
+                if (ids.length < this.limit)
+                    break;
+                // Hand the loop back between batches: a large drain must not starve the
+                // submits and claims sharing this process. Ref'd — see sleep().
+                await this.sleep(0, true);
+            }
         }
         return deleted;
     }

package/dist/store/base.d.ts

@@ -248,6 +248,25 @@ export declare abstract class TaskStore {
      * a producer; `QueueDepthGate` builds the blocking form on top.
      */
     queueDepth(queue: string, maxDepth: number): Promise<number>;
+    /**
+     * Task counts grouped by queue, keyed by status and zero-filled —
+     * `(await stats()).default.queued` is the backlog of a queue. A queue appears
+     * only while it has rows; terminal tasks keep counting until `purge` removes
+     * them.
+     *
+     * `queue` restricts the aggregate to one queue, which is also what stops the
+     * caller paying for every other queue's rows: one installation carrying two
+     * workloads is the coordination this project recommends, and the unfiltered
+     * form reads the whole table. A named queue is always present in the result,
+     * zero-filled if it has no rows at all — asking about a specific queue and
+     * getting `undefined` back would make every caller write the same fallback.
+     *
+     * Filtered or not, this COUNTS, so it costs what it counts: a whole queue,
+     * terminal rows included. Right for a dashboard, wrong on an interval — poll
+     * `queueDepth`, which is bounded, and keep this for when the real numbers are
+     * the point.
+     */
+    stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>>;
     /**
      * Take up to `limit` claimable tasks. `names` restricts the claim to task names
      * this caller can actually run — a worker passes its registered handlers.

package/dist/store/base.js

@@ -509,6 +509,37 @@ export class TaskStore {
         const rows = await this.fetch("queue_depth", { queue, max_depth: maxDepth });
         return Number(rows[0]?.headroom ?? 0);
     }
+    /**
+     * Task counts grouped by queue, keyed by status and zero-filled —
+     * `(await stats()).default.queued` is the backlog of a queue. A queue appears
+     * only while it has rows; terminal tasks keep counting until `purge` removes
+     * them.
+     *
+     * `queue` restricts the aggregate to one queue, which is also what stops the
+     * caller paying for every other queue's rows: one installation carrying two
+     * workloads is the coordination this project recommends, and the unfiltered
+     * form reads the whole table. A named queue is always present in the result,
+     * zero-filled if it has no rows at all — asking about a specific queue and
+     * getting `undefined` back would make every caller write the same fallback.
+     *
+     * Filtered or not, this COUNTS, so it costs what it counts: a whole queue,
+     * terminal rows included. Right for a dashboard, wrong on an interval — poll
+     * `queueDepth`, which is bounded, and keep this for when the real numbers are
+     * the point.
+     */
+    async stats(queue) {
+        const zeros = () => Object.fromEntries(STATUSES.map((s) => [s, 0]));
+        const out = {};
+        // Seed before the query, not after: a named queue with no rows returns no
+        // rows to seed from, and that is exactly the case the promise is about.
+        if (queue != null)
+            out[queue] = zeros();
+        for (const row of await this.fetch("stats", { queue: queue ?? null })) {
+            const per = (out[row.queue] ??= zeros());
+            per[row.status] = Number(row.count);
+        }
+        return out;
+    }
     // ------------------------------------------------------------- worker side
     /**
      * Take up to `limit` claimable tasks. `names` restricts the claim to task names

package/dist/worker.d.ts

@@ -87,6 +87,8 @@ export declare class Worker {
     private readonly backoffMaxMs;
     private stopped;
     private stopWake;
+    private freed$;
+    private freedWake;
     private readonly stopped$;
     private ownsStore;
     constructor(store: TaskStore, queues: string[], opts?: WorkerOptions);
@@ -279,6 +281,9 @@ export declare class Worker {
      * cuts it short when a task on this worker's queues becomes claimable;
      * stop() interrupts it either way, and sleepOrStop bounds it at `ms` so the
      * poll fallback — which also drives lease recovery — never stretches.
+     *
+     * `freed` is the third way out: a resource unit coming back frees a claim the
+     * store has no reason to announce (see `freed$`).
      */
     private idle;
     private sleepOrStop;

package/dist/worker.js

@@ -162,6 +162,13 @@ export class Worker {
     backoffMaxMs;
     stopped = false;
     stopWake;
+    // Re-armed before each claim, resolved when a call gives a resource unit
+    // back. A claim can come back empty because a resource is at capacity rather
+    // than the queue, and that release is local — no store notification covers
+    // it, so the idle sleep waits on this too. Armed before the claim, so a
+    // release landing *during* it still shortens the sleep that follows.
+    freed$ = Promise.resolve();
+    freedWake = () => { };
     // Resolved once by stop(); every sleep races against it. A stopped worker
     // never restarts, so one promise serves the instance's lifetime.
     stopped$ = new Promise((r) => (this.stopWake = r));
@@ -353,6 +360,7 @@ export class Worker {
                 await this.idle(pollMs);
                 continue;
             }
+            this.freed$ = new Promise((r) => (this.freedWake = r));
             let drawn;
             try {
                 drawn = await this.store.claimSession(
@@ -369,7 +377,7 @@ export class Worker {
                 continue;
             }
             if (!drawn?.length) {
-                await this.idle(pollMs);
+                await this.idle(pollMs, this.freed$);
                 continue;
             }
             for (const { src, calls } of drawn) {
@@ -384,6 +392,8 @@ export class Worker {
                     const p = call.finally(() => {
                         this.planner.release(resource);
                         running.delete(p);
+                        if (resource != null)
+                            this.freedWake();
                     });
                     running.add(p);
                 }
@@ -746,9 +756,15 @@ export class Worker {
      * cuts it short when a task on this worker's queues becomes claimable;
      * stop() interrupts it either way, and sleepOrStop bounds it at `ms` so the
      * poll fallback — which also drives lease recovery — never stretches.
+     *
+     * `freed` is the third way out: a resource unit coming back frees a claim the
+     * store has no reason to announce (see `freed$`).
      */
-    idle(ms) {
-        return Promise.race([this.sleepOrStop(ms), this.store.claimWake(this.queues, ms)]);
+    idle(ms, freed) {
+        const racers = [this.sleepOrStop(ms), this.store.claimWake(this.queues, ms)];
+        if (freed)
+            racers.push(freed);
+        return Promise.race(racers);
     }
     sleepOrStop(ms) {
         if (this.stopped)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cairnq",
-  "version": "0.14.0",
+  "version": "0.15.1",
   "description": "SQLite-first, cross-language, storage-centered durable task runtime",
   "license": "MIT",
   "author": "Jannchie <jannchie@gmail.com>",

package/src/client.ts

@@ -1,6 +1,6 @@
-import { RetentionSweeper } from "./retention.js";
+import { type RetentionOptions, RetentionSweeper } from "./retention.js";
 import { TaskCanceled, TaskFailed } from "./errors.js";
-import { isFailed, isSucceeded, type Task } from "./models.js";
+import { isFailed, isSucceeded, type Task, type TaskRef, type TaskStatus } from "./models.js";
 import { SQLiteStore } from "./store/sqlite.js";
 import { PostgresStore } from "./store/postgres.js";
 import type { PgExecutor } from "./store/pg-executor.js";
@@ -40,18 +40,20 @@ export interface ClientOptions {
   /** How long a blocked submit waits before raising QueueFull. Default 600_000. */
   maxQueueWaitMs?: number;
   /**
-   * Delete terminal tasks this many ms after they finished, on a schedule, for
-   * as long as this handle is open. Off unless set — and off means rows
-   * accumulate forever, because nothing else in CairnQ removes them. Tiered
-   * retention (per queue, per status) is `purge()` with filters, from your own
-   * scheduler.
+   * Delete terminal tasks on a schedule, for as long as this handle is open.
+   * Off unless set — and off means rows accumulate forever, because nothing
+   * else in CairnQ removes them.
+   *
+   * A number keeps every terminal row that many ms. The option form is the
+   * same cutoff in its tiered shapes — per status, or per anything `purge`
+   * can filter on — plus the sweep's own knobs; see RetentionOptions.
    */
-  retentionMs?: number;
+  retention?: number | RetentionOptions;
 }
 
 /** API-side handle. Thin wrapper over a TaskStore + SDK-orchestrated wait/call. */
 export class CairnQ {
-  /** null unless `retentionMs` was configured. */
+  /** null unless `retention` was configured. */
   private readonly sweeper: RetentionSweeper | null;
 
   constructor(
@@ -71,7 +73,13 @@ export class CairnQ {
     // Started here rather than in connect(), which is optional — every other
     // path connects lazily, and retention that silently depends on an optional
     // call is retention that silently does not happen.
-    this.sweeper = opts.retentionMs != null ? new RetentionSweeper(_store, opts.retentionMs) : null;
+    this.sweeper =
+      opts.retention != null
+        ? new RetentionSweeper(
+            _store,
+            typeof opts.retention === "number" ? { olderThanMs: opts.retention } : opts.retention,
+          )
+        : null;
     this.sweeper?.start();
   }
 
@@ -145,6 +153,17 @@ export class CairnQ {
     return this._store.getByKey(key);
   }
 
+  /** The status-only probe wait polls on: id + status, no payload. Public for
+   * the same reason it exists — a dashboard or poller that only asks "is it
+   * finished yet" should not drag the payload back per ask. */
+  getStatus(taskId: string): Promise<TaskRef | null> {
+    return this._store.getStatus(taskId);
+  }
+
+  getStatusByKey(key: string): Promise<TaskRef | null> {
+    return this._store.getStatusByKey(key);
+  }
+
   list(input?: ListInput): Promise<Task[]> {
     return this._store.list(input);
   }
@@ -167,7 +186,7 @@ export class CairnQ {
 
   /** Delete terminal tasks that finished more than `olderThanMs` ago and return
    * their ids. Nothing else in CairnQ removes rows, so a long-lived database
-   * needs this on a schedule — `retentionMs` is this call on a timer. Each call
+   * needs this on a schedule — `retention` is this call on a timer. Each call
    * is bounded by `limit` to keep the write short; loop until it returns fewer
    * than `limit`.
    *
@@ -177,6 +196,18 @@ export class CairnQ {
     return this._store.purge(input);
   }
 
+  /** Task counts per queue, keyed by status and zero-filled across all statuses
+   * — `(await stats()).default.queued` is the backlog of a queue. `queue` narrows
+   * the aggregate to one queue, which is also what keeps a caller from paying for
+   * the other workloads sharing the installation; a named queue is always
+   * present, zero-filled if it has no rows.
+   *
+   * This counts rows, so it costs what it counts — use it for a dashboard, and
+   * poll `queueDepth()` instead, which is bounded. */
+  stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>> {
+    return this._store.stats(queue);
+  }
+
   /** Wait for a task to finish. Resolves with the terminal Task (any status);
    * throws TaskTimeout without stopping the task, so `wait(err.taskId)` picks the
    * same wait back up — from another process, or after a longer deadline. */

package/src/index.ts

@@ -1,5 +1,6 @@
 export { CairnQ } from "./client.js";
 export type { CallOptions, ClientOptions, SubmitOptions, WaitOptions } from "./client.js";
+export type { RetentionCutoffs, RetentionOptions, RetentionRule } from "./retention.js";
 export { Worker } from "./worker.js";
 export type { BatchHandler, Handler, TypedHandler, WorkerOptions } from "./worker.js";
 export { TaskContext } from "./context.js";

package/src/retention.ts

@@ -1,4 +1,5 @@
-import type { TaskStore } from "./store/base.js";
+import type { TerminalStatus } from "./models.js";
+import { validatePurgeInput, type PurgeInput, type TaskStore } from "./store/base.js";
 
 /** Sweep every hour unless asked otherwise — often enough that a queue with a
  * day of retention never carries more than an hour of extra rows, rare enough
@@ -8,17 +9,83 @@ const DEFAULT_INTERVAL_MS = 3_600_000;
  * a backlog drains in few statements, small enough that each is a short write. */
 const DEFAULT_LIMIT = 1_000;
 
+/** Per-status cutoffs. A status left out is never swept — granular retention is
+ * an explicit statement of what may go, not a default for what wasn't named. */
+export type RetentionCutoffs = Partial<Record<TerminalStatus, number>>;
+
 /**
- * Deletes terminal tasks on a schedule, for as long as the handle is open —
- * `purge(olderThanMs)` on a timer, which is the whole mechanism.
+ * One "these rows may go after this long" statement. Each field left out widens
+ * what the rule covers; `olderThanMs` is the only required one.
+ *
+ * A rule is one `purge` call's filters, so the fields are exactly `PurgeInput`'s
+ * — deliberately, since a rule the sweeper can express but the store cannot
+ * enforce would be a lie about what is being deleted.
+ */
+export interface RetentionRule {
+  /** Only this queue. Absent means every queue. */
+  queue?: string;
+  /** Only this terminal status. Absent means all three. */
+  status?: TerminalStatus;
+  /** Only this task name. Absent means every name. */
+  name?: string;
+  /** How long a row matching this rule is kept after it finished. */
+  olderThanMs: number;
+}
+
+export interface RetentionOptions {
+  /**
+   * How long a terminal task is kept after it finished. Required: there is no
+   * safe default for how long someone else's results stay readable.
+   *
+   * Three forms, widening as the deployment does:
+   *
+   * - A number keeps every terminal row the same time.
+   * - A per-status map tiers by outcome — a succeeded row is spent once its
+   *   result is consumed, a failed one is worth keeping for diagnosis:
+   *   `{ succeeded: 300_000, failed: 86_400_000 }`. A status left out is never
+   *   swept.
+   * - An array of rules tiers by anything `purge` can filter on, which is what a
+   *   store shared by two workloads needs — the recommended way for two
+   *   languages to coordinate is one installation, and an RPC queue read once
+   *   has nothing in common with a durable queue kept for a week:
+   *   `[{ queue: "rpc", olderThanMs: 300_000 },
+   *      { queue: "jobs", status: "failed", olderThanMs: 604_800_000 }]`.
+   *   Rules are independent, each its own sweep — nothing a rule does not match
+   *   is swept, and rules that overlap simply delete the same row once.
+   */
+  olderThanMs: number | RetentionCutoffs | RetentionRule[];
+  /** Time between sweeps. Default 3_600_000 (one hour). */
+  intervalMs?: number;
+  /** Rows deleted per statement while draining. Default 1_000. */
+  limit?: number;
+  /**
+   * Called for a sweep that threw. The next sweep runs on schedule regardless —
+   * a purge that failed because the database was busy is not a reason to stop
+   * retaining — so without this a store quietly stops being swept. Must not throw.
+   */
+  onError?: (err: unknown) => void;
+}
+
+/** The three `olderThanMs` forms as the one form the sweep runs on. The number
+ * and the per-status map are the rule array's common cases spelled shorter, so
+ * they are widened here rather than handled separately downstream. */
+function toRules(spec: number | RetentionCutoffs | RetentionRule[]): RetentionRule[] {
+  if (typeof spec === "number") return [{ olderThanMs: spec }];
+  if (Array.isArray(spec)) return spec;
+  return (Object.entries(spec) as [TerminalStatus, number][]).map(([status, olderThanMs]) => ({
+    status,
+    olderThanMs,
+  }));
+}
+
+/**
+ * Deletes terminal tasks on a schedule, for as long as the handle is open.
  *
  * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
  * payloads carry real data — an image, a document, a batch of embeddings — turns
  * that into a disk leak measured in gigabytes per backfill. Every deployment
  * that runs longer than a demo needs the sweep; leaving it to an external
  * scheduler means the leak is the default and remembering is the opt-in.
- * A deployment whose retention is tiered (per queue, per status) calls `purge`
- * with those filters from its own scheduler instead.
  *
  * It sweeps in bounded batches with a yield between them, so draining a backlog
  * that accumulated while nothing was sweeping stays a sequence of short writes
@@ -45,20 +112,33 @@ export class RetentionSweeper {
   /** The loop itself, awaited by stop() so no purge outlives the store. */
   private loop: Promise<void> | null = null;
   private readonly intervalMs: number;
+  /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
+  private readonly limit: number;
+  /** One purge per rule: a lone entry for a number, one per status for a map,
+   * one per element for an array. */
+  private readonly purgeInputs: PurgeInput[];
 
   constructor(
     private readonly store: TaskStore,
-    private readonly olderThanMs: number,
-    opts: { intervalMs?: number } = {},
+    private readonly opts: RetentionOptions,
   ) {
-    if (!Number.isFinite(olderThanMs) || olderThanMs < 0) {
-      throw new Error(`retentionMs must be >= 0, got ${olderThanMs}`);
-    }
     this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
     if (!Number.isFinite(this.intervalMs) || this.intervalMs < 1) {
-      throw new Error(`retention intervalMs must be >= 1, got ${this.intervalMs}`);
+      throw new Error(`retention.intervalMs must be >= 1, got ${this.intervalMs}`);
+    }
+    this.limit = opts.limit ?? DEFAULT_LIMIT;
+    const rules = toRules(opts.olderThanMs);
+    // An empty map or array retains nothing and sweeps nothing — almost
+    // certainly a bug upstream of this call, so refuse it rather than silently
+    // never purging.
+    if (!rules.length) {
+      throw new Error("retention.olderThanMs must name at least one rule");
     }
     this.arm();
+    this.purgeInputs = rules.map((rule) => ({ ...rule, limit: this.limit }));
+    // Fail fast on the store's own purge rules (terminal status, cutoff >= 0):
+    // the sweep runs an hour from now, and its errors only surface via onError.
+    for (const input of this.purgeInputs) validatePurgeInput(input);
   }
 
   start(): void {
@@ -102,9 +182,13 @@ export class RetentionSweeper {
       if (this.stopping) return;
       try {
         await this.sweep();
-      } catch {
-        // A purge that failed because the database was busy is not a reason to
-        // stop retaining — the next sweep runs on schedule regardless.
+      } catch (err) {
+        try {
+          this.opts.onError?.(err);
+        } catch {
+          // A reporting hook must never take the sweep down with it — the same
+          // rule the worker's onError follows.
+        }
       }
     }
   }
@@ -116,14 +200,16 @@ export class RetentionSweeper {
    */
   async sweep(): Promise<number> {
     let deleted = 0;
-    for (;;) {
-      const ids = await this.store.purge({ olderThanMs: this.olderThanMs, limit: DEFAULT_LIMIT });
-      deleted += ids.length;
-      if (this.stopping) return deleted;
-      if (ids.length < DEFAULT_LIMIT) break;
-      // Hand the loop back between batches: a large drain must not starve the
-      // submits and claims sharing this process. Ref'd — see sleep().
-      await this.sleep(0, true);
+    for (const input of this.purgeInputs) {
+      for (;;) {
+        const ids = await this.store.purge(input);
+        deleted += ids.length;
+        if (this.stopping) return deleted;
+        if (ids.length < this.limit) break;
+        // Hand the loop back between batches: a large drain must not starve the
+        // submits and claims sharing this process. Ref'd — see sleep().
+        await this.sleep(0, true);
+      }
     }
     return deleted;
   }

package/src/store/base.ts

@@ -652,6 +652,38 @@ export abstract class TaskStore {
     return Number(rows[0]?.headroom ?? 0);
   }
 
+  /**
+   * Task counts grouped by queue, keyed by status and zero-filled —
+   * `(await stats()).default.queued` is the backlog of a queue. A queue appears
+   * only while it has rows; terminal tasks keep counting until `purge` removes
+   * them.
+   *
+   * `queue` restricts the aggregate to one queue, which is also what stops the
+   * caller paying for every other queue's rows: one installation carrying two
+   * workloads is the coordination this project recommends, and the unfiltered
+   * form reads the whole table. A named queue is always present in the result,
+   * zero-filled if it has no rows at all — asking about a specific queue and
+   * getting `undefined` back would make every caller write the same fallback.
+   *
+   * Filtered or not, this COUNTS, so it costs what it counts: a whole queue,
+   * terminal rows included. Right for a dashboard, wrong on an interval — poll
+   * `queueDepth`, which is bounded, and keep this for when the real numbers are
+   * the point.
+   */
+  async stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>> {
+    const zeros = (): Record<TaskStatus, number> =>
+      Object.fromEntries(STATUSES.map((s) => [s, 0])) as Record<TaskStatus, number>;
+    const out: Record<string, Record<TaskStatus, number>> = {};
+    // Seed before the query, not after: a named queue with no rows returns no
+    // rows to seed from, and that is exactly the case the promise is about.
+    if (queue != null) out[queue] = zeros();
+    for (const row of await this.fetch("stats", { queue: queue ?? null })) {
+      const per = (out[row.queue] ??= zeros());
+      per[row.status as TaskStatus] = Number(row.count);
+    }
+    return out;
+  }
+
   // ------------------------------------------------------------- worker side
   /**
    * Take up to `limit` claimable tasks. `names` restricts the claim to task names

package/src/worker.ts

@@ -281,6 +281,13 @@ export class Worker {
   private readonly backoffMaxMs: number;
   private stopped = false;
   private stopWake!: () => void;
+  // Re-armed before each claim, resolved when a call gives a resource unit
+  // back. A claim can come back empty because a resource is at capacity rather
+  // than the queue, and that release is local — no store notification covers
+  // it, so the idle sleep waits on this too. Armed before the claim, so a
+  // release landing *during* it still shortens the sleep that follows.
+  private freed$ = Promise.resolve();
+  private freedWake: () => void = () => {};
   // Resolved once by stop(); every sleep races against it. A stopped worker
   // never restarts, so one promise serves the instance's lifetime.
   private readonly stopped$ = new Promise<void>((r) => (this.stopWake = r));
@@ -519,6 +526,7 @@ export class Worker {
         await this.idle(pollMs);
         continue;
       }
+      this.freed$ = new Promise<void>((r) => (this.freedWake = r));
       let drawn: Draw[] | undefined;
       try {
         drawn = await this.store.claimSession(
@@ -536,7 +544,7 @@ export class Worker {
         continue;
       }
       if (!drawn?.length) {
-        await this.idle(pollMs);
+        await this.idle(pollMs, this.freed$);
         continue;
       }
       for (const { src, calls } of drawn) {
@@ -552,6 +560,7 @@ export class Worker {
           const p = call.finally(() => {
             this.planner.release(resource);
             running.delete(p);
+            if (resource != null) this.freedWake();
           });
           running.add(p);
         }
@@ -929,9 +938,14 @@ export class Worker {
    * cuts it short when a task on this worker's queues becomes claimable;
    * stop() interrupts it either way, and sleepOrStop bounds it at `ms` so the
    * poll fallback — which also drives lease recovery — never stretches.
+   *
+   * `freed` is the third way out: a resource unit coming back frees a claim the
+   * store has no reason to announce (see `freed$`).
    */
-  private idle(ms: number): Promise<void> {
-    return Promise.race([this.sleepOrStop(ms), this.store.claimWake(this.queues, ms)]);
+  private idle(ms: number, freed?: Promise<void>): Promise<void> {
+    const racers = [this.sleepOrStop(ms), this.store.claimWake(this.queues, ms)];
+    if (freed) racers.push(freed);
+    return Promise.race(racers);
   }
 
   private sleepOrStop(ms: number): Promise<void> {