cairnq 0.13.0 → 0.14.0

package/dist/errors.js

@@ -140,38 +140,6 @@ export class TaskCanceled extends CairnQError {
         this.name = "TaskCanceled";
     }
 }
-/**
- * A heartbeat beat came back later than its own interval allowed.
- *
- * The heartbeat shares the event loop with the handlers whose leases it renews,
- * so a handler that blocks the loop stops the renewal with it: the lease expires,
- * the task is recovered and redelivered, and a second worker starts computing
- * what the first is still computing — one task, billed twice, with no error
- * anywhere. Nothing inside the blocked handler can observe that, which is why it
- * is reported through `onError` alongside the other things the run loop survived.
- *
- * The usual cause is synchronous work in a handler: a tight loop, a large
- * JSON.parse, a `*Sync` filesystem or crypto call. Node has one loop and no way
- * to preempt it — move the work to a worker thread, a child process, or an async
- * API that yields. The other cause is a worker simply oversubscribed for its
- * `leaseMs` — nothing is blocking, there is just more work than turns — which the
- * same report covers, because the lease is at equal risk either way.
- */
-export class EventLoopBlocked extends CairnQError {
-    lateMs;
-    intervalMs;
-    leaseMs;
-    constructor(lateMs, intervalMs, leaseMs) {
-        super(`heartbeat beat was ${lateMs}ms late (interval ${intervalMs}ms, lease ${leaseMs}ms): ` +
-            `the event loop was blocked long enough to miss a beat, so this worker's leases ` +
-            `are at risk. Usually synchronous work in a handler (move it off the loop); ` +
-            `otherwise the worker is oversubscribed for its leaseMs.`);
-        this.lateMs = lateMs;
-        this.intervalMs = intervalMs;
-        this.leaseMs = leaseMs;
-        this.name = "EventLoopBlocked";
-    }
-}
 /** A worker write affected 0 rows: the lease expired and was reclaimed. */
 export class LostLease extends CairnQError {
     taskId;
@@ -221,10 +189,30 @@ export class SchemaMismatch extends CairnQError {
         this.name = "SchemaMismatch";
     }
 }
+/**
+ * The store was closing when this operation asked for it.
+ *
+ * `close()` waits for the work already accepted — a group commit still holding
+ * writes, a transaction with a BEGIN IMMEDIATE open — and turns away everything
+ * that arrives after, so that wait cannot be extended indefinitely by a producer
+ * that keeps submitting. An operation that lands in that window gets this rather
+ * than a driver error about a connection that vanished underneath it.
+ *
+ * It does not mean the store is finished for good: connecting is lazy, so a
+ * store used again after `close()` has returned simply reopens. The Python SDK
+ * raises the same named error.
+ */
+export class StoreClosed extends CairnQError {
+    constructor(message = "store is closing") {
+        super(message);
+        this.name = "StoreClosed";
+    }
+}
 /** A value could not be encoded for a protocol JSON column (non-finite number,
- * BigInt, circular structure, …). Raised at the boundary — submit rejects with
- * it, and a worker records a handler result that triggers it as a permanent
- * `unserializable_result` failure. The Python SDK raises the same named error. */
+ * BigInt, circular structure, an opaque built-in like Map or Set, …). Raised at
+ * the boundary — submit rejects with it, and a worker records a handler result
+ * that triggers it as a permanent `unserializable_result` failure. The Python
+ * SDK raises the same named error. */
 export class SerializationError extends CairnQError {
     constructor(message) {
         super(message);

package/dist/index.d.ts

@@ -1,24 +1,18 @@
 export { CairnQ } from "./client.js";
 export type { CallOptions, ClientOptions, SubmitOptions, WaitOptions } from "./client.js";
-export { QueueDepthGate } from "./backpressure.js";
-export type { BackpressureOptions, QueueDepthLimit } from "./backpressure.js";
-export { RetentionSweeper } from "./retention.js";
-export type { RetentionCutoffs, RetentionOptions, RetentionRule } from "./retention.js";
 export { Worker } from "./worker.js";
 export type { BatchHandler, Handler, TypedHandler, WorkerOptions } from "./worker.js";
 export { TaskContext } from "./context.js";
-export type { TaskContextOptions } from "./context.js";
 export { defineTask } from "./task.js";
 export type { TaskDef } from "./task.js";
 export { SQLiteStore } from "./store/sqlite.js";
 export { PostgresStore } from "./store/postgres.js";
+export { TaskStore } from "./store/base.js";
+export type { Conflict, ListInput, PurgeInput, SubmitInput } from "./store/base.js";
 export { ListenUnavailable } from "./store/pg-executor.js";
 export type { PgExecutor, PgSession, Row } from "./store/pg-executor.js";
 export { createPoolExecutor } from "./store/pg-pool.js";
-export { TaskStore } from "./store/base.js";
-export type { ListInput, PurgeInput, SubmitInput, Conflict, WatchOptions, WatchSignal, } from "./store/base.js";
-export { DEFAULT_WATCH_POLL_MS } from "./store/base.js";
+export { STATUSES, isTerminalStatus } from "./models.js";
 export type { Task, TaskRef, TaskStatus, TerminalStatus } from "./models.js";
-export { STATUSES, isTerminal, isTerminalStatus, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
-export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SchemaMismatch, UnsupportedBackend, SerializationError, } from "./errors.js";
+export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, ProtocolVersionMismatch, SchemaMismatch, UnsupportedBackend, SerializationError, StoreClosed, } from "./errors.js";
 export type { FailReason } from "./errors.js";

package/dist/index.js

@@ -1,14 +1,11 @@
 export { CairnQ } from "./client.js";
-export { QueueDepthGate } from "./backpressure.js";
-export { RetentionSweeper } from "./retention.js";
 export { Worker } from "./worker.js";
 export { TaskContext } from "./context.js";
 export { defineTask } from "./task.js";
 export { SQLiteStore } from "./store/sqlite.js";
 export { PostgresStore } from "./store/postgres.js";
+export { TaskStore } from "./store/base.js";
 export { ListenUnavailable } from "./store/pg-executor.js";
 export { createPoolExecutor } from "./store/pg-pool.js";
-export { TaskStore } from "./store/base.js";
-export { DEFAULT_WATCH_POLL_MS } from "./store/base.js";
-export { STATUSES, isTerminal, isTerminalStatus, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
-export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SchemaMismatch, UnsupportedBackend, SerializationError, } from "./errors.js";
+export { STATUSES, isTerminalStatus } from "./models.js";
+export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, ProtocolVersionMismatch, SchemaMismatch, UnsupportedBackend, SerializationError, StoreClosed, } from "./errors.js";

package/dist/models.d.ts

@@ -39,7 +39,20 @@ export declare function isTerminalStatus(status: TaskStatus): status is Terminal
 /** Map a probe row (see get_status.sql) to a TaskRef — the ref twin of
  * rowToTask, so the row shape stays models' knowledge alone. */
 export declare function rowToRef(row: Record<string, unknown>): TaskRef;
-export declare function rowToTask(row: Record<string, unknown>): Task;
+/**
+ * Map a row to a Task.
+ *
+ * `jsonIsText` says how this driver delivers a JSON column, and it has to be
+ * told rather than inferred. SQLite (TEXT) hands back the JSON text to parse; a
+ * jsonb-aware driver (`pg`) hands back an already-decoded value. Those two are
+ * distinguishable for an object or an array — but not for a JSON *string*, where
+ * the text form (`"hello"`, quotes included) and the decoded form (`hello`) are
+ * both, simply, strings. Guessing from the value used to double-parse the
+ * decoded one: `"s3://…"` threw a SyntaxError, and `"42"` came back as the
+ * NUMBER 42 — silent corruption for any payload or result that is a top-level
+ * string. The store knows which driver it has; models does not, so it asks.
+ */
+export declare function rowToTask(row: Record<string, unknown>, jsonIsText: boolean): Task;
 /** Accepts anything carrying a status — a Task or a TaskRef probe. */
 export declare function isTerminal(task: Pick<Task, "status">): boolean;
 export declare function cancelRequested(task: Task): boolean;

package/dist/models.js

@@ -28,14 +28,26 @@ export function isTerminalStatus(status) {
 export function rowToRef(row) {
     return { id: row.id, status: row.status };
 }
-export function rowToTask(row) {
+/**
+ * Map a row to a Task.
+ *
+ * `jsonIsText` says how this driver delivers a JSON column, and it has to be
+ * told rather than inferred. SQLite (TEXT) hands back the JSON text to parse; a
+ * jsonb-aware driver (`pg`) hands back an already-decoded value. Those two are
+ * distinguishable for an object or an array — but not for a JSON *string*, where
+ * the text form (`"hello"`, quotes included) and the decoded form (`hello`) are
+ * both, simply, strings. Guessing from the value used to double-parse the
+ * decoded one: `"s3://…"` threw a SyntaxError, and `"42"` came back as the
+ * NUMBER 42 — silent corruption for any payload or result that is a top-level
+ * string. The store knows which driver it has; models does not, so it asks.
+ */
+export function rowToTask(row, jsonIsText) {
     const t = { ...row };
     for (const col of JSON_COLUMNS) {
         const v = row[col];
-        // The driver decides a JSON column's wire form: SQLite (TEXT) hands back a
-        // string to parse; a jsonb-aware driver (Postgres `pg`) hands back an
-        // already-decoded object. Parse only a string — never assume one backend.
-        t[col] = typeof v === "string" ? JSON.parse(v) : (v ?? null);
+        // A SQL NULL and a decoded JSON `null` both arrive as null and both mean
+        // "no value"; only a real text form is parsed.
+        t[col] = v == null ? null : jsonIsText ? JSON.parse(v) : v;
     }
     for (const col of MS_COLUMNS) {
         const v = row[col];

package/dist/retention.d.ts

@@ -1,67 +1,15 @@
-import type { TerminalStatus } from "./models.js";
-import { type TaskStore } from "./store/base.js";
-/** Per-status cutoffs. A status left out is never swept — granular retention is
- * an explicit statement of what may go, not a default for what wasn't named. */
-export type RetentionCutoffs = Partial<Record<TerminalStatus, number>>;
+import type { TaskStore } from "./store/base.js";
 /**
- * One "these rows may go after this long" statement. Each field left out widens
- * what the rule covers; `olderThanMs` is the only required one.
- *
- * A rule is one `purge` call's filters, so the fields are exactly `PurgeInput`'s
- * — deliberately, since a rule the sweeper can express but the store cannot
- * enforce would be a lie about what is being deleted.
- */
-export interface RetentionRule {
-    /** Only this queue. Absent means every queue. */
-    queue?: string;
-    /** Only this terminal status. Absent means all three. */
-    status?: TerminalStatus;
-    /** Only this task name. Absent means every name. */
-    name?: string;
-    /** How long a row matching this rule is kept after it finished. */
-    olderThanMs: number;
-}
-export interface RetentionOptions {
-    /**
-     * How long a terminal task is kept after it finished. Required: there is no
-     * safe default for how long someone else's results stay readable.
-     *
-     * Three forms, widening as the deployment does:
-     *
-     * - A number keeps every terminal row the same time.
-     * - A per-status map tiers by outcome — a succeeded row is spent once its
-     *   result is consumed, a failed one is worth keeping for diagnosis:
-     *   `{ succeeded: 300_000, failed: 86_400_000 }`. A status left out is never
-     *   swept.
-     * - An array of rules tiers by anything `purge` can filter on, which is what a
-     *   store shared by two workloads needs — the recommended way for two
-     *   languages to coordinate is one installation, and an RPC queue read once
-     *   has nothing in common with a durable queue kept for a week:
-     *   `[{ queue: "rpc", olderThanMs: 300_000 },
-     *      { queue: "jobs", status: "failed", olderThanMs: 604_800_000 }]`.
-     *   Rules are independent, each its own sweep — nothing a rule does not match
-     *   is swept, and rules that overlap simply delete the same row once.
-     */
-    olderThanMs: number | RetentionCutoffs | RetentionRule[];
-    /** Time between sweeps. Default 3_600_000 (one hour). */
-    intervalMs?: number;
-    /** Rows deleted per statement while draining. Default 1_000. */
-    limit?: number;
-    /**
-     * Called for a sweep that threw. The next sweep runs on schedule regardless —
-     * a purge that failed because the database was busy is not a reason to stop
-     * retaining — so without this a store quietly stops being swept. Must not throw.
-     */
-    onError?: (err: unknown) => void;
-}
-/**
- * Deletes terminal tasks on a schedule, for as long as the handle is open.
+ * Deletes terminal tasks on a schedule, for as long as the handle is open —
+ * `purge(olderThanMs)` on a timer, which is the whole mechanism.
  *
  * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
  * payloads carry real data — an image, a document, a batch of embeddings — turns
  * that into a disk leak measured in gigabytes per backfill. Every deployment
  * that runs longer than a demo needs the sweep; leaving it to an external
  * scheduler means the leak is the default and remembering is the opt-in.
+ * A deployment whose retention is tiered (per queue, per status) calls `purge`
+ * with those filters from its own scheduler instead.
  *
  * It sweeps in bounded batches with a yield between them, so draining a backlog
  * that accumulated while nothing was sweeping stays a sequence of short writes
@@ -70,7 +18,7 @@ export interface RetentionOptions {
  */
 export declare class RetentionSweeper {
     private readonly store;
-    private readonly opts;
+    private readonly olderThanMs;
     /** Whether the scheduled loop is running. */
     private active;
     /** Set by stop(), so a drain in progress can cut itself short too. */
@@ -90,12 +38,9 @@ export declare class RetentionSweeper {
     /** The loop itself, awaited by stop() so no purge outlives the store. */
     private loop;
     private readonly intervalMs;
-    /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
-    private readonly limit;
-    /** One purge per rule: a lone entry for a number, one per status for a map,
-     * one per element for an array. */
-    private readonly purgeInputs;
-    constructor(store: TaskStore, opts: RetentionOptions);
+    constructor(store: TaskStore, olderThanMs: number, opts?: {
+        intervalMs?: number;
+    });
     start(): void;
     /** Mint a fresh stop signal. */
     private arm;
@@ -108,7 +53,18 @@ export declare class RetentionSweeper {
      * demand — after a backfill, or from a maintenance command.
      */
     sweep(): Promise<number>;
-    /** Sleep, interruptible by stop(). Unref'd: retention is housekeeping, and a
-     * pending sweep must never be the reason a process refuses to exit. */
+    /**
+     * Sleep, interruptible by stop().
+     *
+     * `holdProcess` is the same distinction the store draws between `claimWake`
+     * and `taskDoneWake`: a wait nobody is awaiting must not hold the process
+     * open, and a wait somebody IS awaiting must. The scheduled loop's interval is
+     * the first — retention is housekeeping, and a pending sweep must never be the
+     * reason a process refuses to exit. `sweep()`'s between-batches yield is the
+     * second: its caller is awaiting the drain, and an unref'd timer there let
+     * Node decide the loop was idle and exit mid-drain, leaving that promise
+     * unsettled forever (a maintenance command that swept two rows of seven and
+     * exited 13). Python's twin never had this, `asyncio.sleep` having no unref.
+     */
     private sleep;
 }

package/dist/retention.js

@@ -1,4 +1,3 @@
-import { validatePurgeInput } from "./store/base.js";
 /** Sweep every hour unless asked otherwise — often enough that a queue with a
  * day of retention never carries more than an hour of extra rows, rare enough
  * that the sweep is invisible next to the task traffic. */
@@ -6,27 +5,17 @@ const DEFAULT_INTERVAL_MS = 3_600_000;
 /** Rows per purge statement. The same bound `purge` defaults to: big enough that
  * a backlog drains in few statements, small enough that each is a short write. */
 const DEFAULT_LIMIT = 1_000;
-/** The three `olderThanMs` forms as the one form the sweep runs on. The number
- * and the per-status map are the rule array's common cases spelled shorter, so
- * they are widened here rather than handled separately downstream. */
-function toRules(spec) {
-    if (typeof spec === "number")
-        return [{ olderThanMs: spec }];
-    if (Array.isArray(spec))
-        return spec;
-    return Object.entries(spec).map(([status, olderThanMs]) => ({
-        status,
-        olderThanMs,
-    }));
-}
 /**
- * Deletes terminal tasks on a schedule, for as long as the handle is open.
+ * Deletes terminal tasks on a schedule, for as long as the handle is open —
+ * `purge(olderThanMs)` on a timer, which is the whole mechanism.
  *
  * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
  * payloads carry real data — an image, a document, a batch of embeddings — turns
  * that into a disk leak measured in gigabytes per backfill. Every deployment
  * that runs longer than a demo needs the sweep; leaving it to an external
  * scheduler means the leak is the default and remembering is the opt-in.
+ * A deployment whose retention is tiered (per queue, per status) calls `purge`
+ * with those filters from its own scheduler instead.
  *
  * It sweeps in bounded batches with a yield between them, so draining a backlog
  * that accumulated while nothing was sweeping stays a sequence of short writes
@@ -35,7 +24,7 @@ function toRules(spec) {
  */
 export class RetentionSweeper {
     store;
-    opts;
+    olderThanMs;
     /** Whether the scheduled loop is running. */
     active = false;
     /** Set by stop(), so a drain in progress can cut itself short too. */
@@ -55,40 +44,26 @@ export class RetentionSweeper {
     /** The loop itself, awaited by stop() so no purge outlives the store. */
     loop = null;
     intervalMs;
-    /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
-    limit;
-    /** One purge per rule: a lone entry for a number, one per status for a map,
-     * one per element for an array. */
-    purgeInputs;
-    constructor(store, opts) {
+    constructor(store, olderThanMs, opts = {}) {
         this.store = store;
-        this.opts = opts;
+        this.olderThanMs = olderThanMs;
+        if (!Number.isFinite(olderThanMs) || olderThanMs < 0) {
+            throw new Error(`retentionMs must be >= 0, got ${olderThanMs}`);
+        }
         this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
         if (!Number.isFinite(this.intervalMs) || this.intervalMs < 1) {
-            throw new Error(`retention.intervalMs must be >= 1, got ${this.intervalMs}`);
-        }
-        this.limit = opts.limit ?? DEFAULT_LIMIT;
-        const rules = toRules(opts.olderThanMs);
-        // An empty map or array retains nothing and sweeps nothing — almost
-        // certainly a bug upstream of this call, so refuse it rather than silently
-        // never purging.
-        if (!rules.length) {
-            throw new Error("retention.olderThanMs must name at least one rule");
+            throw new Error(`retention intervalMs must be >= 1, got ${this.intervalMs}`);
         }
         this.arm();
-        this.purgeInputs = rules.map((rule) => ({ ...rule, limit: this.limit }));
-        // Fail fast on the store's own purge rules (terminal status, cutoff >= 0):
-        // the sweep runs an hour from now, and its errors only surface via onError.
-        for (const input of this.purgeInputs)
-            validatePurgeInput(input);
     }
     start() {
         if (this.active)
             return;
         this.active = true;
-        this.stopping = false;
-        // A stopped sweeper can be started again, and the old signal is spent.
-        this.arm();
+        // No reset here: stop() clears `stopping` and re-arms the signal before it
+        // returns, and the constructor arms the first one — so a sweeper reaching
+        // this line always already has a fresh signal and a clear flag. (The Python
+        // twin has never had a reset here, for the same reason.)
         this.loop = this.run();
     }
     /** Mint a fresh stop signal. */
@@ -102,6 +77,15 @@ export class RetentionSweeper {
         this.wake();
         await this.loop;
         this.loop = null;
+        // Cleared, and the signal re-armed, now that the loop is provably gone.
+        // `stopping` is how a sweep in flight cuts itself short, so leaving it set
+        // would silently truncate a later on-demand sweep() — a supported call — to
+        // its first batch. Leaving the signal spent would be the subtler half of the
+        // same bug: that sweep's between-batches yield would resolve on a microtask
+        // instead of handing the event loop back, so a long drain would starve the
+        // submits and claims sharing this process — exactly what the yield is for.
+        this.stopping = false;
+        this.arm();
     }
     async run() {
         // Sleep first: a process that restarts often would otherwise purge on every
@@ -113,14 +97,9 @@ export class RetentionSweeper {
             try {
                 await this.sweep();
             }
-            catch (err) {
-                try {
-                    this.opts.onError?.(err);
-                }
-                catch {
-                    // A reporting hook must never take the sweep down with it — the same
-                    // rule the worker's onError follows.
-                }
+            catch {
+                // A purge that failed because the database was busy is not a reason to
+                // stop retaining — the next sweep runs on schedule regardless.
             }
         }
     }
@@ -131,28 +110,38 @@ export class RetentionSweeper {
      */
     async sweep() {
         let deleted = 0;
-        for (const input of this.purgeInputs) {
-            for (;;) {
-                const ids = await this.store.purge(input);
-                deleted += ids.length;
-                if (this.stopping)
-                    return deleted;
-                if (ids.length < this.limit)
-                    break;
-                // Hand the loop back between batches: a large drain must not starve the
-                // submits and claims sharing this process.
-                await this.sleep(0);
-            }
+        for (;;) {
+            const ids = await this.store.purge({ olderThanMs: this.olderThanMs, limit: DEFAULT_LIMIT });
+            deleted += ids.length;
+            if (this.stopping)
+                return deleted;
+            if (ids.length < DEFAULT_LIMIT)
+                break;
+            // Hand the loop back between batches: a large drain must not starve the
+            // submits and claims sharing this process. Ref'd — see sleep().
+            await this.sleep(0, true);
         }
         return deleted;
     }
-    /** Sleep, interruptible by stop(). Unref'd: retention is housekeeping, and a
-     * pending sweep must never be the reason a process refuses to exit. */
-    sleep(ms) {
+    /**
+     * Sleep, interruptible by stop().
+     *
+     * `holdProcess` is the same distinction the store draws between `claimWake`
+     * and `taskDoneWake`: a wait nobody is awaiting must not hold the process
+     * open, and a wait somebody IS awaiting must. The scheduled loop's interval is
+     * the first — retention is housekeeping, and a pending sweep must never be the
+     * reason a process refuses to exit. `sweep()`'s between-batches yield is the
+     * second: its caller is awaiting the drain, and an unref'd timer there let
+     * Node decide the loop was idle and exit mid-drain, leaving that promise
+     * unsettled forever (a maintenance command that swept two rows of seven and
+     * exited 13). Python's twin never had this, `asyncio.sleep` having no unref.
+     */
+    sleep(ms, holdProcess = false) {
         let timer;
         const nap = new Promise((resolve) => {
             timer = setTimeout(resolve, ms);
-            timer.unref?.();
+            if (!holdProcess)
+                timer.unref?.();
         });
         // Clear the timer whichever side wins, so a stop is never followed by a
         // leftover sweep timer.