cairnq 0.1.0 → 0.2.0

package/dist/store/base.js

@@ -1 +1,370 @@
-export {};
+import { newId } from "../ids.js";
+import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch, SerializationError, } from "../errors.js";
+import { rowToTask, STATUSES } from "../models.js";
+const rejectMangled = function (_key, v) {
+    if (typeof v === "number" && !Number.isFinite(v)) {
+        throw new SerializationError(`non-finite number ${v} is not JSON-serializable`);
+    }
+    // In an array these become the literal `null` (in an object they are merely
+    // omitted, the JS idiom for "absent") — the twin SDK would read back a null
+    // the caller never wrote.
+    if (Array.isArray(this) && (v === undefined || typeof v === "function" || typeof v === "symbol")) {
+        throw new SerializationError(`${typeof v} inside an array is not JSON-serializable`);
+    }
+    return v;
+};
+/** Encode a value for a protocol JSON column, raising SerializationError on
+ * anything JSON cannot represent. Refuses what JSON.stringify would silently
+ * mangle into `null`: NaN/Infinity anywhere, undefined/function/symbol inside an
+ * array, and a top-level undefined that disappears entirely — either way the
+ * twin SDK reads back something other than what the caller meant (the Python
+ * SDK rejects the same values, via allow_nan=False). */
+export function dumpJson(value) {
+    let text;
+    try {
+        text = JSON.stringify(value);
+    }
+    catch (err) {
+        // BigInt or a circular structure.
+        throw new SerializationError(err instanceof Error ? err.message : String(err));
+    }
+    if (text === undefined) {
+        throw new SerializationError(`value of type ${typeof value} is not JSON-serializable`);
+    }
+    // Every mangled value reaches the output as the literal `null`, so a
+    // null-free result needs no strict pass — this keeps the replacer (which
+    // forfeits V8's native stringifier) off the hot path.
+    if (text.includes("null"))
+        JSON.stringify(value, rejectMangled);
+    return text;
+}
+const SUPPORTED_PROTOCOL_MAJOR = 1;
+/** Refuse to run against a store whose protocol major this SDK does not speak.
+ * The supported major is a protocol fact, not a dialect one — every backend
+ * checks it here so the constant can't fork per store. */
+export function checkProtocolVersion(version) {
+    if (version !== SUPPORTED_PROTOCOL_MAJOR) {
+        throw new ProtocolVersionMismatch(`storage protocol_version=${version}, SDK supports ${SUPPORTED_PROTOCOL_MAJOR}`);
+    }
+}
+// CONFLICTS is the canonical declaration; the type derives from it so the
+// runtime guard in submit() and the type can't drift apart (same pattern as
+// STATUSES/TaskStatus in models.ts).
+const CONFLICTS = ["reuse", "reject", "replace"];
+export const LEASE_EXPIRED_ERROR_JSON = dumpJson(errorEnvelope({
+    type: "LeaseExpired",
+    code: "lease_expired",
+    message: "task lease expired and max attempts reached",
+    retryable: false,
+}));
+/** Strips SQL line comments, so a `:name` in a header comment isn't a parameter. */
+export const COMMENT = /--[^\n]*/g;
+/** A `:name` placeholder. The lookbehind spares Postgres `::type` casts. */
+export const NAMED = /(?<!:):(\w+)/g;
+// Statement text is loaded once at construction and never varies, so the parse is
+// memoized on it: every dialect's binding path runs on each query, and re-scanning
+// the SQL each time would put a regex sweep on the worker's poll loop.
+const paramCache = new Map();
+/**
+ * The parameter names a statement binds, in first-appearance order.
+ *
+ * Callers pass a superset of parameters and each dialect takes what its own SQL
+ * asks for — that is what lets one call site serve both dialects even though e.g.
+ * SQLite binds `:lease_until_ms` where Postgres binds `:lease_ms`. This is the one
+ * place that decides what counts as a parameter; both dialects' binding goes
+ * through it.
+ */
+export function statementParams(sql) {
+    let names = paramCache.get(sql);
+    if (!names) {
+        const seen = new Set();
+        for (const m of sql.replace(COMMENT, "").matchAll(NAMED))
+            seen.add(m[1]);
+        names = [...seen];
+        paramCache.set(sql, names);
+    }
+    return names;
+}
+/**
+ * The storage seam.
+ *
+ * A backend supplies three things: how to run one protocol statement, how to run
+ * several inside a transaction, and how its dialect binds parameters. Everything
+ * above that — the submit conflict branches, the *_by_key lookups, the
+ * recover-then-claim sequence, the ownership-checked writes — lives here once,
+ * because those are protocol decisions rather than storage decisions. Keeping
+ * them in one place is what stops SQLite and Postgres from drifting apart in
+ * behavior; the shared SQL already stops them from drifting in wording.
+ */
+export class TaskStore {
+    /**
+     * Whether it is worth opening the claim transaction at all. SQLite gates its
+     * single write lock behind a read-only probe; Postgres readers don't block
+     * writers, so it just says yes.
+     */
+    async hasClaimableWork(_params) {
+        return true;
+    }
+    // ------------------------------------------------------------ wake channel
+    // Wake-or-timeout contract (PROTOCOL.md "Push wakeups"): resolve when the
+    // watched event may have happened, or after timeoutMs at the latest. The
+    // default is a plain sleep — polling IS the wake mechanism; a dialect with a
+    // push channel (PostgresStore, LISTEN/NOTIFY) resolves earlier.
+    /** Resolves when a task may have become claimable on one of `queues`. The
+     * timer is unref'd: the worker races this against its own stop-aware, ref'd
+     * sleep, so it must neither hold the process open nor need clearing. */
+    claimWake(_queues, timeoutMs) {
+        return new Promise((resolve) => setTimeout(resolve, timeoutMs).unref?.());
+    }
+    /** Resolves when `taskId` may have gone terminal. Plain ref'd sleep —
+     * pollWait awaits it directly, so it is what keeps the process alive. */
+    taskDoneWake(_taskId, timeoutMs) {
+        return new Promise((resolve) => setTimeout(resolve, timeoutMs));
+    }
+    // --------------------------------------------------------------- internals
+    /**
+     * An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
+     * Each statement's WHERE pins worker_id + a live lease, so 0 rows back means
+     * the lease was lost — every such write reports it the same way.
+     */
+    async ownedWrite(name, taskId, params) {
+        const rows = await this.fetch(name, params);
+        if (!rows.length)
+            throw new LostLease(taskId);
+        return rowToTask(rows[0]);
+    }
+    static one(rows) {
+        return rows.length ? rowToTask(rows[0]) : null;
+    }
+    // ------------------------------------------------------------- client side
+    async submit(input) {
+        const id = newId("task");
+        const ins = {
+            id,
+            name: input.name,
+            queue: input.queue ?? "default",
+            payload: dumpJson(input.payload ?? {}),
+            metadata: dumpJson(input.metadata ?? {}),
+            max_attempts: input.maxAttempts ?? 3,
+            priority: input.priority ?? 0,
+            delay_ms: input.runAtDelayMs ?? 0,
+            parent_id: input.parentId ?? null,
+            root_id: input.rootId ?? id,
+            correlation_id: input.correlationId ?? null,
+        };
+        const key = input.key ?? null;
+        const conflict = input.conflict ?? "reuse";
+        // Validate up front: untyped callers otherwise only hit the strategy branch
+        // on the second submit of a key, deep inside the transaction.
+        if (!CONFLICTS.includes(conflict)) {
+            throw new Error(`unknown conflict strategy: ${conflict}`);
+        }
+        // maxAttempts < 1 would still run once (claim increments before the check),
+        // a silently different meaning than the number says; a negative delay is
+        // always a mistake. Both fail loudly instead. Only supplied values are
+        // checked — the defaults live in the params object alone.
+        if (input.maxAttempts != null && input.maxAttempts < 1) {
+            throw new Error(`maxAttempts must be >= 1, got ${input.maxAttempts}`);
+        }
+        if (input.runAtDelayMs != null && input.runAtDelayMs < 0) {
+            throw new Error(`runAtDelayMs must be >= 0, got ${input.runAtDelayMs}`);
+        }
+        if (key === null)
+            return rowToTask((await this.fetch("insert_task", ins))[0]);
+        // A key makes submit a read-then-write, so it has to be one transaction —
+        // opened by taking the key's lock, because on Postgres the transaction alone
+        // is not enough: concurrent same-key submits must not both see "no existing
+        // task" (see lock_key.sql; on SQLite it is a no-op).
+        return this.tx(async (fetch) => {
+            await fetch("lock_key", { key });
+            const existing = (await fetch("get_key", { key }));
+            if (existing.length) {
+                // Read the task itself before branching: a concurrent purge (which
+                // takes no key lock) may have deleted it — cascading the key row away —
+                // between our statements' snapshots. A vanished task means the key is
+                // free after all, whatever the strategy.
+                const current = (await fetch("get", { id: existing[0].task_id }))[0];
+                if (current) {
+                    if (conflict === "reuse")
+                        return rowToTask(current);
+                    if (conflict === "reject")
+                        throw new AlreadyExists(key);
+                    // "replace": cancel the recorded task, then repoint the key below.
+                    await fetch("cancel", { id: existing[0].task_id });
+                }
+            }
+            const row = (await fetch("insert_task", ins))[0];
+            await fetch("upsert_key", { key, task_id: id });
+            return rowToTask(row);
+        });
+    }
+    async get(taskId) {
+        return TaskStore.one(await this.fetch("get", { id: taskId }));
+    }
+    async getByKey(key) {
+        return TaskStore.one(await this.fetch("get_by_key", { key }));
+    }
+    async list(input = {}) {
+        // Validate up front, like submit's conflict guard: a typo'd status otherwise
+        // matches nothing and returns [] indistinguishably from "no such tasks".
+        if (input.status != null && !STATUSES.includes(input.status)) {
+            throw new Error(`unknown status filter: ${input.status}`);
+        }
+        if ((input.limit != null && input.limit < 0) || (input.offset != null && input.offset < 0)) {
+            throw new Error(`limit/offset must be >= 0, got limit=${input.limit} offset=${input.offset}`);
+        }
+        const rows = await this.fetch("list", {
+            status: input.status ?? null,
+            queue: input.queue ?? null,
+            name: input.name ?? null,
+            root_id: input.rootId ?? null,
+            correlation_id: input.correlationId ?? null,
+            limit: input.limit ?? 100,
+            offset: input.offset ?? 0,
+        });
+        return rows.map(rowToTask);
+    }
+    async cancel(taskId) {
+        return TaskStore.one(await this.fetch("cancel", { id: taskId }));
+    }
+    async retry(taskId, opts = {}) {
+        return TaskStore.one(await this.fetch("retry", { id: taskId, reset_attempt: opts.resetAttempt ?? false }));
+    }
+    async cancelByKey(key) {
+        return this.byKey("cancel", key, {});
+    }
+    async retryByKey(key, opts = {}) {
+        return this.byKey("retry", key, { reset_attempt: opts.resetAttempt ?? false });
+    }
+    /**
+     * Resolve a key to the task it currently points at, then act on that task —
+     * under the key's lock, so a concurrent `replace` can't repoint the key
+     * between the lookup and the write (the transaction alone is not enough on
+     * Postgres; see lock_key.sql).
+     */
+    async byKey(name, key, params) {
+        return this.tx(async (fetch) => {
+            await fetch("lock_key", { key });
+            const existing = (await fetch("get_key", { key }));
+            if (!existing.length)
+                return null;
+            return TaskStore.one(await fetch(name, { id: existing[0].task_id, ...params }));
+        });
+    }
+    /**
+     * Delete terminal tasks that completed more than `olderThanMs` ago and return
+     * their ids. Nothing else removes rows, so a long-lived database needs this
+     * called periodically. Bounded by `limit` to keep each sweep a short write;
+     * call it in a loop until it returns fewer than `limit`.
+     */
+    async purge(input = {}) {
+        if (input.olderThanMs != null && input.olderThanMs < 0) {
+            throw new Error(`olderThanMs must be >= 0, got ${input.olderThanMs}`);
+        }
+        if (input.limit != null && input.limit < 1) {
+            throw new Error(`limit must be >= 1, got ${input.limit}`);
+        }
+        const rows = await this.fetch("purge", {
+            older_than_ms: input.olderThanMs ?? 0,
+            limit: input.limit ?? 1_000,
+        });
+        return rows.map((r) => r.id);
+    }
+    /**
+     * Task counts per queue, keyed by status and zero-filled across all statuses —
+     * `(await stats()).default.queued` is the backlog of a queue. A queue appears
+     * only while it has rows; terminal tasks keep counting until `purge` removes
+     * them.
+     */
+    async stats() {
+        const out = {};
+        for (const row of await this.fetch("stats", {})) {
+            const per = (out[row.queue] ??= Object.fromEntries(STATUSES.map((s) => [s, 0])));
+            per[row.status] = Number(row.count);
+        }
+        return out;
+    }
+    // ------------------------------------------------------------- worker side
+    /**
+     * Take up to `limit` claimable tasks. `names` restricts the claim to task names
+     * this caller can actually run — a worker passes its registered handlers.
+     * Queues alone do not partition work, so without it a worker claims a task it
+     * cannot run and fails it permanently. Undefined means no filter; an empty
+     * array claims nothing.
+     */
+    async claim(input) {
+        const params = {
+            queues: input.queues,
+            names: input.names ?? null,
+            worker_id: input.workerId,
+            lease_ms: input.leaseMs ?? 30_000,
+            limit: input.limit ?? 1,
+            lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
+        };
+        if (!(await this.hasClaimableWork(params)))
+            return [];
+        // Recovery must share the claim's transaction: a lease reclaimed here has to
+        // be visible to the claim that follows, and to nobody in between.
+        return this.tx(async (fetch) => {
+            await fetch("recover_leases", params);
+            return (await fetch("claim", params)).map(rowToTask);
+        });
+    }
+    async heartbeat(input) {
+        return this.ownedWrite("heartbeat", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            lease_ms: input.leaseMs ?? 30_000,
+        });
+    }
+    async progress(input) {
+        return this.ownedWrite("progress", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            progress: input.progress,
+            message: input.message,
+        });
+    }
+    async succeed(input) {
+        return this.ownedWrite("succeed", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            result: input.result == null ? null : dumpJson(input.result),
+            message: null,
+        });
+    }
+    async complete(input) {
+        return this.ownedWrite("complete", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            result: input.result == null ? null : dumpJson(input.result),
+        });
+    }
+    async fail(input) {
+        let error;
+        try {
+            error = dumpJson(input.error ?? {});
+        }
+        catch (err) {
+            if (!(err instanceof SerializationError))
+                throw err;
+            // A failure record must never itself fail to serialize (a TaskError
+            // carrying exotic details would otherwise strand the task until lease
+            // expiry). Strip the envelope to its string fields and record that.
+            const e = (input.error ?? {});
+            error = dumpJson(errorEnvelope({
+                type: String(e.type ?? "TaskError"),
+                code: String(e.code ?? "task_error"),
+                message: String(e.message ?? ""),
+                retryable: input.retryable !== false,
+            }));
+        }
+        return this.ownedWrite("fail", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            error,
+            retryable: input.retryable !== false,
+            delay_ms: input.delayMs ?? 0,
+        });
+    }
+}

package/dist/store/postgres.d.ts

@@ -1,29 +1,62 @@
-import { type Task } from "../models.js";
-import type { ListInput, SubmitInput, TaskStore } from "./base.js";
+import { type Fetch, type Params, TaskStore } from "./base.js";
 /**
- * Translate the protocol's named-parameter SQL (`:name`) into Postgres positional
+ * The rewritten SQL and the order its `$n` slots must be filled in.
+ *
+ * Translates the protocol's named-parameter SQL (`:name`) into Postgres positional
  * placeholders (`$1`), collapsing each DISTINCT name to ONE slot — statements reuse
- * a name across CASE branches / IS NULL guards (e.g. list.sql). SQL comments are
- * stripped first so a `:name` mentioned in a header comment (e.g. "now + :lease_ms")
- * never leaks into the parameter list. Exported for unit testing.
+ * a name across CASE branches / IS NULL guards (e.g. list.sql). Which names count
+ * as parameters is `statementParams`' decision, shared with the SQLite binding path
+ * so the two can't disagree about, say, a `::type` cast.
+ *
+ * Memoized on the statement text, which is loaded once and never varies: this runs
+ * on every query, including the worker's poll loop, for a result that cannot have
+ * changed. Exported for unit testing.
  */
-export declare function toPositional(sql: string, params: Record<string, unknown>): {
+export declare function positionalStatement(sql: string): {
+    text: string;
+    order: readonly string[];
+};
+/**
+ * The statement's rewritten text plus this call's values, in slot order. Names the
+ * statement does not use are simply not bound, so callers may pass a superset.
+ */
+export declare function toPositional(sql: string, params: Params): {
     text: string;
     values: unknown[];
 };
 /**
- * PostgresStore — asyncpg's TS counterpart: a `pg` Pool executing the shared
- * cairnq-protocol SQL (postgres dialect). Multi-host capable — unlike SQLite this
- * coordinates API and worker processes across machines through one database. Time
- * comes from the DB clock (now()), claim uses FOR UPDATE SKIP LOCKED, JSON columns
- * are jsonb (bound as JSON text, read back as objects by rowToTask's adaptive parse).
+ * PostgresStore — the Postgres dialect of the shared cairnq-protocol SQL.
+ *
+ * Everything protocol-shaped lives in TaskStore; this file is only what Postgres
+ * does differently: a `pg` Pool, `:name` -> `$n` translation, and time taken from
+ * the DB clock (`now()`) instead of from the SDK, which is what makes this backend
+ * multi-host — unlike SQLite it coordinates API and worker processes across
+ * machines, with no shared clock to agree on. claim uses FOR UPDATE SKIP LOCKED
+ * and needs no claimable_probe, because PG readers don't block writers. JSON
+ * columns are jsonb (bound as JSON text, read back as objects by rowToTask).
  */
-export declare class PostgresStore implements TaskStore {
+export declare class PostgresStore extends TaskStore {
     private readonly dsn;
     private readonly opts;
     private pool;
     private connecting;
     private readonly statements;
+    private listener;
+    private listenerConnecting;
+    /** LISTEN is off for good: the store was closed, or the server accepted a
+     * connection but refused LISTEN (e.g. a transaction-mode pooler) —
+     * deterministic, so retrying would fail the same way every time. */
+    private listenerUnavailable;
+    /** A failure to even connect is transient (network blip, server restarting):
+     * retry, but not before this time, backing off so a down server is not
+     * hammered from the poll loop. */
+    private listenerRetryAt;
+    private listenerBackoffMs;
+    /** Queues notified while nobody was waiting; consumed by the next claimWake
+     * so a wake that lands between polls is not lost. */
+    private readonly pendingQueues;
+    /** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
+    private readonly waiters;
     constructor(dsn: string, opts?: {
         max?: number;
     });
@@ -34,54 +67,20 @@ export declare class PostgresStore implements TaskStore {
     private applyMigrations;
     private readProtocolVersion;
     protocolVersion(): Promise<number>;
-    private run;
-    private runOn;
-    private tx;
-    private ownedWrite;
-    submit(input: SubmitInput): Promise<Task>;
-    get(taskId: string): Promise<Task | null>;
-    getByKey(key: string): Promise<Task | null>;
-    list(input?: ListInput): Promise<Task[]>;
-    cancel(taskId: string): Promise<Task | null>;
-    cancelByKey(key: string): Promise<Task | null>;
-    retry(taskId: string, opts?: {
-        resetAttempt?: boolean;
-    }): Promise<Task | null>;
-    retryByKey(key: string, opts?: {
-        resetAttempt?: boolean;
-    }): Promise<Task | null>;
-    claim(input: {
-        queues: string[];
-        workerId: string;
-        leaseMs?: number;
-        limit?: number;
-    }): Promise<Task[]>;
-    heartbeat(input: {
-        taskId: string;
-        workerId: string;
-        leaseMs?: number;
-    }): Promise<Task>;
-    progress(input: {
-        taskId: string;
-        workerId: string;
-        progress: number | null;
-        message: string | null;
-    }): Promise<Task>;
-    succeed(input: {
-        taskId: string;
-        workerId: string;
-        result: unknown;
-    }): Promise<Task>;
-    complete(input: {
-        taskId: string;
-        workerId: string;
-        result: unknown;
-    }): Promise<Task>;
-    fail(input: {
-        taskId: string;
-        workerId: string;
-        error: unknown;
-        retryable?: boolean;
-        delayMs?: number;
-    }): Promise<Task>;
+    claimWake(queues: string[], timeoutMs: number): Promise<void>;
+    taskDoneWake(taskId: string, timeoutMs: number): Promise<void>;
+    /** A promise resolving on notification-or-timeout, deregistering either way.
+     * The timer is unref'd: while a listener exists its socket keeps the process
+     * alive, and dropListener wakes every waiter the moment it goes away. */
+    private wakeOn;
+    /** True once the LISTEN connection is up; starts connecting it otherwise
+     * (respecting the transient-failure backoff). Callers fall back to plain
+     * polling until it is ready (or forever, if it can't be established) —
+     * correctness never depends on it. */
+    private listenerReady;
+    private startListener;
+    private onNotification;
+    private dropListener;
+    protected fetch(name: string, params: Params): Promise<any[]>;
+    protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
 }