cairnq 0.8.0 → 0.10.0

package/dist/store/base.js

@@ -122,17 +122,8 @@ export function statementParams(sql) {
     }
     return names;
 }
-/**
- * The storage seam.
- *
- * A backend supplies three things: how to run one protocol statement, how to run
- * several inside a transaction, and how its dialect binds parameters. Everything
- * above that — the submit conflict branches, the *_by_key lookups, the
- * recover-then-claim sequence, the ownership-checked writes — lives here once,
- * because those are protocol decisions rather than storage decisions. Keeping
- * them in one place is what stops SQLite and Postgres from drifting apart in
- * behavior; the shared SQL already stops them from drifting in wording.
- */
+/** See WatchOptions.pollMs. */
+export const DEFAULT_WATCH_POLL_MS = 2_000;
 export class TaskStore {
     /** Set by useBackpressure; null means submit is ungated. */
     gate = null;
@@ -354,6 +345,57 @@ export class TaskStore {
         }
         return out;
     }
+    /**
+     * Call `onSignal` when the tasks on `queues` may have changed — something was
+     * queued, or something finished.
+     *
+     * This is notify-ACCELERATED POLLING, not an event log, and the difference is
+     * the whole contract. Where a push channel is available (Postgres LISTEN) an
+     * idle watch costs nothing and a signal arrives within milliseconds of the
+     * event. Where it is not — a transaction-mode pooler refuses LISTEN, SQLite has
+     * no channel at all — the timer alone still delivers `poll` signals, so a
+     * consumer that re-reads on every signal is correct in both cases and merely
+     * less prompt in one.
+     *
+     * What it will NOT do is promise that a signal means something happened, or
+     * that every event produces its own signal. Treat a signal as "re-read now"
+     * and take the truth from `stats()` / `list()` / `get()`, which is where it
+     * lives. `reason` is a hint for reading less: a `done` signal names the task,
+     * so a dashboard can refresh that row instead of the list.
+     *
+     * Returns an unsubscribe. The timer is unref'd — watching does not hold a
+     * process open.
+     */
+    watch(opts, onSignal) {
+        const pollMs = Math.max(1, opts.pollMs ?? DEFAULT_WATCH_POLL_MS);
+        const queues = opts.queues ?? null;
+        let live = true;
+        const emit = (signal) => {
+            // A signal delivered after unsubscribe would have the consumer re-reading
+            // a store it has stopped caring about, possibly a closed one.
+            if (live)
+                onSignal(signal);
+        };
+        const unsubscribe = this.subscribePush?.((signal) => {
+            // A queued signal names its queue, so a watch scoped to some queues can
+            // drop the rest. A done signal names only the task — which queue it was on
+            // is not in the notification, so it is never filtered out.
+            if (signal.reason === "queued" && queues && signal.queue && !queues.includes(signal.queue)) {
+                return;
+            }
+            emit(signal);
+        });
+        const timer = setInterval(() => {
+            this.warmPush?.();
+            emit({ reason: "poll" });
+        }, pollMs);
+        timer.unref?.();
+        return () => {
+            live = false;
+            unsubscribe?.();
+            clearInterval(timer);
+        };
+    }
     /**
      * How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
      *
@@ -504,6 +546,40 @@ export class TaskStore {
             result: input.result == null ? null : dumpJson(input.result),
         });
     }
+    /**
+     * `complete`, with the caller's own writes committed in the same transaction.
+     *
+     * `fn` runs first and whatever it returns becomes the task's result; the
+     * settlement is the last statement in the transaction. So a lost lease — the
+     * settlement matching no row — rolls the caller's writes back with it, and
+     * there is no ordering in which the work is recorded but the task is not.
+     *
+     * The settlement runs LAST rather than checking ownership up front on purpose:
+     * the ownership predicate lives in the protocol's complete.sql, and a
+     * fail-fast pre-check here would be a second copy of it, free to drift. The
+     * cost of that choice is that a doomed attempt does its work before finding
+     * out, which is the rare path.
+     *
+     * `fn` must be replayable for the same reason `tx`'s callback must be.
+     */
+    async completeIn(input, fn) {
+        if (!this.txWithSession) {
+            throw new Error("this store cannot share a transaction with the caller — " +
+                "completeIn requires a Postgres store (see PgExecutor)");
+        }
+        return this.txWithSession(async (fetch, session) => {
+            const value = await fn(session);
+            const rows = await fetch("complete", {
+                id: input.taskId,
+                worker_id: input.workerId,
+                result: value == null ? null : dumpJson(value),
+            });
+            // Rolls back `fn`'s writes along with the settlement that did not land.
+            if (!rows.length)
+                throw new LostLease(input.taskId);
+            return { task: rowToTask(rows[0]), value };
+        });
+    }
     async fail(input) {
         let error;
         try {

package/dist/store/pg-executor.d.ts

@@ -0,0 +1,78 @@
+/**
+ * The seam between PostgresStore and whatever actually talks to Postgres.
+ *
+ * PostgresStore owns the *dialect* — the protocol's named-parameter SQL rewritten
+ * to `$n`, the migration ledger, the LISTEN policy. It does not own the
+ * *connection*. Applications that already run a Postgres driver (an ORM, a pool
+ * they size themselves) pass their own executor and cairnq joins that session
+ * instead of opening a second one, which is what makes a task's settlement
+ * commit in the same transaction as the rows the task produced.
+ *
+ * Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
+ * reference implementation over `pg`. An adapter passes rows through as its
+ * driver produced them: cairnq normalizes both column types the drivers disagree
+ * about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
+ * adapter has to reconfigure its driver — and none has to change how the
+ * application's OWN columns come back in order to satisfy cairnq.
+ */
+/** A row as the driver hands it back: column name -> value. */
+export type Row = Record<string, unknown>;
+/**
+ * Somewhere statements can run. The same shape whether it is a pool (each call
+ * on some connection) or one transaction's dedicated connection — the store's
+ * statements do not care, and this is what lets `tx` hand the same interface to
+ * its callback.
+ */
+export interface PgSession {
+    /**
+     * One parameterised statement, `$1`-style. `values` is positional and may
+     * legitimately contain nulls — a null parameter is "this filter is off" in
+     * several protocol statements, not a missing argument.
+     */
+    query(text: string, values: readonly unknown[]): Promise<Row[]>;
+    /**
+     * Parameterless SQL that may hold several statements, for migration DDL.
+     * Separate from `query` because it must go over the simple query protocol:
+     * the extended protocol a parameterised call uses accepts only one statement,
+     * and every migration is a script.
+     */
+    exec(sql: string): Promise<void>;
+}
+/** A session that can also open transactions, listen, and be shut down. */
+export interface PgExecutor extends PgSession {
+    /**
+     * Run `fn` inside one transaction on one dedicated connection, committing if
+     * it returns and rolling back if it throws. The store relies on both halves:
+     * a claim that cannot see its own `recover_leases` is a double-dispatch, and a
+     * keyed submit that commits half way poisons the key.
+     */
+    tx<T>(fn: (session: PgSession) => Promise<T>): Promise<T>;
+    /**
+     * Subscribe a dedicated connection to `channels`. Optional: an executor that
+     * omits it (or a Postgres that refuses LISTEN) costs latency, never
+     * correctness — the store falls back to plain polling, which is the contract
+     * PROTOCOL.md gives for push wakeups.
+     *
+     * Resolves with a function that stops listening. `onClose` reports a
+     * connection that dropped on its own, so the store can degrade and retry.
+     *
+     * Throw `ListenUnavailable` when this Postgres will never accept LISTEN — a
+     * transaction-mode pooler, say. Any other rejection is read as transient and
+     * retried with backoff, so a permanent condition raised as a plain Error
+     * becomes a reconnect loop that cannot succeed.
+     */
+    listen?(channels: readonly string[], onNotify: (channel: string, payload: string | undefined) => void, onClose: () => void): Promise<() => void>;
+    /**
+     * Release this executor's resources. Called by PostgresStore.close() ONLY for
+     * an executor the store created itself: an injected one belongs to the caller,
+     * whose other work would not survive cairnq closing it.
+     */
+    close(): Promise<void>;
+}
+/**
+ * LISTEN will not work on this connection, and retrying cannot change that.
+ * See `PgExecutor.listen`.
+ */
+export declare class ListenUnavailable extends Error {
+    constructor(message?: string);
+}

package/dist/store/pg-executor.js

@@ -0,0 +1,27 @@
+/**
+ * The seam between PostgresStore and whatever actually talks to Postgres.
+ *
+ * PostgresStore owns the *dialect* — the protocol's named-parameter SQL rewritten
+ * to `$n`, the migration ledger, the LISTEN policy. It does not own the
+ * *connection*. Applications that already run a Postgres driver (an ORM, a pool
+ * they size themselves) pass their own executor and cairnq joins that session
+ * instead of opening a second one, which is what makes a task's settlement
+ * commit in the same transaction as the rows the task produced.
+ *
+ * Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
+ * reference implementation over `pg`. An adapter passes rows through as its
+ * driver produced them: cairnq normalizes both column types the drivers disagree
+ * about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
+ * adapter has to reconfigure its driver — and none has to change how the
+ * application's OWN columns come back in order to satisfy cairnq.
+ */
+/**
+ * LISTEN will not work on this connection, and retrying cannot change that.
+ * See `PgExecutor.listen`.
+ */
+export class ListenUnavailable extends Error {
+    constructor(message = "LISTEN is not available on this connection") {
+        super(message);
+        this.name = "ListenUnavailable";
+    }
+}

package/dist/store/pg-pool.d.ts

@@ -0,0 +1,16 @@
+import { type PgExecutor } from "./pg-executor.js";
+/**
+ * The built-in executor: a `pg.Pool` over a libpq DSN. What `CairnQ.postgres(dsn)`
+ * uses, and the reference for what an adapter over another driver must do.
+ *
+ * Creating it does not connect — `pg.Pool` is lazy, and the store's first
+ * statement (the migration ledger) is what proves the database is reachable.
+ *
+ * `schema` puts cairnq's tables in a schema of their own rather than in whatever
+ * the connection's search_path leads with. The protocol's SQL names no schema, so
+ * this is a per-connection `search_path` and not one statement changes.
+ */
+export declare function createPoolExecutor(dsn: string, opts?: {
+    max?: number;
+    schema?: string;
+}): Promise<PgExecutor>;

package/dist/store/pg-pool.js

@@ -0,0 +1,148 @@
+import { ListenUnavailable } from "./pg-executor.js";
+// `pg` is an optional dependency: the SDK is SQLite-first, so it's loaded lazily
+// the first time a pool-backed executor is built. Absent -> a clear install hint.
+// An application that brings its own executor never reaches this.
+let pgModule = null;
+async function loadPg() {
+    if (pgModule)
+        return pgModule;
+    let mod;
+    try {
+        mod = (await import("pg"));
+    }
+    catch {
+        throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
+    }
+    const pg = (mod.default ?? mod);
+    // Deliberately NOT setting a global int8 type parser here. It would be the
+    // shortest fix for Postgres sending bigint as text, and it is what this file
+    // used to do — but pg's type parsers are process-global, so a library that
+    // installs one silently changes how the APPLICATION's own bigint columns come
+    // back. rowToTask normalizes instead, which costs nothing and leaves the
+    // caller's driver as they configured it.
+    pgModule = pg;
+    return pg;
+}
+/**
+ * Roll back on the way out of a failed transaction, without letting the rollback
+ * become the error the caller sees. A dropped connection fails both the statement
+ * and the rollback, and it is the first one that says what went wrong.
+ */
+async function rollbackQuietly(client) {
+    try {
+        await client.query("rollback");
+    }
+    catch {
+        // Already rolled back, or the connection is gone. Either way the original
+        // error is the one worth propagating.
+    }
+}
+/** The PgSession face of one pg client or pool. */
+function session(q) {
+    return {
+        async query(text, values) {
+            return (await q.query(text, values)).rows;
+        },
+        async exec(sql) {
+            // No values: pg sends this over the simple query protocol, which is what
+            // makes a multi-statement migration script legal here and not in query().
+            await q.query(sql);
+        },
+    };
+}
+/**
+ * A Postgres identifier that is safe to interpolate — cairnq quotes the schema
+ * name, and a name that could close that quote could rewrite the statement.
+ * Deliberately narrower than what Postgres accepts: a schema cairnq is asked to
+ * live in is a deployment decision, not a place to be clever.
+ */
+const PLAIN_IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/;
+/**
+ * The built-in executor: a `pg.Pool` over a libpq DSN. What `CairnQ.postgres(dsn)`
+ * uses, and the reference for what an adapter over another driver must do.
+ *
+ * Creating it does not connect — `pg.Pool` is lazy, and the store's first
+ * statement (the migration ledger) is what proves the database is reachable.
+ *
+ * `schema` puts cairnq's tables in a schema of their own rather than in whatever
+ * the connection's search_path leads with. The protocol's SQL names no schema, so
+ * this is a per-connection `search_path` and not one statement changes.
+ */
+export async function createPoolExecutor(dsn, opts = {}) {
+    const pg = await loadPg();
+    const schema = opts.schema;
+    if (schema !== undefined && !PLAIN_IDENT.test(schema)) {
+        throw new Error(`schema must be a plain identifier (letters, digits, _ and $, not starting ` +
+            `with a digit), got ${JSON.stringify(schema)}`);
+    }
+    const pool = new pg.Pool({ connectionString: dsn, max: opts.max });
+    if (schema) {
+        // Queued on the connection before it is handed out, so every statement this
+        // pool ever runs — migrations included — resolves in the right schema. Pooled
+        // connections come and go, which is why this is per-connection rather than a
+        // one-off at startup.
+        pool.on("connect", (client) => {
+            void client.query(`set search_path to "${schema}"`);
+        });
+    }
+    if (schema) {
+        // Created once, here, rather than from the connect handler (which would ask
+        // for CREATE privilege on every new connection) or from the migrations
+        // (which name no schema, by design). Without it the first `create table`
+        // fails with "no schema has been selected to create in", which says nothing
+        // about the cause. This is the one thing that makes building the executor
+        // connect; without `schema` it stays lazy.
+        try {
+            await pool.query(`create schema if not exists "${schema}"`);
+        }
+        catch (e) {
+            await pool.end();
+            throw e;
+        }
+    }
+    const poolSession = session(pool);
+    return {
+        query: poolSession.query,
+        exec: poolSession.exec,
+        async tx(fn) {
+            const client = await pool.connect();
+            try {
+                await client.query("begin");
+                const out = await fn(session(client));
+                await client.query("commit");
+                return out;
+            }
+            catch (e) {
+                await rollbackQuietly(client);
+                throw e;
+            }
+            finally {
+                client.release();
+            }
+        },
+        async listen(channels, onNotify, onClose) {
+            // Built from the raw DSN rather than taken from the pool: a listener holds
+            // its connection for its whole life, and a pooled one would be a slot the
+            // store never gives back. If `opts` ever grows connection-level settings
+            // (ssl, application_name), the listener must receive them too.
+            const client = new pg.Client({ connectionString: dsn });
+            await client.connect(); // failure here is transient: caller retries with backoff
+            client.on("notification", (msg) => onNotify(msg.channel, msg.payload));
+            // A dropped listener degrades to polling; the store reconnects on the next wake.
+            client.on("error", () => onClose());
+            try {
+                await client.query(channels.map((c) => `listen ${c}`).join("; "));
+            }
+            catch (e) {
+                void client.end().catch(() => { });
+                // Connected, but LISTEN was refused (e.g. a transaction-mode pooler) —
+                // deterministic, so tell the store not to retry.
+                throw new ListenUnavailable(e instanceof Error ? e.message : undefined);
+            }
+            return () => void client.end().catch(() => { });
+        },
+        async close() {
+            await pool.end();
+        },
+    };
+}

package/dist/store/postgres.d.ts

@@ -1,4 +1,5 @@
-import { type Fetch, type Params, TaskStore } from "./base.js";
+import { type Fetch, type Params, TaskStore, type WatchSignal } from "./base.js";
+import { type PgExecutor, type PgSession } from "./pg-executor.js";
 /**
  * The rewritten SQL and the order its `$n` slots must be filled in.
  *
@@ -28,23 +29,32 @@ export declare function toPositional(sql: string, params: Params): {
  * PostgresStore — the Postgres dialect of the shared cairnq-protocol SQL.
  *
  * Everything protocol-shaped lives in TaskStore; this file is only what Postgres
- * does differently: a `pg` Pool, `:name` -> `$n` translation, and time taken from
- * the DB clock (`now()`) instead of from the SDK, which is what makes this backend
- * multi-host — unlike SQLite it coordinates API and worker processes across
- * machines, with no shared clock to agree on. claim uses FOR UPDATE SKIP LOCKED
- * and needs no claimable_probe, because PG readers don't block writers. JSON
- * columns are jsonb (bound as JSON text, read back as objects by rowToTask).
+ * does differently: `:name` -> `$n` translation, the migration ledger, LISTEN
+ * policy, and time taken from the DB clock (`now()`) instead of from the SDK,
+ * which is what makes this backend multi-host — unlike SQLite it coordinates API
+ * and worker processes across machines, with no shared clock to agree on. claim
+ * uses FOR UPDATE SKIP LOCKED and needs no claimable_probe, because PG readers
+ * don't block writers. JSON columns are jsonb (bound as JSON text, read back as
+ * objects by rowToTask).
+ *
+ * What it deliberately does NOT own is the connection. Given a DSN it builds a
+ * `pg` pool (see pg-pool.ts); given a PgExecutor it runs inside the caller's
+ * session instead — no second driver, no second pool, and the caller's writes and
+ * cairnq's can share one transaction.
  */
 export declare class PostgresStore extends TaskStore {
-    private readonly dsn;
     private readonly opts;
-    private pool;
+    private readonly dsn;
+    /** The caller's executor, if one was injected — never closed by this store. */
+    private readonly provided;
+    /** Set once migrations have run and the protocol version checked out. */
+    private executor;
     private connecting;
     private readonly statements;
     private listener;
     private listenerConnecting;
-    /** LISTEN is off for good: the store was closed, or the server accepted a
-     * connection but refused LISTEN (e.g. a transaction-mode pooler) —
+    /** LISTEN is off for good: the store was closed, the executor does not support
+     * it, or the server refused it (e.g. a transaction-mode pooler) —
      * deterministic, so retrying would fail the same way every time. */
     private listenerUnavailable;
     /** A failure to even connect is transient (network blip, server restarting):
@@ -57,30 +67,71 @@ export declare class PostgresStore extends TaskStore {
     private readonly pendingQueues;
     /** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
     private readonly waiters;
-    constructor(dsn: string, opts?: {
+    /** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
+     * consumes the notification, a subscriber is standing and only observes. */
+    private readonly subscribers;
+    /**
+     * `source` is either a libpq connection string — this store then owns a `pg`
+     * pool and requires the optional `pg` package — or a PgExecutor the caller
+     * already has, which this store uses and never closes.
+     */
+    constructor(source: string | PgExecutor, opts?: {
         max?: number;
+        schema?: string;
     });
     connect(): Promise<void>;
     close(): Promise<void>;
     private ensure;
     private doConnect;
+    /**
+     * Refuse a connection pointed somewhere other than the deployment's cairnq.
+     *
+     * Two shapes, because `schema` means "the schema cairnq's tables live in" and
+     * cairnq can either arrange that (it built the connection) or only check it
+     * (the caller's executor did):
+     *
+     * - `schema` configured -> assert the connection actually resolves there. On
+     *   the DSN path this is a cheap self-check; on an injected executor it is the
+     *   only way to state the expectation at all.
+     * - `schema` not configured -> the dangerous case is being about to create a
+     *   SECOND installation while one already exists elsewhere in this database,
+     *   which is exactly what a mismatched pair of SDKs does. Joining an existing
+     *   installation is fine no matter what else is around, so the check is
+     *   deliberately narrow: it fires only when this schema has no cairnq and some
+     *   other schema does.
+     *
+     * That narrowness is what keeps it from crying wolf. Two applications each
+     * running their own cairnq in their own schema are legitimate; the second one
+     * to be set up trips this once, and saying `schema` explicitly — which such a
+     * deployment should be doing anyway — is both the fix and the confirmation.
+     */
+    private checkSchema;
     private applyMigrations;
     private readProtocolVersion;
     protocolVersion(): Promise<number>;
     claimWake(queues: string[], timeoutMs: number): Promise<void>;
+    protected subscribePush(onSignal: (signal: WatchSignal) => void): () => void;
+    protected warmPush(): void;
     taskDoneWake(taskId: string, timeoutMs: number): Promise<void>;
     /** A promise resolving on notification-or-timeout, deregistering either way.
      * The timer is unref'd: while a listener exists its socket keeps the process
      * alive, and dropListener wakes every waiter the moment it goes away. */
     private wakeOn;
-    /** True once the LISTEN connection is up; starts connecting it otherwise
+    /** True once the LISTEN subscription is up; starts establishing it otherwise
      * (respecting the transient-failure backoff). Callers fall back to plain
      * polling until it is ready (or forever, if it can't be established) —
      * correctness never depends on it. */
     private listenerReady;
     private startListener;
     private onNotification;
+    /** Hand a notification to every watch() subscriber. A throwing subscriber is
+     * its own problem: it must not cost the others their signal, nor take down the
+     * listener connection that delivered it. */
+    private publish;
     private dropListener;
     protected fetch(name: string, params: Params): Promise<any[]>;
     protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
+    protected txWithSession<T>(fn: (fetch: Fetch, session: PgSession) => Promise<T>): Promise<T>;
+    /** A Fetch that runs the protocol's statements on one particular session. */
+    private boundFetch;
 }