cairnq 0.1.0

package/dist/store/sqlite.js

@@ -0,0 +1,297 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import Database from "better-sqlite3";
+import { newId, nowMs } from "../ids.js";
+import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch } from "../errors.js";
+import { rowToTask } from "../models.js";
+import { loadMigrations, loadStatements } from "../sql.js";
+const SUPPORTED_PROTOCOL_MAJOR = 1;
+const LEASE_EXPIRED_ERROR = errorEnvelope({
+    type: "LeaseExpired",
+    code: "lease_expired",
+    message: "task lease expired and max attempts reached",
+    retryable: false,
+});
+// Serialized once: it's an immutable constant bound on every claim that finds work.
+const LEASE_EXPIRED_ERROR_JSON = JSON.stringify(LEASE_EXPIRED_ERROR);
+/**
+ * SQLiteStore — better-sqlite3 backend executing the shared cairnq-protocol SQL.
+ *
+ * The driver is synchronous, which suits SQLite's single writer: claim is one
+ * short transaction, the handler runs outside any transaction, and
+ * progress/heartbeat/succeed/fail are each their own short write. JS being
+ * single-threaded means sync DB calls never interleave. Cross-process contention
+ * (deployment mode B) is absorbed by busy_timeout.
+ */
+export class SQLiteStore {
+    path;
+    opts;
+    db = null;
+    stmts = {};
+    statements;
+    constructor(path, opts = {}) {
+        this.path = path;
+        this.opts = opts;
+        this.statements = loadStatements("sqlite");
+    }
+    async connect() {
+        this.ensure();
+    }
+    async close() {
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+            this.stmts = {};
+        }
+    }
+    ensure() {
+        if (this.db)
+            return this.db;
+        if (this.path !== ":memory:")
+            mkdirSync(dirname(this.path), { recursive: true });
+        const db = new Database(this.path);
+        db.pragma("journal_mode = WAL");
+        db.pragma("foreign_keys = ON");
+        db.pragma(`busy_timeout = ${this.opts.busyTimeoutMs ?? 5000}`);
+        this.applyMigrations(db);
+        for (const [name, sql] of Object.entries(this.statements)) {
+            this.stmts[name] = db.prepare(sql);
+        }
+        this.db = db;
+        this.checkVersion();
+        return db;
+    }
+    applyMigrations(db) {
+        db.exec("create table if not exists cairnq_migrations " +
+            "(name text primary key, applied_at_ms integer not null)");
+        const applied = new Set(db.prepare("select name from cairnq_migrations").all().map((r) => r.name));
+        // `or ignore`: another process may apply the same migration concurrently on a
+        // fresh shared db (mode B cold start). Migrations are idempotent.
+        const insert = db.prepare("insert or ignore into cairnq_migrations (name, applied_at_ms) values (?, ?)");
+        for (const { name, sql } of loadMigrations("sqlite")) {
+            if (applied.has(name))
+                continue;
+            db.transaction(() => {
+                db.exec(sql);
+                insert.run(name, nowMs());
+            })();
+        }
+    }
+    checkVersion() {
+        const version = this.readProtocolVersion();
+        if (version !== SUPPORTED_PROTOCOL_MAJOR) {
+            throw new ProtocolVersionMismatch(`storage protocol_version=${version}, SDK supports ${SUPPORTED_PROTOCOL_MAJOR}`);
+        }
+    }
+    readProtocolVersion() {
+        const row = this.db
+            .prepare("select value from cairnq_meta where key = 'protocol_version'")
+            .get();
+        return row ? Number(row.value) : 0;
+    }
+    async protocolVersion() {
+        this.ensure();
+        return this.readProtocolVersion();
+    }
+    all(name, params) {
+        return this.stmts[name].all(params);
+    }
+    run(name, params) {
+        this.stmts[name].run(params);
+    }
+    // An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
+    // Each statement's WHERE pins worker_id + a live lease, so 0 rows back means the
+    // lease was lost — every such write reports it the same way.
+    ownedWrite(name, taskId, params) {
+        const rows = this.all(name, params);
+        if (!rows.length)
+            throw new LostLease(taskId);
+        return rowToTask(rows[0]);
+    }
+    // ------------------------------------------------------------- client side
+    async submit(input) {
+        this.ensure();
+        const now = nowMs();
+        const id = newId("task");
+        const ins = {
+            id,
+            name: input.name,
+            queue: input.queue ?? "default",
+            payload: JSON.stringify(input.payload ?? {}),
+            metadata: JSON.stringify(input.metadata ?? {}),
+            max_attempts: input.maxAttempts ?? 3,
+            priority: input.priority ?? 0,
+            run_at_ms: now + (input.runAtDelayMs ?? 0),
+            parent_id: input.parentId ?? null,
+            root_id: input.rootId ?? id,
+            correlation_id: input.correlationId ?? null,
+            now_ms: now,
+        };
+        const key = input.key ?? null;
+        const conflict = input.conflict ?? "reuse";
+        const txn = this.db.transaction(() => {
+            if (key === null)
+                return this.all("insert_task", ins)[0];
+            const existing = this.all("get_key", { key });
+            if (existing.length) {
+                const exId = existing[0].task_id;
+                if (conflict === "reuse")
+                    return this.all("get", { id: exId })[0];
+                if (conflict === "reject")
+                    throw new AlreadyExists(key);
+                if (conflict === "replace") {
+                    this.all("cancel", { id: exId, now_ms: now });
+                    const row = this.all("insert_task", ins)[0];
+                    this.run("upsert_key", { key, task_id: id, now_ms: now });
+                    return row;
+                }
+                throw new Error(`unknown conflict strategy: ${conflict}`);
+            }
+            const row = this.all("insert_task", ins)[0];
+            this.run("upsert_key", { key, task_id: id, now_ms: now });
+            return row;
+        });
+        return rowToTask(txn.immediate());
+    }
+    async get(taskId) {
+        this.ensure();
+        const rows = this.all("get", { id: taskId });
+        return rows.length ? rowToTask(rows[0]) : null;
+    }
+    async getByKey(key) {
+        this.ensure();
+        const rows = this.all("get_by_key", { key });
+        return rows.length ? rowToTask(rows[0]) : null;
+    }
+    async list(input = {}) {
+        this.ensure();
+        const rows = this.all("list", {
+            status: input.status ?? null,
+            queue: input.queue ?? null,
+            name: input.name ?? null,
+            root_id: input.rootId ?? null,
+            correlation_id: input.correlationId ?? null,
+            limit: input.limit ?? 100,
+            offset: input.offset ?? 0,
+        });
+        return rows.map(rowToTask);
+    }
+    async cancel(taskId) {
+        this.ensure();
+        const rows = this.all("cancel", { id: taskId, now_ms: nowMs() });
+        return rows.length ? rowToTask(rows[0]) : null;
+    }
+    async cancelByKey(key) {
+        this.ensure();
+        const txn = this.db.transaction(() => {
+            const existing = this.all("get_key", { key });
+            if (!existing.length)
+                return null;
+            const rows = this.all("cancel", { id: existing[0].task_id, now_ms: nowMs() });
+            return rows.length ? rows[0] : null;
+        });
+        const row = txn.immediate();
+        return row ? rowToTask(row) : null;
+    }
+    async retry(taskId, opts = {}) {
+        this.ensure();
+        const rows = this.all("retry", {
+            id: taskId,
+            now_ms: nowMs(),
+            reset_attempt: opts.resetAttempt ? 1 : 0,
+        });
+        return rows.length ? rowToTask(rows[0]) : null;
+    }
+    async retryByKey(key, opts = {}) {
+        this.ensure();
+        const txn = this.db.transaction(() => {
+            const existing = this.all("get_key", { key });
+            if (!existing.length)
+                return null;
+            const rows = this.all("retry", {
+                id: existing[0].task_id,
+                now_ms: nowMs(),
+                reset_attempt: opts.resetAttempt ? 1 : 0,
+            });
+            return rows.length ? rows[0] : null;
+        });
+        const row = txn.immediate();
+        return row ? rowToTask(row) : null;
+    }
+    // ------------------------------------------------------------- worker side
+    async claim(input) {
+        this.ensure();
+        const now = nowMs();
+        const queues = JSON.stringify(input.queues);
+        const leaseMs = input.leaseMs ?? 30_000;
+        const limit = input.limit ?? 1;
+        // Read-only probe first: skip the write lock entirely when idle.
+        const probe = this.all("claimable_probe", { queues, now_ms: now })[0];
+        if (!probe || !probe.has_work)
+            return [];
+        const txn = this.db.transaction(() => {
+            this.all("recover_leases", {
+                now_ms: now,
+                lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
+            });
+            return this.all("claim", {
+                queues,
+                now_ms: now,
+                worker_id: input.workerId,
+                lease_until_ms: now + leaseMs,
+                limit,
+            });
+        });
+        return txn.immediate().map(rowToTask);
+    }
+    async heartbeat(input) {
+        this.ensure();
+        const now = nowMs();
+        return this.ownedWrite("heartbeat", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            now_ms: now,
+            lease_until_ms: now + (input.leaseMs ?? 30_000),
+        });
+    }
+    async progress(input) {
+        this.ensure();
+        return this.ownedWrite("progress", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            now_ms: nowMs(),
+            progress: input.progress,
+            message: input.message,
+        });
+    }
+    async succeed(input) {
+        this.ensure();
+        return this.ownedWrite("succeed", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            now_ms: nowMs(),
+            result: input.result == null ? null : JSON.stringify(input.result),
+            message: null,
+        });
+    }
+    async complete(input) {
+        this.ensure();
+        return this.ownedWrite("complete", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            now_ms: nowMs(),
+            result: input.result == null ? null : JSON.stringify(input.result),
+        });
+    }
+    async fail(input) {
+        this.ensure();
+        return this.ownedWrite("fail", input.taskId, {
+            id: input.taskId,
+            worker_id: input.workerId,
+            now_ms: nowMs(),
+            error: JSON.stringify(input.error ?? {}),
+            retryable: input.retryable === false ? 0 : 1,
+            delay_ms: input.delayMs ?? 0,
+        });
+    }
+}

package/dist/task.d.ts

@@ -0,0 +1,21 @@
+/**
+ * A typed task handle. `defineTask<Payload, Result>("name")` gives one symbol that
+ * the worker (`worker.task(def, handler)`) and the client (`tasks.submit/call(def, …)`)
+ * both reference — so the name lives in exactly one place (no string drift), editors
+ * autocomplete it and find every caller, and `call(def, …)` infers its `Result` type.
+ *
+ * It's purely opt-in: every API still accepts a plain name string, and cross-language
+ * callers keep using the string (only the name crosses the DB boundary). The `__payload`
+ * / `__result` fields are phantom type carriers — they let TypeScript infer `P`/`R` from
+ * a `TaskDef<P, R>` argument and are never set or read at runtime.
+ */
+export interface TaskDef<P = unknown, R = unknown> {
+    readonly name: string;
+    /** @internal phantom — type inference only, never present at runtime. */
+    readonly __payload?: P;
+    /** @internal phantom — type inference only, never present at runtime. */
+    readonly __result?: R;
+}
+export declare function defineTask<P = unknown, R = unknown>(name: string): TaskDef<P, R>;
+/** Resolve a name from either a plain string or a TaskDef. */
+export declare function taskName(task: string | TaskDef): string;

package/dist/task.js

@@ -0,0 +1,7 @@
+export function defineTask(name) {
+    return { name };
+}
+/** Resolve a name from either a plain string or a TaskDef. */
+export function taskName(task) {
+    return typeof task === "string" ? task : task.name;
+}

package/dist/wait.d.ts

@@ -0,0 +1,8 @@
+import { type Task } from "./models.js";
+import type { TaskStore } from "./store/base.js";
+/** Poll get() until terminal or timeout. Returns the terminal Task (any status).
+ * Throws TaskTimeout, leaving the task running. */
+export declare function pollWait(store: TaskStore, taskId: string, { timeoutMs, pollMs }: {
+    timeoutMs: number;
+    pollMs?: number;
+}): Promise<Task>;

package/dist/wait.js

@@ -0,0 +1,18 @@
+import { TaskTimeout } from "./errors.js";
+import { nowMs } from "./ids.js";
+import { isTerminal } from "./models.js";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Poll get() until terminal or timeout. Returns the terminal Task (any status).
+ * Throws TaskTimeout, leaving the task running. */
+export async function pollWait(store, taskId, { timeoutMs, pollMs = 150 }) {
+    const deadline = nowMs() + timeoutMs;
+    for (;;) {
+        const task = await store.get(taskId);
+        if (task && isTerminal(task))
+            return task;
+        const remaining = deadline - nowMs();
+        if (remaining <= 0)
+            throw new TaskTimeout(taskId);
+        await sleep(Math.min(pollMs, remaining));
+    }
+}

package/dist/worker.d.ts

@@ -0,0 +1,60 @@
+import { TaskContext } from "./context.js";
+import type { TaskStore } from "./store/base.js";
+import { type TaskDef } from "./task.js";
+export type Handler = (ctx: TaskContext, payload: any) => unknown | Promise<unknown>;
+/** Handler typed against a TaskDef<P, R>: payload is P, the return is R. */
+export type TypedHandler<P, R> = (ctx: TaskContext, payload: P) => R | Promise<R>;
+export interface WorkerOptions {
+    concurrency?: number;
+    leaseMs?: number;
+    heartbeatIntervalMs?: number;
+    pollIntervalMs?: number;
+    claimBatch?: number;
+}
+export declare class Worker {
+    private readonly store;
+    private readonly queues;
+    private readonly opts;
+    private readonly handlers;
+    private readonly workerId;
+    private stopped;
+    private stopResolvers;
+    private ownsStore;
+    constructor(store: TaskStore, queues: string[], opts?: WorkerOptions);
+    static sqlite(path: string, opts?: WorkerOptions & {
+        queues?: string[];
+        busyTimeoutMs?: number;
+    }): Worker;
+    /** Multi-host backend. `dsn` is a libpq connection string; requires the
+     * optional `pg` package. */
+    static postgres(dsn: string, opts?: WorkerOptions & {
+        queues?: string[];
+        max?: number;
+    }): Worker;
+    get id(): string;
+    task(handler: Handler): this;
+    task(name: string, handler: Handler): this;
+    task<P, R>(def: TaskDef<P, R>, handler: TypedHandler<P, R>): this;
+    stop(): void;
+    /** Close the underlying store connection. Call after run() returns. */
+    close(): Promise<void>;
+    private closeIfOwned;
+    run(opts?: {
+        concurrency?: number;
+    }): Promise<void>;
+    /** Blocking-style entry point for a standalone worker process: run until
+     * SIGINT/SIGTERM, then close the store. Use this at a script's top level;
+     * use run() / background() when you manage the event loop yourself. */
+    serve(opts?: {
+        concurrency?: number;
+    }): Promise<void>;
+    /** Run the worker in the same process for the duration of fn (deployment mode A). */
+    background<T>(fn: () => Promise<T>, opts?: {
+        concurrency?: number;
+    }): Promise<T>;
+    private execute;
+    private startHeartbeat;
+    private safeFail;
+    private sleepOrStop;
+    private installSignals;
+}

package/dist/worker.js

@@ -0,0 +1,252 @@
+import { TaskContext } from "./context.js";
+import { errorEnvelope, LostLease, TaskError } from "./errors.js";
+import { newId } from "./ids.js";
+import { SQLiteStore } from "./store/sqlite.js";
+import { PostgresStore } from "./store/postgres.js";
+import { taskName } from "./task.js";
+function exceptionEnvelope(err) {
+    const e = err;
+    return errorEnvelope({
+        type: e?.name ?? "Error",
+        code: "handler_error",
+        message: String(e?.message ?? err),
+        retryable: true,
+    });
+}
+export class Worker {
+    store;
+    queues;
+    opts;
+    handlers = new Map();
+    workerId = newId("worker");
+    stopped = false;
+    stopResolvers = [];
+    // True only when this worker created its own store (via Worker.sqlite); an
+    // injected store may be shared, so serve()/background() must not close it.
+    ownsStore = false;
+    constructor(store, queues, opts = {}) {
+        this.store = store;
+        this.queues = queues;
+        this.opts = opts;
+    }
+    static sqlite(path, opts = {}) {
+        const { queues = ["default"], busyTimeoutMs, ...rest } = opts;
+        const worker = new Worker(new SQLiteStore(path, { busyTimeoutMs }), queues, rest);
+        worker.ownsStore = true;
+        return worker;
+    }
+    /** Multi-host backend. `dsn` is a libpq connection string; requires the
+     * optional `pg` package. */
+    static postgres(dsn, opts = {}) {
+        const { queues = ["default"], max, ...rest } = opts;
+        const worker = new Worker(new PostgresStore(dsn, { max }), queues, rest);
+        worker.ownsStore = true;
+        return worker;
+    }
+    get id() {
+        return this.workerId;
+    }
+    task(arg, handler) {
+        let name;
+        let fn;
+        if (typeof arg === "function") {
+            // Bare form: worker.task(fn) — registered under the function's name.
+            fn = arg;
+            name = fn.name;
+            if (!name) {
+                throw new Error("worker.task(fn): the handler is anonymous; pass a name explicitly, " +
+                    "e.g. worker.task('summary.create', fn)");
+            }
+        }
+        else {
+            // Named string or a TaskDef — resolve the name the one way everything does.
+            name = taskName(arg);
+            fn = handler;
+        }
+        this.handlers.set(name, fn);
+        return this;
+    }
+    stop() {
+        this.stopped = true;
+        const resolvers = this.stopResolvers;
+        this.stopResolvers = [];
+        for (const r of resolvers)
+            r();
+    }
+    /** Close the underlying store connection. Call after run() returns. */
+    async close() {
+        await this.store.close();
+    }
+    // serve()/background() only close a store the worker created itself (via
+    // Worker.sqlite). An injected store may be shared with a CairnQ client, so
+    // closing it here would pull the connection out from under it.
+    async closeIfOwned() {
+        if (this.ownsStore)
+            await this.close();
+    }
+    async run(opts = {}) {
+        const concurrency = opts.concurrency ?? this.opts.concurrency ?? 1;
+        const leaseMs = this.opts.leaseMs ?? 30_000;
+        const pollMs = this.opts.pollIntervalMs ?? 500;
+        const batch = this.opts.claimBatch ?? concurrency;
+        await this.store.connect();
+        this.installSignals();
+        const running = new Set();
+        while (!this.stopped) {
+            const free = concurrency - running.size;
+            if (free <= 0) {
+                await this.sleepOrStop(5);
+                continue;
+            }
+            const claimed = await this.store.claim({
+                queues: this.queues,
+                workerId: this.workerId,
+                leaseMs,
+                limit: Math.min(batch, free),
+            });
+            if (claimed.length === 0) {
+                await this.sleepOrStop(pollMs);
+                continue;
+            }
+            for (const task of claimed) {
+                const p = this.execute(task, leaseMs).finally(() => running.delete(p));
+                running.add(p);
+            }
+        }
+        await Promise.all([...running]);
+    }
+    /** Blocking-style entry point for a standalone worker process: run until
+     * SIGINT/SIGTERM, then close the store. Use this at a script's top level;
+     * use run() / background() when you manage the event loop yourself. */
+    async serve(opts = {}) {
+        try {
+            await this.run(opts);
+        }
+        finally {
+            await this.closeIfOwned();
+        }
+    }
+    /** Run the worker in the same process for the duration of fn (deployment mode A). */
+    async background(fn, opts = {}) {
+        const runner = this.run(opts);
+        try {
+            return await fn();
+        }
+        finally {
+            this.stop();
+            await runner;
+            await this.closeIfOwned();
+        }
+    }
+    async execute(task, leaseMs) {
+        const ctx = new TaskContext(this.store, task, this.workerId, leaseMs);
+        const hb = this.startHeartbeat(ctx, leaseMs);
+        try {
+            const handler = this.handlers.get(task.name);
+            if (!handler) {
+                await this.safeFail(task.id, errorEnvelope({
+                    type: "NoHandler",
+                    code: "no_handler",
+                    message: `no handler registered for ${task.name}`,
+                    retryable: false,
+                }), false);
+                return;
+            }
+            let result;
+            try {
+                result = await handler(ctx, task.payload);
+            }
+            catch (err) {
+                if (err instanceof LostLease)
+                    return;
+                if (err instanceof TaskError) {
+                    await this.safeFail(task.id, err.envelope(), err.retryable);
+                }
+                else {
+                    await this.safeFail(task.id, exceptionEnvelope(err), true);
+                }
+                return;
+            }
+            try {
+                // complete (not succeed): finalizes as canceled if a cancel was
+                // requested while the handler ran, else succeeded.
+                await this.store.complete({ taskId: task.id, workerId: this.workerId, result });
+            }
+            catch (err) {
+                if (err instanceof LostLease)
+                    return;
+                throw err;
+            }
+        }
+        finally {
+            hb.cancel();
+            await hb.done;
+        }
+    }
+    startHeartbeat(ctx, leaseMs) {
+        let active = true;
+        let wake = null;
+        const interval = this.opts.heartbeatIntervalMs ?? Math.max(1_000, Math.floor(leaseMs / 3));
+        const done = (async () => {
+            while (active) {
+                // Cancellable sleep: cancel() resolves this immediately and clears the
+                // timer, so done never hangs on a pending timeout.
+                await new Promise((resolve) => {
+                    const timer = setTimeout(resolve, interval);
+                    wake = () => {
+                        clearTimeout(timer);
+                        resolve();
+                    };
+                });
+                wake = null;
+                if (!active)
+                    break;
+                try {
+                    await ctx.heartbeat();
+                }
+                catch (err) {
+                    if (err instanceof LostLease)
+                        break;
+                }
+            }
+        })();
+        return {
+            cancel: () => {
+                active = false;
+                if (wake)
+                    wake();
+            },
+            done,
+        };
+    }
+    async safeFail(taskId, envelope, retryable) {
+        try {
+            await this.store.fail({ taskId, workerId: this.workerId, error: envelope, retryable });
+        }
+        catch (err) {
+            if (!(err instanceof LostLease))
+                throw err;
+        }
+    }
+    sleepOrStop(ms) {
+        if (this.stopped)
+            return Promise.resolve();
+        return new Promise((resolve) => {
+            let done = false;
+            const finish = () => {
+                if (done)
+                    return;
+                done = true;
+                clearTimeout(timer);
+                resolve();
+            };
+            const timer = setTimeout(finish, ms);
+            this.stopResolvers.push(finish);
+        });
+    }
+    installSignals() {
+        const handler = () => this.stop();
+        process.once("SIGINT", handler);
+        process.once("SIGTERM", handler);
+    }
+}