cairnq 0.1.0 → 0.2.0

package/src/store/sqlite.ts

@@ -1,47 +1,121 @@
 import { mkdirSync } from "node:fs";
-import { dirname } from "node:path";
+import { dirname, resolve } from "node:path";
 
 import Database from "better-sqlite3";
 
-import { newId, nowMs } from "../ids.js";
-import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch } from "../errors.js";
-import { rowToTask, type Task } from "../models.js";
+import { nowMs } from "../ids.js";
 import { loadMigrations, loadStatements } from "../sql.js";
-import type { ListInput, SubmitInput, TaskStore } from "./base.js";
+import {
+  checkProtocolVersion,
+  type Fetch,
+  type Params,
+  statementParams,
+  TaskStore,
+} from "./base.js";
 
 type DB = Database.Database;
 type Stmt = Database.Statement;
 
-const SUPPORTED_PROTOCOL_MAJOR = 1;
+const WAL_RETRY_DELAY_MS = 50;
+const WAL_RETRY_BUDGET_MS = 5_000;
 
-const LEASE_EXPIRED_ERROR = errorEnvelope({
-  type: "LeaseExpired",
-  code: "lease_expired",
-  message: "task lease expired and max attempts reached",
-  retryable: false,
-});
-// Serialized once: it's an immutable constant bound on every claim that finds work.
-const LEASE_EXPIRED_ERROR_JSON = JSON.stringify(LEASE_EXPIRED_ERROR);
+/** Sleep without yielding — the whole open path is synchronous already. */
+function sleepSync(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/** Whether this path names an in-memory database rather than a file. */
+function isMemory(path: string): boolean {
+  return path === ":memory:" || path.includes("mode=memory");
+}
 
 /**
- * SQLiteStore — better-sqlite3 backend executing the shared cairnq-protocol SQL.
+ * Serializes every SQLiteStore on one database file, process-wide.
  *
- * The driver is synchronous, which suits SQLite's single writer: claim is one
- * short transaction, the handler runs outside any transaction, and
- * progress/heartbeat/succeed/fail are each their own short write. JS being
- * single-threaded means sync DB calls never interleave. Cross-process contention
- * (deployment mode B) is absorbed by busy_timeout.
+ * better-sqlite3 is synchronous, and a transaction holds SQLite's write lock
+ * across `await`s (the callback seam is shared with Postgres, so it is async). A
+ * second connection in this process then blocks the only thread waiting for that
+ * lock, and the holder can never reach COMMIT — reaching it needs the thread the
+ * waiter is sitting on. busy_timeout cannot break that inversion, being one
+ * thread; the wait just burns the timeout and throws "database is locked". So the
+ * two must not overlap at all.
+ *
+ * Keyed by database, not by store: what the lock protects is the file. Across
+ * processes there is no inversion (the holder keeps its own thread) and
+ * busy_timeout still applies. An in-memory database is private to one connection
+ * and gets a key of its own.
  */
-export class SQLiteStore implements TaskStore {
+const fileLocks = new Map<string, Promise<unknown>>();
+let memoryDbSeq = 0;
+
+/**
+ * Put the database in WAL mode, waiting out a concurrent cold start.
+ *
+ * journal_mode is a persistent property of the file, so only the first connection
+ * to a new database actually switches it — and that switch needs an exclusive
+ * lock. `busy_timeout` does not cover it: SQLite returns SQLITE_BUSY for a
+ * journal_mode change rather than invoking the busy handler, so several processes
+ * opening the same new database at once would otherwise get an instant "database
+ * is locked". Retry briefly instead; the window is only as long as one other
+ * opener's switch.
+ *
+ * Callers must skip in-memory databases: those report journal_mode = "memory" and
+ * can never be WAL, so waiting for one is waiting for something that will not
+ * happen.
+ */
+function enableWal(db: DB): void {
+  const deadline = Date.now() + WAL_RETRY_BUDGET_MS;
+  for (;;) {
+    try {
+      const rows = db.pragma("journal_mode = WAL") as { journal_mode?: string }[];
+      if (rows[0]?.journal_mode?.toLowerCase() === "wal") return;
+    } catch (err) {
+      const message = String((err as Error).message ?? err);
+      if (!/locked|busy/i.test(message)) throw err;
+    }
+    if (Date.now() >= deadline) {
+      throw new Error(
+        "could not switch the database to WAL mode: it stayed locked by another connection",
+      );
+    }
+    sleepSync(WAL_RETRY_DELAY_MS);
+  }
+}
+
+/**
+ * SQLiteStore — the SQLite dialect of the shared cairnq-protocol SQL.
+ *
+ * Everything protocol-shaped lives in TaskStore; this file is only what SQLite
+ * does differently: better-sqlite3's synchronous driver, BEGIN IMMEDIATE
+ * transactions, a read-only probe in front of the write lock, and time supplied
+ * by the SDK (`:now_ms`) rather than by the database.
+ *
+ * The driver being synchronous suits SQLite's single writer: claim is one short
+ * transaction, the handler runs outside any transaction, and
+ * progress/heartbeat/succeed/fail are each their own short write. Cross-process
+ * contention is absorbed by busy_timeout.
+ */
+export class SQLiteStore extends TaskStore {
   private db: DB | null = null;
   private stmts: Record<string, Stmt> = {};
   private readonly statements: Record<string, string>;
+  /** This store's entry in `fileLocks` — see there for why it is per-database. */
+  private readonly lockKey: string;
 
   constructor(
     private readonly path: string,
     private readonly opts: { busyTimeoutMs?: number } = {},
   ) {
+    super();
     this.statements = loadStatements("sqlite");
+    // Only a bare ":memory:" is guaranteed private to its connection, so only
+    // it gets a lock of its own. A "mode=memory" URI stays path-keyed: with
+    // cache=shared it names ONE shared database, and on a build without URI
+    // filenames it is a literal file — in both cases two stores on that string
+    // must share a lock. Over-serializing a private URI-memory database is
+    // harmless; skipping the lock on a shared one is the deadlock this map
+    // exists to prevent.
+    this.lockKey = path === ":memory:" ? `memory#${memoryDbSeq++}` : resolve(path);
   }
 
   async connect(): Promise<void> {
@@ -58,17 +132,22 @@ export class SQLiteStore implements TaskStore {
 
   private ensure(): DB {
     if (this.db) return this.db;
-    if (this.path !== ":memory:") mkdirSync(dirname(this.path), { recursive: true });
+    const memory = isMemory(this.path);
+    if (!memory) mkdirSync(dirname(this.path), { recursive: true });
     const db = new Database(this.path);
-    db.pragma("journal_mode = WAL");
-    db.pragma("foreign_keys = ON");
+    // busy_timeout first, so every later statement waits out contention instead
+    // of failing instantly.
     db.pragma(`busy_timeout = ${this.opts.busyTimeoutMs ?? 5000}`);
+    // WAL exists so several processes can share one file. An in-memory database
+    // is private to this connection, so there is nothing to share or wait for.
+    if (!memory) enableWal(db);
+    db.pragma("foreign_keys = ON");
     this.applyMigrations(db);
     for (const [name, sql] of Object.entries(this.statements)) {
       this.stmts[name] = db.prepare(sql);
     }
     this.db = db;
-    this.checkVersion();
+    checkProtocolVersion(this.readProtocolVersion());
     return db;
   }
 
@@ -77,275 +156,142 @@ export class SQLiteStore implements TaskStore {
       "create table if not exists cairnq_migrations " +
         "(name text primary key, applied_at_ms integer not null)",
     );
-    const applied = new Set(
-      (db.prepare("select name from cairnq_migrations").all() as { name: string }[]).map(
-        (r) => r.name,
-      ),
-    );
-    // `or ignore`: another process may apply the same migration concurrently on a
-    // fresh shared db (mode B cold start). Migrations are idempotent.
+    const isApplied = db.prepare("select 1 from cairnq_migrations where name = ?");
     const insert = db.prepare(
-      "insert or ignore into cairnq_migrations (name, applied_at_ms) values (?, ?)",
+      "insert into cairnq_migrations (name, applied_at_ms) values (?, ?)",
     );
     for (const { name, sql } of loadMigrations("sqlite")) {
-      if (applied.has(name)) continue;
+      // Check and apply under one write lock. Two processes cold-starting on a
+      // shared database would otherwise both see a migration as unapplied and
+      // both run it — harmless for the idempotent ones, not for a future ALTER.
+      // `immediate` takes the write lock up front; the loser sees it applied.
       db.transaction(() => {
+        if (isApplied.get(name)) return;
         db.exec(sql);
         insert.run(name, nowMs());
-      })();
-    }
-  }
-
-  private checkVersion(): void {
-    const version = this.readProtocolVersion();
-    if (version !== SUPPORTED_PROTOCOL_MAJOR) {
-      throw new ProtocolVersionMismatch(
-        `storage protocol_version=${version}, SDK supports ${SUPPORTED_PROTOCOL_MAJOR}`,
-      );
+      }).immediate();
     }
   }
 
   private readProtocolVersion(): number {
-    const row = this.db!
-      .prepare("select value from cairnq_meta where key = 'protocol_version'")
-      .get() as { value: string } | undefined;
-    return row ? Number(row.value) : 0;
+    const rows = this.runNow("protocol_version", {});
+    return rows.length ? Number(rows[0].value) : 0;
   }
 
   async protocolVersion(): Promise<number> {
     this.ensure();
-    return this.readProtocolVersion();
-  }
-
-  private all(name: string, params: Record<string, unknown>): any[] {
-    return this.stmts[name].all(params) as any[];
-  }
-
-  private run(name: string, params: Record<string, unknown>): void {
-    this.stmts[name].run(params);
+    // Under the store lock: this public read must not slip a statement into
+    // another operation's open transaction on the shared connection.
+    return this.withLock(() => this.readProtocolVersion());
   }
 
-  // An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
-  // Each statement's WHERE pins worker_id + a live lease, so 0 rows back means the
-  // lease was lost — every such write reports it the same way.
-  private ownedWrite(name: string, taskId: string, params: Record<string, unknown>): Task {
-    const rows = this.all(name, params);
-    if (!rows.length) throw new LostLease(taskId);
-    return rowToTask(rows[0]);
-  }
-
-  // ------------------------------------------------------------- client side
-  async submit(input: SubmitInput): Promise<Task> {
-    this.ensure();
+  // ------------------------------------------------------------ dialect seam
+  /**
+   * Adapt the dialect-neutral parameters to what this statement binds.
+   *
+   * SQLite statements carry no DB clock, so every absolute `*_ms` is derived here
+   * from one `now`, and booleans cross as 0/1. The result is narrowed to the
+   * names the SQL actually uses, which is what makes it safe for a caller to pass
+   * one superset of parameters for both dialects.
+   *
+   * Each derivation writes a name Postgres does not use (`lease_until_ms` from
+   * `lease_ms`, and so on), so a statement binds one or the other, never both —
+   * which is why the derived values can be computed unconditionally and left for
+   * the narrowing step to discard.
+   */
+  private bind(sql: string, params: Params): Params {
     const now = nowMs();
-    const id = newId("task");
-    const ins = {
-      id,
-      name: input.name,
-      queue: input.queue ?? "default",
-      payload: JSON.stringify(input.payload ?? {}),
-      metadata: JSON.stringify(input.metadata ?? {}),
-      max_attempts: input.maxAttempts ?? 3,
-      priority: input.priority ?? 0,
-      run_at_ms: now + (input.runAtDelayMs ?? 0),
-      parent_id: input.parentId ?? null,
-      root_id: input.rootId ?? id,
-      correlation_id: input.correlationId ?? null,
-      now_ms: now,
-    };
-    const key = input.key ?? null;
-    const conflict = input.conflict ?? "reuse";
-
-    const txn = this.db!.transaction(() => {
-      if (key === null) return this.all("insert_task", ins)[0];
-      const existing = this.all("get_key", { key }) as { task_id: string }[];
-      if (existing.length) {
-        const exId = existing[0].task_id;
-        if (conflict === "reuse") return this.all("get", { id: exId })[0];
-        if (conflict === "reject") throw new AlreadyExists(key);
-        if (conflict === "replace") {
-          this.all("cancel", { id: exId, now_ms: now });
-          const row = this.all("insert_task", ins)[0];
-          this.run("upsert_key", { key, task_id: id, now_ms: now });
-          return row;
-        }
-        throw new Error(`unknown conflict strategy: ${conflict}`);
+    const bound: Params = {};
+    for (const name of statementParams(sql)) {
+      switch (name) {
+        case "now_ms":
+          bound[name] = now;
+          break;
+        case "lease_until_ms":
+          bound[name] = now + (params.lease_ms as number);
+          break;
+        case "run_at_ms":
+          bound[name] = now + (params.delay_ms as number);
+          break;
+        case "before_ms":
+          bound[name] = now - (params.older_than_ms as number);
+          break;
+        case "queues":
+          bound[name] = JSON.stringify(params.queues);
+          break;
+        case "names":
+          // json_each needs a JSON array; null stays null so the SQL's
+          // `:names is null` arm means "no filter".
+          bound[name] = params.names == null ? null : JSON.stringify(params.names);
+          break;
+        case "retryable":
+        case "reset_attempt":
+          bound[name] = params[name] ? 1 : 0;
+          break;
+        default:
+          bound[name] = params[name];
       }
-      const row = this.all("insert_task", ins)[0];
-      this.run("upsert_key", { key, task_id: id, now_ms: now });
-      return row;
-    });
-    return rowToTask(txn.immediate());
-  }
-
-  async get(taskId: string): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("get", { id: taskId });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async getByKey(key: string): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("get_by_key", { key });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async list(input: ListInput = {}): Promise<Task[]> {
-    this.ensure();
-    const rows = this.all("list", {
-      status: input.status ?? null,
-      queue: input.queue ?? null,
-      name: input.name ?? null,
-      root_id: input.rootId ?? null,
-      correlation_id: input.correlationId ?? null,
-      limit: input.limit ?? 100,
-      offset: input.offset ?? 0,
-    });
-    return rows.map(rowToTask);
-  }
-
-  async cancel(taskId: string): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("cancel", { id: taskId, now_ms: nowMs() });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async cancelByKey(key: string): Promise<Task | null> {
-    this.ensure();
-    const txn = this.db!.transaction(() => {
-      const existing = this.all("get_key", { key }) as { task_id: string }[];
-      if (!existing.length) return null;
-      const rows = this.all("cancel", { id: existing[0].task_id, now_ms: nowMs() });
-      return rows.length ? rows[0] : null;
-    });
-    const row = txn.immediate();
-    return row ? rowToTask(row) : null;
-  }
-
-  async retry(taskId: string, opts: { resetAttempt?: boolean } = {}): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("retry", {
-      id: taskId,
-      now_ms: nowMs(),
-      reset_attempt: opts.resetAttempt ? 1 : 0,
-    });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async retryByKey(key: string, opts: { resetAttempt?: boolean } = {}): Promise<Task | null> {
-    this.ensure();
-    const txn = this.db!.transaction(() => {
-      const existing = this.all("get_key", { key }) as { task_id: string }[];
-      if (!existing.length) return null;
-      const rows = this.all("retry", {
-        id: existing[0].task_id,
-        now_ms: nowMs(),
-        reset_attempt: opts.resetAttempt ? 1 : 0,
-      });
-      return rows.length ? rows[0] : null;
-    });
-    const row = txn.immediate();
-    return row ? rowToTask(row) : null;
-  }
-
-  // ------------------------------------------------------------- worker side
-  async claim(input: {
-    queues: string[];
-    workerId: string;
-    leaseMs?: number;
-    limit?: number;
-  }): Promise<Task[]> {
-    this.ensure();
-    const now = nowMs();
-    const queues = JSON.stringify(input.queues);
-    const leaseMs = input.leaseMs ?? 30_000;
-    const limit = input.limit ?? 1;
-    // Read-only probe first: skip the write lock entirely when idle.
-    const probe = this.all("claimable_probe", { queues, now_ms: now })[0] as { has_work: number };
-    if (!probe || !probe.has_work) return [];
-    const txn = this.db!.transaction(() => {
-      this.all("recover_leases", {
-        now_ms: now,
-        lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
-      });
-      return this.all("claim", {
-        queues,
-        now_ms: now,
-        worker_id: input.workerId,
-        lease_until_ms: now + leaseMs,
-        limit,
-      });
-    });
-    return (txn.immediate() as any[]).map(rowToTask);
+    }
+    return bound;
   }
 
-  async heartbeat(input: {
-    taskId: string;
-    workerId: string;
-    leaseMs?: number;
-  }): Promise<Task> {
-    this.ensure();
-    const now = nowMs();
-    return this.ownedWrite("heartbeat", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: now,
-      lease_until_ms: now + (input.leaseMs ?? 30_000),
-    });
+  private runNow(name: string, params: Params): any[] {
+    const stmt = this.stmts[name];
+    const bound = this.bind(this.statements[name], params);
+    // Nearly every protocol statement ends in RETURNING; upsert_key does not, and
+    // better-sqlite3 refuses .all() on a statement that yields no rows.
+    if (!stmt.reader) {
+      stmt.run(bound);
+      return [];
+    }
+    return stmt.all(bound) as any[];
   }
 
-  async progress(input: {
-    taskId: string;
-    workerId: string;
-    progress: number | null;
-    message: string | null;
-  }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("progress", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      progress: input.progress,
-      message: input.message,
-    });
+  /** Serialize an operation against every other operation on this database. */
+  private withLock<T>(fn: () => T | Promise<T>): Promise<T> {
+    const previous = fileLocks.get(this.lockKey) ?? Promise.resolve();
+    const run = previous.then(fn, fn) as Promise<T>;
+    fileLocks.set(
+      this.lockKey,
+      run.then(
+        () => undefined,
+        () => undefined,
+      ),
+    );
+    return run;
   }
 
-  async succeed(input: { taskId: string; workerId: string; result: unknown }): Promise<Task> {
+  protected async fetch(name: string, params: Params): Promise<any[]> {
     this.ensure();
-    return this.ownedWrite("succeed", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      result: input.result == null ? null : JSON.stringify(input.result),
-      message: null,
-    });
+    return this.withLock(() => this.runNow(name, params));
   }
 
-  async complete(input: { taskId: string; workerId: string; result: unknown }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("complete", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      result: input.result == null ? null : JSON.stringify(input.result),
+  protected async tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T> {
+    const db = this.ensure();
+    // BEGIN IMMEDIATE by hand rather than db.transaction(): the callback is async
+    // (the seam is shared with Postgres), and better-sqlite3's wrapper only takes
+    // a synchronous one. The lock above makes the manual version safe.
+    return this.withLock(async () => {
+      db.exec("BEGIN IMMEDIATE");
+      try {
+        const out = await fn(async (name, params) => this.runNow(name, params));
+        db.exec("COMMIT");
+        return out;
+      } catch (err) {
+        try {
+          db.exec("ROLLBACK");
+        } catch {
+          // Already rolled back by SQLite (e.g. a constraint abort).
+        }
+        throw err;
+      }
     });
   }
 
-  async fail(input: {
-    taskId: string;
-    workerId: string;
-    error: unknown;
-    retryable?: boolean;
-    delayMs?: number;
-  }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("fail", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      error: JSON.stringify(input.error ?? {}),
-      retryable: input.retryable === false ? 0 : 1,
-      delay_ms: input.delayMs ?? 0,
-    });
+  protected async hasClaimableWork(params: Params): Promise<boolean> {
+    // Read-only probe first: an idle worker never takes SQLite's single write
+    // lock, so idle workers don't serialize against each other.
+    const rows = await this.fetch("claimable_probe", params);
+    return Boolean(rows[0]?.has_work);
   }
 }

package/src/wait.ts

@@ -3,21 +3,44 @@ import { nowMs } from "./ids.js";
 import { isTerminal, type Task } from "./models.js";
 import type { TaskStore } from "./store/base.js";
 
-const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+export const DEFAULT_POLL_MS = 100;
+export const MAX_POLL_MS = 500;
+const GROWTH = 1.5;
+
+/**
+ * Grow the polling interval towards the ceiling.
+ *
+ * wait() has no idea whether the task takes 50ms or an hour. Starting tight keeps
+ * short tasks snappy; growing keeps a long wait from costing a read every 100ms
+ * for its whole duration. The +1 keeps truncation from pinning tiny intervals:
+ * Math.floor(1 * 1.5) === 1 would otherwise never grow past 1.
+ */
+export function nextPollMs(current: number, maxMs: number): number {
+  return Math.min(maxMs, Math.max(current + 1, Math.floor(current * GROWTH)));
+}
 
 /** Poll get() until terminal or timeout. Returns the terminal Task (any status).
- * Throws TaskTimeout, leaving the task running. */
+ * Throws TaskTimeout, leaving the task running. `pollMs` is the *first* interval;
+ * it backs off towards `maxPollMs`. */
 export async function pollWait(
   store: TaskStore,
   taskId: string,
-  { timeoutMs, pollMs = 150 }: { timeoutMs: number; pollMs?: number },
+  {
+    timeoutMs,
+    pollMs = DEFAULT_POLL_MS,
+    maxPollMs = MAX_POLL_MS,
+  }: { timeoutMs: number; pollMs?: number; maxPollMs?: number },
 ): Promise<Task> {
   const deadline = nowMs() + timeoutMs;
+  let interval = pollMs;
   for (;;) {
     const task = await store.get(taskId);
     if (task && isTerminal(task)) return task;
     const remaining = deadline - nowMs();
-    if (remaining <= 0) throw new TaskTimeout(taskId);
-    await sleep(Math.min(pollMs, remaining));
+    if (remaining <= 0) throw new TaskTimeout(taskId, { timeoutMs, task });
+    // A store with a push channel (Postgres) cuts the sleep short when the task
+    // goes terminal; the re-get above stays the source of truth either way.
+    await store.taskDoneWake(taskId, Math.min(interval, remaining));
+    interval = nextPollMs(interval, maxPollMs);
   }
 }