cairnq 0.1.0 → 0.3.0

package/src/store/sqlite.ts

@@ -1,47 +1,210 @@
 import { mkdirSync } from "node:fs";
-import { dirname } from "node:path";
+import { dirname, resolve } from "node:path";
 
 import Database from "better-sqlite3";
 
-import { newId, nowMs } from "../ids.js";
-import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch } from "../errors.js";
-import { rowToTask, type Task } from "../models.js";
+import { nowMs } from "../ids.js";
 import { loadMigrations, loadStatements } from "../sql.js";
-import type { ListInput, SubmitInput, TaskStore } from "./base.js";
+import {
+  checkProtocolVersion,
+  type Fetch,
+  type Params,
+  statementParams,
+  TaskStore,
+} from "./base.js";
 
 type DB = Database.Database;
 type Stmt = Database.Statement;
 
-const SUPPORTED_PROTOCOL_MAJOR = 1;
+const WAL_RETRY_DELAY_MS = 50;
+const WAL_RETRY_BUDGET_MS = 5_000;
 
-const LEASE_EXPIRED_ERROR = errorEnvelope({
-  type: "LeaseExpired",
-  code: "lease_expired",
-  message: "task lease expired and max attempts reached",
-  retryable: false,
-});
-// Serialized once: it's an immutable constant bound on every claim that finds work.
-const LEASE_EXPIRED_ERROR_JSON = JSON.stringify(LEASE_EXPIRED_ERROR);
+const BUSY_RETRY_BASE_MS = 1;
+const BUSY_RETRY_MAX_DELAY_MS = 50;
 
 /**
- * SQLiteStore — better-sqlite3 backend executing the shared cairnq-protocol SQL.
+ * How often a live connection revisits its planner statistics.
  *
- * The driver is synchronous, which suits SQLite's single writer: claim is one
- * short transaction, the handler runs outside any transaction, and
- * progress/heartbeat/succeed/fail are each their own short write. JS being
- * single-threaded means sync DB calls never interleave. Cross-process contention
- * (deployment mode B) is absorbed by busy_timeout.
+ * Bounds how long the planner can work from a stale table shape; a minute is
+ * arbitrary but small next to the days a worker holds its connection. It does not
+ * set how often an ANALYZE actually runs — SQLite decides that itself, and only
+ * when a table has outgrown its statistics by roughly 24x, so a shorter interval
+ * costs more no-ops (a few microseconds each) rather than more analyzing.
  */
-export class SQLiteStore implements TaskStore {
+const STATS_REFRESH_INTERVAL_MS = 60_000;
+
+/** Sleep without yielding — the whole open path is synchronous already. */
+function sleepSync(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/** Sleep by yielding to the event loop — the point of the busy retry loop. */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Whether this error is SQLite refusing to wait for the write lock.
+ *
+ * Prefix match: the code carries detail suffixes (`SQLITE_BUSY_SNAPSHOT`). Only
+ * SQLITE_BUSY qualifies — SQLITE_LOCKED is same-connection table contention,
+ * which the per-file lock prevents and a retry could not resolve anyway.
+ */
+function isBusy(err: unknown): boolean {
+  if (!err || typeof err !== "object") return false;
+  const code = (err as { code?: unknown }).code;
+  return typeof code === "string" && code.startsWith("SQLITE_BUSY");
+}
+
+/** Whether this path names an in-memory database rather than a file. */
+function isMemory(path: string): boolean {
+  return path === ":memory:" || path.includes("mode=memory");
+}
+
+/**
+ * Whether cairnq_tasks has been analyzed at all.
+ *
+ * Two steps because sqlite_stat1 does not exist until something runs ANALYZE, and
+ * querying a missing table is an error rather than an empty result.
+ */
+function hasStatistics(db: DB): boolean {
+  const table = db
+    .prepare("select 1 from sqlite_master where type = 'table' and name = 'sqlite_stat1'")
+    .get();
+  if (!table) return false;
+  return Boolean(
+    db.prepare("select 1 from sqlite_stat1 where tbl = 'cairnq_tasks'").get(),
+  );
+}
+
+/**
+ * Bring cairnq_tasks' statistics up to date, cheaply enough to call on a timer.
+ *
+ * Without them the planner misreads `status = 'running'` as a large fraction of the
+ * table and passes over the partial cairnq_tasks_lease_idx that lease recovery is
+ * indexed for.
+ *
+ * The explicit bootstrap is not redundant with `PRAGMA optimize`. Before SQLite
+ * 3.46 the pragma skips a table that has no sqlite_stat1 entry entirely — no mask
+ * changes that, verified on 3.45.1 — so on those builds it can never produce the
+ * *first* statistics, and the index stays unused for the life of the database.
+ * Distro Pythons link exactly those builds (Ubuntu 24.04 ships 3.45.1), while
+ * better-sqlite3 bundles its own newer one, so this is also what keeps the two SDKs
+ * behaving alike rather than by luck of packaging.
+ *
+ * Once an entry exists, every version's pragma applies its own growth heuristic,
+ * which is the part worth deferring to: it is a few microseconds when there is
+ * nothing to do, where a bare ANALYZE would rescan the table every time.
+ */
+function refreshStatistics(db: DB): void {
+  if (hasStatistics(db)) db.pragma("optimize");
+  // Scoped to the one table whose shape the planner gets wrong; the key and meta
+  // tables are read by primary key, where statistics change nothing. A database
+  // this one shares with the caller's own tables is left alone.
+  else db.exec("ANALYZE cairnq_tasks");
+}
+
+/**
+ * Serializes every SQLiteStore on one database file, process-wide.
+ *
+ * better-sqlite3 is synchronous, and a transaction holds SQLite's write lock
+ * across `await`s (the callback seam is shared with Postgres, so it is async). Two
+ * connections in this process would then contend for that lock the expensive way:
+ * every loser spends SQLITE_BUSY retries and backoff on a holder it could simply
+ * have queued behind.
+ *
+ * Keyed by database, not by store: what the lock protects is the file. Across
+ * processes there is nothing to serialize from here — each holder has its own
+ * thread, and `withLock`'s retry absorbs that contention. An in-memory database is
+ * private to one connection and gets a key of its own.
+ */
+const fileLocks = new Map<string, Promise<unknown>>();
+let memoryDbSeq = 0;
+
+/**
+ * Put the database in WAL mode, waiting out a concurrent cold start.
+ *
+ * journal_mode is a persistent property of the file, so only the first connection
+ * to a new database actually switches it — and that switch needs an exclusive
+ * lock. `busy_timeout` does not cover it: SQLite returns SQLITE_BUSY for a
+ * journal_mode change rather than invoking the busy handler, so several processes
+ * opening the same new database at once would otherwise get an instant "database
+ * is locked". Retry briefly instead; the window is only as long as one other
+ * opener's switch.
+ *
+ * Callers must skip in-memory databases: those report journal_mode = "memory" and
+ * can never be WAL, so waiting for one is waiting for something that will not
+ * happen.
+ */
+function enableWal(db: DB): void {
+  const deadline = Date.now() + WAL_RETRY_BUDGET_MS;
+  for (;;) {
+    try {
+      const rows = db.pragma("journal_mode = WAL") as { journal_mode?: string }[];
+      if (rows[0]?.journal_mode?.toLowerCase() === "wal") return;
+    } catch (err) {
+      if (!isBusy(err)) throw err;
+    }
+    if (Date.now() >= deadline) {
+      throw new Error(
+        "could not switch the database to WAL mode: it stayed locked by another connection",
+      );
+    }
+    sleepSync(WAL_RETRY_DELAY_MS);
+  }
+}
+
+/**
+ * SQLiteStore — the SQLite dialect of the shared cairnq-protocol SQL.
+ *
+ * Everything protocol-shaped lives in TaskStore; this file is only what SQLite
+ * does differently: better-sqlite3's synchronous driver, BEGIN IMMEDIATE
+ * transactions, a read-only probe in front of the write lock, and time supplied
+ * by the SDK (`:now_ms`) rather than by the database.
+ *
+ * The driver being synchronous suits SQLite's single writer: claim is one short
+ * transaction, the handler runs outside any transaction, and
+ * progress/heartbeat/succeed/fail are each their own short write.
+ *
+ * Cross-process contention is absorbed by retrying in JavaScript, not by
+ * busy_timeout. The two cost the same wait but not the same blocking: a nonzero
+ * busy_timeout waits *inside* the synchronous driver, so a caller that loses the
+ * write lock stalls this process's event loop for up to the whole timeout — the
+ * P99 of an HTTP server that submits tasks. Executing a statement takes
+ * microseconds; waiting for a lock takes milliseconds to seconds, and only the
+ * second part needs to happen off the thread. So busy_timeout goes to 0 (fail
+ * immediately) and the wait becomes an awaited backoff, which the event loop runs
+ * through. The budget is the same either way — `busyTimeoutMs`.
+ *
+ * The open path keeps a real busy_timeout: it is synchronous by nature (WAL
+ * switch, migrations) and happens once, under the caller's `connect()`.
+ */
+export class SQLiteStore extends TaskStore {
   private db: DB | null = null;
   private stmts: Record<string, Stmt> = {};
   private readonly statements: Record<string, string>;
+  /** This store's entry in `fileLocks` — see there for why it is per-database. */
+  private readonly lockKey: string;
+  /** How long a single operation may keep retrying a lost write lock. */
+  private readonly busyBudgetMs: number;
+  /** When this connection may next revisit its planner statistics. */
+  private nextStatsRefreshAt = 0;
 
   constructor(
     private readonly path: string,
-    private readonly opts: { busyTimeoutMs?: number } = {},
+    opts: { busyTimeoutMs?: number } = {},
   ) {
+    super();
+    this.busyBudgetMs = opts.busyTimeoutMs ?? 5000;
     this.statements = loadStatements("sqlite");
+    // Only a bare ":memory:" is guaranteed private to its connection, so only
+    // it gets a lock of its own. A "mode=memory" URI stays path-keyed: with
+    // cache=shared it names ONE shared database, and on a build without URI
+    // filenames it is a literal file — in both cases two stores on that string
+    // must share a lock. Over-serializing a private URI-memory database is
+    // harmless; skipping the lock on a shared one is the deadlock this map
+    // exists to prevent.
+    this.lockKey = path === ":memory:" ? `memory#${memoryDbSeq++}` : resolve(path);
   }
 
   async connect(): Promise<void> {
@@ -58,17 +221,35 @@ export class SQLiteStore implements TaskStore {
 
   private ensure(): DB {
     if (this.db) return this.db;
-    if (this.path !== ":memory:") mkdirSync(dirname(this.path), { recursive: true });
+    const memory = isMemory(this.path);
+    if (!memory) mkdirSync(dirname(this.path), { recursive: true });
     const db = new Database(this.path);
-    db.pragma("journal_mode = WAL");
+    // Only the synchronous part of the open path gets a real busy_timeout: the WAL
+    // switch and the migrations cannot await a retry. See the class comment.
+    db.pragma(`busy_timeout = ${this.busyBudgetMs}`);
+    // WAL exists so several processes can share one file. An in-memory database
+    // is private to this connection, so there is nothing to share or wait for.
+    if (!memory) enableWal(db);
     db.pragma("foreign_keys = ON");
-    db.pragma(`busy_timeout = ${this.opts.busyTimeoutMs ?? 5000}`);
     this.applyMigrations(db);
+    // Everything past here either awaits its retry or is optional, so stop blocking.
+    db.pragma("busy_timeout = 0");
+    // Give the query planner statistics (see refreshStatistics), repeated on a timer
+    // from here on (see maybeRefreshStatistics).
+    try {
+      refreshStatistics(db);
+    } catch (err) {
+      // Statistics are an optimization, never correctness, so losing them to a
+      // concurrent writer must not fail the open — the next one gets another
+      // chance. Anything else is a real fault and belongs to the caller.
+      if (!isBusy(err)) throw err;
+    }
+    this.nextStatsRefreshAt = Date.now() + STATS_REFRESH_INTERVAL_MS;
     for (const [name, sql] of Object.entries(this.statements)) {
       this.stmts[name] = db.prepare(sql);
     }
     this.db = db;
-    this.checkVersion();
+    checkProtocolVersion(this.readProtocolVersion());
     return db;
   }
 
@@ -77,275 +258,209 @@ export class SQLiteStore implements TaskStore {
       "create table if not exists cairnq_migrations " +
         "(name text primary key, applied_at_ms integer not null)",
     );
-    const applied = new Set(
-      (db.prepare("select name from cairnq_migrations").all() as { name: string }[]).map(
-        (r) => r.name,
-      ),
-    );
-    // `or ignore`: another process may apply the same migration concurrently on a
-    // fresh shared db (mode B cold start). Migrations are idempotent.
+    const isApplied = db.prepare("select 1 from cairnq_migrations where name = ?");
     const insert = db.prepare(
-      "insert or ignore into cairnq_migrations (name, applied_at_ms) values (?, ?)",
+      "insert into cairnq_migrations (name, applied_at_ms) values (?, ?)",
     );
     for (const { name, sql } of loadMigrations("sqlite")) {
-      if (applied.has(name)) continue;
+      // Check and apply under one write lock. Two processes cold-starting on a
+      // shared database would otherwise both see a migration as unapplied and
+      // both run it — harmless for the idempotent ones, not for a future ALTER.
+      // `immediate` takes the write lock up front; the loser sees it applied.
       db.transaction(() => {
+        if (isApplied.get(name)) return;
         db.exec(sql);
         insert.run(name, nowMs());
-      })();
-    }
-  }
-
-  private checkVersion(): void {
-    const version = this.readProtocolVersion();
-    if (version !== SUPPORTED_PROTOCOL_MAJOR) {
-      throw new ProtocolVersionMismatch(
-        `storage protocol_version=${version}, SDK supports ${SUPPORTED_PROTOCOL_MAJOR}`,
-      );
+      }).immediate();
     }
   }
 
   private readProtocolVersion(): number {
-    const row = this.db!
-      .prepare("select value from cairnq_meta where key = 'protocol_version'")
-      .get() as { value: string } | undefined;
-    return row ? Number(row.value) : 0;
+    const rows = this.runNow("protocol_version", {});
+    return rows.length ? Number(rows[0].value) : 0;
   }
 
   async protocolVersion(): Promise<number> {
     this.ensure();
-    return this.readProtocolVersion();
-  }
-
-  private all(name: string, params: Record<string, unknown>): any[] {
-    return this.stmts[name].all(params) as any[];
+    // Under the store lock: this public read must not slip a statement into
+    // another operation's open transaction on the shared connection.
+    return this.withLock(() => this.readProtocolVersion());
   }
 
-  private run(name: string, params: Record<string, unknown>): void {
-    this.stmts[name].run(params);
-  }
-
-  // An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
-  // Each statement's WHERE pins worker_id + a live lease, so 0 rows back means the
-  // lease was lost — every such write reports it the same way.
-  private ownedWrite(name: string, taskId: string, params: Record<string, unknown>): Task {
-    const rows = this.all(name, params);
-    if (!rows.length) throw new LostLease(taskId);
-    return rowToTask(rows[0]);
-  }
-
-  // ------------------------------------------------------------- client side
-  async submit(input: SubmitInput): Promise<Task> {
-    this.ensure();
+  // ------------------------------------------------------------ dialect seam
+  /**
+   * Adapt the dialect-neutral parameters to what this statement binds.
+   *
+   * SQLite statements carry no DB clock, so every absolute `*_ms` is derived here
+   * from one `now`, and booleans cross as 0/1. The result is narrowed to the
+   * names the SQL actually uses, which is what makes it safe for a caller to pass
+   * one superset of parameters for both dialects.
+   *
+   * Each derivation writes a name Postgres does not use (`lease_until_ms` from
+   * `lease_ms`, and so on), so a statement binds one or the other, never both —
+   * which is why the derived values can be computed unconditionally and left for
+   * the narrowing step to discard.
+   */
+  private bind(sql: string, params: Params): Params {
     const now = nowMs();
-    const id = newId("task");
-    const ins = {
-      id,
-      name: input.name,
-      queue: input.queue ?? "default",
-      payload: JSON.stringify(input.payload ?? {}),
-      metadata: JSON.stringify(input.metadata ?? {}),
-      max_attempts: input.maxAttempts ?? 3,
-      priority: input.priority ?? 0,
-      run_at_ms: now + (input.runAtDelayMs ?? 0),
-      parent_id: input.parentId ?? null,
-      root_id: input.rootId ?? id,
-      correlation_id: input.correlationId ?? null,
-      now_ms: now,
-    };
-    const key = input.key ?? null;
-    const conflict = input.conflict ?? "reuse";
-
-    const txn = this.db!.transaction(() => {
-      if (key === null) return this.all("insert_task", ins)[0];
-      const existing = this.all("get_key", { key }) as { task_id: string }[];
-      if (existing.length) {
-        const exId = existing[0].task_id;
-        if (conflict === "reuse") return this.all("get", { id: exId })[0];
-        if (conflict === "reject") throw new AlreadyExists(key);
-        if (conflict === "replace") {
-          this.all("cancel", { id: exId, now_ms: now });
-          const row = this.all("insert_task", ins)[0];
-          this.run("upsert_key", { key, task_id: id, now_ms: now });
-          return row;
-        }
-        throw new Error(`unknown conflict strategy: ${conflict}`);
+    const bound: Params = {};
+    for (const name of statementParams(sql)) {
+      switch (name) {
+        case "now_ms":
+          bound[name] = now;
+          break;
+        case "lease_until_ms":
+          bound[name] = now + (params.lease_ms as number);
+          break;
+        case "run_at_ms":
+          bound[name] = now + (params.delay_ms as number);
+          break;
+        case "before_ms":
+          bound[name] = now - (params.older_than_ms as number);
+          break;
+        case "queues":
+          bound[name] = JSON.stringify(params.queues);
+          break;
+        case "names":
+          // json_each needs a JSON array; null stays null so the SQL's
+          // `:names is null` arm means "no filter".
+          bound[name] = params.names == null ? null : JSON.stringify(params.names);
+          break;
+        case "retryable":
+        case "reset_attempt":
+          bound[name] = params[name] ? 1 : 0;
+          break;
+        default:
+          bound[name] = params[name];
       }
-      const row = this.all("insert_task", ins)[0];
-      this.run("upsert_key", { key, task_id: id, now_ms: now });
-      return row;
-    });
-    return rowToTask(txn.immediate());
-  }
-
-  async get(taskId: string): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("get", { id: taskId });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async getByKey(key: string): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("get_by_key", { key });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async list(input: ListInput = {}): Promise<Task[]> {
-    this.ensure();
-    const rows = this.all("list", {
-      status: input.status ?? null,
-      queue: input.queue ?? null,
-      name: input.name ?? null,
-      root_id: input.rootId ?? null,
-      correlation_id: input.correlationId ?? null,
-      limit: input.limit ?? 100,
-      offset: input.offset ?? 0,
-    });
-    return rows.map(rowToTask);
-  }
-
-  async cancel(taskId: string): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("cancel", { id: taskId, now_ms: nowMs() });
-    return rows.length ? rowToTask(rows[0]) : null;
-  }
-
-  async cancelByKey(key: string): Promise<Task | null> {
-    this.ensure();
-    const txn = this.db!.transaction(() => {
-      const existing = this.all("get_key", { key }) as { task_id: string }[];
-      if (!existing.length) return null;
-      const rows = this.all("cancel", { id: existing[0].task_id, now_ms: nowMs() });
-      return rows.length ? rows[0] : null;
-    });
-    const row = txn.immediate();
-    return row ? rowToTask(row) : null;
-  }
-
-  async retry(taskId: string, opts: { resetAttempt?: boolean } = {}): Promise<Task | null> {
-    this.ensure();
-    const rows = this.all("retry", {
-      id: taskId,
-      now_ms: nowMs(),
-      reset_attempt: opts.resetAttempt ? 1 : 0,
-    });
-    return rows.length ? rowToTask(rows[0]) : null;
+    }
+    return bound;
   }
 
-  async retryByKey(key: string, opts: { resetAttempt?: boolean } = {}): Promise<Task | null> {
-    this.ensure();
-    const txn = this.db!.transaction(() => {
-      const existing = this.all("get_key", { key }) as { task_id: string }[];
-      if (!existing.length) return null;
-      const rows = this.all("retry", {
-        id: existing[0].task_id,
-        now_ms: nowMs(),
-        reset_attempt: opts.resetAttempt ? 1 : 0,
-      });
-      return rows.length ? rows[0] : null;
-    });
-    const row = txn.immediate();
-    return row ? rowToTask(row) : null;
+  private runNow(name: string, params: Params): any[] {
+    const stmt = this.stmts[name];
+    const bound = this.bind(this.statements[name], params);
+    // Nearly every protocol statement ends in RETURNING; upsert_key does not, and
+    // better-sqlite3 refuses .all() on a statement that yields no rows.
+    if (!stmt.reader) {
+      stmt.run(bound);
+      return [];
+    }
+    return stmt.all(bound) as any[];
   }
 
-  // ------------------------------------------------------------- worker side
-  async claim(input: {
-    queues: string[];
-    workerId: string;
-    leaseMs?: number;
-    limit?: number;
-  }): Promise<Task[]> {
-    this.ensure();
-    const now = nowMs();
-    const queues = JSON.stringify(input.queues);
-    const leaseMs = input.leaseMs ?? 30_000;
-    const limit = input.limit ?? 1;
-    // Read-only probe first: skip the write lock entirely when idle.
-    const probe = this.all("claimable_probe", { queues, now_ms: now })[0] as { has_work: number };
-    if (!probe || !probe.has_work) return [];
-    const txn = this.db!.transaction(() => {
-      this.all("recover_leases", {
-        now_ms: now,
-        lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
-      });
-      return this.all("claim", {
-        queues,
-        now_ms: now,
-        worker_id: input.workerId,
-        lease_until_ms: now + leaseMs,
-        limit,
-      });
-    });
-    return (txn.immediate() as any[]).map(rowToTask);
+  /** Queue an operation behind every other operation on this database. */
+  private enqueue<T>(fn: () => T | Promise<T>): Promise<T> {
+    const previous = fileLocks.get(this.lockKey) ?? Promise.resolve();
+    const run = previous.then(fn, fn) as Promise<T>;
+    fileLocks.set(
+      this.lockKey,
+      run.then(
+        () => undefined,
+        () => undefined,
+      ),
+    );
+    return run;
   }
 
-  async heartbeat(input: {
-    taskId: string;
-    workerId: string;
-    leaseMs?: number;
-  }): Promise<Task> {
-    this.ensure();
-    const now = nowMs();
-    return this.ownedWrite("heartbeat", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: now,
-      lease_until_ms: now + (input.leaseMs ?? 30_000),
-    });
+  /**
+   * Serialize an operation against this database, waiting out a lost write lock on
+   * a jittered backoff. Replaces busy_timeout's synchronous wait (see the class
+   * comment); on exhausting the budget the original SQLITE_BUSY surfaces, which is
+   * what a nonzero busy_timeout would have thrown too.
+   *
+   * Each attempt re-queues rather than backing off while holding its turn: the
+   * contention left to retry is cross-process, and under WAL a *reader* never sees
+   * SQLITE_BUSY at all — so sleeping in place would stall this process's reads
+   * (including the worker's own poll) on a lock they were never waiting for.
+   *
+   * Retrying is safe because an attempt is one statement, or one transaction that
+   * has already rolled back: nothing partially applied survives it. `fn` may
+   * therefore run more than once and must not carry effects of its own — the
+   * callers in TaskStore build their ids and payloads before opening one.
+   */
+  private async withLock<T>(fn: () => T | Promise<T>): Promise<T> {
+    const deadline = Date.now() + this.busyBudgetMs;
+    let delay = BUSY_RETRY_BASE_MS;
+    for (;;) {
+      try {
+        return await this.enqueue(fn);
+      } catch (err) {
+        if (!isBusy(err) || Date.now() >= deadline) throw err;
+        // Jitter so several losers don't wake together and collide again.
+        await sleep(delay * (0.5 + Math.random()));
+        delay = Math.min(delay * 2, BUSY_RETRY_MAX_DELAY_MS);
+      }
+    }
   }
 
-  async progress(input: {
-    taskId: string;
-    workerId: string;
-    progress: number | null;
-    message: string | null;
-  }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("progress", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      progress: input.progress,
-      message: input.message,
-    });
+  /**
+   * Revisit this connection's planner statistics, at most once per
+   * STATS_REFRESH_INTERVAL_MS.
+   *
+   * A connection lives for days, and the statements were prepared against whatever
+   * the table looked like when it opened — a worker started against an empty
+   * database plans as if it were still empty however large the backlog grows. The
+   * prepared statements do pick the refreshed plans up: ANALYZE bumps the schema
+   * cookie, so SQLite silently re-prepares them on next use. That is what makes
+   * this worth doing rather than a restart-only concern.
+   *
+   * Queued rather than run under `withLock`: statistics are best-effort, so losing
+   * the write lock to another process should cost nothing — skip and let the next
+   * interval try, instead of spending an operation's whole retry budget on them.
+   */
+  private async maybeRefreshStatistics(db: DB): Promise<void> {
+    const now = Date.now();
+    if (now < this.nextStatsRefreshAt) return;
+    // Claim the slot before running, not after: otherwise a burst of concurrent
+    // operations all see it due and queue an ANALYZE apiece.
+    this.nextStatsRefreshAt = now + STATS_REFRESH_INTERVAL_MS;
+    try {
+      await this.enqueue(() => refreshStatistics(db));
+    } catch (err) {
+      if (!isBusy(err)) throw err;
+    }
   }
 
-  async succeed(input: { taskId: string; workerId: string; result: unknown }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("succeed", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      result: input.result == null ? null : JSON.stringify(input.result),
-      message: null,
-    });
+  protected async fetch(name: string, params: Params): Promise<any[]> {
+    const db = this.ensure();
+    await this.maybeRefreshStatistics(db);
+    return this.withLock(() => this.runNow(name, params));
   }
 
-  async complete(input: { taskId: string; workerId: string; result: unknown }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("complete", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      result: input.result == null ? null : JSON.stringify(input.result),
+  protected async tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T> {
+    const db = this.ensure();
+    await this.maybeRefreshStatistics(db);
+    // BEGIN IMMEDIATE by hand rather than db.transaction(): the callback is async
+    // (the seam is shared with Postgres), and better-sqlite3's wrapper only takes
+    // a synchronous one. The lock above makes the manual version safe.
+    return this.withLock(async () => {
+      // With busy_timeout at 0 this is where a lost write lock surfaces, and it
+      // fails before the transaction exists — so the retry re-runs `fn` cleanly.
+      db.exec("BEGIN IMMEDIATE");
+      try {
+        const out = await fn(async (name, params) => this.runNow(name, params));
+        db.exec("COMMIT");
+        return out;
+      } catch (err) {
+        // Nothing to roll back when BEGIN was what failed — the common case under
+        // contention — or when SQLite already did it (a constraint abort).
+        if (db.inTransaction) {
+          try {
+            db.exec("ROLLBACK");
+          } catch {
+            // Raced with SQLite's own rollback; the transaction is gone either way.
+          }
+        }
+        throw err;
+      }
     });
   }
 
-  async fail(input: {
-    taskId: string;
-    workerId: string;
-    error: unknown;
-    retryable?: boolean;
-    delayMs?: number;
-  }): Promise<Task> {
-    this.ensure();
-    return this.ownedWrite("fail", input.taskId, {
-      id: input.taskId,
-      worker_id: input.workerId,
-      now_ms: nowMs(),
-      error: JSON.stringify(input.error ?? {}),
-      retryable: input.retryable === false ? 0 : 1,
-      delay_ms: input.delayMs ?? 0,
-    });
+  protected async hasClaimableWork(params: Params): Promise<boolean> {
+    // Read-only probe first: an idle worker never takes SQLite's single write
+    // lock, so idle workers don't serialize against each other.
+    const rows = await this.fetch("claimable_probe", params);
+    return Boolean(rows[0]?.has_work);
   }
 }