cairnq 0.2.0 → 0.4.0

package/dist/_protocol/migrations/postgres/0001_init.sql

@@ -46,7 +46,9 @@ create table if not exists cairnq_tasks (
 );
 
 -- Serves the claim query: WHERE queue=? AND status='queued' ORDER BY priority
--- desc, created_at_ms asc (run_at_ms applied as a residual filter).
+-- desc, created_at_ms asc (run_at_ms applied as a residual filter). Only
+-- claim_one_queue.sql can read it in claim order; claim.sql's array-valued queue
+-- filter forces a sort, and past a few thousand queued rows a sequential scan.
 create index if not exists cairnq_tasks_claim_idx
     on cairnq_tasks (queue, status, priority desc, created_at_ms);
 create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);

package/dist/_protocol/migrations/postgres/0004_lease_index.sql

@@ -0,0 +1,16 @@
+-- Serves recover_leases.sql: find tasks whose lease expired. That statement runs
+-- on every worker's every poll, inside the claim transaction, so it is the one
+-- recovery read on the hot path. See the SQLite twin for the reasoning; it holds
+-- here too, and additionally keeps the FOR UPDATE SKIP LOCKED subquery from
+-- taking row locks it will immediately skip.
+--
+-- Not CONCURRENTLY: migrations run inside the same transaction as their
+-- bookkeeping insert (see PROTOCOL.md), and CREATE INDEX CONCURRENTLY cannot run
+-- in one. On a table this size the plain form's lock is brief; a deployment large
+-- enough to care can build it by hand ahead of the upgrade, and IF NOT EXISTS
+-- makes this migration a no-op then.
+create index if not exists cairnq_tasks_lease_idx
+    on cairnq_tasks (lease_until_ms)
+    where status = 'running' and lease_until_ms is not null;
+
+update cairnq_meta set value = '4' where key = 'schema_version';

package/dist/_protocol/migrations/postgres/0005_clear_terminal_lease.sql

@@ -0,0 +1,17 @@
+-- Make the lease invariant true of the whole table, not just of rows written from
+-- here on: `lease_until_ms is not null` if and only if `status = 'running'`.
+--
+-- succeed.sql and complete.sql used to leave the dead attempt's lease behind, so a
+-- succeeded task reported a lease into the future and nobody owned it. The crash
+-- path (recover_leases.sql) always cleared it, so the two ways out of 'running'
+-- disagreed on what a terminal row looks like. The statements now agree; this
+-- catches up the rows they already wrote.
+--
+-- Scoped to `status <> 'running'` rather than to the three terminal states: that is
+-- the invariant itself, and it also covers a 'queued' row should one ever be left
+-- holding a lease.
+update cairnq_tasks
+set lease_until_ms = null
+where status <> 'running' and lease_until_ms is not null;
+
+update cairnq_meta set value = '5' where key = 'schema_version';

package/dist/_protocol/migrations/sqlite/0001_init.sql

@@ -45,7 +45,9 @@ create table if not exists cairnq_tasks (
 
 -- Serves the claim query: WHERE queue=? AND status='queued' ORDER BY priority
 -- desc, created_at_ms asc (run_at_ms applied as a residual filter). Leading with
--- queue+status then the ORDER BY columns avoids a sort for single-queue claims.
+-- queue+status then the ORDER BY columns is what lets claim_one_queue.sql read
+-- rows in claim order; claim.sql's list-valued queue filter has to merge several
+-- ranges of this index, so it sorts instead.
 create index if not exists cairnq_tasks_claim_idx
     on cairnq_tasks (queue, status, priority desc, created_at_ms);
 create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);

package/dist/_protocol/migrations/sqlite/0004_lease_index.sql

@@ -0,0 +1,22 @@
+-- Serves recover_leases.sql: find tasks whose lease expired. That statement runs
+-- on every worker's every poll, inside the claim transaction, so it is the one
+-- recovery read on the hot path.
+--
+-- Partial, because the rows it looks for are a tiny slice of the table: only
+-- 'running' rows can hold a lease, and their count is bounded by total worker
+-- concurrency, while terminal rows accumulate until purge. A partial index stays
+-- that small no matter how large the table grows, and — unlike the plain
+-- cairnq_tasks_status_idx it replaces on this path — it carries lease_until_ms,
+-- so the expiry test is a range scan rather than a row visit per running task.
+--
+-- Both predicate terms appear verbatim in recover_leases.sql's WHERE clause:
+-- SQLite only uses a partial index when it can match each term syntactically.
+-- It also needs statistics to prefer it, which is why both SDKs run
+-- `PRAGMA optimize` when they open a database.
+--
+-- On the missing 0003 and the schema_version jump, see PROTOCOL.md §Versioning.
+create index if not exists cairnq_tasks_lease_idx
+    on cairnq_tasks (lease_until_ms)
+    where status = 'running' and lease_until_ms is not null;
+
+update cairnq_meta set value = '4' where key = 'schema_version';

package/dist/_protocol/migrations/sqlite/0005_clear_terminal_lease.sql

@@ -0,0 +1,17 @@
+-- Make the lease invariant true of the whole table, not just of rows written from
+-- here on: `lease_until_ms is not null` if and only if `status = 'running'`.
+--
+-- succeed.sql and complete.sql used to leave the dead attempt's lease behind, so a
+-- succeeded task reported a lease into the future and nobody owned it. The crash
+-- path (recover_leases.sql) always cleared it, so the two ways out of 'running'
+-- disagreed on what a terminal row looks like. The statements now agree; this
+-- catches up the rows they already wrote.
+--
+-- Scoped to `status <> 'running'` rather than to the three terminal states: that is
+-- the invariant itself, and it also covers a 'queued' row should one ever be left
+-- holding a lease.
+update cairnq_tasks
+set lease_until_ms = null
+where status <> 'running' and lease_until_ms is not null;
+
+update cairnq_meta set value = '5' where key = 'schema_version';

package/dist/_protocol/sql/postgres/claim_one_queue.sql

@@ -0,0 +1,35 @@
+-- claim, for a caller watching exactly ONE queue. Byte-for-byte claim.sql except
+-- that the queue filter is an equality on :queue instead of `= any(:queues)` — a
+-- drift-guard test asserts precisely that, so treat claim.sql as the source and
+-- re-derive this file when it changes.
+--
+-- It exists because the array form cannot be read in claim order: with ORDER BY
+-- + LIMIT over `= any(...)` the planner falls back to a sequential scan and a
+-- full sort of every claimable row (measured on 20k queued: Seq Scan 20000 rows,
+-- quicksort 1861kB), inside the transaction that holds the claim. The equality
+-- form index-scans cairnq_tasks_claim_idx and only incrementally sorts the id
+-- tie-break — 33 rows read for the same query.
+--
+-- params: queue, names (text[] or null), worker_id, lease_ms, limit
+update cairnq_tasks t
+set
+    status = 'running',
+    worker_id = :worker_id,
+    lease_until_ms = (extract(epoch from clock_timestamp()) * 1000)::bigint + :lease_ms,
+    attempt = attempt + 1,
+    updated_at_ms = (extract(epoch from clock_timestamp()) * 1000)::bigint
+from (
+    select id from cairnq_tasks
+    where status = 'queued'
+      and queue = :queue
+      and (:names::text[] is null or name = any(:names::text[]))
+      and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
+    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
+    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- the id's random half decides, stably but not in submit order.
+    order by priority desc, created_at_ms asc, id asc
+    limit :limit
+    for update skip locked
+) sel
+where t.id = sel.id
+returning t.*;

package/dist/_protocol/sql/postgres/complete.sql

@@ -8,6 +8,9 @@ set
     status = case when cancel_requested_at_ms is not null then 'canceled' else 'succeeded' end,
     result = case when cancel_requested_at_ms is not null then result else :result::jsonb end,
     progress = case when cancel_requested_at_ms is not null then progress else 1.0 end,
+    -- Terminal on both branches, so unconditional: a lease describes an attempt in
+    -- flight and this one just ended (see succeed.sql).
+    lease_until_ms = null,
     completed_at_ms = (extract(epoch from now()) * 1000)::bigint,
     updated_at_ms = (extract(epoch from now()) * 1000)::bigint
 where id = :id

package/dist/_protocol/sql/postgres/fail.sql

@@ -23,8 +23,10 @@ set
     error = :error::jsonb,
     worker_id = case when cancel_requested_at_ms is null and :retryable and attempt < max_attempts
                      then null else worker_id end,
-    lease_until_ms = case when cancel_requested_at_ms is null and :retryable and attempt < max_attempts
-                          then null else lease_until_ms end,
+    -- Unconditional, unlike worker_id above: all three branches end the attempt
+    -- that held the lease — two terminally, one to wait for redelivery — and none
+    -- of them leaves anyone owning it (see succeed.sql).
+    lease_until_ms = null,
     run_at_ms = case when cancel_requested_at_ms is null and :retryable and attempt < max_attempts
                      then (extract(epoch from now()) * 1000)::bigint + :delay_ms else run_at_ms end,
     completed_at_ms = case when cancel_requested_at_ms is null and :retryable and attempt < max_attempts

package/dist/_protocol/sql/postgres/recover_leases.sql

@@ -38,7 +38,18 @@ where id in (
     select id from cairnq_tasks
     where status = 'running'
       and lease_until_ms is not null
-      and lease_until_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
+      -- The scalar subselect is load-bearing, not noise: clock_timestamp() is
+      -- VOLATILE, so inlined here it becomes a per-row filter and this scan
+      -- degrades to reading every running task (20k in flight: Seq Scan 20000
+      -- rows, 19997 removed by filter). Wrapped, it is an InitPlan evaluated once
+      -- and usable as an index bound on cairnq_tasks_lease_idx — same 20k table,
+      -- 3 rows read. Do not "simplify" it away.
+      --
+      -- Deliberately not now(): that is STABLE and would also index, but it
+      -- freezes at BEGIN, and this runs in a transaction that may have waited on
+      -- row locks — the cutoff would then be older than the caller thinks and
+      -- leave expired leases behind for another poll.
+      and lease_until_ms <= (select (extract(epoch from clock_timestamp()) * 1000)::bigint)
     for update skip locked
 )
 returning *;

package/dist/_protocol/sql/postgres/succeed.sql

@@ -7,6 +7,10 @@ set
     result = :result::jsonb,
     progress = 1.0,
     message = coalesce(:message, message),
+    -- A lease describes an attempt in flight; this one just ended. worker_id is
+    -- what carries the audit trail (who ran it), so nothing is lost by clearing
+    -- it, and the terminal-lease invariant holds — see PROTOCOL.md §Lease model.
+    lease_until_ms = null,
     completed_at_ms = (extract(epoch from now()) * 1000)::bigint,
     updated_at_ms = (extract(epoch from now()) * 1000)::bigint
 where id = :id

package/dist/_protocol/sql/sqlite/claim_one_queue.sql

@@ -0,0 +1,36 @@
+-- claim, for a caller watching exactly ONE queue. Byte-for-byte claim.sql except
+-- that the queue filter is an equality on :queue instead of an IN over :queues —
+-- a drift-guard test asserts precisely that, so treat claim.sql as the source and
+-- re-derive this file when it changes.
+--
+-- It exists because the IN form costs a full sort. json_each() hides the list's
+-- length from the planner, so SQLite must merge several index ranges and can no
+-- longer read rows in claim order; it materializes every claimable row into a
+-- temp B-tree just to take LIMIT of them. Cost then grows with the queued
+-- backlog, inside the write transaction, on every claim: measured at 21us / 239us
+-- / 1792us for a backlog of 50 / 2000 / 20000. The equality form keeps
+-- cairnq_tasks_claim_idx in claim order, needs only a partial sort for the id
+-- tie-break, and stays flat at ~12us.
+--
+-- params: queue, names (JSON array text or null), now_ms, worker_id,
+--         lease_until_ms, limit
+update cairnq_tasks
+set
+    status = 'running',
+    worker_id = :worker_id,
+    lease_until_ms = :lease_until_ms,
+    attempt = attempt + 1,
+    updated_at_ms = :now_ms
+where id in (
+    select id from cairnq_tasks
+    where status = 'queued'
+      and queue = :queue
+      and (:names is null or name in (select value from json_each(:names)))
+      and run_at_ms <= :now_ms
+    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
+    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- the id's random half decides, stably but not in submit order.
+    order by priority desc, created_at_ms asc, id asc
+    limit :limit
+)
+returning *;

package/dist/_protocol/sql/sqlite/complete.sql

@@ -9,6 +9,9 @@ set
     status = case when cancel_requested_at_ms is not null then 'canceled' else 'succeeded' end,
     result = case when cancel_requested_at_ms is not null then result else :result end,
     progress = case when cancel_requested_at_ms is not null then progress else 1.0 end,
+    -- Terminal on both branches, so unconditional: a lease describes an attempt in
+    -- flight and this one just ended (see succeed.sql).
+    lease_until_ms = null,
     completed_at_ms = :now_ms,
     updated_at_ms = :now_ms
 where id = :id

package/dist/_protocol/sql/sqlite/fail.sql

@@ -23,8 +23,10 @@ set
     error = :error,
     worker_id = case when cancel_requested_at_ms is null and :retryable = 1 and attempt < max_attempts
                      then null else worker_id end,
-    lease_until_ms = case when cancel_requested_at_ms is null and :retryable = 1 and attempt < max_attempts
-                          then null else lease_until_ms end,
+    -- Unconditional, unlike worker_id above: all three branches end the attempt
+    -- that held the lease — two terminally, one to wait for redelivery — and none
+    -- of them leaves anyone owning it (see succeed.sql).
+    lease_until_ms = null,
     run_at_ms = case when cancel_requested_at_ms is null and :retryable = 1 and attempt < max_attempts
                      then :now_ms + :delay_ms else run_at_ms end,
     completed_at_ms = case when cancel_requested_at_ms is null and :retryable = 1 and attempt < max_attempts

package/dist/_protocol/sql/sqlite/succeed.sql

@@ -6,6 +6,10 @@ set
     result = :result,
     progress = 1.0,
     message = coalesce(:message, message),
+    -- A lease describes an attempt in flight; this one just ended. worker_id is
+    -- what carries the audit trail (who ran it), so nothing is lost by clearing
+    -- it, and the terminal-lease invariant holds — see PROTOCOL.md §Lease model.
+    lease_until_ms = null,
     completed_at_ms = :now_ms,
     updated_at_ms = :now_ms
 where id = :id

package/dist/store/base.d.ts

@@ -74,7 +74,14 @@ export declare abstract class TaskStore {
     abstract protocolVersion(): Promise<number>;
     /** Run one protocol statement outside a transaction, connecting if needed. */
     protected abstract fetch(name: string, params: Params): Promise<any[]>;
-    /** Run several statements atomically; `fn` receives a Fetch bound to the txn. */
+    /**
+     * Run several statements atomically; `fn` receives a Fetch bound to the txn.
+     *
+     * A backend may invoke `fn` more than once, retrying the transaction after a
+     * transient failure (SQLite does, on write-lock contention). So `fn` must be
+     * replayable: derive nothing inside it that the caller cannot derive twice —
+     * build ids and payloads before opening the transaction, not within it.
+     */
     protected abstract tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
     /**
      * Whether it is worth opening the claim transaction at all. SQLite gates its

package/dist/store/base.js

@@ -293,8 +293,14 @@ export class TaskStore {
      * array claims nothing.
      */
     async claim(input) {
+        // One queue is the common case and gets its own statement: a list-valued queue
+        // filter cannot be read in claim order, so the planner sorts every claimable
+        // row to take LIMIT of them, and claim's cost grows with the queued backlog
+        // while it holds the claim transaction. See claim_one_queue.sql.
+        const oneQueue = input.queues.length === 1;
         const params = {
             queues: input.queues,
+            queue: oneQueue ? input.queues[0] : null,
             names: input.names ?? null,
             worker_id: input.workerId,
             lease_ms: input.leaseMs ?? 30_000,
@@ -307,7 +313,7 @@ export class TaskStore {
         // be visible to the claim that follows, and to nobody in between.
         return this.tx(async (fetch) => {
             await fetch("recover_leases", params);
-            return (await fetch("claim", params)).map(rowToTask);
+            return (await fetch(oneQueue ? "claim_one_queue" : "claim", params)).map(rowToTask);
         });
     }
     async heartbeat(input) {

package/dist/store/sqlite.d.ts

@@ -9,17 +9,32 @@ import { type Fetch, type Params, TaskStore } from "./base.js";
  *
  * The driver being synchronous suits SQLite's single writer: claim is one short
  * transaction, the handler runs outside any transaction, and
- * progress/heartbeat/succeed/fail are each their own short write. Cross-process
- * contention is absorbed by busy_timeout.
+ * progress/heartbeat/succeed/fail are each their own short write.
+ *
+ * Cross-process contention is absorbed by retrying in JavaScript, not by
+ * busy_timeout. The two cost the same wait but not the same blocking: a nonzero
+ * busy_timeout waits *inside* the synchronous driver, so a caller that loses the
+ * write lock stalls this process's event loop for up to the whole timeout — the
+ * P99 of an HTTP server that submits tasks. Executing a statement takes
+ * microseconds; waiting for a lock takes milliseconds to seconds, and only the
+ * second part needs to happen off the thread. So busy_timeout goes to 0 (fail
+ * immediately) and the wait becomes an awaited backoff, which the event loop runs
+ * through. The budget is the same either way — `busyTimeoutMs`.
+ *
+ * The open path keeps a real busy_timeout: it is synchronous by nature (WAL
+ * switch, migrations) and happens once, under the caller's `connect()`.
  */
 export declare class SQLiteStore extends TaskStore {
     private readonly path;
-    private readonly opts;
     private db;
     private stmts;
     private readonly statements;
     /** This store's entry in `fileLocks` — see there for why it is per-database. */
     private readonly lockKey;
+    /** How long a single operation may keep retrying a lost write lock. */
+    private readonly busyBudgetMs;
+    /** When this connection may next revisit its planner statistics. */
+    private nextStatsRefreshAt;
     constructor(path: string, opts?: {
         busyTimeoutMs?: number;
     });
@@ -44,8 +59,41 @@ export declare class SQLiteStore extends TaskStore {
      */
     private bind;
     private runNow;
-    /** Serialize an operation against every other operation on this database. */
+    /** Queue an operation behind every other operation on this database. */
+    private enqueue;
+    /**
+     * Serialize an operation against this database, waiting out a lost write lock on
+     * a jittered backoff. Replaces busy_timeout's synchronous wait (see the class
+     * comment); on exhausting the budget the original SQLITE_BUSY surfaces, which is
+     * what a nonzero busy_timeout would have thrown too.
+     *
+     * Each attempt re-queues rather than backing off while holding its turn: the
+     * contention left to retry is cross-process, and under WAL a *reader* never sees
+     * SQLITE_BUSY at all — so sleeping in place would stall this process's reads
+     * (including the worker's own poll) on a lock they were never waiting for.
+     *
+     * Retrying is safe because an attempt is one statement, or one transaction that
+     * has already rolled back: nothing partially applied survives it. `fn` may
+     * therefore run more than once and must not carry effects of its own — the
+     * callers in TaskStore build their ids and payloads before opening one.
+     */
     private withLock;
+    /**
+     * Revisit this connection's planner statistics, at most once per
+     * STATS_REFRESH_INTERVAL_MS.
+     *
+     * A connection lives for days, and the statements were prepared against whatever
+     * the table looked like when it opened — a worker started against an empty
+     * database plans as if it were still empty however large the backlog grows. The
+     * prepared statements do pick the refreshed plans up: ANALYZE bumps the schema
+     * cookie, so SQLite silently re-prepares them on next use. That is what makes
+     * this worth doing rather than a restart-only concern.
+     *
+     * Queued rather than run under `withLock`: statistics are best-effort, so losing
+     * the write lock to another process should cost nothing — skip and let the next
+     * interval try, instead of spending an operation's whole retry budget on them.
+     */
+    private maybeRefreshStatistics;
     protected fetch(name: string, params: Params): Promise<any[]>;
     protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
     protected hasClaimableWork(params: Params): Promise<boolean>;

package/dist/store/sqlite.js

@@ -6,29 +6,98 @@ import { loadMigrations, loadStatements } from "../sql.js";
 import { checkProtocolVersion, statementParams, TaskStore, } from "./base.js";
 const WAL_RETRY_DELAY_MS = 50;
 const WAL_RETRY_BUDGET_MS = 5_000;
+const BUSY_RETRY_BASE_MS = 1;
+const BUSY_RETRY_MAX_DELAY_MS = 50;
+/**
+ * How often a live connection revisits its planner statistics.
+ *
+ * Bounds how long the planner can work from a stale table shape; a minute is
+ * arbitrary but small next to the days a worker holds its connection. It does not
+ * set how often an ANALYZE actually runs — SQLite decides that itself, and only
+ * once the table has diverged from its statistics by 10x, so a shorter interval
+ * costs more no-ops (a few microseconds each) rather than more analyzing.
+ */
+const STATS_REFRESH_INTERVAL_MS = 60_000;
 /** Sleep without yielding — the whole open path is synchronous already. */
 function sleepSync(ms) {
     Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
+/** Sleep by yielding to the event loop — the point of the busy retry loop. */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Whether this error is SQLite refusing to wait for the write lock.
+ *
+ * Prefix match: the code carries detail suffixes (`SQLITE_BUSY_SNAPSHOT`). Only
+ * SQLITE_BUSY qualifies — SQLITE_LOCKED is same-connection table contention,
+ * which the per-file lock prevents and a retry could not resolve anyway.
+ */
+function isBusy(err) {
+    if (!err || typeof err !== "object")
+        return false;
+    const code = err.code;
+    return typeof code === "string" && code.startsWith("SQLITE_BUSY");
+}
 /** Whether this path names an in-memory database rather than a file. */
 function isMemory(path) {
     return path === ":memory:" || path.includes("mode=memory");
 }
+/**
+ * Whether cairnq_tasks has been analyzed at all.
+ *
+ * Two steps because sqlite_stat1 does not exist until something runs ANALYZE, and
+ * querying a missing table is an error rather than an empty result.
+ */
+function hasStatistics(db) {
+    const table = db
+        .prepare("select 1 from sqlite_master where type = 'table' and name = 'sqlite_stat1'")
+        .get();
+    if (!table)
+        return false;
+    return Boolean(db.prepare("select 1 from sqlite_stat1 where tbl = 'cairnq_tasks'").get());
+}
+/**
+ * Bring cairnq_tasks' statistics up to date, cheaply enough to call on a timer.
+ *
+ * Without them the planner misreads `status = 'running'` as a large fraction of the
+ * table and passes over the partial cairnq_tasks_lease_idx that lease recovery is
+ * indexed for.
+ *
+ * The explicit bootstrap is not redundant with `PRAGMA optimize`. Before SQLite
+ * 3.46 the pragma skips a table that has no sqlite_stat1 entry entirely — no mask
+ * changes that, verified on 3.45.1 — so on those builds it can never produce the
+ * *first* statistics, and the index stays unused for the life of the database.
+ * Distro Pythons link exactly those builds (Ubuntu 24.04 ships 3.45.1), while
+ * better-sqlite3 bundles its own newer one, so this is also what keeps the two SDKs
+ * behaving alike rather than by luck of packaging.
+ *
+ * Once an entry exists, every version's pragma applies its own growth heuristic,
+ * which is the part worth deferring to: it is a few microseconds when there is
+ * nothing to do, where a bare ANALYZE would rescan the table every time.
+ */
+function refreshStatistics(db) {
+    if (hasStatistics(db))
+        db.pragma("optimize");
+    // Scoped to the one table whose shape the planner gets wrong; the key and meta
+    // tables are read by primary key, where statistics change nothing. A database
+    // this one shares with the caller's own tables is left alone.
+    else
+        db.exec("ANALYZE cairnq_tasks");
+}
 /**
  * Serializes every SQLiteStore on one database file, process-wide.
  *
  * better-sqlite3 is synchronous, and a transaction holds SQLite's write lock
- * across `await`s (the callback seam is shared with Postgres, so it is async). A
- * second connection in this process then blocks the only thread waiting for that
- * lock, and the holder can never reach COMMIT — reaching it needs the thread the
- * waiter is sitting on. busy_timeout cannot break that inversion, being one
- * thread; the wait just burns the timeout and throws "database is locked". So the
- * two must not overlap at all.
+ * across `await`s (the callback seam is shared with Postgres, so it is async). Two
+ * connections in this process would then contend for that lock the expensive way:
+ * every loser spends SQLITE_BUSY retries and backoff on a holder it could simply
+ * have queued behind.
  *
  * Keyed by database, not by store: what the lock protects is the file. Across
- * processes there is no inversion (the holder keeps its own thread) and
- * busy_timeout still applies. An in-memory database is private to one connection
- * and gets a key of its own.
+ * processes there is nothing to serialize from here — each holder has its own
+ * thread, and `withLock`'s retry absorbs that contention. An in-memory database is
+ * private to one connection and gets a key of its own.
  */
 const fileLocks = new Map();
 let memoryDbSeq = 0;
@@ -56,8 +125,7 @@ function enableWal(db) {
                 return;
         }
         catch (err) {
-            const message = String(err.message ?? err);
-            if (!/locked|busy/i.test(message))
+            if (!isBusy(err))
                 throw err;
         }
         if (Date.now() >= deadline) {
@@ -76,21 +144,36 @@ function enableWal(db) {
  *
  * The driver being synchronous suits SQLite's single writer: claim is one short
  * transaction, the handler runs outside any transaction, and
- * progress/heartbeat/succeed/fail are each their own short write. Cross-process
- * contention is absorbed by busy_timeout.
+ * progress/heartbeat/succeed/fail are each their own short write.
+ *
+ * Cross-process contention is absorbed by retrying in JavaScript, not by
+ * busy_timeout. The two cost the same wait but not the same blocking: a nonzero
+ * busy_timeout waits *inside* the synchronous driver, so a caller that loses the
+ * write lock stalls this process's event loop for up to the whole timeout — the
+ * P99 of an HTTP server that submits tasks. Executing a statement takes
+ * microseconds; waiting for a lock takes milliseconds to seconds, and only the
+ * second part needs to happen off the thread. So busy_timeout goes to 0 (fail
+ * immediately) and the wait becomes an awaited backoff, which the event loop runs
+ * through. The budget is the same either way — `busyTimeoutMs`.
+ *
+ * The open path keeps a real busy_timeout: it is synchronous by nature (WAL
+ * switch, migrations) and happens once, under the caller's `connect()`.
  */
 export class SQLiteStore extends TaskStore {
     path;
-    opts;
     db = null;
     stmts = {};
     statements;
     /** This store's entry in `fileLocks` — see there for why it is per-database. */
     lockKey;
+    /** How long a single operation may keep retrying a lost write lock. */
+    busyBudgetMs;
+    /** When this connection may next revisit its planner statistics. */
+    nextStatsRefreshAt = 0;
     constructor(path, opts = {}) {
         super();
         this.path = path;
-        this.opts = opts;
+        this.busyBudgetMs = opts.busyTimeoutMs ?? 5000;
         this.statements = loadStatements("sqlite");
         // Only a bare ":memory:" is guaranteed private to its connection, so only
         // it gets a lock of its own. A "mode=memory" URI stays path-keyed: with
@@ -118,15 +201,30 @@ export class SQLiteStore extends TaskStore {
         if (!memory)
             mkdirSync(dirname(this.path), { recursive: true });
         const db = new Database(this.path);
-        // busy_timeout first, so every later statement waits out contention instead
-        // of failing instantly.
-        db.pragma(`busy_timeout = ${this.opts.busyTimeoutMs ?? 5000}`);
+        // Only the synchronous part of the open path gets a real busy_timeout: the WAL
+        // switch and the migrations cannot await a retry. See the class comment.
+        db.pragma(`busy_timeout = ${this.busyBudgetMs}`);
         // WAL exists so several processes can share one file. An in-memory database
         // is private to this connection, so there is nothing to share or wait for.
         if (!memory)
             enableWal(db);
         db.pragma("foreign_keys = ON");
         this.applyMigrations(db);
+        // Everything past here either awaits its retry or is optional, so stop blocking.
+        db.pragma("busy_timeout = 0");
+        // Give the query planner statistics (see refreshStatistics), repeated on a timer
+        // from here on (see maybeRefreshStatistics).
+        try {
+            refreshStatistics(db);
+        }
+        catch (err) {
+            // Statistics are an optimization, never correctness, so losing them to a
+            // concurrent writer must not fail the open — the next one gets another
+            // chance. Anything else is a real fault and belongs to the caller.
+            if (!isBusy(err))
+                throw err;
+        }
+        this.nextStatsRefreshAt = Date.now() + STATS_REFRESH_INTERVAL_MS;
         for (const [name, sql] of Object.entries(this.statements)) {
             this.stmts[name] = db.prepare(sql);
         }
@@ -222,23 +320,89 @@ export class SQLiteStore extends TaskStore {
         }
         return stmt.all(bound);
     }
-    /** Serialize an operation against every other operation on this database. */
-    withLock(fn) {
+    /** Queue an operation behind every other operation on this database. */
+    enqueue(fn) {
         const previous = fileLocks.get(this.lockKey) ?? Promise.resolve();
         const run = previous.then(fn, fn);
         fileLocks.set(this.lockKey, run.then(() => undefined, () => undefined));
         return run;
     }
+    /**
+     * Serialize an operation against this database, waiting out a lost write lock on
+     * a jittered backoff. Replaces busy_timeout's synchronous wait (see the class
+     * comment); on exhausting the budget the original SQLITE_BUSY surfaces, which is
+     * what a nonzero busy_timeout would have thrown too.
+     *
+     * Each attempt re-queues rather than backing off while holding its turn: the
+     * contention left to retry is cross-process, and under WAL a *reader* never sees
+     * SQLITE_BUSY at all — so sleeping in place would stall this process's reads
+     * (including the worker's own poll) on a lock they were never waiting for.
+     *
+     * Retrying is safe because an attempt is one statement, or one transaction that
+     * has already rolled back: nothing partially applied survives it. `fn` may
+     * therefore run more than once and must not carry effects of its own — the
+     * callers in TaskStore build their ids and payloads before opening one.
+     */
+    async withLock(fn) {
+        const deadline = Date.now() + this.busyBudgetMs;
+        let delay = BUSY_RETRY_BASE_MS;
+        for (;;) {
+            try {
+                return await this.enqueue(fn);
+            }
+            catch (err) {
+                if (!isBusy(err) || Date.now() >= deadline)
+                    throw err;
+                // Jitter so several losers don't wake together and collide again.
+                await sleep(delay * (0.5 + Math.random()));
+                delay = Math.min(delay * 2, BUSY_RETRY_MAX_DELAY_MS);
+            }
+        }
+    }
+    /**
+     * Revisit this connection's planner statistics, at most once per
+     * STATS_REFRESH_INTERVAL_MS.
+     *
+     * A connection lives for days, and the statements were prepared against whatever
+     * the table looked like when it opened — a worker started against an empty
+     * database plans as if it were still empty however large the backlog grows. The
+     * prepared statements do pick the refreshed plans up: ANALYZE bumps the schema
+     * cookie, so SQLite silently re-prepares them on next use. That is what makes
+     * this worth doing rather than a restart-only concern.
+     *
+     * Queued rather than run under `withLock`: statistics are best-effort, so losing
+     * the write lock to another process should cost nothing — skip and let the next
+     * interval try, instead of spending an operation's whole retry budget on them.
+     */
+    async maybeRefreshStatistics(db) {
+        const now = Date.now();
+        if (now < this.nextStatsRefreshAt)
+            return;
+        // Claim the slot before running, not after: otherwise a burst of concurrent
+        // operations all see it due and queue an ANALYZE apiece.
+        this.nextStatsRefreshAt = now + STATS_REFRESH_INTERVAL_MS;
+        try {
+            await this.enqueue(() => refreshStatistics(db));
+        }
+        catch (err) {
+            if (!isBusy(err))
+                throw err;
+        }
+    }
     async fetch(name, params) {
-        this.ensure();
+        const db = this.ensure();
+        await this.maybeRefreshStatistics(db);
         return this.withLock(() => this.runNow(name, params));
     }
     async tx(fn) {
         const db = this.ensure();
+        await this.maybeRefreshStatistics(db);
         // BEGIN IMMEDIATE by hand rather than db.transaction(): the callback is async
         // (the seam is shared with Postgres), and better-sqlite3's wrapper only takes
         // a synchronous one. The lock above makes the manual version safe.
         return this.withLock(async () => {
+            // With busy_timeout at 0 this is where a lost write lock surfaces, and it
+            // fails before the transaction exists — so the retry re-runs `fn` cleanly.
             db.exec("BEGIN IMMEDIATE");
             try {
                 const out = await fn(async (name, params) => this.runNow(name, params));
@@ -246,11 +410,15 @@ export class SQLiteStore extends TaskStore {
                 return out;
             }
             catch (err) {
-                try {
-                    db.exec("ROLLBACK");
-                }
-                catch {
-                    // Already rolled back by SQLite (e.g. a constraint abort).
+                // Nothing to roll back when BEGIN was what failed — the common case under
+                // contention — or when SQLite already did it (a constraint abort).
+                if (db.inTransaction) {
+                    try {
+                        db.exec("ROLLBACK");
+                    }
+                    catch {
+                        // Raced with SQLite's own rollback; the transaction is gone either way.
+                    }
                 }
                 throw err;
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cairnq",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "SQLite-first, cross-language, storage-centered durable task runtime",
   "license": "MIT",
   "author": "Jannchie <jannchie@gmail.com>",
@@ -31,7 +31,7 @@
     }
   },
   "files": ["dist", "src"],
-  "engines": { "node": ">=20" },
+  "engines": { "node": ">=22" },
   "scripts": {
     "bench": "tsx bench/run.ts",
     "build": "tsc -p tsconfig.json",
@@ -39,7 +39,7 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "better-sqlite3": "^11.3.0"
+    "better-sqlite3": "^13.0.2"
   },
   "peerDependencies": {
     "pg": "^8.13.0"
@@ -55,8 +55,8 @@
     "@types/pg": "^8.11.10",
     "pg": "^8.13.0",
     "tsx": "^4.19.0",
-    "typescript": "^5.6.0",
-    "vitest": "^2.1.0"
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10"
   },
   "pnpm": {
     "onlyBuiltDependencies": ["better-sqlite3", "esbuild"]

package/src/store/base.ts

@@ -156,7 +156,14 @@ export abstract class TaskStore {
 
   /** Run one protocol statement outside a transaction, connecting if needed. */
   protected abstract fetch(name: string, params: Params): Promise<any[]>;
-  /** Run several statements atomically; `fn` receives a Fetch bound to the txn. */
+  /**
+   * Run several statements atomically; `fn` receives a Fetch bound to the txn.
+   *
+   * A backend may invoke `fn` more than once, retrying the transaction after a
+   * transient failure (SQLite does, on write-lock contention). So `fn` must be
+   * replayable: derive nothing inside it that the caller cannot derive twice —
+   * build ids and payloads before opening the transaction, not within it.
+   */
   protected abstract tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
 
   /**
@@ -378,8 +385,14 @@ export abstract class TaskStore {
     limit?: number;
     names?: string[];
   }): Promise<Task[]> {
+    // One queue is the common case and gets its own statement: a list-valued queue
+    // filter cannot be read in claim order, so the planner sorts every claimable
+    // row to take LIMIT of them, and claim's cost grows with the queued backlog
+    // while it holds the claim transaction. See claim_one_queue.sql.
+    const oneQueue = input.queues.length === 1;
     const params: Params = {
       queues: input.queues,
+      queue: oneQueue ? input.queues[0] : null,
       names: input.names ?? null,
       worker_id: input.workerId,
       lease_ms: input.leaseMs ?? 30_000,
@@ -391,7 +404,7 @@ export abstract class TaskStore {
     // be visible to the claim that follows, and to nobody in between.
     return this.tx(async (fetch) => {
       await fetch("recover_leases", params);
-      return (await fetch("claim", params)).map(rowToTask);
+      return (await fetch(oneQueue ? "claim_one_queue" : "claim", params)).map(rowToTask);
     });
   }
 

package/src/store/sqlite.ts

@@ -19,31 +19,104 @@ type Stmt = Database.Statement;
 const WAL_RETRY_DELAY_MS = 50;
 const WAL_RETRY_BUDGET_MS = 5_000;
 
+const BUSY_RETRY_BASE_MS = 1;
+const BUSY_RETRY_MAX_DELAY_MS = 50;
+
+/**
+ * How often a live connection revisits its planner statistics.
+ *
+ * Bounds how long the planner can work from a stale table shape; a minute is
+ * arbitrary but small next to the days a worker holds its connection. It does not
+ * set how often an ANALYZE actually runs — SQLite decides that itself, and only
+ * once the table has diverged from its statistics by 10x, so a shorter interval
+ * costs more no-ops (a few microseconds each) rather than more analyzing.
+ */
+const STATS_REFRESH_INTERVAL_MS = 60_000;
+
 /** Sleep without yielding — the whole open path is synchronous already. */
 function sleepSync(ms: number): void {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+/** Sleep by yielding to the event loop — the point of the busy retry loop. */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Whether this error is SQLite refusing to wait for the write lock.
+ *
+ * Prefix match: the code carries detail suffixes (`SQLITE_BUSY_SNAPSHOT`). Only
+ * SQLITE_BUSY qualifies — SQLITE_LOCKED is same-connection table contention,
+ * which the per-file lock prevents and a retry could not resolve anyway.
+ */
+function isBusy(err: unknown): boolean {
+  if (!err || typeof err !== "object") return false;
+  const code = (err as { code?: unknown }).code;
+  return typeof code === "string" && code.startsWith("SQLITE_BUSY");
+}
+
 /** Whether this path names an in-memory database rather than a file. */
 function isMemory(path: string): boolean {
   return path === ":memory:" || path.includes("mode=memory");
 }
 
+/**
+ * Whether cairnq_tasks has been analyzed at all.
+ *
+ * Two steps because sqlite_stat1 does not exist until something runs ANALYZE, and
+ * querying a missing table is an error rather than an empty result.
+ */
+function hasStatistics(db: DB): boolean {
+  const table = db
+    .prepare("select 1 from sqlite_master where type = 'table' and name = 'sqlite_stat1'")
+    .get();
+  if (!table) return false;
+  return Boolean(
+    db.prepare("select 1 from sqlite_stat1 where tbl = 'cairnq_tasks'").get(),
+  );
+}
+
+/**
+ * Bring cairnq_tasks' statistics up to date, cheaply enough to call on a timer.
+ *
+ * Without them the planner misreads `status = 'running'` as a large fraction of the
+ * table and passes over the partial cairnq_tasks_lease_idx that lease recovery is
+ * indexed for.
+ *
+ * The explicit bootstrap is not redundant with `PRAGMA optimize`. Before SQLite
+ * 3.46 the pragma skips a table that has no sqlite_stat1 entry entirely — no mask
+ * changes that, verified on 3.45.1 — so on those builds it can never produce the
+ * *first* statistics, and the index stays unused for the life of the database.
+ * Distro Pythons link exactly those builds (Ubuntu 24.04 ships 3.45.1), while
+ * better-sqlite3 bundles its own newer one, so this is also what keeps the two SDKs
+ * behaving alike rather than by luck of packaging.
+ *
+ * Once an entry exists, every version's pragma applies its own growth heuristic,
+ * which is the part worth deferring to: it is a few microseconds when there is
+ * nothing to do, where a bare ANALYZE would rescan the table every time.
+ */
+function refreshStatistics(db: DB): void {
+  if (hasStatistics(db)) db.pragma("optimize");
+  // Scoped to the one table whose shape the planner gets wrong; the key and meta
+  // tables are read by primary key, where statistics change nothing. A database
+  // this one shares with the caller's own tables is left alone.
+  else db.exec("ANALYZE cairnq_tasks");
+}
+
 /**
  * Serializes every SQLiteStore on one database file, process-wide.
  *
  * better-sqlite3 is synchronous, and a transaction holds SQLite's write lock
- * across `await`s (the callback seam is shared with Postgres, so it is async). A
- * second connection in this process then blocks the only thread waiting for that
- * lock, and the holder can never reach COMMIT — reaching it needs the thread the
- * waiter is sitting on. busy_timeout cannot break that inversion, being one
- * thread; the wait just burns the timeout and throws "database is locked". So the
- * two must not overlap at all.
+ * across `await`s (the callback seam is shared with Postgres, so it is async). Two
+ * connections in this process would then contend for that lock the expensive way:
+ * every loser spends SQLITE_BUSY retries and backoff on a holder it could simply
+ * have queued behind.
  *
  * Keyed by database, not by store: what the lock protects is the file. Across
- * processes there is no inversion (the holder keeps its own thread) and
- * busy_timeout still applies. An in-memory database is private to one connection
- * and gets a key of its own.
+ * processes there is nothing to serialize from here — each holder has its own
+ * thread, and `withLock`'s retry absorbs that contention. An in-memory database is
+ * private to one connection and gets a key of its own.
  */
 const fileLocks = new Map<string, Promise<unknown>>();
 let memoryDbSeq = 0;
@@ -70,8 +143,7 @@ function enableWal(db: DB): void {
       const rows = db.pragma("journal_mode = WAL") as { journal_mode?: string }[];
       if (rows[0]?.journal_mode?.toLowerCase() === "wal") return;
     } catch (err) {
-      const message = String((err as Error).message ?? err);
-      if (!/locked|busy/i.test(message)) throw err;
+      if (!isBusy(err)) throw err;
     }
     if (Date.now() >= deadline) {
       throw new Error(
@@ -92,8 +164,20 @@ function enableWal(db: DB): void {
  *
  * The driver being synchronous suits SQLite's single writer: claim is one short
  * transaction, the handler runs outside any transaction, and
- * progress/heartbeat/succeed/fail are each their own short write. Cross-process
- * contention is absorbed by busy_timeout.
+ * progress/heartbeat/succeed/fail are each their own short write.
+ *
+ * Cross-process contention is absorbed by retrying in JavaScript, not by
+ * busy_timeout. The two cost the same wait but not the same blocking: a nonzero
+ * busy_timeout waits *inside* the synchronous driver, so a caller that loses the
+ * write lock stalls this process's event loop for up to the whole timeout — the
+ * P99 of an HTTP server that submits tasks. Executing a statement takes
+ * microseconds; waiting for a lock takes milliseconds to seconds, and only the
+ * second part needs to happen off the thread. So busy_timeout goes to 0 (fail
+ * immediately) and the wait becomes an awaited backoff, which the event loop runs
+ * through. The budget is the same either way — `busyTimeoutMs`.
+ *
+ * The open path keeps a real busy_timeout: it is synchronous by nature (WAL
+ * switch, migrations) and happens once, under the caller's `connect()`.
  */
 export class SQLiteStore extends TaskStore {
   private db: DB | null = null;
@@ -101,12 +185,17 @@ export class SQLiteStore extends TaskStore {
   private readonly statements: Record<string, string>;
   /** This store's entry in `fileLocks` — see there for why it is per-database. */
   private readonly lockKey: string;
+  /** How long a single operation may keep retrying a lost write lock. */
+  private readonly busyBudgetMs: number;
+  /** When this connection may next revisit its planner statistics. */
+  private nextStatsRefreshAt = 0;
 
   constructor(
     private readonly path: string,
-    private readonly opts: { busyTimeoutMs?: number } = {},
+    opts: { busyTimeoutMs?: number } = {},
   ) {
     super();
+    this.busyBudgetMs = opts.busyTimeoutMs ?? 5000;
     this.statements = loadStatements("sqlite");
     // Only a bare ":memory:" is guaranteed private to its connection, so only
     // it gets a lock of its own. A "mode=memory" URI stays path-keyed: with
@@ -135,14 +224,27 @@ export class SQLiteStore extends TaskStore {
     const memory = isMemory(this.path);
     if (!memory) mkdirSync(dirname(this.path), { recursive: true });
     const db = new Database(this.path);
-    // busy_timeout first, so every later statement waits out contention instead
-    // of failing instantly.
-    db.pragma(`busy_timeout = ${this.opts.busyTimeoutMs ?? 5000}`);
+    // Only the synchronous part of the open path gets a real busy_timeout: the WAL
+    // switch and the migrations cannot await a retry. See the class comment.
+    db.pragma(`busy_timeout = ${this.busyBudgetMs}`);
     // WAL exists so several processes can share one file. An in-memory database
     // is private to this connection, so there is nothing to share or wait for.
     if (!memory) enableWal(db);
     db.pragma("foreign_keys = ON");
     this.applyMigrations(db);
+    // Everything past here either awaits its retry or is optional, so stop blocking.
+    db.pragma("busy_timeout = 0");
+    // Give the query planner statistics (see refreshStatistics), repeated on a timer
+    // from here on (see maybeRefreshStatistics).
+    try {
+      refreshStatistics(db);
+    } catch (err) {
+      // Statistics are an optimization, never correctness, so losing them to a
+      // concurrent writer must not fail the open — the next one gets another
+      // chance. Anything else is a real fault and belongs to the caller.
+      if (!isBusy(err)) throw err;
+    }
+    this.nextStatsRefreshAt = Date.now() + STATS_REFRESH_INTERVAL_MS;
     for (const [name, sql] of Object.entries(this.statements)) {
       this.stmts[name] = db.prepare(sql);
     }
@@ -247,8 +349,8 @@ export class SQLiteStore extends TaskStore {
     return stmt.all(bound) as any[];
   }
 
-  /** Serialize an operation against every other operation on this database. */
-  private withLock<T>(fn: () => T | Promise<T>): Promise<T> {
+  /** Queue an operation behind every other operation on this database. */
+  private enqueue<T>(fn: () => T | Promise<T>): Promise<T> {
     const previous = fileLocks.get(this.lockKey) ?? Promise.resolve();
     const run = previous.then(fn, fn) as Promise<T>;
     fileLocks.set(
@@ -261,27 +363,94 @@ export class SQLiteStore extends TaskStore {
     return run;
   }
 
+  /**
+   * Serialize an operation against this database, waiting out a lost write lock on
+   * a jittered backoff. Replaces busy_timeout's synchronous wait (see the class
+   * comment); on exhausting the budget the original SQLITE_BUSY surfaces, which is
+   * what a nonzero busy_timeout would have thrown too.
+   *
+   * Each attempt re-queues rather than backing off while holding its turn: the
+   * contention left to retry is cross-process, and under WAL a *reader* never sees
+   * SQLITE_BUSY at all — so sleeping in place would stall this process's reads
+   * (including the worker's own poll) on a lock they were never waiting for.
+   *
+   * Retrying is safe because an attempt is one statement, or one transaction that
+   * has already rolled back: nothing partially applied survives it. `fn` may
+   * therefore run more than once and must not carry effects of its own — the
+   * callers in TaskStore build their ids and payloads before opening one.
+   */
+  private async withLock<T>(fn: () => T | Promise<T>): Promise<T> {
+    const deadline = Date.now() + this.busyBudgetMs;
+    let delay = BUSY_RETRY_BASE_MS;
+    for (;;) {
+      try {
+        return await this.enqueue(fn);
+      } catch (err) {
+        if (!isBusy(err) || Date.now() >= deadline) throw err;
+        // Jitter so several losers don't wake together and collide again.
+        await sleep(delay * (0.5 + Math.random()));
+        delay = Math.min(delay * 2, BUSY_RETRY_MAX_DELAY_MS);
+      }
+    }
+  }
+
+  /**
+   * Revisit this connection's planner statistics, at most once per
+   * STATS_REFRESH_INTERVAL_MS.
+   *
+   * A connection lives for days, and the statements were prepared against whatever
+   * the table looked like when it opened — a worker started against an empty
+   * database plans as if it were still empty however large the backlog grows. The
+   * prepared statements do pick the refreshed plans up: ANALYZE bumps the schema
+   * cookie, so SQLite silently re-prepares them on next use. That is what makes
+   * this worth doing rather than a restart-only concern.
+   *
+   * Queued rather than run under `withLock`: statistics are best-effort, so losing
+   * the write lock to another process should cost nothing — skip and let the next
+   * interval try, instead of spending an operation's whole retry budget on them.
+   */
+  private async maybeRefreshStatistics(db: DB): Promise<void> {
+    const now = Date.now();
+    if (now < this.nextStatsRefreshAt) return;
+    // Claim the slot before running, not after: otherwise a burst of concurrent
+    // operations all see it due and queue an ANALYZE apiece.
+    this.nextStatsRefreshAt = now + STATS_REFRESH_INTERVAL_MS;
+    try {
+      await this.enqueue(() => refreshStatistics(db));
+    } catch (err) {
+      if (!isBusy(err)) throw err;
+    }
+  }
+
   protected async fetch(name: string, params: Params): Promise<any[]> {
-    this.ensure();
+    const db = this.ensure();
+    await this.maybeRefreshStatistics(db);
     return this.withLock(() => this.runNow(name, params));
   }
 
   protected async tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T> {
     const db = this.ensure();
+    await this.maybeRefreshStatistics(db);
     // BEGIN IMMEDIATE by hand rather than db.transaction(): the callback is async
     // (the seam is shared with Postgres), and better-sqlite3's wrapper only takes
     // a synchronous one. The lock above makes the manual version safe.
     return this.withLock(async () => {
+      // With busy_timeout at 0 this is where a lost write lock surfaces, and it
+      // fails before the transaction exists — so the retry re-runs `fn` cleanly.
       db.exec("BEGIN IMMEDIATE");
       try {
         const out = await fn(async (name, params) => this.runNow(name, params));
         db.exec("COMMIT");
         return out;
       } catch (err) {
-        try {
-          db.exec("ROLLBACK");
-        } catch {
-          // Already rolled back by SQLite (e.g. a constraint abort).
+        // Nothing to roll back when BEGIN was what failed — the common case under
+        // contention — or when SQLite already did it (a constraint abort).
+        if (db.inTransaction) {
+          try {
+            db.exec("ROLLBACK");
+          } catch {
+            // Raced with SQLite's own rollback; the transaction is gone either way.
+          }
         }
         throw err;
       }