cairnq 0.12.0 → 0.13.0

package/dist/_protocol/migrations/postgres/0010_drop_status_index.sql

@@ -0,0 +1,29 @@
+-- Drop cairnq_tasks_status_idx, which nothing can use.
+--
+-- 0001 shipped it for list's status filter. 0007 later added
+-- (status, completed_at_ms) for purge, and a one-column index on the same
+-- leading column is strictly contained in that: any lookup the narrow one could
+-- serve, the wider one serves from its prefix. Confirmed on the statements as
+-- they run — with the status filter specialized to an equality, SQLite picks
+-- cairnq_tasks_status_completed_idx for list and for purge, and never the
+-- narrow index, on a 20k-row table with statistics.
+--
+-- Until specialization existed neither was reachable at all, so this was
+-- invisible: both indexes looked equally unused, and dropping either looked
+-- equally safe or unsafe. It is only once the filters reach an index that one
+-- of them is demonstrably the one reached.
+--
+-- The write cost is what makes it worth removing rather than leaving: every
+-- insert and every status transition — submit, claim, each settle, each retry —
+-- maintains it, which is the hottest write path there is, for a structure no
+-- read consults. The other three of 0001's filter indexes (name, root_id,
+-- correlation_id) stay: each is the only index that serves its filter, and each
+-- is demonstrably read now.
+--
+-- Dropping an index needs no CONCURRENTLY on either dialect and holds a lock
+-- only long enough to unlink it, so unlike 0008 this is not an upgrade window
+-- to plan around. An older SDK is unaffected: it never named the index, and the
+-- statement that used to want it could not reach it anyway.
+drop index if exists cairnq_tasks_status_idx;
+
+update cairnq_meta set value = '10' where key = 'schema_version';

package/dist/_protocol/migrations/sqlite/0010_drop_status_index.sql

@@ -0,0 +1,29 @@
+-- Drop cairnq_tasks_status_idx, which nothing can use.
+--
+-- 0001 shipped it for list's status filter. 0007 later added
+-- (status, completed_at_ms) for purge, and a one-column index on the same
+-- leading column is strictly contained in that: any lookup the narrow one could
+-- serve, the wider one serves from its prefix. Confirmed on the statements as
+-- they run — with the status filter specialized to an equality, SQLite picks
+-- cairnq_tasks_status_completed_idx for list and for purge, and never the
+-- narrow index, on a 20k-row table with statistics.
+--
+-- Until specialization existed neither was reachable at all, so this was
+-- invisible: both indexes looked equally unused, and dropping either looked
+-- equally safe or unsafe. It is only once the filters reach an index that one
+-- of them is demonstrably the one reached.
+--
+-- The write cost is what makes it worth removing rather than leaving: every
+-- insert and every status transition — submit, claim, each settle, each retry —
+-- maintains it, which is the hottest write path there is, for a structure no
+-- read consults. The other three of 0001's filter indexes (name, root_id,
+-- correlation_id) stay: each is the only index that serves its filter, and each
+-- is demonstrably read now.
+--
+-- Dropping an index needs no CONCURRENTLY on either dialect and holds a lock
+-- only long enough to unlink it, so unlike 0008 this is not an upgrade window
+-- to plan around. An older SDK is unaffected: it never named the index, and the
+-- statement that used to want it could not reach it anyway.
+drop index if exists cairnq_tasks_status_idx;
+
+update cairnq_meta set value = '10' where key = 'schema_version';

package/dist/_protocol/sql/postgres/purge.sql

@@ -21,6 +21,20 @@
 -- durable job's log kept for a week. Migration 0009 adds the index that makes
 -- the queue filter read only its own queue's rows rather than skipping past
 -- every other queue's.
+-- Three specializations exist for the two filters that DO have indexes —
+-- purge_one_queue.sql, purge_one_status.sql, purge_one_queue_one_status.sql —
+-- and the SDK picks one per call. This file's optional form is the one it uses
+-- when neither filter is set, because `(:p is null or col = :p)` is planned
+-- before the parameter has a value: SQLite must plan both branches, reaches no
+-- index at all, and walks every row past the cutoff in completion order.
+--
+-- Both dialects ship every variant. Postgres does not need them — it re-plans
+-- with the parameter values for a statement's first executions and folds the
+-- null branch away — but a caller that had to know which dialect indexes which
+-- form would be a worse contract than four extra files.
+--
+-- :name has no specialization: no index covers it, so it is a residual predicate
+-- either way and an equality form would buy nothing.
 -- params: older_than_ms, queue, status, name, limit
 delete from cairnq_tasks
 where id in (

package/dist/_protocol/sql/postgres/stats.sql

@@ -13,6 +13,8 @@
 -- counted, which is the whole queue, terminal rows included. That is fine for a
 -- dashboard and wrong for a poll loop — queue_depth.sql is the bounded question,
 -- and the one to ask on an interval.
+-- stats_one_queue.sql is the equality form the SDK uses when a queue is
+-- named; see purge.sql for why the optional filter here cannot be indexed.
 -- params: queue
 select queue, status, count(*) as count
 from cairnq_tasks

package/dist/_protocol/sql/sqlite/purge.sql

@@ -15,6 +15,20 @@
 -- result read once and a durable job's log kept for a week. Migration 0009 adds
 -- the index that makes the queue filter read only its own queue's rows rather
 -- than skipping past every other queue's.
+-- Three specializations exist for the two filters that DO have indexes —
+-- purge_one_queue.sql, purge_one_status.sql, purge_one_queue_one_status.sql —
+-- and the SDK picks one per call. This file's optional form is the one it uses
+-- when neither filter is set, because `(:p is null or col = :p)` is planned
+-- before the parameter has a value: SQLite must plan both branches, reaches no
+-- index at all, and walks every row past the cutoff in completion order.
+--
+-- Both dialects ship every variant. Postgres does not need them — it re-plans
+-- with the parameter values for a statement's first executions and folds the
+-- null branch away — but a caller that had to know which dialect indexes which
+-- form would be a worse contract than four extra files.
+--
+-- :name has no specialization: no index covers it, so it is a residual predicate
+-- either way and an equality form would buy nothing.
 -- params: before_ms, queue, status, name, limit
 delete from cairnq_tasks
 where id in (

package/dist/_protocol/sql/sqlite/stats.sql

@@ -13,6 +13,8 @@
 -- counted, which is the whole queue, terminal rows included. That is fine for a
 -- dashboard and wrong for a poll loop — queue_depth.sql is the bounded question,
 -- and the one to ask on an interval.
+-- stats_one_queue.sql is the equality form the SDK uses when a queue is
+-- named; see purge.sql for why the optional filter here cannot be indexed.
 -- params: queue
 select queue, status, count(*) as count
 from cairnq_tasks

package/dist/store/base.d.ts

@@ -70,6 +70,24 @@ export declare const LEASE_EXPIRED_ERROR_JSON: string;
 export declare const COMMENT: RegExp;
 /** A `:name` placeholder. The lookbehind spares Postgres `::type` casts. */
 export declare const NAMED: RegExp;
+/**
+ * The statement as it should run for these arguments: every optional filter the
+ * caller actually supplied rewritten to an equality.
+ *
+ * `(:p is null or col = :p)` cannot use an index. SQLite plans a statement when
+ * it is prepared, before any parameter has a value, so it must plan for both
+ * branches and settles for a scan; that is not a tuning detail but the whole
+ * difference between `list(root_id=…)` seeking cairnq_tasks_root_idx and reading
+ * the table. Postgres re-plans with the values for a statement's first
+ * executions and folds the branch away on its own, so this is a no-op there —
+ * but it costs nothing, and one behaviour is easier to reason about than two.
+ *
+ * Filters the caller did NOT supply are left alone rather than removed: a
+ * constant-true term costs a per-row evaluation the planner mostly discards, and
+ * leaving them keeps the parameter set identical to the file's, so the binding
+ * path below needs to know nothing about any of this.
+ */
+export declare function specialize(sql: string, params: Params): string;
 /**
  * The parameter names a statement binds, in first-appearance order.
  *

package/dist/store/base.js

@@ -98,10 +98,56 @@ export const LEASE_EXPIRED_ERROR_JSON = dumpJson(errorEnvelope({
 export const COMMENT = /--[^\n]*/g;
 /** A `:name` placeholder. The lookbehind spares Postgres `::type` casts. */
 export const NAMED = /(?<!:):(\w+)/g;
+/**
+ * An optional filter: `(:p is null or col = :p)`, with the `::type` cast the
+ * Postgres dialect adds to pin the parameter's type. The back-reference is what
+ * keeps it from matching anything else — both halves must name the same
+ * parameter. claim's `(:names is null or name in (…))` deliberately does not
+ * match: a list-valued filter is a different problem, and claim_one_name.sql is
+ * its answer.
+ */
+const OPTIONAL_FILTER = /\(:(\w+)(?:::[\w[\]]+)? is null or (\S+) = :\1\)/g;
 // Statement text is loaded once at construction and never varies, so the parse is
 // memoized on it: every dialect's binding path runs on each query, and re-scanning
 // the SQL each time would put a regex sweep on the worker's poll loop.
 const paramCache = new Map();
+// Same argument for the specialized texts, which vary only by WHICH filters a
+// caller supplied — a small, bounded set per statement, reached within the first
+// few calls and constant thereafter.
+const specialCache = new Map();
+/**
+ * The statement as it should run for these arguments: every optional filter the
+ * caller actually supplied rewritten to an equality.
+ *
+ * `(:p is null or col = :p)` cannot use an index. SQLite plans a statement when
+ * it is prepared, before any parameter has a value, so it must plan for both
+ * branches and settles for a scan; that is not a tuning detail but the whole
+ * difference between `list(root_id=…)` seeking cairnq_tasks_root_idx and reading
+ * the table. Postgres re-plans with the values for a statement's first
+ * executions and folds the branch away on its own, so this is a no-op there —
+ * but it costs nothing, and one behaviour is easier to reason about than two.
+ *
+ * Filters the caller did NOT supply are left alone rather than removed: a
+ * constant-true term costs a per-row evaluation the planner mostly discards, and
+ * leaving them keeps the parameter set identical to the file's, so the binding
+ * path below needs to know nothing about any of this.
+ */
+export function specialize(sql, params) {
+    let active = "";
+    for (const [, name] of sql.matchAll(OPTIONAL_FILTER)) {
+        if (params[name] != null)
+            active += name + ",";
+    }
+    if (!active)
+        return sql;
+    const key = active + sql;
+    let out = specialCache.get(key);
+    if (out === undefined) {
+        out = sql.replace(OPTIONAL_FILTER, (whole, name, column) => params[name] != null ? `${column} = :${name}` : whole);
+        specialCache.set(key, out);
+    }
+    return out;
+}
 /**
  * The parameter names a statement binds, in first-appearance order.
  *
@@ -335,20 +381,7 @@ export class TaskStore {
      */
     async purge(input = {}) {
         validatePurgeInput(input);
-        // Each optional filter has an equality form, picked here — the same trade
-        // claimSession makes between claim and its specializations, for the same
-        // reason. `(:queue is null or queue = :queue)` is planned before any
-        // parameter has a value, so on SQLite it can use no index at all and the
-        // sweep walks every row past the cutoff, whichever queue it belongs to.
-        // See purge_one_queue.sql. `name` is not specialized: nothing indexes it.
-        const statement = input.queue != null
-            ? input.status != null
-                ? "purge_one_queue_one_status"
-                : "purge_one_queue"
-            : input.status != null
-                ? "purge_one_status"
-                : "purge";
-        const rows = await this.fetch(statement, {
+        const rows = await this.fetch("purge", {
             older_than_ms: input.olderThanMs ?? 0,
             queue: input.queue ?? null,
             status: input.status ?? null,
@@ -382,11 +415,7 @@ export class TaskStore {
         // rows to seed from, and that is exactly the case the promise is about.
         if (queue != null)
             out[queue] = zeros();
-        // Equality form when a queue was named — see stats_one_queue.sql: the
-        // optional filter cannot be indexed, so the "narrowed" form would read the
-        // whole table anyway and narrowing would buy nothing.
-        const statement = queue != null ? "stats_one_queue" : "stats";
-        for (const row of await this.fetch(statement, { queue: queue ?? null })) {
+        for (const row of await this.fetch("stats", { queue: queue ?? null })) {
             const per = (out[row.queue] ??= zeros());
             per[row.status] = Number(row.count);
         }

package/dist/store/pg-executor.d.ts

@@ -17,18 +17,33 @@
  *
  * ONE thing an adapter does have to get right: enable the driver's
  * prepared-statement path if it is not already the default. The store issues a
- * small, FIXED set of statement texts — every one is loaded from a file at
- * startup and the `:name` -> `$n` rewrite is memoized on it, so the same handful
- * of strings is submitted for the life of the process, and nothing here ever
- * interpolates a value into SQL. That is exactly the shape a server-side
- * prepared statement is for, and the worker's poll loop reruns those texts
- * forever. A driver that instead describes each statement before binding pays an
- * extra round trip and a re-parse on every call: measured on one such adapter
- * (postgres.js, whose `unsafe()` defaults to `prepare: false`), the same hot
- * path went 137ms -> 17ms per round trip once preparation was turned on. The
- * reference implementation happens not to be affected — `pg` uses the extended
- * protocol by default — which is why this is stated here rather than left to be
- * discovered.
+ * small, FIXED set of statement texts — each is loaded from a file at startup,
+ * `specialize` may return one variant of it per set of optional filters a caller
+ * supplies, and the `:name` -> `$n` rewrite is memoized on the result. So the
+ * same handful of strings is submitted for the life of the process, reached
+ * within the first few calls, and nothing here ever interpolates a value into
+ * SQL. That is exactly the shape a server-side
+ * prepared statement is for. A driver that instead describes each statement
+ * before binding pays an extra round trip and a re-parse every time.
+ *
+ * The cost is **per statement**, and that is the number to reason with: measured
+ * on postgres.js, whose `unsafe()` defaults to `prepare: false`, one statement
+ * run 2000 times went 0.52ms -> 0.19ms once preparation was on — about 2.8x, or
+ * ~0.33ms a statement. Per task it disappears: the same measurement through a
+ * whole `call` round trip moved 104.9ms -> 102.9ms, and against a real handler
+ * 17ms -> 16ms. A task costs what its handler costs, and a couple of statements
+ * either way is noise next to that.
+ *
+ * Where it does land is the bookkeeping that runs whether or not there is any
+ * work — the claim loop, `recover_leases`, heartbeats, `wait`'s polling. Those
+ * are pure statement cost with no handler to hide behind, which is also why this
+ * multiplies with `claimable_probe` rather than being independent of it: the
+ * probe cuts how many statements an idle poll issues, this cuts what each one
+ * costs. A fleet that is mostly idle pays almost nothing else.
+ *
+ * The reference implementation happens not to be affected — `pg` uses the
+ * extended protocol by default — which is why this is stated here rather than
+ * left to be discovered.
  */
 /** A row as the driver hands it back: column name -> value. */
 export type Row = Record<string, unknown>;

package/dist/store/pg-executor.js

@@ -17,18 +17,33 @@
  *
  * ONE thing an adapter does have to get right: enable the driver's
  * prepared-statement path if it is not already the default. The store issues a
- * small, FIXED set of statement texts — every one is loaded from a file at
- * startup and the `:name` -> `$n` rewrite is memoized on it, so the same handful
- * of strings is submitted for the life of the process, and nothing here ever
- * interpolates a value into SQL. That is exactly the shape a server-side
- * prepared statement is for, and the worker's poll loop reruns those texts
- * forever. A driver that instead describes each statement before binding pays an
- * extra round trip and a re-parse on every call: measured on one such adapter
- * (postgres.js, whose `unsafe()` defaults to `prepare: false`), the same hot
- * path went 137ms -> 17ms per round trip once preparation was turned on. The
- * reference implementation happens not to be affected — `pg` uses the extended
- * protocol by default — which is why this is stated here rather than left to be
- * discovered.
+ * small, FIXED set of statement texts — each is loaded from a file at startup,
+ * `specialize` may return one variant of it per set of optional filters a caller
+ * supplies, and the `:name` -> `$n` rewrite is memoized on the result. So the
+ * same handful of strings is submitted for the life of the process, reached
+ * within the first few calls, and nothing here ever interpolates a value into
+ * SQL. That is exactly the shape a server-side
+ * prepared statement is for. A driver that instead describes each statement
+ * before binding pays an extra round trip and a re-parse every time.
+ *
+ * The cost is **per statement**, and that is the number to reason with: measured
+ * on postgres.js, whose `unsafe()` defaults to `prepare: false`, one statement
+ * run 2000 times went 0.52ms -> 0.19ms once preparation was on — about 2.8x, or
+ * ~0.33ms a statement. Per task it disappears: the same measurement through a
+ * whole `call` round trip moved 104.9ms -> 102.9ms, and against a real handler
+ * 17ms -> 16ms. A task costs what its handler costs, and a couple of statements
+ * either way is noise next to that.
+ *
+ * Where it does land is the bookkeeping that runs whether or not there is any
+ * work — the claim loop, `recover_leases`, heartbeats, `wait`'s polling. Those
+ * are pure statement cost with no handler to hide behind, which is also why this
+ * multiplies with `claimable_probe` rather than being independent of it: the
+ * probe cuts how many statements an idle poll issues, this cuts what each one
+ * costs. A fleet that is mostly idle pays almost nothing else.
+ *
+ * The reference implementation happens not to be affected — `pg` uses the
+ * extended protocol by default — which is why this is stated here rather than
+ * left to be discovered.
  */
 /**
  * LISTEN will not work on this connection, and retrying cannot change that.

package/dist/store/postgres.js

@@ -1,6 +1,6 @@
 import { SchemaMismatch } from "../errors.js";
 import { loadMigrations, loadStatements } from "../sql.js";
-import { checkProtocolVersion, COMMENT, NAMED, statementParams, TaskStore, } from "./base.js";
+import { checkProtocolVersion, COMMENT, NAMED, specialize, statementParams, TaskStore, } from "./base.js";
 import { ListenUnavailable } from "./pg-executor.js";
 import { createPoolExecutor } from "./pg-pool.js";
 // Notification channels, emitted by the 0003_notify trigger.
@@ -461,7 +461,7 @@ export class PostgresStore extends TaskStore {
     // ------------------------------------------------------------ dialect seam
     async fetch(name, params) {
         await this.ensure();
-        const { text, values } = toPositional(this.statements[name], params);
+        const { text, values } = toPositional(specialize(this.statements[name], params), params);
         return this.executor.query(text, values);
     }
     async tx(fn) {
@@ -475,7 +475,7 @@ export class PostgresStore extends TaskStore {
     /** A Fetch that runs the protocol's statements on one particular session. */
     boundFetch(s) {
         return async (name, params) => {
-            const { text, values } = toPositional(this.statements[name], params);
+            const { text, values } = toPositional(specialize(this.statements[name], params), params);
             return s.query(text, values);
         };
     }

package/dist/store/sqlite.d.ts

@@ -27,6 +27,13 @@ import { type Fetch, type Params, TaskStore } from "./base.js";
 export declare class SQLiteStore extends TaskStore {
     private readonly path;
     private db;
+    /**
+     * Prepared statements, keyed by the SQL they were prepared from rather than by
+     * statement name: `specialize` gives a statement one text per set of optional
+     * filters a caller supplies, and each of those is its own prepared plan — which
+     * is the entire point, since the plan is what the specialization changes.
+     * Bounded by the statement set times the filter combinations actually used.
+     */
     private stmts;
     private readonly statements;
     /** This store's entry in `fileLocks` — see there for why it is per-database. */

package/dist/store/sqlite.js

@@ -3,7 +3,7 @@ import { dirname, resolve } from "node:path";
 import { createRequire } from "node:module";
 import { nowMs } from "../ids.js";
 import { loadMigrations, loadStatements } from "../sql.js";
-import { checkProtocolVersion, COMMENT, statementParams, TaskStore, } from "./base.js";
+import { checkProtocolVersion, COMMENT, specialize, statementParams, TaskStore, } from "./base.js";
 // `better-sqlite3` is an optional dependency, matching `pg` on the Postgres side:
 // a Postgres-only deployment should not have to build a native module it never
 // loads, and importing this file must not pull one in. Required (not imported)
@@ -194,7 +194,14 @@ function enableWal(db) {
 export class SQLiteStore extends TaskStore {
     path;
     db = null;
-    stmts = {};
+    /**
+     * Prepared statements, keyed by the SQL they were prepared from rather than by
+     * statement name: `specialize` gives a statement one text per set of optional
+     * filters a caller supplies, and each of those is its own prepared plan — which
+     * is the entire point, since the plan is what the specialization changes.
+     * Bounded by the statement set times the filter combinations actually used.
+     */
+    stmts = new Map();
     statements;
     /** This store's entry in `fileLocks` — see there for why it is per-database. */
     lockKey;
@@ -230,7 +237,7 @@ export class SQLiteStore extends TaskStore {
         if (this.db) {
             this.db.close();
             this.db = null;
-            this.stmts = {};
+            this.stmts.clear();
         }
     }
     ensure() {
@@ -264,8 +271,10 @@ export class SQLiteStore extends TaskStore {
                 throw err;
         }
         this.nextStatsRefreshAt = Date.now() + STATS_REFRESH_INTERVAL_MS;
-        for (const [name, sql] of Object.entries(this.statements)) {
-            this.stmts[name] = db.prepare(sql);
+        // Warm the unfiltered texts, which every statement has and most callers use.
+        // The specialized ones prepare on first use; see `stmts`.
+        for (const sql of Object.values(this.statements)) {
+            this.stmts.set(sql, db.prepare(sql));
         }
         this.db = db;
         checkProtocolVersion(this.readProtocolVersion());
@@ -352,8 +361,13 @@ export class SQLiteStore extends TaskStore {
         return bound;
     }
     runNow(name, params) {
-        const stmt = this.stmts[name];
-        const bound = this.bind(this.statements[name], params);
+        const sql = specialize(this.statements[name], params);
+        let stmt = this.stmts.get(sql);
+        if (!stmt) {
+            stmt = this.db.prepare(sql);
+            this.stmts.set(sql, stmt);
+        }
+        const bound = this.bind(sql, params);
         // Nearly every protocol statement ends in RETURNING; upsert_key does not, and
         // better-sqlite3 refuses .all() on a statement that yields no rows.
         if (!stmt.reader) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cairnq",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "SQLite-first, cross-language, storage-centered durable task runtime",
   "license": "MIT",
   "author": "Jannchie <jannchie@gmail.com>",

package/src/store/base.ts

@@ -174,10 +174,58 @@ export const COMMENT = /--[^\n]*/g;
 /** A `:name` placeholder. The lookbehind spares Postgres `::type` casts. */
 export const NAMED = /(?<!:):(\w+)/g;
 
+/**
+ * An optional filter: `(:p is null or col = :p)`, with the `::type` cast the
+ * Postgres dialect adds to pin the parameter's type. The back-reference is what
+ * keeps it from matching anything else — both halves must name the same
+ * parameter. claim's `(:names is null or name in (…))` deliberately does not
+ * match: a list-valued filter is a different problem, and claim_one_name.sql is
+ * its answer.
+ */
+const OPTIONAL_FILTER = /\(:(\w+)(?:::[\w[\]]+)? is null or (\S+) = :\1\)/g;
+
 // Statement text is loaded once at construction and never varies, so the parse is
 // memoized on it: every dialect's binding path runs on each query, and re-scanning
 // the SQL each time would put a regex sweep on the worker's poll loop.
 const paramCache = new Map<string, readonly string[]>();
+// Same argument for the specialized texts, which vary only by WHICH filters a
+// caller supplied — a small, bounded set per statement, reached within the first
+// few calls and constant thereafter.
+const specialCache = new Map<string, string>();
+
+/**
+ * The statement as it should run for these arguments: every optional filter the
+ * caller actually supplied rewritten to an equality.
+ *
+ * `(:p is null or col = :p)` cannot use an index. SQLite plans a statement when
+ * it is prepared, before any parameter has a value, so it must plan for both
+ * branches and settles for a scan; that is not a tuning detail but the whole
+ * difference between `list(root_id=…)` seeking cairnq_tasks_root_idx and reading
+ * the table. Postgres re-plans with the values for a statement's first
+ * executions and folds the branch away on its own, so this is a no-op there —
+ * but it costs nothing, and one behaviour is easier to reason about than two.
+ *
+ * Filters the caller did NOT supply are left alone rather than removed: a
+ * constant-true term costs a per-row evaluation the planner mostly discards, and
+ * leaving them keeps the parameter set identical to the file's, so the binding
+ * path below needs to know nothing about any of this.
+ */
+export function specialize(sql: string, params: Params): string {
+  let active = "";
+  for (const [, name] of sql.matchAll(OPTIONAL_FILTER)) {
+    if (params[name] != null) active += name + ",";
+  }
+  if (!active) return sql;
+  const key = active + sql;
+  let out = specialCache.get(key);
+  if (out === undefined) {
+    out = sql.replace(OPTIONAL_FILTER, (whole, name: string, column: string) =>
+      params[name] != null ? `${column} = :${name}` : whole,
+    );
+    specialCache.set(key, out);
+  }
+  return out;
+}
 
 /**
  * The parameter names a statement binds, in first-appearance order.
@@ -522,21 +570,7 @@ export abstract class TaskStore {
    */
   async purge(input: PurgeInput = {}): Promise<string[]> {
     validatePurgeInput(input);
-    // Each optional filter has an equality form, picked here — the same trade
-    // claimSession makes between claim and its specializations, for the same
-    // reason. `(:queue is null or queue = :queue)` is planned before any
-    // parameter has a value, so on SQLite it can use no index at all and the
-    // sweep walks every row past the cutoff, whichever queue it belongs to.
-    // See purge_one_queue.sql. `name` is not specialized: nothing indexes it.
-    const statement =
-      input.queue != null
-        ? input.status != null
-          ? "purge_one_queue_one_status"
-          : "purge_one_queue"
-        : input.status != null
-          ? "purge_one_status"
-          : "purge";
-    const rows = await this.fetch(statement, {
+    const rows = await this.fetch("purge", {
       older_than_ms: input.olderThanMs ?? 0,
       queue: input.queue ?? null,
       status: input.status ?? null,
@@ -571,11 +605,7 @@ export abstract class TaskStore {
     // Seed before the query, not after: a named queue with no rows returns no
     // rows to seed from, and that is exactly the case the promise is about.
     if (queue != null) out[queue] = zeros();
-    // Equality form when a queue was named — see stats_one_queue.sql: the
-    // optional filter cannot be indexed, so the "narrowed" form would read the
-    // whole table anyway and narrowing would buy nothing.
-    const statement = queue != null ? "stats_one_queue" : "stats";
-    for (const row of await this.fetch(statement, { queue: queue ?? null })) {
+    for (const row of await this.fetch("stats", { queue: queue ?? null })) {
       const per = (out[row.queue] ??= zeros());
       per[row.status as TaskStatus] = Number(row.count);
     }

package/src/store/pg-executor.ts

@@ -17,18 +17,33 @@
  *
  * ONE thing an adapter does have to get right: enable the driver's
  * prepared-statement path if it is not already the default. The store issues a
- * small, FIXED set of statement texts — every one is loaded from a file at
- * startup and the `:name` -> `$n` rewrite is memoized on it, so the same handful
- * of strings is submitted for the life of the process, and nothing here ever
- * interpolates a value into SQL. That is exactly the shape a server-side
- * prepared statement is for, and the worker's poll loop reruns those texts
- * forever. A driver that instead describes each statement before binding pays an
- * extra round trip and a re-parse on every call: measured on one such adapter
- * (postgres.js, whose `unsafe()` defaults to `prepare: false`), the same hot
- * path went 137ms -> 17ms per round trip once preparation was turned on. The
- * reference implementation happens not to be affected — `pg` uses the extended
- * protocol by default — which is why this is stated here rather than left to be
- * discovered.
+ * small, FIXED set of statement texts — each is loaded from a file at startup,
+ * `specialize` may return one variant of it per set of optional filters a caller
+ * supplies, and the `:name` -> `$n` rewrite is memoized on the result. So the
+ * same handful of strings is submitted for the life of the process, reached
+ * within the first few calls, and nothing here ever interpolates a value into
+ * SQL. That is exactly the shape a server-side
+ * prepared statement is for. A driver that instead describes each statement
+ * before binding pays an extra round trip and a re-parse every time.
+ *
+ * The cost is **per statement**, and that is the number to reason with: measured
+ * on postgres.js, whose `unsafe()` defaults to `prepare: false`, one statement
+ * run 2000 times went 0.52ms -> 0.19ms once preparation was on — about 2.8x, or
+ * ~0.33ms a statement. Per task it disappears: the same measurement through a
+ * whole `call` round trip moved 104.9ms -> 102.9ms, and against a real handler
+ * 17ms -> 16ms. A task costs what its handler costs, and a couple of statements
+ * either way is noise next to that.
+ *
+ * Where it does land is the bookkeeping that runs whether or not there is any
+ * work — the claim loop, `recover_leases`, heartbeats, `wait`'s polling. Those
+ * are pure statement cost with no handler to hide behind, which is also why this
+ * multiplies with `claimable_probe` rather than being independent of it: the
+ * probe cuts how many statements an idle poll issues, this cuts what each one
+ * costs. A fleet that is mostly idle pays almost nothing else.
+ *
+ * The reference implementation happens not to be affected — `pg` uses the
+ * extended protocol by default — which is why this is stated here rather than
+ * left to be discovered.
  */
 
 /** A row as the driver hands it back: column name -> value. */

package/src/store/postgres.ts

@@ -6,6 +6,7 @@ import {
   type Fetch,
   NAMED,
   type Params,
+  specialize,
   statementParams,
   TaskStore,
   type WatchSignal,
@@ -488,7 +489,7 @@ export class PostgresStore extends TaskStore {
   // ------------------------------------------------------------ dialect seam
   protected async fetch(name: string, params: Params): Promise<any[]> {
     await this.ensure();
-    const { text, values } = toPositional(this.statements[name], params);
+    const { text, values } = toPositional(specialize(this.statements[name], params), params);
     return this.executor!.query(text, values);
   }
 
@@ -507,7 +508,7 @@ export class PostgresStore extends TaskStore {
   /** A Fetch that runs the protocol's statements on one particular session. */
   private boundFetch(s: PgSession): Fetch {
     return async (name, params) => {
-      const { text, values } = toPositional(this.statements[name], params);
+      const { text, values } = toPositional(specialize(this.statements[name], params), params);
       return s.query(text, values);
     };
   }

package/src/store/sqlite.ts

@@ -12,6 +12,7 @@ import {
   COMMENT,
   type Fetch,
   type Params,
+  specialize,
   statementParams,
   TaskStore,
 } from "./base.js";
@@ -231,7 +232,14 @@ function enableWal(db: DB): void {
  */
 export class SQLiteStore extends TaskStore {
   private db: DB | null = null;
-  private stmts: Record<string, Stmt> = {};
+  /**
+   * Prepared statements, keyed by the SQL they were prepared from rather than by
+   * statement name: `specialize` gives a statement one text per set of optional
+   * filters a caller supplies, and each of those is its own prepared plan — which
+   * is the entire point, since the plan is what the specialization changes.
+   * Bounded by the statement set times the filter combinations actually used.
+   */
+  private stmts = new Map<string, Stmt>();
   private readonly statements: Record<string, string>;
   /** This store's entry in `fileLocks` — see there for why it is per-database. */
   private readonly lockKey: string;
@@ -274,7 +282,7 @@ export class SQLiteStore extends TaskStore {
     if (this.db) {
       this.db.close();
       this.db = null;
-      this.stmts = {};
+      this.stmts.clear();
     }
   }
 
@@ -304,8 +312,10 @@ export class SQLiteStore extends TaskStore {
       if (!isBusy(err)) throw err;
     }
     this.nextStatsRefreshAt = Date.now() + STATS_REFRESH_INTERVAL_MS;
-    for (const [name, sql] of Object.entries(this.statements)) {
-      this.stmts[name] = db.prepare(sql);
+    // Warm the unfiltered texts, which every statement has and most callers use.
+    // The specialized ones prepare on first use; see `stmts`.
+    for (const sql of Object.values(this.statements)) {
+      this.stmts.set(sql, db.prepare(sql));
     }
     this.db = db;
     checkProtocolVersion(this.readProtocolVersion());
@@ -400,8 +410,13 @@ export class SQLiteStore extends TaskStore {
   }
 
   private runNow(name: string, params: Params): any[] {
-    const stmt = this.stmts[name];
-    const bound = this.bind(this.statements[name], params);
+    const sql = specialize(this.statements[name], params);
+    let stmt = this.stmts.get(sql);
+    if (!stmt) {
+      stmt = this.db!.prepare(sql);
+      this.stmts.set(sql, stmt);
+    }
+    const bound = this.bind(sql, params);
     // Nearly every protocol statement ends in RETURNING; upsert_key does not, and
     // better-sqlite3 refuses .all() on a statement that yields no rows.
     if (!stmt.reader) {

package/dist/_protocol/sql/postgres/purge_one_queue.sql

@@ -1,42 +0,0 @@
--- purge, for a sweep bounded to ONE queue. Byte-for-byte purge.sql except that
--- the queue filter is an equality instead of an optional `is null or` — a drift-
--- guard test asserts precisely that, so treat purge.sql as the source and re-
--- derive this file when it changes.
---
--- It exists because the optional-filter form cannot be indexed. SQLite plans a
--- statement when it is prepared, before any parameter has a value, so `(:queue
--- is null or queue = :queue)` has to be planned for BOTH branches and the
--- planner falls back to cairnq_tasks_completed_idx, walking every row past the
--- cutoff in completion order and discarding the ones belonging to another queue.
--- `limit` bounds what comes back, never what is read, so the cost grows with
--- exactly the rows the filter was meant to skip — and the deployment the filter
--- exists for (one installation, two workloads on different clocks) is the one
--- where those rows are most numerous. Measured on 20k rows over two queues and
--- four statuses: the optional form chooses cairnq_tasks_completed_idx for every
--- filter combination, the equality form chooses cairnq_tasks_queue_completed_idx
--- (0009) and reads only its own range. Same reason claim.sql has
--- specializations, same shape.
---
--- Postgres does not have SQLite's problem — it re-plans with the parameter
--- values for the first executions and folds the null branch away — but it ships
--- the variant too, because both dialects carry the same statement set and a
--- caller that had to know which dialect indexes which form would be a worse
--- contract.
---
--- :name stays optional in every variant: nothing indexes it, so it is a residual
--- predicate either way and specializing it would buy nothing.
--- params: older_than_ms, queue, status, name, limit
-delete from cairnq_tasks
-where id in (
-    select id from cairnq_tasks
-    where status in ('succeeded', 'failed', 'canceled')
-      and queue = :queue
-      and (:status::text is null or status = :status)
-      and (:name::text is null or name = :name)
-      and completed_at_ms is not null
-      and completed_at_ms < (extract(epoch from now()) * 1000)::bigint - :older_than_ms
-    order by completed_at_ms asc
-    limit :limit
-    for update skip locked
-)
-returning id;

package/dist/_protocol/sql/postgres/purge_one_queue_one_status.sql

@@ -1,42 +0,0 @@
--- purge, for a sweep bounded to one queue AND one terminal status. Byte-for-byte
--- purge.sql except that the queue and status filters are an equality instead of
--- an optional `is null or` — a drift-guard test asserts precisely that, so treat
--- purge.sql as the source and re-derive this file when it changes.
---
--- It exists because the optional-filter form cannot be indexed. SQLite plans a
--- statement when it is prepared, before any parameter has a value, so `(:queue
--- is null or queue = :queue)` has to be planned for BOTH branches and the
--- planner falls back to cairnq_tasks_completed_idx, walking every row past the
--- cutoff in completion order and discarding the ones belonging to another queue.
--- `limit` bounds what comes back, never what is read, so the cost grows with
--- exactly the rows the filter was meant to skip — and the deployment the filter
--- exists for (one installation, two workloads on different clocks) is the one
--- where those rows are most numerous. Measured on 20k rows over two queues and
--- four statuses: the optional form chooses cairnq_tasks_completed_idx for every
--- filter combination, the equality form chooses cairnq_tasks_queue_completed_idx
--- (0009) and reads only its own range. Same reason claim.sql has
--- specializations, same shape.
---
--- Postgres does not have SQLite's problem — it re-plans with the parameter
--- values for the first executions and folds the null branch away — but it ships
--- the variant too, because both dialects carry the same statement set and a
--- caller that had to know which dialect indexes which form would be a worse
--- contract.
---
--- :name stays optional in every variant: nothing indexes it, so it is a residual
--- predicate either way and specializing it would buy nothing.
--- params: older_than_ms, queue, status, name, limit
-delete from cairnq_tasks
-where id in (
-    select id from cairnq_tasks
-    where status in ('succeeded', 'failed', 'canceled')
-      and queue = :queue
-      and status = :status
-      and (:name::text is null or name = :name)
-      and completed_at_ms is not null
-      and completed_at_ms < (extract(epoch from now()) * 1000)::bigint - :older_than_ms
-    order by completed_at_ms asc
-    limit :limit
-    for update skip locked
-)
-returning id;

package/dist/_protocol/sql/postgres/purge_one_status.sql

@@ -1,42 +0,0 @@
--- purge, for a sweep bounded to ONE terminal status. Byte-for-byte purge.sql
--- except that the status filter is an equality instead of an optional `is null
--- or` — a drift-guard test asserts precisely that, so treat purge.sql as the
--- source and re-derive this file when it changes.
---
--- It exists because the optional-filter form cannot be indexed. SQLite plans a
--- statement when it is prepared, before any parameter has a value, so `(:status
--- is null or status = :status)` has to be planned for BOTH branches and the
--- planner falls back to cairnq_tasks_completed_idx, walking every row past the
--- cutoff in completion order and discarding the ones belonging to another
--- status. `limit` bounds what comes back, never what is read, so the cost grows
--- with exactly the rows the filter was meant to skip — and the deployment the
--- filter exists for (one installation, two workloads on different clocks) is the
--- one where those rows are most numerous. Measured on 20k rows over two queues
--- and four statuses: the optional form chooses cairnq_tasks_completed_idx for
--- every filter combination, the equality form chooses
--- cairnq_tasks_status_completed_idx (0007) and reads only its own range. Same
--- reason claim.sql has specializations, same shape.
---
--- Postgres does not have SQLite's problem — it re-plans with the parameter
--- values for the first executions and folds the null branch away — but it ships
--- the variant too, because both dialects carry the same statement set and a
--- caller that had to know which dialect indexes which form would be a worse
--- contract.
---
--- :name stays optional in every variant: nothing indexes it, so it is a residual
--- predicate either way and specializing it would buy nothing.
--- params: older_than_ms, queue, status, name, limit
-delete from cairnq_tasks
-where id in (
-    select id from cairnq_tasks
-    where status in ('succeeded', 'failed', 'canceled')
-      and (:queue::text is null or queue = :queue)
-      and status = :status
-      and (:name::text is null or name = :name)
-      and completed_at_ms is not null
-      and completed_at_ms < (extract(epoch from now()) * 1000)::bigint - :older_than_ms
-    order by completed_at_ms asc
-    limit :limit
-    for update skip locked
-)
-returning id;

package/dist/_protocol/sql/postgres/stats_one_queue.sql

@@ -1,21 +0,0 @@
--- stats, for a caller asking about ONE queue. Byte-for-byte stats.sql except
--- that the queue filter is an equality instead of an optional `is null or` — a
--- drift-guard test asserts precisely that, so treat stats.sql as the source and
--- re-derive this file when it changes.
---
--- It exists for the same reason purge_one_queue.sql does. SQLite plans a
--- statement before its parameters have values, so the optional form has to be
--- planned for both branches: it reads the whole table (as a covering index
--- scan) and groups it, which is exactly the cost narrowing to one queue was
--- meant to avoid. The equality form seeks the (queue, status) prefix of
--- cairnq_tasks_claim_idx and reads only that queue's entries.
---
--- This still COUNTS what it reports, so it costs what it counts: one queue's
--- rows, terminal ones included. Narrower than the unfiltered form, still not a
--- poll-loop question — queue_depth.sql is the bounded one.
--- params: queue
-select queue, status, count(*) as count
-from cairnq_tasks
-where queue = :queue
-group by queue, status
-order by queue asc, status asc;

package/dist/_protocol/sql/sqlite/purge_one_queue.sql

@@ -1,41 +0,0 @@
--- purge, for a sweep bounded to ONE queue. Byte-for-byte purge.sql except that
--- the queue filter is an equality instead of an optional `is null or` — a drift-
--- guard test asserts precisely that, so treat purge.sql as the source and re-
--- derive this file when it changes.
---
--- It exists because the optional-filter form cannot be indexed. SQLite plans a
--- statement when it is prepared, before any parameter has a value, so `(:queue
--- is null or queue = :queue)` has to be planned for BOTH branches and the
--- planner falls back to cairnq_tasks_completed_idx, walking every row past the
--- cutoff in completion order and discarding the ones belonging to another queue.
--- `limit` bounds what comes back, never what is read, so the cost grows with
--- exactly the rows the filter was meant to skip — and the deployment the filter
--- exists for (one installation, two workloads on different clocks) is the one
--- where those rows are most numerous. Measured on 20k rows over two queues and
--- four statuses: the optional form chooses cairnq_tasks_completed_idx for every
--- filter combination, the equality form chooses cairnq_tasks_queue_completed_idx
--- (0009) and reads only its own range. Same reason claim.sql has
--- specializations, same shape.
---
--- Postgres does not have SQLite's problem — it re-plans with the parameter
--- values for the first executions and folds the null branch away — but it ships
--- the variant too, because both dialects carry the same statement set and a
--- caller that had to know which dialect indexes which form would be a worse
--- contract.
---
--- :name stays optional in every variant: nothing indexes it, so it is a residual
--- predicate either way and specializing it would buy nothing.
--- params: before_ms, queue, status, name, limit
-delete from cairnq_tasks
-where id in (
-    select id from cairnq_tasks
-    where status in ('succeeded', 'failed', 'canceled')
-      and queue = :queue
-      and (:status is null or status = :status)
-      and (:name is null or name = :name)
-      and completed_at_ms is not null
-      and completed_at_ms < :before_ms
-    order by completed_at_ms asc
-    limit :limit
-)
-returning id;

package/dist/_protocol/sql/sqlite/purge_one_queue_one_status.sql

@@ -1,41 +0,0 @@
--- purge, for a sweep bounded to one queue AND one terminal status. Byte-for-byte
--- purge.sql except that the queue and status filters are an equality instead of
--- an optional `is null or` — a drift-guard test asserts precisely that, so treat
--- purge.sql as the source and re-derive this file when it changes.
---
--- It exists because the optional-filter form cannot be indexed. SQLite plans a
--- statement when it is prepared, before any parameter has a value, so `(:queue
--- is null or queue = :queue)` has to be planned for BOTH branches and the
--- planner falls back to cairnq_tasks_completed_idx, walking every row past the
--- cutoff in completion order and discarding the ones belonging to another queue.
--- `limit` bounds what comes back, never what is read, so the cost grows with
--- exactly the rows the filter was meant to skip — and the deployment the filter
--- exists for (one installation, two workloads on different clocks) is the one
--- where those rows are most numerous. Measured on 20k rows over two queues and
--- four statuses: the optional form chooses cairnq_tasks_completed_idx for every
--- filter combination, the equality form chooses cairnq_tasks_queue_completed_idx
--- (0009) and reads only its own range. Same reason claim.sql has
--- specializations, same shape.
---
--- Postgres does not have SQLite's problem — it re-plans with the parameter
--- values for the first executions and folds the null branch away — but it ships
--- the variant too, because both dialects carry the same statement set and a
--- caller that had to know which dialect indexes which form would be a worse
--- contract.
---
--- :name stays optional in every variant: nothing indexes it, so it is a residual
--- predicate either way and specializing it would buy nothing.
--- params: before_ms, queue, status, name, limit
-delete from cairnq_tasks
-where id in (
-    select id from cairnq_tasks
-    where status in ('succeeded', 'failed', 'canceled')
-      and queue = :queue
-      and status = :status
-      and (:name is null or name = :name)
-      and completed_at_ms is not null
-      and completed_at_ms < :before_ms
-    order by completed_at_ms asc
-    limit :limit
-)
-returning id;

package/dist/_protocol/sql/sqlite/purge_one_status.sql

@@ -1,41 +0,0 @@
--- purge, for a sweep bounded to ONE terminal status. Byte-for-byte purge.sql
--- except that the status filter is an equality instead of an optional `is null
--- or` — a drift-guard test asserts precisely that, so treat purge.sql as the
--- source and re-derive this file when it changes.
---
--- It exists because the optional-filter form cannot be indexed. SQLite plans a
--- statement when it is prepared, before any parameter has a value, so `(:status
--- is null or status = :status)` has to be planned for BOTH branches and the
--- planner falls back to cairnq_tasks_completed_idx, walking every row past the
--- cutoff in completion order and discarding the ones belonging to another
--- status. `limit` bounds what comes back, never what is read, so the cost grows
--- with exactly the rows the filter was meant to skip — and the deployment the
--- filter exists for (one installation, two workloads on different clocks) is the
--- one where those rows are most numerous. Measured on 20k rows over two queues
--- and four statuses: the optional form chooses cairnq_tasks_completed_idx for
--- every filter combination, the equality form chooses
--- cairnq_tasks_status_completed_idx (0007) and reads only its own range. Same
--- reason claim.sql has specializations, same shape.
---
--- Postgres does not have SQLite's problem — it re-plans with the parameter
--- values for the first executions and folds the null branch away — but it ships
--- the variant too, because both dialects carry the same statement set and a
--- caller that had to know which dialect indexes which form would be a worse
--- contract.
---
--- :name stays optional in every variant: nothing indexes it, so it is a residual
--- predicate either way and specializing it would buy nothing.
--- params: before_ms, queue, status, name, limit
-delete from cairnq_tasks
-where id in (
-    select id from cairnq_tasks
-    where status in ('succeeded', 'failed', 'canceled')
-      and (:queue is null or queue = :queue)
-      and status = :status
-      and (:name is null or name = :name)
-      and completed_at_ms is not null
-      and completed_at_ms < :before_ms
-    order by completed_at_ms asc
-    limit :limit
-)
-returning id;

package/dist/_protocol/sql/sqlite/stats_one_queue.sql

@@ -1,21 +0,0 @@
--- stats, for a caller asking about ONE queue. Byte-for-byte stats.sql except
--- that the queue filter is an equality instead of an optional `is null or` — a
--- drift-guard test asserts precisely that, so treat stats.sql as the source and
--- re-derive this file when it changes.
---
--- It exists for the same reason purge_one_queue.sql does. SQLite plans a
--- statement before its parameters have values, so the optional form has to be
--- planned for both branches: it reads the whole table (as a covering index
--- scan) and groups it, which is exactly the cost narrowing to one queue was
--- meant to avoid. The equality form seeks the (queue, status) prefix of
--- cairnq_tasks_claim_idx and reads only that queue's entries.
---
--- This still COUNTS what it reports, so it costs what it counts: one queue's
--- rows, terminal ones included. Narrower than the unfiltered form, still not a
--- poll-loop question — queue_depth.sql is the bounded one.
--- params: queue
-select queue, status, count(*) as count
-from cairnq_tasks
-where queue = :queue
-group by queue, status
-order by queue asc, status asc;