cairnq 0.10.0 → 0.12.0

package/dist/_protocol/migrations/postgres/0001_init.sql

@@ -49,6 +49,8 @@ create table if not exists cairnq_tasks (
 -- desc, created_at_ms asc (run_at_ms applied as a residual filter). Only
 -- claim_one_queue.sql can read it in claim order; claim.sql's array-valued queue
 -- filter forces a sort, and past a few thousand queued rows a sequential scan.
+-- The shape as first shipped; migration 0008 rebuilds it around run_at_ms and
+-- says why.
 create index if not exists cairnq_tasks_claim_idx
     on cairnq_tasks (queue, status, priority desc, created_at_ms);
 create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);

package/dist/_protocol/migrations/postgres/0006_claim_name_index.sql

@@ -19,6 +19,8 @@
 -- existing large cairnq_tasks this takes a write lock for the build. Deploying
 -- into a busy database is the case to watch; build it by hand with CONCURRENTLY
 -- first if that matters, and this statement then becomes a no-op.
+-- The measurements above belong to the shape defined here; migration 0008
+-- rebuilds it around run_at_ms and carries the current numbers.
 create index if not exists cairnq_tasks_claim_name_idx
     on cairnq_tasks (queue, status, name, priority desc, created_at_ms);
 

package/dist/_protocol/migrations/postgres/0008_claim_due_index.sql

@@ -0,0 +1,68 @@
+-- Serve the claim's whole ORDER BY from the index, and order by when a task
+-- became DUE rather than by when it was created.
+--
+-- The two halves are one fix. `run_at_ms <= :now_ms` was a residual on the old
+-- (queue, status, priority desc, created_at_ms) index, and a not-yet-due row —
+-- a retry waiting out its backoff, a task submitted with a delay — sorts by the
+-- created_at it has always had, which puts it AHEAD of the rows that are
+-- actually claimable. So every draw walked the whole backoff pile before
+-- reaching anything it could take, inside the transaction that holds the claim.
+-- The pile is largest exactly when a downstream dependency has just failed and
+-- thousands of tasks are backing off together.
+--
+-- Ordering by run_at_ms instead puts every not-yet-due row AFTER the due ones,
+-- so the scan reaches its rows first, and appending `id` lets the index carry the
+-- tie-break as well, which is what lets the planner satisfy the ORDER BY from the
+-- index alone.
+--
+-- The measurements behind this live in the SQLite twin of this file, and they are
+-- SQLite numbers: no Postgres was available where this was written, so this side
+-- is reasoned from the same index shape rather than benchmarked. What holds
+-- across both is the ordering — the due rows are now at the front of the index
+-- range instead of behind every not-yet-due one.
+--
+-- What this does NOT fix, in either dialect: an empty draw still walks the whole
+-- range looking for rows that are not there, and `claimable_probe` — whose
+-- two-armed EXISTS cannot use the index at all — is untouched. A queue deep in
+-- backoff still costs a worker real time per poll.
+--
+-- For a task that was never delayed or retried the change is invisible in
+-- practice: insert_task sets run_at_ms = now + delay, so the two columns hold the
+-- same millisecond for an ordinary submit. (On Postgres "the same millisecond" is
+-- not quite an identity — insert_task evaluates clock_timestamp() once per column
+-- and it advances within a statement, so a submit landing on a millisecond
+-- boundary can differ by 1ms. That reorders two tasks submitted in the same
+-- millisecond, which claim.sql already calls decided by the id's random half
+-- rather than by submit order.) It changes real ordering only for tasks whose
+-- delivery was deferred, and for those "oldest due first" is the fairer answer —
+-- a task that failed and backed off should not cut ahead of everything submitted
+-- while it waited, which is what its original created_at_ms bought it.
+--
+-- Both indexes are rebuilt inside the migration's transaction, which holds a
+-- lock that blocks writes to cairnq_tasks while they build — seconds on a large
+-- table. There is no CONCURRENTLY here: it cannot run inside a transaction, and
+-- the ledger's check-and-apply is one. Upgrade a large deployment in a window
+-- where that pause is affordable.
+--
+-- Same names, new definitions: these indexes ARE "the ones the claim reads", and
+-- every reference to them in the SQL comments and PROTOCOL.md still points at
+-- the right object.
+--
+-- An SDK older than this migration keeps working: it orders by created_at_ms,
+-- finds no index in that order, and sorts. Slower, exactly where this migration
+-- is faster — but note the sharper consequence while a fleet is mixed. Two SDK
+-- versions against one database then disagree about which task is NEXT for any
+-- delayed or retried work: each takes a valid claimable task, no task is lost or
+-- run twice, but the documented claim order holds only within one version. That
+-- is a difference in fairness, not in correctness, which is why protocol_version
+-- stays at 1 — and it is a reason to keep the mixed window short.
+drop index if exists cairnq_tasks_claim_idx;
+create index cairnq_tasks_claim_idx
+    on cairnq_tasks (queue, status, priority desc, run_at_ms, id);
+
+-- The per-name twin (see 0006), rebuilt on the same principle.
+drop index if exists cairnq_tasks_claim_name_idx;
+create index cairnq_tasks_claim_name_idx
+    on cairnq_tasks (queue, status, name, priority desc, run_at_ms, id);
+
+update cairnq_meta set value = '8' where key = 'schema_version';

package/dist/_protocol/migrations/postgres/0009_purge_queue_index.sql

@@ -0,0 +1,39 @@
+-- Give the retention sweep a queue dimension it can actually read.
+--
+-- purge gained an optional `:queue` filter, and without an index for it queue is
+-- a residual on cairnq_tasks_completed_idx (completed_at_ms) or
+-- cairnq_tasks_status_completed_idx (status, completed_at_ms): the scan walks in
+-- completion order and throws away every row belonging to another queue. `limit`
+-- bounds what comes back, not what is read — and the shape that makes the filter
+-- worth having in the first place is exactly the worst case for it. One store
+-- carrying an RPC queue kept for minutes and a durable queue kept for a week is
+-- the cross-language coordination this project recommends; sweeping the RPC
+-- queue then means walking the week's worth of older rows the other queue is
+-- deliberately holding onto, over and over, once per batch.
+--
+-- Partial, on the terminal statuses, for two reasons. It is the smaller half of
+-- the table — a busy queue's live rows never enter it — and, more to the point,
+-- rows enter it only when a task settles, so the index is not a tax on the claim
+-- path the way a full index on (queue, ...) would be. purge.sql always carries
+-- the literal `status in ('succeeded','failed','canceled')`, whether or not the
+-- caller narrowed to one status, so the predicate matches exactly and the
+-- planner never has to prove anything subtler to use it.
+--
+-- Measured on SQLite 3.39.4, 20k rows over two queues and four statuses, with
+-- 0002's and 0007's indexes present alongside it: every filter combination purge
+-- can issue is served by one index scan with the ORDER BY satisfied from the
+-- index (no temp b-tree), and the two pre-existing shapes — unfiltered, and
+-- status-only — still choose their old indexes, so nothing that worked before
+-- got slower. Postgres is reasoned from the same shape rather than benchmarked;
+-- see 0008 for why that caveat keeps appearing.
+--
+-- Unlike 0008 this only CREATES an index, so an older SDK is unaffected: it
+-- never passes `:queue`, its statements are unchanged, and it pays only the
+-- write-side cost of an index it does not read. The build still holds a lock for
+-- as long as it takes (no CONCURRENTLY — the ledger's check-and-apply is one
+-- transaction), but over terminal rows only, which is the smaller set.
+create index if not exists cairnq_tasks_queue_completed_idx
+    on cairnq_tasks (queue, completed_at_ms)
+    where status in ('succeeded', 'failed', 'canceled');
+
+update cairnq_meta set value = '9' where key = 'schema_version';

package/dist/_protocol/migrations/sqlite/0001_init.sql

@@ -48,6 +48,8 @@ create table if not exists cairnq_tasks (
 -- queue+status then the ORDER BY columns is what lets claim_one_queue.sql read
 -- rows in claim order; claim.sql's list-valued queue filter has to merge several
 -- ranges of this index, so it sorts instead.
+-- The shape as first shipped; migration 0008 rebuilds it around run_at_ms and
+-- says why.
 create index if not exists cairnq_tasks_claim_idx
     on cairnq_tasks (queue, status, priority desc, created_at_ms);
 create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);

package/dist/_protocol/migrations/sqlite/0006_claim_name_index.sql

@@ -20,6 +20,8 @@
 -- bloom filter over the subquery and falls back to cairnq_tasks_claim_idx
 -- (measured 1446us, i.e. no improvement at all). That is why the per-name
 -- statements exist as separate files rather than the shared one being reused.
+-- The measurements above belong to the shape defined here; migration 0008
+-- rebuilds it around run_at_ms and carries the current numbers.
 create index if not exists cairnq_tasks_claim_name_idx
     on cairnq_tasks (queue, status, name, priority desc, created_at_ms);
 

package/dist/_protocol/migrations/sqlite/0008_claim_due_index.sql

@@ -0,0 +1,84 @@
+-- Serve the claim's whole ORDER BY from the index, and order by when a task
+-- became DUE rather than by when it was created.
+--
+-- The two halves are one fix. `run_at_ms <= :now_ms` was a residual on the old
+-- (queue, status, priority desc, created_at_ms) index, and a not-yet-due row —
+-- a retry waiting out its backoff, a task submitted with a delay — sorts by the
+-- created_at it has always had, which puts it AHEAD of the rows that are
+-- actually claimable. So every draw walked the whole backoff pile before
+-- reaching anything it could take, inside the transaction that holds the claim.
+-- The pile is largest exactly when a downstream dependency has just failed and
+-- thousands of tasks are backing off together.
+--
+-- Ordering by run_at_ms instead puts every not-yet-due row AFTER the due ones,
+-- so the scan reaches its rows first, and appending `id` lets the index carry the
+-- tie-break as well, which is what lets the planner satisfy the ORDER BY from the
+-- index alone. Measured through the real statement (`claim_one_queue`, one queue,
+-- four names, 20k queued rows backing off), one claim of one task, on the three
+-- SQLite builds that happened to be on the authoring machine:
+--
+--     SQLite   linked by                       one due row       an empty draw
+--     3.39.4   a system python3.11             1166us ->   22us  1110us ->  350us
+--     3.47.1   a python3.13 (this repo's venv)  3378us ->  116us  3128us -> 1006us
+--     3.53.4   better-sqlite3 13 (bundled)     1100us ->   72us  1060us ->   50us
+--
+-- Read the columns, not the rows: the win holds on every build, and the absolute
+-- cost swings more between SQLite versions than between before and after on some
+-- of them. better-sqlite3 bundles its own SQLite, so the TypeScript SDK's version
+-- is a dependency; the Python SDK links whatever the interpreter was built
+-- against, so its version belongs to the user's Python and cannot be assumed.
+--
+-- The plan differs by SQLite version and the win does not come from one single
+-- mechanism: 3.39 drops the sorter outright, while 3.53 keeps one and instead
+-- reaches the due rows through a skip-scan. What both have in common is the
+-- ordering — the due rows are now at the front of the range instead of behind
+-- every not-yet-due one. Numbers are SQLite; Postgres was not measured (no
+-- server in the authoring environment), so treat the Postgres side as reasoned
+-- rather than benchmarked.
+--
+-- What this does NOT fix: an empty draw still walks the whole range looking for
+-- rows that are not there (350us above), and `claimable_probe` — whose two-armed
+-- EXISTS cannot use the index at all — is unchanged at ~2.2ms on that backlog.
+-- A queue deep in backoff still costs a worker real time per poll.
+--
+-- For a task that was never delayed or retried the change is invisible in
+-- practice: insert_task sets run_at_ms = now + delay, so the two columns hold the
+-- same millisecond for an ordinary submit. (On Postgres "the same millisecond" is
+-- not quite an identity — insert_task evaluates clock_timestamp() once per column
+-- and it advances within a statement, so a submit landing on a millisecond
+-- boundary can differ by 1ms. That reorders two tasks submitted in the same
+-- millisecond, which claim.sql already calls decided by the id's random half
+-- rather than by submit order.) It changes real ordering only for tasks whose
+-- delivery was deferred, and for those "oldest due first" is the fairer answer —
+-- a task that failed and backed off should not cut ahead of everything submitted
+-- while it waited, which is what its original created_at_ms bought it.
+--
+-- Both indexes are rebuilt inside the migration's write transaction, so every
+-- other process's writes wait while they build — and they wait only as long as
+-- their busy budget (`busyTimeoutMs` / `busy_timeout_ms`, 5s by default) before
+-- failing. Measured on a warm local SSD: ~2.5s + ~3.4s for a 1M-row, 500MB
+-- database, i.e. past that default. Small databases do not notice; see
+-- "Upgrading" in the README for what to do about a large one.
+--
+-- Same names, new definitions: these indexes ARE "the ones the claim reads", and
+-- every reference to them in the SQL comments and PROTOCOL.md still points at
+-- the right object.
+--
+-- An SDK older than this migration keeps working: it orders by created_at_ms,
+-- finds no index in that order, and sorts. Slower, exactly where this migration
+-- is faster — but note the sharper consequence while a fleet is mixed. Two SDK
+-- versions against one database then disagree about which task is NEXT for any
+-- delayed or retried work: each takes a valid claimable task, no task is lost or
+-- run twice, but the documented claim order holds only within one version. That
+-- is a difference in fairness, not in correctness, which is why protocol_version
+-- stays at 1 — and it is a reason to keep the mixed window short.
+drop index if exists cairnq_tasks_claim_idx;
+create index cairnq_tasks_claim_idx
+    on cairnq_tasks (queue, status, priority desc, run_at_ms, id);
+
+-- The per-name twin (see 0006), rebuilt on the same principle.
+drop index if exists cairnq_tasks_claim_name_idx;
+create index cairnq_tasks_claim_name_idx
+    on cairnq_tasks (queue, status, name, priority desc, run_at_ms, id);
+
+update cairnq_meta set value = '8' where key = 'schema_version';

package/dist/_protocol/migrations/sqlite/0009_purge_queue_index.sql

@@ -0,0 +1,39 @@
+-- Give the retention sweep a queue dimension it can actually read.
+--
+-- purge gained an optional `:queue` filter, and without an index for it queue is
+-- a residual on cairnq_tasks_completed_idx (completed_at_ms) or
+-- cairnq_tasks_status_completed_idx (status, completed_at_ms): the scan walks in
+-- completion order and throws away every row belonging to another queue. `limit`
+-- bounds what comes back, not what is read — and the shape that makes the filter
+-- worth having in the first place is exactly the worst case for it. One store
+-- carrying an RPC queue kept for minutes and a durable queue kept for a week is
+-- the cross-language coordination this project recommends; sweeping the RPC
+-- queue then means walking the week's worth of older rows the other queue is
+-- deliberately holding onto, over and over, once per batch.
+--
+-- Partial, on the terminal statuses, for two reasons. It is the smaller half of
+-- the table — a busy queue's live rows never enter it — and, more to the point,
+-- rows enter it only when a task settles, so the index is not a tax on the claim
+-- path the way a full index on (queue, ...) would be. purge.sql always carries
+-- the literal `status in ('succeeded','failed','canceled')`, whether or not the
+-- caller narrowed to one status, so the predicate matches exactly and the
+-- planner never has to prove anything subtler to use it.
+--
+-- Measured on SQLite 3.39.4, 20k rows over two queues and four statuses, with
+-- 0002's and 0007's indexes present alongside it: every filter combination purge
+-- can issue is served by one index scan with the ORDER BY satisfied from the
+-- index (no temp b-tree), and the two pre-existing shapes — unfiltered, and
+-- status-only — still choose their old indexes, so nothing that worked before
+-- got slower. Postgres is reasoned from the same shape rather than benchmarked;
+-- see 0008 for why that caveat keeps appearing.
+--
+-- Unlike 0008 this only CREATES an index, so an older SDK is unaffected: it
+-- never passes `:queue`, its statements are unchanged, and it pays only the
+-- write-side cost of an index it does not read. The build still holds a lock for
+-- as long as it takes (no CONCURRENTLY — the ledger's check-and-apply is one
+-- transaction), but over terminal rows only, which is the smaller set.
+create index if not exists cairnq_tasks_queue_completed_idx
+    on cairnq_tasks (queue, completed_at_ms)
+    where status in ('succeeded', 'failed', 'canceled');
+
+update cairnq_meta set value = '9' where key = 'schema_version';

package/dist/_protocol/sql/postgres/claim.sql

@@ -29,10 +29,13 @@ from (
       and queue = any(:queues::text[])
       and (:names::text[] is null or name = any(:names::text[]))
       and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
-    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
-    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- Ordered by when a task became DUE, not when it was created: see migration
+    -- 0008 for why, for what it costs a mixed-version fleet, and for which of
+    -- these four statements the index can serve without a sort.
+    -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
+    -- deterministic: FIFO at millisecond granularity; within one millisecond
     -- the id's random half decides, stably but not in submit order.
-    order by priority desc, created_at_ms asc, id asc
+    order by priority desc, run_at_ms asc, id asc
     limit :limit
     for update skip locked
 ) sel

package/dist/_protocol/sql/postgres/claim_one_name.sql

@@ -26,10 +26,13 @@ from (
       and queue = any(:queues::text[])
       and name = :name
       and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
-    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
-    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- Ordered by when a task became DUE, not when it was created: see migration
+    -- 0008 for why, for what it costs a mixed-version fleet, and for which of
+    -- these four statements the index can serve without a sort.
+    -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
+    -- deterministic: FIFO at millisecond granularity; within one millisecond
     -- the id's random half decides, stably but not in submit order.
-    order by priority desc, created_at_ms asc, id asc
+    order by priority desc, run_at_ms asc, id asc
     limit :limit
     for update skip locked
 ) sel

package/dist/_protocol/sql/postgres/claim_one_queue.sql

@@ -7,8 +7,9 @@
 -- + LIMIT over `= any(...)` the planner falls back to a sequential scan and a
 -- full sort of every claimable row (measured on 20k queued: Seq Scan 20000 rows,
 -- quicksort 1861kB), inside the transaction that holds the claim. The equality
--- form index-scans cairnq_tasks_claim_idx and only incrementally sorts the id
--- tie-break — 33 rows read for the same query.
+-- form index-scans cairnq_tasks_claim_idx, which since migration 0008 carries the
+-- id tie-break too and so needs no sort node at all — 33 rows read for the same
+-- query.
 --
 -- params: queue, names (text[] or null), worker_id, lease_ms, limit
 update cairnq_tasks t
@@ -24,10 +25,13 @@ from (
       and queue = :queue
       and (:names::text[] is null or name = any(:names::text[]))
       and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
-    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
-    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- Ordered by when a task became DUE, not when it was created: see migration
+    -- 0008 for why, for what it costs a mixed-version fleet, and for which of
+    -- these four statements the index can serve without a sort.
+    -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
+    -- deterministic: FIFO at millisecond granularity; within one millisecond
     -- the id's random half decides, stably but not in submit order.
-    order by priority desc, created_at_ms asc, id asc
+    order by priority desc, run_at_ms asc, id asc
     limit :limit
     for update skip locked
 ) sel

package/dist/_protocol/sql/postgres/claim_one_queue_one_name.sql

@@ -23,10 +23,13 @@ from (
       and queue = :queue
       and name = :name
       and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
-    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
-    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- Ordered by when a task became DUE, not when it was created: see migration
+    -- 0008 for why, for what it costs a mixed-version fleet, and for which of
+    -- these four statements the index can serve without a sort.
+    -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
+    -- deterministic: FIFO at millisecond granularity; within one millisecond
     -- the id's random half decides, stably but not in submit order.
-    order by priority desc, created_at_ms asc, id asc
+    order by priority desc, run_at_ms asc, id asc
     limit :limit
     for update skip locked
 ) sel

package/dist/_protocol/sql/postgres/claimable_probe.sql

@@ -0,0 +1,47 @@
+-- Read-only check: is there anything worth opening the claim transaction for?
+-- Run before claim so an idle worker's poll costs one statement instead of a
+-- transaction. Mirrors claim.sql's filters, so the probe never promises work
+-- claim will skip. The expired-lease arm stays unfiltered on purpose: recovering
+-- a dead worker's task is every worker's job, whatever names it happens to
+-- handle.
+--
+-- The SQLite twin exists for a reason that does not apply here — keeping idle
+-- workers off the single write lock — and this dialect went without one on that
+-- basis. What survives the difference is the rest of the poll: without a probe
+-- every empty poll still opens a transaction, runs recover_leases, and then runs
+-- one claim statement per self-limiting name. A worker declaring a dozen such
+-- names pays a dozen statements to learn there is nothing to do, and on this
+-- dialect the empty poll is the COMMON case precisely because LISTEN wakes the
+-- worker on the rare one.
+--
+-- Two separate EXISTS, not one select with an OR: an OR across two different
+-- index shapes gets no index at all (see migration 0008's note on the SQLite
+-- twin), while each EXISTS here chooses its own — cairnq_tasks_claim_idx for the
+-- queued arm, cairnq_tasks_lease_idx (0004, partial on running rows) for the
+-- lease arm — and stops at the first row it finds.
+--
+-- Time is clock_timestamp() wrapped in a scalar subselect, for the reason spelled
+-- out at length in recover_leases.sql: inlined, a VOLATILE function becomes a
+-- per-row filter and the scan degrades to reading every candidate row. Wrapped,
+-- it is an InitPlan evaluated once and usable as an index bound.
+--
+-- What this does NOT make free: the queued arm still walks the (queue, status)
+-- range looking for a due row when every row in it is backing off, the same cost
+-- an empty claim draw pays. The saving is the transaction and the other N-1
+-- statements, not the range scan.
+-- params: queues (text[]), names (text[] or null)
+select (
+    exists (
+        select 1 from cairnq_tasks
+        where status = 'queued'
+          and queue = any(:queues::text[])
+          and (:names::text[] is null or name = any(:names::text[]))
+          and run_at_ms <= (select (extract(epoch from clock_timestamp()) * 1000)::bigint)
+    )
+    or exists (
+        select 1 from cairnq_tasks
+        where status = 'running'
+          and lease_until_ms is not null
+          and lease_until_ms <= (select (extract(epoch from clock_timestamp()) * 1000)::bigint)
+    )
+) as has_work;

package/dist/_protocol/sql/postgres/purge.sql

@@ -11,16 +11,22 @@
 -- live task. Locking the rows in the subselect freezes them terminal until the
 -- delete commits; a concurrent retry then re-evaluates against the deleted row
 -- and correctly finds nothing.
--- The status/name filters are optional (pass NULL to skip; `::text` pins the
--- param's type, as in list.sql): retention needs are tiered — a succeeded row
--- is spent once its result is consumed, while a failed one is worth keeping
--- for diagnosis — and without them the shortest-lived tier sets the retention
--- for every row.
--- params: older_than_ms, status, name, limit
+-- The queue/status/name filters are optional (pass NULL to skip; `::text` pins
+-- the param's type, as in list.sql): retention needs are tiered — a succeeded row is spent once its result is
+-- consumed, while a failed one is worth keeping for diagnosis — and without them
+-- the shortest-lived tier sets the retention for every row. `queue` is the same
+-- argument one level up: a single installation is how this project recommends
+-- two languages coordinate, so it routinely carries two workloads whose rows
+-- have nothing to do with each other's lifetimes — an RPC result read once and a
+-- durable job's log kept for a week. Migration 0009 adds the index that makes
+-- the queue filter read only its own queue's rows rather than skipping past
+-- every other queue's.
+-- params: older_than_ms, queue, status, name, limit
 delete from cairnq_tasks
 where id in (
     select id from cairnq_tasks
     where status in ('succeeded', 'failed', 'canceled')
+      and (:queue::text is null or queue = :queue)
       and (:status::text is null or status = :status)
       and (:name::text is null or name = :name)
       and completed_at_ms is not null

package/dist/_protocol/sql/postgres/purge_one_queue.sql

@@ -0,0 +1,42 @@
+-- purge, for a sweep bounded to ONE queue. Byte-for-byte purge.sql except that
+-- the queue filter is an equality instead of an optional `is null or` — a drift-
+-- guard test asserts precisely that, so treat purge.sql as the source and re-
+-- derive this file when it changes.
+--
+-- It exists because the optional-filter form cannot be indexed. SQLite plans a
+-- statement when it is prepared, before any parameter has a value, so `(:queue
+-- is null or queue = :queue)` has to be planned for BOTH branches and the
+-- planner falls back to cairnq_tasks_completed_idx, walking every row past the
+-- cutoff in completion order and discarding the ones belonging to another queue.
+-- `limit` bounds what comes back, never what is read, so the cost grows with
+-- exactly the rows the filter was meant to skip — and the deployment the filter
+-- exists for (one installation, two workloads on different clocks) is the one
+-- where those rows are most numerous. Measured on 20k rows over two queues and
+-- four statuses: the optional form chooses cairnq_tasks_completed_idx for every
+-- filter combination, the equality form chooses cairnq_tasks_queue_completed_idx
+-- (0009) and reads only its own range. Same reason claim.sql has
+-- specializations, same shape.
+--
+-- Postgres does not have SQLite's problem — it re-plans with the parameter
+-- values for the first executions and folds the null branch away — but it ships
+-- the variant too, because both dialects carry the same statement set and a
+-- caller that had to know which dialect indexes which form would be a worse
+-- contract.
+--
+-- :name stays optional in every variant: nothing indexes it, so it is a residual
+-- predicate either way and specializing it would buy nothing.
+-- params: older_than_ms, queue, status, name, limit
+delete from cairnq_tasks
+where id in (
+    select id from cairnq_tasks
+    where status in ('succeeded', 'failed', 'canceled')
+      and queue = :queue
+      and (:status::text is null or status = :status)
+      and (:name::text is null or name = :name)
+      and completed_at_ms is not null
+      and completed_at_ms < (extract(epoch from now()) * 1000)::bigint - :older_than_ms
+    order by completed_at_ms asc
+    limit :limit
+    for update skip locked
+)
+returning id;

package/dist/_protocol/sql/postgres/purge_one_queue_one_status.sql

@@ -0,0 +1,42 @@
+-- purge, for a sweep bounded to one queue AND one terminal status. Byte-for-byte
+-- purge.sql except that the queue and status filters are an equality instead of
+-- an optional `is null or` — a drift-guard test asserts precisely that, so treat
+-- purge.sql as the source and re-derive this file when it changes.
+--
+-- It exists because the optional-filter form cannot be indexed. SQLite plans a
+-- statement when it is prepared, before any parameter has a value, so `(:queue
+-- is null or queue = :queue)` has to be planned for BOTH branches and the
+-- planner falls back to cairnq_tasks_completed_idx, walking every row past the
+-- cutoff in completion order and discarding the ones belonging to another queue.
+-- `limit` bounds what comes back, never what is read, so the cost grows with
+-- exactly the rows the filter was meant to skip — and the deployment the filter
+-- exists for (one installation, two workloads on different clocks) is the one
+-- where those rows are most numerous. Measured on 20k rows over two queues and
+-- four statuses: the optional form chooses cairnq_tasks_completed_idx for every
+-- filter combination, the equality form chooses cairnq_tasks_queue_completed_idx
+-- (0009) and reads only its own range. Same reason claim.sql has
+-- specializations, same shape.
+--
+-- Postgres does not have SQLite's problem — it re-plans with the parameter
+-- values for the first executions and folds the null branch away — but it ships
+-- the variant too, because both dialects carry the same statement set and a
+-- caller that had to know which dialect indexes which form would be a worse
+-- contract.
+--
+-- :name stays optional in every variant: nothing indexes it, so it is a residual
+-- predicate either way and specializing it would buy nothing.
+-- params: older_than_ms, queue, status, name, limit
+delete from cairnq_tasks
+where id in (
+    select id from cairnq_tasks
+    where status in ('succeeded', 'failed', 'canceled')
+      and queue = :queue
+      and status = :status
+      and (:name::text is null or name = :name)
+      and completed_at_ms is not null
+      and completed_at_ms < (extract(epoch from now()) * 1000)::bigint - :older_than_ms
+    order by completed_at_ms asc
+    limit :limit
+    for update skip locked
+)
+returning id;

package/dist/_protocol/sql/postgres/purge_one_status.sql

@@ -0,0 +1,42 @@
+-- purge, for a sweep bounded to ONE terminal status. Byte-for-byte purge.sql
+-- except that the status filter is an equality instead of an optional `is null
+-- or` — a drift-guard test asserts precisely that, so treat purge.sql as the
+-- source and re-derive this file when it changes.
+--
+-- It exists because the optional-filter form cannot be indexed. SQLite plans a
+-- statement when it is prepared, before any parameter has a value, so `(:status
+-- is null or status = :status)` has to be planned for BOTH branches and the
+-- planner falls back to cairnq_tasks_completed_idx, walking every row past the
+-- cutoff in completion order and discarding the ones belonging to another
+-- status. `limit` bounds what comes back, never what is read, so the cost grows
+-- with exactly the rows the filter was meant to skip — and the deployment the
+-- filter exists for (one installation, two workloads on different clocks) is the
+-- one where those rows are most numerous. Measured on 20k rows over two queues
+-- and four statuses: the optional form chooses cairnq_tasks_completed_idx for
+-- every filter combination, the equality form chooses
+-- cairnq_tasks_status_completed_idx (0007) and reads only its own range. Same
+-- reason claim.sql has specializations, same shape.
+--
+-- Postgres does not have SQLite's problem — it re-plans with the parameter
+-- values for the first executions and folds the null branch away — but it ships
+-- the variant too, because both dialects carry the same statement set and a
+-- caller that had to know which dialect indexes which form would be a worse
+-- contract.
+--
+-- :name stays optional in every variant: nothing indexes it, so it is a residual
+-- predicate either way and specializing it would buy nothing.
+-- params: older_than_ms, queue, status, name, limit
+delete from cairnq_tasks
+where id in (
+    select id from cairnq_tasks
+    where status in ('succeeded', 'failed', 'canceled')
+      and (:queue::text is null or queue = :queue)
+      and status = :status
+      and (:name::text is null or name = :name)
+      and completed_at_ms is not null
+      and completed_at_ms < (extract(epoch from now()) * 1000)::bigint - :older_than_ms
+    order by completed_at_ms asc
+    limit :limit
+    for update skip locked
+)
+returning id;

package/dist/_protocol/sql/postgres/stats.sql

@@ -1,8 +1,21 @@
--- Queue depth at a glance: task counts grouped by queue and status. Read-only.
+-- Task counts grouped by queue and status. Read-only.
 -- A queue appears only while it has rows — terminal tasks count until purge
 -- removes them. The SDK zero-fills the statuses a queue has no rows in.
--- params: (none)
+--
+-- :queue is optional (pass NULL for every queue; `::text` pins the param's type,
+-- as in list.sql). Unfiltered, this reads every row in the table, so its cost
+-- grows with everything the installation has ever run — and one store carrying
+-- two workloads is the coordination cairnq recommends, so a caller asking about
+-- its own queue should not pay for the other's backlog. Filtered to one queue it
+-- can be served from cairnq_tasks_claim_idx's (queue, status) prefix instead.
+--
+-- Filtered or not, this still COUNTS: the cost is proportional to the rows being
+-- counted, which is the whole queue, terminal rows included. That is fine for a
+-- dashboard and wrong for a poll loop — queue_depth.sql is the bounded question,
+-- and the one to ask on an interval.
+-- params: queue
 select queue, status, count(*) as count
 from cairnq_tasks
+where (:queue::text is null or queue = :queue)
 group by queue, status
 order by queue asc, status asc;

package/dist/_protocol/sql/postgres/stats_one_queue.sql

@@ -0,0 +1,21 @@
+-- stats, for a caller asking about ONE queue. Byte-for-byte stats.sql except
+-- that the queue filter is an equality instead of an optional `is null or` — a
+-- drift-guard test asserts precisely that, so treat stats.sql as the source and
+-- re-derive this file when it changes.
+--
+-- It exists for the same reason purge_one_queue.sql does. SQLite plans a
+-- statement before its parameters have values, so the optional form has to be
+-- planned for both branches: it reads the whole table (as a covering index
+-- scan) and groups it, which is exactly the cost narrowing to one queue was
+-- meant to avoid. The equality form seeks the (queue, status) prefix of
+-- cairnq_tasks_claim_idx and reads only that queue's entries.
+--
+-- This still COUNTS what it reports, so it costs what it counts: one queue's
+-- rows, terminal ones included. Narrower than the unfiltered form, still not a
+-- poll-loop question — queue_depth.sql is the bounded one.
+-- params: queue
+select queue, status, count(*) as count
+from cairnq_tasks
+where queue = :queue
+group by queue, status
+order by queue asc, status asc;

package/dist/_protocol/sql/sqlite/claim.sql

@@ -22,10 +22,13 @@ where id in (
       and queue in (select value from json_each(:queues))
       and (:names is null or name in (select value from json_each(:names)))
       and run_at_ms <= :now_ms
-    -- id breaks created_at_ms ties (same-millisecond submits), so claim order
-    -- is deterministic: FIFO at millisecond granularity; within one millisecond
+    -- Ordered by when a task became DUE, not when it was created: see migration
+    -- 0008 for why, for what it costs a mixed-version fleet, and for which of
+    -- these four statements the index can serve without a sort.
+    -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
+    -- deterministic: FIFO at millisecond granularity; within one millisecond
     -- the id's random half decides, stably but not in submit order.
-    order by priority desc, created_at_ms asc, id asc
+    order by priority desc, run_at_ms asc, id asc
     limit :limit
 )
 returning *;