cairnq 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +10 -2
  2. package/dist/_protocol/migrations/postgres/0001_init.sql +2 -0
  3. package/dist/_protocol/migrations/postgres/0006_claim_name_index.sql +2 -0
  4. package/dist/_protocol/migrations/postgres/0008_claim_due_index.sql +68 -0
  5. package/dist/_protocol/migrations/sqlite/0001_init.sql +2 -0
  6. package/dist/_protocol/migrations/sqlite/0006_claim_name_index.sql +2 -0
  7. package/dist/_protocol/migrations/sqlite/0008_claim_due_index.sql +84 -0
  8. package/dist/_protocol/sql/postgres/claim.sql +6 -3
  9. package/dist/_protocol/sql/postgres/claim_one_name.sql +6 -3
  10. package/dist/_protocol/sql/postgres/claim_one_queue.sql +9 -5
  11. package/dist/_protocol/sql/postgres/claim_one_queue_one_name.sql +6 -3
  12. package/dist/_protocol/sql/postgres/installations.sql +33 -0
  13. package/dist/_protocol/sql/sqlite/claim.sql +6 -3
  14. package/dist/_protocol/sql/sqlite/claim_one_name.sql +6 -3
  15. package/dist/_protocol/sql/sqlite/claim_one_queue.sql +9 -6
  16. package/dist/_protocol/sql/sqlite/claim_one_queue_one_name.sql +9 -4
  17. package/dist/context.d.ts +4 -2
  18. package/dist/context.js +12 -8
  19. package/dist/errors.d.ts +28 -0
  20. package/dist/errors.js +34 -0
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.js +1 -1
  23. package/dist/models.d.ts +2 -0
  24. package/dist/models.js +28 -1
  25. package/dist/retention.d.ts +13 -1
  26. package/dist/retention.js +26 -11
  27. package/dist/store/base.d.ts +12 -3
  28. package/dist/store/base.js +21 -6
  29. package/dist/store/pg-executor.d.ts +6 -5
  30. package/dist/store/pg-executor.js +6 -5
  31. package/dist/store/pg-pool.js +37 -11
  32. package/dist/store/postgres.d.ts +64 -0
  33. package/dist/store/postgres.js +121 -1
  34. package/dist/worker.d.ts +4 -2
  35. package/dist/worker.js +10 -8
  36. package/package.json +1 -1
  37. package/src/context.ts +10 -6
  38. package/src/errors.ts +36 -0
  39. package/src/index.ts +2 -0
  40. package/src/models.ts +28 -1
  41. package/src/retention.ts +27 -11
  42. package/src/store/base.ts +27 -5
  43. package/src/store/pg-executor.ts +6 -5
  44. package/src/store/pg-pool.ts +38 -11
  45. package/src/store/postgres.ts +125 -1
  46. package/src/worker.ts +10 -8
package/README.md CHANGED
@@ -114,10 +114,18 @@ const executor: PgExecutor = { /* ~30 lines over your driver */ };
114
114
  const tasks = CairnQ.postgres(executor);
115
115
  ```
116
116
 
117
- `schema` (DSN form only) puts cairnq's tables in a schema of their own:
117
+ `schema` puts cairnq's tables in a schema of their own:
118
118
  `CairnQ.postgres(dsn, { schema: "cairnq" })` creates it if absent and sets
119
119
  `search_path` per connection. The protocol's SQL names no schema, so nothing else
120
- changes. With your own executor, the search_path is yours to set.
120
+ changes. With your own executor the search_path is yours to set, and `schema`
121
+ becomes an assertion about where it lands.
122
+
123
+ **Every process in a deployment must agree on it.** A queue whose API and worker
124
+ resolve to different schemas is two empty queues, and — because every migration is
125
+ `create table if not exists` — both sides come up healthy, pass their protocol
126
+ version check, and never see each other's tasks. cairnq refuses to connect where
127
+ it can see that about to happen (`SchemaMismatch`); the Python SDK applies the
128
+ same rule.
121
129
 
122
130
  Two things an adapter must get right: `int8` has to come back as a JS number
123
131
  (every cairnq `*_ms` is an epoch or a counter, all inside the safe range), and
@@ -49,6 +49,8 @@ create table if not exists cairnq_tasks (
49
49
  -- desc, created_at_ms asc (run_at_ms applied as a residual filter). Only
50
50
  -- claim_one_queue.sql can read it in claim order; claim.sql's array-valued queue
51
51
  -- filter forces a sort, and past a few thousand queued rows a sequential scan.
52
+ -- The shape as first shipped; migration 0008 rebuilds it around run_at_ms and
53
+ -- says why.
52
54
  create index if not exists cairnq_tasks_claim_idx
53
55
  on cairnq_tasks (queue, status, priority desc, created_at_ms);
54
56
  create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);
@@ -19,6 +19,8 @@
19
19
  -- existing large cairnq_tasks this takes a write lock for the build. Deploying
20
20
  -- into a busy database is the case to watch; build it by hand with CONCURRENTLY
21
21
  -- first if that matters, and this statement then becomes a no-op.
22
+ -- The measurements above belong to the shape defined here; migration 0008
23
+ -- rebuilds it around run_at_ms and carries the current numbers.
22
24
  create index if not exists cairnq_tasks_claim_name_idx
23
25
  on cairnq_tasks (queue, status, name, priority desc, created_at_ms);
24
26
 
@@ -0,0 +1,68 @@
1
+ -- Serve the claim's whole ORDER BY from the index, and order by when a task
2
+ -- became DUE rather than by when it was created.
3
+ --
4
+ -- The two halves are one fix. `run_at_ms <= :now_ms` was a residual on the old
5
+ -- (queue, status, priority desc, created_at_ms) index, and a not-yet-due row —
6
+ -- a retry waiting out its backoff, a task submitted with a delay — sorts by the
7
+ -- created_at it has always had, which puts it AHEAD of the rows that are
8
+ -- actually claimable. So every draw walked the whole backoff pile before
9
+ -- reaching anything it could take, inside the transaction that holds the claim.
10
+ -- The pile is largest exactly when a downstream dependency has just failed and
11
+ -- thousands of tasks are backing off together.
12
+ --
13
+ -- Ordering by run_at_ms instead puts every not-yet-due row AFTER the due ones,
14
+ -- so the scan reaches its rows first, and appending `id` lets the index carry the
15
+ -- tie-break as well, which is what lets the planner satisfy the ORDER BY from the
16
+ -- index alone.
17
+ --
18
+ -- The measurements behind this live in the SQLite twin of this file, and they are
19
+ -- SQLite numbers: no Postgres was available where this was written, so this side
20
+ -- is reasoned from the same index shape rather than benchmarked. What holds
21
+ -- across both is the ordering — the due rows are now at the front of the index
22
+ -- range instead of behind every not-yet-due one.
23
+ --
24
+ -- What this does NOT fix, in either dialect: an empty draw still walks the whole
25
+ -- range looking for rows that are not there, and `claimable_probe` — whose
26
+ -- two-armed EXISTS cannot use the index at all — is untouched. A queue deep in
27
+ -- backoff still costs a worker real time per poll.
28
+ --
29
+ -- For a task that was never delayed or retried the change is invisible in
30
+ -- practice: insert_task sets run_at_ms = now + delay, so the two columns hold the
31
+ -- same millisecond for an ordinary submit. (On Postgres "the same millisecond" is
32
+ -- not quite an identity — insert_task evaluates clock_timestamp() once per column
33
+ -- and it advances within a statement, so a submit landing on a millisecond
34
+ -- boundary can differ by 1ms. That reorders two tasks submitted in the same
35
+ -- millisecond, which claim.sql already calls decided by the id's random half
36
+ -- rather than by submit order.) It changes real ordering only for tasks whose
37
+ -- delivery was deferred, and for those "oldest due first" is the fairer answer —
38
+ -- a task that failed and backed off should not cut ahead of everything submitted
39
+ -- while it waited, which is what its original created_at_ms bought it.
40
+ --
41
+ -- Both indexes are rebuilt inside the migration's transaction, which holds a
42
+ -- lock that blocks writes to cairnq_tasks while they build — seconds on a large
43
+ -- table. There is no CONCURRENTLY here: it cannot run inside a transaction, and
44
+ -- the ledger's check-and-apply is one. Upgrade a large deployment in a window
45
+ -- where that pause is affordable.
46
+ --
47
+ -- Same names, new definitions: these indexes ARE "the ones the claim reads", and
48
+ -- every reference to them in the SQL comments and PROTOCOL.md still points at
49
+ -- the right object.
50
+ --
51
+ -- An SDK older than this migration keeps working: it orders by created_at_ms,
52
+ -- finds no index in that order, and sorts. Slower, exactly where this migration
53
+ -- is faster — but note the sharper consequence while a fleet is mixed. Two SDK
54
+ -- versions against one database then disagree about which task is NEXT for any
55
+ -- delayed or retried work: each takes a valid claimable task, no task is lost or
56
+ -- run twice, but the documented claim order holds only within one version. That
57
+ -- is a difference in fairness, not in correctness, which is why protocol_version
58
+ -- stays at 1 — and it is a reason to keep the mixed window short.
59
+ drop index if exists cairnq_tasks_claim_idx;
60
+ create index cairnq_tasks_claim_idx
61
+ on cairnq_tasks (queue, status, priority desc, run_at_ms, id);
62
+
63
+ -- The per-name twin (see 0006), rebuilt on the same principle.
64
+ drop index if exists cairnq_tasks_claim_name_idx;
65
+ create index cairnq_tasks_claim_name_idx
66
+ on cairnq_tasks (queue, status, name, priority desc, run_at_ms, id);
67
+
68
+ update cairnq_meta set value = '8' where key = 'schema_version';
@@ -48,6 +48,8 @@ create table if not exists cairnq_tasks (
48
48
  -- queue+status then the ORDER BY columns is what lets claim_one_queue.sql read
49
49
  -- rows in claim order; claim.sql's list-valued queue filter has to merge several
50
50
  -- ranges of this index, so it sorts instead.
51
+ -- The shape as first shipped; migration 0008 rebuilds it around run_at_ms and
52
+ -- says why.
51
53
  create index if not exists cairnq_tasks_claim_idx
52
54
  on cairnq_tasks (queue, status, priority desc, created_at_ms);
53
55
  create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);
@@ -20,6 +20,8 @@
20
20
  -- bloom filter over the subquery and falls back to cairnq_tasks_claim_idx
21
21
  -- (measured 1446us, i.e. no improvement at all). That is why the per-name
22
22
  -- statements exist as separate files rather than the shared one being reused.
23
+ -- The measurements above belong to the shape defined here; migration 0008
24
+ -- rebuilds it around run_at_ms and carries the current numbers.
23
25
  create index if not exists cairnq_tasks_claim_name_idx
24
26
  on cairnq_tasks (queue, status, name, priority desc, created_at_ms);
25
27
 
@@ -0,0 +1,84 @@
1
+ -- Serve the claim's whole ORDER BY from the index, and order by when a task
2
+ -- became DUE rather than by when it was created.
3
+ --
4
+ -- The two halves are one fix. `run_at_ms <= :now_ms` was a residual on the old
5
+ -- (queue, status, priority desc, created_at_ms) index, and a not-yet-due row —
6
+ -- a retry waiting out its backoff, a task submitted with a delay — sorts by the
7
+ -- created_at it has always had, which puts it AHEAD of the rows that are
8
+ -- actually claimable. So every draw walked the whole backoff pile before
9
+ -- reaching anything it could take, inside the transaction that holds the claim.
10
+ -- The pile is largest exactly when a downstream dependency has just failed and
11
+ -- thousands of tasks are backing off together.
12
+ --
13
+ -- Ordering by run_at_ms instead puts every not-yet-due row AFTER the due ones,
14
+ -- so the scan reaches its rows first, and appending `id` lets the index carry the
15
+ -- tie-break as well, which is what lets the planner satisfy the ORDER BY from the
16
+ -- index alone. Measured through the real statement (`claim_one_queue`, one queue,
17
+ -- four names, 20k queued rows backing off), one claim of one task, on the three
18
+ -- SQLite builds that happened to be on the authoring machine:
19
+ --
20
+ -- SQLite linked by one due row an empty draw
21
+ -- 3.39.4 a system python3.11 1166us -> 22us 1110us -> 350us
22
+ -- 3.47.1 a python3.13 (this repo's venv) 3378us -> 116us 3128us -> 1006us
23
+ -- 3.53.4 better-sqlite3 13 (bundled) 1100us -> 72us 1060us -> 50us
24
+ --
25
+ -- Read the columns, not the rows: the win holds on every build, and the absolute
26
+ -- cost swings more between SQLite versions than between before and after on some
27
+ -- of them. better-sqlite3 bundles its own SQLite, so the TypeScript SDK's version
28
+ -- is a dependency; the Python SDK links whatever the interpreter was built
29
+ -- against, so its version belongs to the user's Python and cannot be assumed.
30
+ --
31
+ -- The plan differs by SQLite version and the win does not come from one single
32
+ -- mechanism: 3.39 drops the sorter outright, while 3.53 keeps one and instead
33
+ -- reaches the due rows through a skip-scan. What both have in common is the
34
+ -- ordering — the due rows are now at the front of the range instead of behind
35
+ -- every not-yet-due one. Numbers are SQLite; Postgres was not measured (no
36
+ -- server in the authoring environment), so treat the Postgres side as reasoned
37
+ -- rather than benchmarked.
38
+ --
39
+ -- What this does NOT fix: an empty draw still walks the whole range looking for
40
+ -- rows that are not there (350us above), and `claimable_probe` — whose two-armed
41
+ -- EXISTS cannot use the index at all — is unchanged at ~2.2ms on that backlog.
42
+ -- A queue deep in backoff still costs a worker real time per poll.
43
+ --
44
+ -- For a task that was never delayed or retried the change is invisible in
45
+ -- practice: insert_task sets run_at_ms = now + delay, so the two columns hold the
46
+ -- same millisecond for an ordinary submit. (On Postgres "the same millisecond" is
47
+ -- not quite an identity — insert_task evaluates clock_timestamp() once per column
48
+ -- and it advances within a statement, so a submit landing on a millisecond
49
+ -- boundary can differ by 1ms. That reorders two tasks submitted in the same
50
+ -- millisecond, which claim.sql already calls decided by the id's random half
51
+ -- rather than by submit order.) It changes real ordering only for tasks whose
52
+ -- delivery was deferred, and for those "oldest due first" is the fairer answer —
53
+ -- a task that failed and backed off should not cut ahead of everything submitted
54
+ -- while it waited, which is what its original created_at_ms bought it.
55
+ --
56
+ -- Both indexes are rebuilt inside the migration's write transaction, so every
57
+ -- other process's writes wait while they build — and they wait only as long as
58
+ -- their busy budget (`busyTimeoutMs` / `busy_timeout_ms`, 5s by default) before
59
+ -- failing. Measured on a warm local SSD: ~2.5s + ~3.4s for a 1M-row, 500MB
60
+ -- database, i.e. past that default. Small databases do not notice; see
61
+ -- "Upgrading" in the README for what to do about a large one.
62
+ --
63
+ -- Same names, new definitions: these indexes ARE "the ones the claim reads", and
64
+ -- every reference to them in the SQL comments and PROTOCOL.md still points at
65
+ -- the right object.
66
+ --
67
+ -- An SDK older than this migration keeps working: it orders by created_at_ms,
68
+ -- finds no index in that order, and sorts. Slower, exactly where this migration
69
+ -- is faster — but note the sharper consequence while a fleet is mixed. Two SDK
70
+ -- versions against one database then disagree about which task is NEXT for any
71
+ -- delayed or retried work: each takes a valid claimable task, no task is lost or
72
+ -- run twice, but the documented claim order holds only within one version. That
73
+ -- is a difference in fairness, not in correctness, which is why protocol_version
74
+ -- stays at 1 — and it is a reason to keep the mixed window short.
75
+ drop index if exists cairnq_tasks_claim_idx;
76
+ create index cairnq_tasks_claim_idx
77
+ on cairnq_tasks (queue, status, priority desc, run_at_ms, id);
78
+
79
+ -- The per-name twin (see 0006), rebuilt on the same principle.
80
+ drop index if exists cairnq_tasks_claim_name_idx;
81
+ create index cairnq_tasks_claim_name_idx
82
+ on cairnq_tasks (queue, status, name, priority desc, run_at_ms, id);
83
+
84
+ update cairnq_meta set value = '8' where key = 'schema_version';
@@ -29,10 +29,13 @@ from (
29
29
  and queue = any(:queues::text[])
30
30
  and (:names::text[] is null or name = any(:names::text[]))
31
31
  and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
32
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
33
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
32
+ -- Ordered by when a task became DUE, not when it was created: see migration
33
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
34
+ -- these four statements the index can serve without a sort.
35
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
36
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
34
37
  -- the id's random half decides, stably but not in submit order.
35
- order by priority desc, created_at_ms asc, id asc
38
+ order by priority desc, run_at_ms asc, id asc
36
39
  limit :limit
37
40
  for update skip locked
38
41
  ) sel
@@ -26,10 +26,13 @@ from (
26
26
  and queue = any(:queues::text[])
27
27
  and name = :name
28
28
  and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
29
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
30
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
29
+ -- Ordered by when a task became DUE, not when it was created: see migration
30
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
31
+ -- these four statements the index can serve without a sort.
32
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
33
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
31
34
  -- the id's random half decides, stably but not in submit order.
32
- order by priority desc, created_at_ms asc, id asc
35
+ order by priority desc, run_at_ms asc, id asc
33
36
  limit :limit
34
37
  for update skip locked
35
38
  ) sel
@@ -7,8 +7,9 @@
7
7
  -- + LIMIT over `= any(...)` the planner falls back to a sequential scan and a
8
8
  -- full sort of every claimable row (measured on 20k queued: Seq Scan 20000 rows,
9
9
  -- quicksort 1861kB), inside the transaction that holds the claim. The equality
10
- -- form index-scans cairnq_tasks_claim_idx and only incrementally sorts the id
11
- -- tie-break — 33 rows read for the same query.
10
+ -- form index-scans cairnq_tasks_claim_idx, which since migration 0008 carries the
11
+ -- id tie-break too and so needs no sort node at all — 33 rows read for the same
12
+ -- query.
12
13
  --
13
14
  -- params: queue, names (text[] or null), worker_id, lease_ms, limit
14
15
  update cairnq_tasks t
@@ -24,10 +25,13 @@ from (
24
25
  and queue = :queue
25
26
  and (:names::text[] is null or name = any(:names::text[]))
26
27
  and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
27
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
28
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
28
+ -- Ordered by when a task became DUE, not when it was created: see migration
29
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
30
+ -- these four statements the index can serve without a sort.
31
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
32
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
29
33
  -- the id's random half decides, stably but not in submit order.
30
- order by priority desc, created_at_ms asc, id asc
34
+ order by priority desc, run_at_ms asc, id asc
31
35
  limit :limit
32
36
  for update skip locked
33
37
  ) sel
@@ -23,10 +23,13 @@ from (
23
23
  and queue = :queue
24
24
  and name = :name
25
25
  and run_at_ms <= (extract(epoch from clock_timestamp()) * 1000)::bigint
26
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
27
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
26
+ -- Ordered by when a task became DUE, not when it was created: see migration
27
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
28
+ -- these four statements the index can serve without a sort.
29
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
30
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
28
31
  -- the id's random half decides, stably but not in submit order.
29
- order by priority desc, created_at_ms asc, id asc
32
+ order by priority desc, run_at_ms asc, id asc
30
33
  limit :limit
31
34
  for update skip locked
32
35
  ) sel
@@ -0,0 +1,33 @@
1
+ -- Where cairnq already lives in this database, and where this connection is
2
+ -- pointing. Read-only; runs once per connect, before migrations.
3
+ --
4
+ -- Exists because `search_path` is out-of-band configuration: two processes given
5
+ -- the same DSN can still resolve to different schemas, and because every
6
+ -- migration is `create table if not exists`, the second one to start does not
7
+ -- fail — it quietly builds a parallel, empty installation. Nothing downstream can
8
+ -- tell: protocol_version reads from whichever cairnq_meta the connection sees, so
9
+ -- the version check passes on both sides while the API's tasks are invisible to
10
+ -- the worker forever. The only way to catch that is to look OUTSIDE the
11
+ -- connection's own search_path, which is what this does.
12
+ --
13
+ -- One row per installation, never zero: the LEFT JOIN keeps `current_schema`
14
+ -- readable on a database that holds no cairnq yet, where `schema` is null.
15
+ -- Deliberately NOT an array column — pg_namespace.nspname is `name`, and which
16
+ -- drivers decode a `name[]` (or a text[]) into a list is exactly the kind of
17
+ -- disagreement this protocol keeps out of the SDKs. Scalar columns behave the
18
+ -- same everywhere.
19
+ --
20
+ -- `current_schema()` is null when the search_path names nothing that exists, in
21
+ -- which case the caller cannot conclude anything.
22
+ -- params: (none)
23
+ select
24
+ current_schema()::text as current_schema,
25
+ found.schema
26
+ from (select 1) one
27
+ left join (
28
+ select n.nspname::text as schema
29
+ from pg_class c
30
+ join pg_namespace n on n.oid = c.relnamespace
31
+ where c.relname = 'cairnq_tasks' and c.relkind = 'r'
32
+ ) found on true
33
+ order by found.schema;
@@ -22,10 +22,13 @@ where id in (
22
22
  and queue in (select value from json_each(:queues))
23
23
  and (:names is null or name in (select value from json_each(:names)))
24
24
  and run_at_ms <= :now_ms
25
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
26
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
25
+ -- Ordered by when a task became DUE, not when it was created: see migration
26
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
27
+ -- these four statements the index can serve without a sort.
28
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
29
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
27
30
  -- the id's random half decides, stably but not in submit order.
28
- order by priority desc, created_at_ms asc, id asc
31
+ order by priority desc, run_at_ms asc, id asc
29
32
  limit :limit
30
33
  )
31
34
  returning *;
@@ -27,10 +27,13 @@ where id in (
27
27
  and queue in (select value from json_each(:queues))
28
28
  and name = :name
29
29
  and run_at_ms <= :now_ms
30
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
31
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
30
+ -- Ordered by when a task became DUE, not when it was created: see migration
31
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
32
+ -- these four statements the index can serve without a sort.
33
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
34
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
32
35
  -- the id's random half decides, stably but not in submit order.
33
- order by priority desc, created_at_ms asc, id asc
36
+ order by priority desc, run_at_ms asc, id asc
34
37
  limit :limit
35
38
  )
36
39
  returning *;
@@ -8,9 +8,9 @@
8
8
  -- longer read rows in claim order; it materializes every claimable row into a
9
9
  -- temp B-tree just to take LIMIT of them. Cost then grows with the queued
10
10
  -- backlog, inside the write transaction, on every claim: measured at 21us / 239us
11
- -- / 1792us for a backlog of 50 / 2000 / 20000. The equality form keeps
12
- -- cairnq_tasks_claim_idx in claim order, needs only a partial sort for the id
13
- -- tie-break, and stays flat at ~12us.
11
+ -- / 1792us for a backlog of 50 / 2000 / 20000. The equality form reads
12
+ -- cairnq_tasks_claim_idx in claim order the whole ORDER BY, id tie-break
13
+ -- included, since migration 0008 — so it needs no sort at all and stays flat.
14
14
  --
15
15
  -- params: queue, names (JSON array text or null), now_ms, worker_id,
16
16
  -- lease_until_ms, limit
@@ -27,10 +27,13 @@ where id in (
27
27
  and queue = :queue
28
28
  and (:names is null or name in (select value from json_each(:names)))
29
29
  and run_at_ms <= :now_ms
30
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
31
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
30
+ -- Ordered by when a task became DUE, not when it was created: see migration
31
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
32
+ -- these four statements the index can serve without a sort.
33
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
34
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
32
35
  -- the id's random half decides, stably but not in submit order.
33
- order by priority desc, created_at_ms asc, id asc
36
+ order by priority desc, run_at_ms asc, id asc
34
37
  limit :limit
35
38
  )
36
39
  returning *;
@@ -7,7 +7,9 @@
7
7
  -- claim_one_name.sql's name equality, and each is there for the reason that file
8
8
  -- gives. Together they let cairnq_tasks_claim_name_idx be read in claim order
9
9
  -- with both leading columns pinned, so the draw is a seek that terminates at
10
- -- :limit rows however deep the backlog is.
10
+ -- :limit rows however deep the backlog is — including a backlog of tasks that are
11
+ -- queued but not yet due, which since migration 0008 sort behind the due ones
12
+ -- rather than ahead of them.
11
13
  -- params: queue, name, now_ms, worker_id, lease_until_ms, limit
12
14
  update cairnq_tasks
13
15
  set
@@ -22,10 +24,13 @@ where id in (
22
24
  and queue = :queue
23
25
  and name = :name
24
26
  and run_at_ms <= :now_ms
25
- -- id breaks created_at_ms ties (same-millisecond submits), so claim order
26
- -- is deterministic: FIFO at millisecond granularity; within one millisecond
27
+ -- Ordered by when a task became DUE, not when it was created: see migration
28
+ -- 0008 for why, for what it costs a mixed-version fleet, and for which of
29
+ -- these four statements the index can serve without a sort.
30
+ -- id breaks run_at_ms ties (same-millisecond submits), so claim order is
31
+ -- deterministic: FIFO at millisecond granularity; within one millisecond
27
32
  -- the id's random half decides, stably but not in submit order.
28
- order by priority desc, created_at_ms asc, id asc
33
+ order by priority desc, run_at_ms asc, id asc
29
34
  limit :limit
30
35
  )
31
36
  returning *;
package/dist/context.d.ts CHANGED
@@ -19,7 +19,7 @@ export interface TaskContextOptions {
19
19
  export declare class TaskContext {
20
20
  private readonly store;
21
21
  private readonly task;
22
- readonly workerId: string;
22
+ private readonly ownerId;
23
23
  private readonly leaseMs;
24
24
  private readonly abort;
25
25
  private leaseLost;
@@ -27,8 +27,10 @@ export declare class TaskContext {
27
27
  private isSettled;
28
28
  private readonly backoffMs;
29
29
  private readonly backoffMaxMs;
30
- constructor(store: TaskStore, task: Task, workerId: string, leaseMs: number, opts?: TaskContextOptions);
30
+ constructor(store: TaskStore, task: Task, ownerId: string, leaseMs: number, opts?: TaskContextOptions);
31
31
  get taskId(): string;
32
+ /** The worker running this task — what `worker_id` on the row points at. */
33
+ get workerId(): string;
32
34
  get name(): string;
33
35
  get queue(): string;
34
36
  get attempt(): number;
package/dist/context.js CHANGED
@@ -14,7 +14,7 @@ import { pollWait } from "./wait.js";
14
14
  export class TaskContext {
15
15
  store;
16
16
  task;
17
- workerId;
17
+ ownerId;
18
18
  leaseMs;
19
19
  abort = new AbortController();
20
20
  leaseLost = false;
@@ -28,10 +28,10 @@ export class TaskContext {
28
28
  isSettled = false;
29
29
  backoffMs;
30
30
  backoffMaxMs;
31
- constructor(store, task, workerId, leaseMs, opts = {}) {
31
+ constructor(store, task, ownerId, leaseMs, opts = {}) {
32
32
  this.store = store;
33
33
  this.task = task;
34
- this.workerId = workerId;
34
+ this.ownerId = ownerId;
35
35
  this.leaseMs = leaseMs;
36
36
  this.backoffMs = opts.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
37
37
  this.backoffMaxMs = opts.retryBackoffMaxMs ?? DEFAULT_RETRY_BACKOFF_MAX_MS;
@@ -39,6 +39,10 @@ export class TaskContext {
39
39
  get taskId() {
40
40
  return this.task.id;
41
41
  }
42
+ /** The worker running this task — what `worker_id` on the row points at. */
43
+ get workerId() {
44
+ return this.ownerId;
45
+ }
42
46
  get name() {
43
47
  return this.task.name;
44
48
  }
@@ -138,7 +142,7 @@ export class TaskContext {
138
142
  async progress(value, message = null) {
139
143
  return this.owned(() => this.store.progress({
140
144
  taskId: this.task.id,
141
- workerId: this.workerId,
145
+ workerId: this.ownerId,
142
146
  progress: value,
143
147
  message,
144
148
  }));
@@ -146,7 +150,7 @@ export class TaskContext {
146
150
  async heartbeat() {
147
151
  return this.owned(() => this.store.heartbeat({
148
152
  taskId: this.task.id,
149
- workerId: this.workerId,
153
+ workerId: this.ownerId,
150
154
  leaseMs: this.leaseMs,
151
155
  }));
152
156
  }
@@ -180,7 +184,7 @@ export class TaskContext {
180
184
  async succeed(result = null) {
181
185
  if (this.isSettled)
182
186
  return null;
183
- const task = await this.owned(() => this.store.complete({ taskId: this.task.id, workerId: this.workerId, result }));
187
+ const task = await this.owned(() => this.store.complete({ taskId: this.task.id, workerId: this.ownerId, result }));
184
188
  this.markSettled();
185
189
  return task;
186
190
  }
@@ -211,7 +215,7 @@ export class TaskContext {
211
215
  if (this.isSettled)
212
216
  return null;
213
217
  const task = await this.owned(async () => {
214
- const { task } = await this.store.completeIn({ taskId: this.task.id, workerId: this.workerId }, write);
218
+ const { task } = await this.store.completeIn({ taskId: this.task.id, workerId: this.ownerId }, write);
215
219
  return task;
216
220
  });
217
221
  this.markSettled();
@@ -229,7 +233,7 @@ export class TaskContext {
229
233
  const [envelope, retryable] = asEnvelope(error, opts.retryable ?? true);
230
234
  const task = await this.owned(() => this.store.fail({
231
235
  taskId: this.task.id,
232
- workerId: this.workerId,
236
+ workerId: this.ownerId,
233
237
  error: envelope,
234
238
  retryable,
235
239
  delayMs: failDelayMs(this.task.attempt, retryable, this.backoffMs, this.backoffMaxMs),
package/dist/errors.d.ts CHANGED
@@ -101,6 +101,34 @@ export declare class LostLease extends CairnQError {
101
101
  export declare class ProtocolVersionMismatch extends CairnQError {
102
102
  constructor(message: string);
103
103
  }
104
+ /**
105
+ * This store cannot do what was asked, and no argument would change that — the
106
+ * capability belongs to the backend, not to the call.
107
+ *
108
+ * Thrown by `completeIn` on a store with no driver session to share (SQLite has
109
+ * none). A CairnQError rather than a bare Error so the same catch works across
110
+ * both SDKs; the Python SDK raises the same named error.
111
+ */
112
+ export declare class UnsupportedBackend extends CairnQError {
113
+ constructor(message: string);
114
+ }
115
+ /**
116
+ * This connection is not pointed at the cairnq installation the rest of the
117
+ * deployment is using — raised at connect, before any task is written.
118
+ *
119
+ * The schema a Postgres connection resolves to is out-of-band configuration
120
+ * (`search_path`, a `schema` option, an ORM's pool settings), so two processes
121
+ * given the same DSN can still land in different schemas. Every migration is
122
+ * `create table if not exists`, so the odd one out does not fail: it builds a
123
+ * second, empty installation and its protocol version check passes against the
124
+ * cairnq_meta it just created. Left undetected, an API and a worker then agree
125
+ * about everything except WHERE, and no task ever crosses.
126
+ *
127
+ * The Python SDK raises the same named error.
128
+ */
129
+ export declare class SchemaMismatch extends CairnQError {
130
+ constructor(message: string);
131
+ }
104
132
  /** A value could not be encoded for a protocol JSON column (non-finite number,
105
133
  * BigInt, circular structure, …). Raised at the boundary — submit rejects with
106
134
  * it, and a worker records a handler result that triggers it as a permanent
package/dist/errors.js CHANGED
@@ -187,6 +187,40 @@ export class ProtocolVersionMismatch extends CairnQError {
187
187
  this.name = "ProtocolVersionMismatch";
188
188
  }
189
189
  }
190
+ /**
191
+ * This store cannot do what was asked, and no argument would change that — the
192
+ * capability belongs to the backend, not to the call.
193
+ *
194
+ * Thrown by `completeIn` on a store with no driver session to share (SQLite has
195
+ * none). A CairnQError rather than a bare Error so the same catch works across
196
+ * both SDKs; the Python SDK raises the same named error.
197
+ */
198
+ export class UnsupportedBackend extends CairnQError {
199
+ constructor(message) {
200
+ super(message);
201
+ this.name = "UnsupportedBackend";
202
+ }
203
+ }
204
+ /**
205
+ * This connection is not pointed at the cairnq installation the rest of the
206
+ * deployment is using — raised at connect, before any task is written.
207
+ *
208
+ * The schema a Postgres connection resolves to is out-of-band configuration
209
+ * (`search_path`, a `schema` option, an ORM's pool settings), so two processes
210
+ * given the same DSN can still land in different schemas. Every migration is
211
+ * `create table if not exists`, so the odd one out does not fail: it builds a
212
+ * second, empty installation and its protocol version check passes against the
213
+ * cairnq_meta it just created. Left undetected, an API and a worker then agree
214
+ * about everything except WHERE, and no task ever crosses.
215
+ *
216
+ * The Python SDK raises the same named error.
217
+ */
218
+ export class SchemaMismatch extends CairnQError {
219
+ constructor(message) {
220
+ super(message);
221
+ this.name = "SchemaMismatch";
222
+ }
223
+ }
190
224
  /** A value could not be encoded for a protocol JSON column (non-finite number,
191
225
  * BigInt, circular structure, …). Raised at the boundary — submit rejects with
192
226
  * it, and a worker records a handler result that triggers it as a permanent
package/dist/index.d.ts CHANGED
@@ -20,5 +20,5 @@ export type { ListInput, PurgeInput, SubmitInput, Conflict, WatchOptions, WatchS
20
20
  export { DEFAULT_WATCH_POLL_MS } from "./store/base.js";
21
21
  export type { Task, TaskRef, TaskStatus, TerminalStatus } from "./models.js";
22
22
  export { STATUSES, isTerminal, isTerminalStatus, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
23
- export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
23
+ export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SchemaMismatch, UnsupportedBackend, SerializationError, } from "./errors.js";
24
24
  export type { FailReason } from "./errors.js";
package/dist/index.js CHANGED
@@ -11,4 +11,4 @@ export { createPoolExecutor } from "./store/pg-pool.js";
11
11
  export { TaskStore } from "./store/base.js";
12
12
  export { DEFAULT_WATCH_POLL_MS } from "./store/base.js";
13
13
  export { STATUSES, isTerminal, isTerminalStatus, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
14
- export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
14
+ export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SchemaMismatch, UnsupportedBackend, SerializationError, } from "./errors.js";
package/dist/models.d.ts CHANGED
@@ -31,6 +31,8 @@ export interface TaskRef {
31
31
  id: string;
32
32
  status: TaskStatus;
33
33
  }
34
+ export declare const JSON_COLUMNS: readonly ["payload", "result", "error", "metadata"];
35
+ export declare const MS_COLUMNS: readonly ["lease_until_ms", "run_at_ms", "cancel_requested_at_ms", "created_at_ms", "updated_at_ms", "completed_at_ms"];
34
36
  export declare const TERMINAL: readonly ["succeeded", "failed", "canceled"];
35
37
  export type TerminalStatus = (typeof TERMINAL)[number];
36
38
  export declare function isTerminalStatus(status: TaskStatus): status is TerminalStatus;