cairnq 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -32
- package/dist/_protocol/migrations/postgres/0010_drop_status_index.sql +29 -0
- package/dist/_protocol/migrations/sqlite/0010_drop_status_index.sql +29 -0
- package/dist/_protocol/sql/postgres/purge.sql +14 -0
- package/dist/_protocol/sql/sqlite/purge.sql +14 -0
- package/dist/backpressure.d.ts +0 -3
- package/dist/backpressure.js +1 -3
- package/dist/client.d.ts +42 -46
- package/dist/client.js +27 -43
- package/dist/context.d.ts +23 -1
- package/dist/context.js +18 -7
- package/dist/errors.d.ts +20 -26
- package/dist/errors.js +23 -35
- package/dist/index.d.ts +4 -10
- package/dist/index.js +3 -6
- package/dist/models.d.ts +14 -1
- package/dist/models.js +17 -5
- package/dist/retention.d.ts +22 -66
- package/dist/retention.js +53 -64
- package/dist/store/base.d.ts +47 -90
- package/dist/store/base.js +182 -140
- package/dist/store/pg-executor.d.ts +38 -12
- package/dist/store/pg-executor.js +38 -12
- package/dist/store/pg-pool.js +14 -0
- package/dist/store/postgres.d.ts +18 -12
- package/dist/store/postgres.js +26 -32
- package/dist/store/sqlite.d.ts +46 -2
- package/dist/store/sqlite.js +128 -35
- package/dist/worker.d.ts +38 -93
- package/dist/worker.js +181 -234
- package/package.json +1 -1
- package/src/backpressure.ts +1 -5
- package/src/client.ts +66 -71
- package/src/context.ts +28 -7
- package/src/errors.ts +25 -36
- package/src/index.ts +4 -27
- package/src/models.ts +17 -5
- package/src/retention.ts +50 -115
- package/src/store/base.ts +179 -184
- package/src/store/pg-executor.ts +38 -12
- package/src/store/pg-pool.ts +14 -0
- package/src/store/postgres.ts +27 -33
- package/src/store/sqlite.ts +128 -32
- package/src/worker.ts +208 -300
- package/dist/_protocol/sql/postgres/purge_one_queue.sql +0 -42
- package/dist/_protocol/sql/postgres/purge_one_queue_one_status.sql +0 -42
- package/dist/_protocol/sql/postgres/purge_one_status.sql +0 -42
- package/dist/_protocol/sql/postgres/stats.sql +0 -21
- package/dist/_protocol/sql/postgres/stats_one_queue.sql +0 -21
- package/dist/_protocol/sql/sqlite/purge_one_queue.sql +0 -41
- package/dist/_protocol/sql/sqlite/purge_one_queue_one_status.sql +0 -41
- package/dist/_protocol/sql/sqlite/purge_one_status.sql +0 -41
- package/dist/_protocol/sql/sqlite/stats.sql +0 -21
- package/dist/_protocol/sql/sqlite/stats_one_queue.sql +0 -21
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ Synchronous call (submit + wait):
|
|
|
34
34
|
import { TaskFailed, TaskTimeout } from "cairnq";
|
|
35
35
|
|
|
36
36
|
try {
|
|
37
|
-
const result = await tasks.call("summary.create", { text }, {
|
|
37
|
+
const result = await tasks.call("summary.create", { text }, { timeoutMs: 10_000 });
|
|
38
38
|
} catch (err) {
|
|
39
39
|
if (err instanceof TaskFailed) log(err.code, err.message, err.retryable); // envelope fields
|
|
40
40
|
else if (err instanceof TaskTimeout) {
|
|
@@ -45,13 +45,11 @@ try {
|
|
|
45
45
|
}
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
-
Inspect a task by id/key
|
|
48
|
+
Inspect a task by id/key:
|
|
49
49
|
|
|
50
50
|
```ts
|
|
51
|
-
import { isSucceeded } from "cairnq"; // also isFailed/isCanceled/isRunning/isQueued/isTerminal
|
|
52
|
-
|
|
53
51
|
const task = await tasks.getByKey(key);
|
|
54
|
-
if (task
|
|
52
|
+
if (task?.status === "succeeded") use(task.result);
|
|
55
53
|
```
|
|
56
54
|
|
|
57
55
|
Optionally define a task once and share the symbol across both ends — no string
|
|
@@ -72,19 +70,17 @@ Opt-in: every API still accepts a plain name string (cross-language callers use
|
|
|
72
70
|
|
|
73
71
|
```ts
|
|
74
72
|
const worker = Worker.sqlite("tasks.db", {
|
|
75
|
-
concurrency: 4, // handler calls at once
|
|
73
|
+
concurrency: 4, // handler calls at once (a batch call counts as one)
|
|
76
74
|
retryBackoffMs: 1_000, // window doubles per attempt, capped at retryBackoffMaxMs (30s),
|
|
77
75
|
// jittered over its upper half; 0 disables
|
|
78
76
|
onError: (err, info) => log.warn({ err, ...info }), // claims/writes the loop survived
|
|
79
77
|
});
|
|
80
78
|
|
|
81
|
-
// Nothing else deletes rows, so give the client a retention
|
|
82
|
-
// terminal tasks in bounded batches for as long as the handle is open.
|
|
83
|
-
// per
|
|
84
|
-
//
|
|
85
|
-
const tasks = CairnQ.sqlite("tasks.db", {
|
|
86
|
-
retention: { olderThanMs: { succeeded: 300_000, failed: 7 * 24 * 3600_000 } },
|
|
87
|
-
});
|
|
79
|
+
// Nothing else deletes rows, so give the client a retention cutoff — it sweeps
|
|
80
|
+
// terminal tasks in bounded batches for as long as the handle is open. Tiered
|
|
81
|
+
// retention (per queue, per status) is purge() with filters, from your own
|
|
82
|
+
// scheduler.
|
|
83
|
+
const tasks = CairnQ.sqlite("tasks.db", { retentionMs: 7 * 24 * 3600_000 });
|
|
88
84
|
```
|
|
89
85
|
|
|
90
86
|
A handler that does real side effects should bail out when it loses its lease —
|
|
@@ -150,24 +146,5 @@ work durable while the task still reads as running — on retry, recomputed. If
|
|
|
150
146
|
the lease turns out to be gone, the settlement matches no row and the caller's
|
|
151
147
|
writes roll back with it.
|
|
152
148
|
|
|
153
|
-
## Watching
|
|
154
|
-
|
|
155
|
-
`watch` calls back when the tasks on a queue may have changed — for a dashboard
|
|
156
|
-
that would otherwise poll:
|
|
157
|
-
|
|
158
|
-
```ts
|
|
159
|
-
const stop = tasks.watch({ queues: ["render"] }, async (signal) => {
|
|
160
|
-
if (signal.reason === "done") return refreshOne(signal.taskId!);
|
|
161
|
-
setCounts(await tasks.stats());
|
|
162
|
-
});
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
It is notify-accelerated polling, not an event log. On Postgres an idle watch
|
|
166
|
-
costs nothing and a signal lands within milliseconds; where LISTEN is
|
|
167
|
-
unavailable — a transaction-mode pooler, or SQLite, which has no channel — the
|
|
168
|
-
timer alone still delivers `poll` signals. So the same consumer is correct either
|
|
169
|
-
way, and only its promptness differs. Treat a signal as "re-read now"; the truth
|
|
170
|
-
is in `stats()` / `list()` / `get()`.
|
|
171
|
-
|
|
172
149
|
The protocol (schema + canonical SQL) lives in `../cairnq-protocol` and is shared
|
|
173
150
|
verbatim with the Python SDK. See `../cairnq-protocol/PROTOCOL.md`.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
-- Drop cairnq_tasks_status_idx, which nothing can use.
|
|
2
|
+
--
|
|
3
|
+
-- 0001 shipped it for list's status filter. 0007 later added
|
|
4
|
+
-- (status, completed_at_ms) for purge, and a one-column index on the same
|
|
5
|
+
-- leading column is strictly contained in that: any lookup the narrow one could
|
|
6
|
+
-- serve, the wider one serves from its prefix. Confirmed on the statements as
|
|
7
|
+
-- they run — with the status filter specialized to an equality, SQLite picks
|
|
8
|
+
-- cairnq_tasks_status_completed_idx for list and for purge, and never the
|
|
9
|
+
-- narrow index, on a 20k-row table with statistics.
|
|
10
|
+
--
|
|
11
|
+
-- Until specialization existed neither was reachable at all, so this was
|
|
12
|
+
-- invisible: both indexes looked equally unused, and dropping either looked
|
|
13
|
+
-- equally safe or unsafe. It is only once the filters reach an index that one
|
|
14
|
+
-- of them is demonstrably the one reached.
|
|
15
|
+
--
|
|
16
|
+
-- The write cost is what makes it worth removing rather than leaving: every
|
|
17
|
+
-- insert and every status transition — submit, claim, each settle, each retry —
|
|
18
|
+
-- maintains it, which is the hottest write path there is, for a structure no
|
|
19
|
+
-- read consults. The other three of 0001's filter indexes (name, root_id,
|
|
20
|
+
-- correlation_id) stay: each is the only index that serves its filter, and each
|
|
21
|
+
-- is demonstrably read now.
|
|
22
|
+
--
|
|
23
|
+
-- Dropping an index needs no CONCURRENTLY on either dialect and holds a lock
|
|
24
|
+
-- only long enough to unlink it, so unlike 0008 this is not an upgrade window
|
|
25
|
+
-- to plan around. An older SDK is unaffected: it never named the index, and the
|
|
26
|
+
-- statement that used to want it could not reach it anyway.
|
|
27
|
+
drop index if exists cairnq_tasks_status_idx;
|
|
28
|
+
|
|
29
|
+
update cairnq_meta set value = '10' where key = 'schema_version';
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
-- Drop cairnq_tasks_status_idx, which nothing can use.
|
|
2
|
+
--
|
|
3
|
+
-- 0001 shipped it for list's status filter. 0007 later added
|
|
4
|
+
-- (status, completed_at_ms) for purge, and a one-column index on the same
|
|
5
|
+
-- leading column is strictly contained in that: any lookup the narrow one could
|
|
6
|
+
-- serve, the wider one serves from its prefix. Confirmed on the statements as
|
|
7
|
+
-- they run — with the status filter specialized to an equality, SQLite picks
|
|
8
|
+
-- cairnq_tasks_status_completed_idx for list and for purge, and never the
|
|
9
|
+
-- narrow index, on a 20k-row table with statistics.
|
|
10
|
+
--
|
|
11
|
+
-- Until specialization existed neither was reachable at all, so this was
|
|
12
|
+
-- invisible: both indexes looked equally unused, and dropping either looked
|
|
13
|
+
-- equally safe or unsafe. It is only once the filters reach an index that one
|
|
14
|
+
-- of them is demonstrably the one reached.
|
|
15
|
+
--
|
|
16
|
+
-- The write cost is what makes it worth removing rather than leaving: every
|
|
17
|
+
-- insert and every status transition — submit, claim, each settle, each retry —
|
|
18
|
+
-- maintains it, which is the hottest write path there is, for a structure no
|
|
19
|
+
-- read consults. The other three of 0001's filter indexes (name, root_id,
|
|
20
|
+
-- correlation_id) stay: each is the only index that serves its filter, and each
|
|
21
|
+
-- is demonstrably read now.
|
|
22
|
+
--
|
|
23
|
+
-- Dropping an index needs no CONCURRENTLY on either dialect and holds a lock
|
|
24
|
+
-- only long enough to unlink it, so unlike 0008 this is not an upgrade window
|
|
25
|
+
-- to plan around. An older SDK is unaffected: it never named the index, and the
|
|
26
|
+
-- statement that used to want it could not reach it anyway.
|
|
27
|
+
drop index if exists cairnq_tasks_status_idx;
|
|
28
|
+
|
|
29
|
+
update cairnq_meta set value = '10' where key = 'schema_version';
|
|
@@ -21,6 +21,20 @@
|
|
|
21
21
|
-- durable job's log kept for a week. Migration 0009 adds the index that makes
|
|
22
22
|
-- the queue filter read only its own queue's rows rather than skipping past
|
|
23
23
|
-- every other queue's.
|
|
24
|
+
-- The optional form written here is not what runs when a filter IS supplied:
|
|
25
|
+
-- `(:p is null or col = :p)` is planned before the parameter has a value, so
|
|
26
|
+
-- SQLite must plan both branches, reaches no index at all, and walks every row
|
|
27
|
+
-- past the cutoff in completion order. The SDK rewrites each supplied filter to
|
|
28
|
+
-- a plain equality before preparing the statement (`specialize`), which is what
|
|
29
|
+
-- lets the queue and status indexes be reached. That rewrite replaced the three
|
|
30
|
+
-- hand-written variant files this comment used to name.
|
|
31
|
+
--
|
|
32
|
+
-- Postgres does not need the rewrite — it re-plans with the parameter values for
|
|
33
|
+
-- a statement's first executions and folds the null branch away — but it costs
|
|
34
|
+
-- nothing there, and one behaviour is easier to reason about than two.
|
|
35
|
+
--
|
|
36
|
+
-- :name has no specialization: no index covers it, so it is a residual predicate
|
|
37
|
+
-- either way and an equality form would buy nothing.
|
|
24
38
|
-- params: older_than_ms, queue, status, name, limit
|
|
25
39
|
delete from cairnq_tasks
|
|
26
40
|
where id in (
|
|
@@ -15,6 +15,20 @@
|
|
|
15
15
|
-- result read once and a durable job's log kept for a week. Migration 0009 adds
|
|
16
16
|
-- the index that makes the queue filter read only its own queue's rows rather
|
|
17
17
|
-- than skipping past every other queue's.
|
|
18
|
+
-- The optional form written here is not what runs when a filter IS supplied:
|
|
19
|
+
-- `(:p is null or col = :p)` is planned before the parameter has a value, so
|
|
20
|
+
-- SQLite must plan both branches, reaches no index at all, and walks every row
|
|
21
|
+
-- past the cutoff in completion order. The SDK rewrites each supplied filter to
|
|
22
|
+
-- a plain equality before preparing the statement (`specialize`), which is what
|
|
23
|
+
-- lets the queue and status indexes be reached. That rewrite replaced the three
|
|
24
|
+
-- hand-written variant files this comment used to name.
|
|
25
|
+
--
|
|
26
|
+
-- Postgres does not need the rewrite — it re-plans with the parameter values for
|
|
27
|
+
-- a statement's first executions and folds the null branch away — but it costs
|
|
28
|
+
-- nothing there, and one behaviour is easier to reason about than two.
|
|
29
|
+
--
|
|
30
|
+
-- :name has no specialization: no index covers it, so it is a residual predicate
|
|
31
|
+
-- either way and an equality form would buy nothing.
|
|
18
32
|
-- params: before_ms, queue, status, name, limit
|
|
19
33
|
delete from cairnq_tasks
|
|
20
34
|
where id in (
|
package/dist/backpressure.d.ts
CHANGED
|
@@ -7,8 +7,6 @@ export interface BackpressureOptions {
|
|
|
7
7
|
maxQueueDepth: QueueDepthLimit;
|
|
8
8
|
/** How long a blocked submit waits before raising QueueFull. Default 600_000. */
|
|
9
9
|
maxQueueWaitMs?: number;
|
|
10
|
-
/** First backoff between depth probes; doubles to a 5s ceiling. Default 250. */
|
|
11
|
-
queuePollIntervalMs?: number;
|
|
12
10
|
}
|
|
13
11
|
/**
|
|
14
12
|
* Blocks `submit` while a queue is at its depth limit.
|
|
@@ -37,7 +35,6 @@ export declare class QueueDepthGate {
|
|
|
37
35
|
private readonly probing;
|
|
38
36
|
private readonly limits;
|
|
39
37
|
private readonly maxWaitMs;
|
|
40
|
-
private readonly initialProbeMs;
|
|
41
38
|
constructor(store: TaskStore, opts: BackpressureOptions);
|
|
42
39
|
private validate;
|
|
43
40
|
/** The limit for `queue`, or null when it is not gated. */
|
package/dist/backpressure.js
CHANGED
|
@@ -44,12 +44,10 @@ export class QueueDepthGate {
|
|
|
44
44
|
probing = new Map();
|
|
45
45
|
limits;
|
|
46
46
|
maxWaitMs;
|
|
47
|
-
initialProbeMs;
|
|
48
47
|
constructor(store, opts) {
|
|
49
48
|
this.store = store;
|
|
50
49
|
this.limits = opts.maxQueueDepth;
|
|
51
50
|
this.maxWaitMs = opts.maxQueueWaitMs ?? DEFAULT_MAX_WAIT_MS;
|
|
52
|
-
this.initialProbeMs = opts.queuePollIntervalMs ?? INITIAL_PROBE_INTERVAL_MS;
|
|
53
51
|
if (typeof this.limits === "number")
|
|
54
52
|
this.validate("*", this.limits);
|
|
55
53
|
else
|
|
@@ -80,7 +78,7 @@ export class QueueDepthGate {
|
|
|
80
78
|
if (limit == null)
|
|
81
79
|
return;
|
|
82
80
|
const startedAt = Date.now();
|
|
83
|
-
let waitMs =
|
|
81
|
+
let waitMs = INITIAL_PROBE_INTERVAL_MS;
|
|
84
82
|
for (;;) {
|
|
85
83
|
const left = this.headroom.get(queue) ?? 0;
|
|
86
84
|
if (left > 0) {
|
package/dist/client.d.ts
CHANGED
|
@@ -1,29 +1,50 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import { type RetentionOptions } from "./retention.js";
|
|
3
|
-
import { type Task, type TaskRef, type TaskStatus } from "./models.js";
|
|
1
|
+
import { type Task } from "./models.js";
|
|
4
2
|
import type { PgExecutor } from "./store/pg-executor.js";
|
|
5
|
-
import type { ListInput, PurgeInput,
|
|
3
|
+
import type { Conflict, ListInput, PurgeInput, TaskStore } from "./store/base.js";
|
|
6
4
|
import { type TaskDef } from "./task.js";
|
|
7
5
|
import { type PollOptions } from "./wait.js";
|
|
8
|
-
|
|
6
|
+
/** Per-task options a submit may carry. Everything else about a task — how it
|
|
7
|
+
* is delivered, retried, batched — is declared once, on the worker. */
|
|
8
|
+
export interface SubmitOptions {
|
|
9
|
+
queue?: string;
|
|
10
|
+
/** Business-stable idempotency key; see `conflict` for what a duplicate means. */
|
|
11
|
+
key?: string | null;
|
|
12
|
+
conflict?: Conflict;
|
|
13
|
+
maxAttempts?: number;
|
|
14
|
+
priority?: number;
|
|
15
|
+
metadata?: unknown;
|
|
16
|
+
/** Run no earlier than this many ms from now. */
|
|
17
|
+
delayMs?: number;
|
|
18
|
+
}
|
|
9
19
|
/** The wait loop's knobs with the timeout optional (default 30s) — the public
|
|
10
20
|
* face of PollOptions, whose comments document each knob. */
|
|
11
21
|
export type WaitOptions = Partial<PollOptions>;
|
|
12
|
-
|
|
13
|
-
|
|
22
|
+
/** submit + wait in one call: the submit's options and the wait's, together.
|
|
23
|
+
* `timeoutMs` bounds the wait, not the task. */
|
|
24
|
+
export interface CallOptions extends SubmitOptions, WaitOptions {
|
|
14
25
|
}
|
|
15
26
|
/** Options this handle configures on the store it wraps, rather than the
|
|
16
27
|
* store's own constructor arguments. */
|
|
17
|
-
export
|
|
18
|
-
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
|
|
22
|
-
|
|
28
|
+
export interface ClientOptions {
|
|
29
|
+
/** Queued tasks a queue may hold before `submit` blocks — a number for every
|
|
30
|
+
* queue, or a per-queue record that leaves unnamed queues unbounded. `submit`
|
|
31
|
+
* raises QueueFull if the queue stays full for `maxQueueWaitMs`. */
|
|
32
|
+
maxQueueDepth?: number | Record<string, number>;
|
|
33
|
+
/** How long a blocked submit waits before raising QueueFull. Default 600_000. */
|
|
34
|
+
maxQueueWaitMs?: number;
|
|
35
|
+
/**
|
|
36
|
+
* Delete terminal tasks this many ms after they finished, on a schedule, for
|
|
37
|
+
* as long as this handle is open. Off unless set — and off means rows
|
|
38
|
+
* accumulate forever, because nothing else in CairnQ removes them. Tiered
|
|
39
|
+
* retention (per queue, per status) is `purge()` with filters, from your own
|
|
40
|
+
* scheduler.
|
|
41
|
+
*/
|
|
42
|
+
retentionMs?: number;
|
|
43
|
+
}
|
|
23
44
|
/** API-side handle. Thin wrapper over a TaskStore + SDK-orchestrated wait/call. */
|
|
24
45
|
export declare class CairnQ {
|
|
25
46
|
private readonly _store;
|
|
26
|
-
/** null unless `
|
|
47
|
+
/** null unless `retentionMs` was configured. */
|
|
27
48
|
private readonly sweeper;
|
|
28
49
|
constructor(_store: TaskStore, opts?: ClientOptions);
|
|
29
50
|
static sqlite(path: string, opts?: {
|
|
@@ -44,22 +65,16 @@ export declare class CairnQ {
|
|
|
44
65
|
close(): Promise<void>;
|
|
45
66
|
/** Enqueue a task. With `maxQueueDepth` configured this blocks while the
|
|
46
67
|
* target queue is at its limit, and raises QueueFull if it stays there for
|
|
47
|
-
* `maxQueueWaitMs` —
|
|
48
|
-
* across several producers. */
|
|
68
|
+
* `maxQueueWaitMs` — a soft limit across several producers. */
|
|
49
69
|
submit(name: string, payload?: unknown, opts?: SubmitOptions): Promise<Task>;
|
|
50
70
|
submit<P, R>(task: TaskDef<P, R>, payload?: P, opts?: SubmitOptions): Promise<Task>;
|
|
51
71
|
/** How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
|
|
52
72
|
* The non-blocking read behind `maxQueueDepth`, for a producer that would
|
|
53
|
-
* rather shed load or pick another queue than wait.
|
|
54
|
-
*
|
|
73
|
+
* rather shed load or pick another queue than wait. Bounded at `maxDepth`
|
|
74
|
+
* index entries, so it stays cheap to ask on every enqueue. */
|
|
55
75
|
queueDepth(queue: string, maxDepth: number): Promise<number>;
|
|
56
76
|
get(taskId: string): Promise<Task | null>;
|
|
57
77
|
getByKey(key: string): Promise<Task | null>;
|
|
58
|
-
/** The status-only probe wait polls on: id + status, no payload. Public for
|
|
59
|
-
* the same reason it exists — a dashboard or poller that only asks "is it
|
|
60
|
-
* finished yet" should not drag the payload back per ask. */
|
|
61
|
-
getStatus(taskId: string): Promise<TaskRef | null>;
|
|
62
|
-
getStatusByKey(key: string): Promise<TaskRef | null>;
|
|
63
78
|
list(input?: ListInput): Promise<Task[]>;
|
|
64
79
|
cancel(taskId: string): Promise<Task | null>;
|
|
65
80
|
cancelByKey(key: string): Promise<Task | null>;
|
|
@@ -71,32 +86,13 @@ export declare class CairnQ {
|
|
|
71
86
|
}): Promise<Task | null>;
|
|
72
87
|
/** Delete terminal tasks that finished more than `olderThanMs` ago and return
|
|
73
88
|
* their ids. Nothing else in CairnQ removes rows, so a long-lived database
|
|
74
|
-
* needs this on a schedule
|
|
75
|
-
* short; loop until it returns fewer
|
|
89
|
+
* needs this on a schedule — `retentionMs` is this call on a timer. Each call
|
|
90
|
+
* is bounded by `limit` to keep the write short; loop until it returns fewer
|
|
91
|
+
* than `limit`.
|
|
76
92
|
*
|
|
77
93
|
* `queue` / `status` / `name` narrow the sweep — one installation carrying two
|
|
78
94
|
* workloads needs a retention per workload, not one for the whole database. */
|
|
79
95
|
purge(input?: PurgeInput): Promise<string[]>;
|
|
80
|
-
/** Task counts per queue, keyed by status and zero-filled across all statuses
|
|
81
|
-
* — `(await stats()).default.queued` is the backlog of a queue. `queue` narrows
|
|
82
|
-
* the aggregate to one queue, which is also what keeps a caller from paying for
|
|
83
|
-
* the other workloads sharing the installation; a named queue is always
|
|
84
|
-
* present, zero-filled if it has no rows.
|
|
85
|
-
*
|
|
86
|
-
* This counts rows, so it costs what it counts — use it for a dashboard, and
|
|
87
|
-
* poll `queueDepth()` instead, which is bounded. */
|
|
88
|
-
stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>>;
|
|
89
|
-
/**
|
|
90
|
-
* Call `onSignal` when the tasks on `queues` may have changed. Returns an
|
|
91
|
-
* unsubscribe.
|
|
92
|
-
*
|
|
93
|
-
* Notify-accelerated polling, not an event log: a signal means "re-read now",
|
|
94
|
-
* and `stats()` / `list()` / `get()` are where the truth is. On Postgres an
|
|
95
|
-
* idle watch costs nothing and signals land in milliseconds; everywhere else
|
|
96
|
-
* the timer alone still delivers, so the same consumer code is correct either
|
|
97
|
-
* way. See TaskStore.watch for the full contract.
|
|
98
|
-
*/
|
|
99
|
-
watch(opts: WatchOptions, onSignal: (signal: WatchSignal) => void): () => void;
|
|
100
96
|
/** Wait for a task to finish. Resolves with the terminal Task (any status);
|
|
101
97
|
* throws TaskTimeout without stopping the task, so `wait(err.taskId)` picks the
|
|
102
98
|
* same wait back up — from another process, or after a longer deadline. */
|
|
@@ -111,7 +107,7 @@ export declare class CairnQ {
|
|
|
111
107
|
* TaskFailed / TaskCanceled / TaskTimeout otherwise. Pass a TaskDef and the
|
|
112
108
|
* resolved value is typed as its Result.
|
|
113
109
|
*
|
|
114
|
-
* `
|
|
110
|
+
* `timeoutMs` bounds the wait, not the task: on timeout the task runs on,
|
|
115
111
|
* and `wait(err.taskId)` — or `waitByKey`, from a process that only has the
|
|
116
112
|
* key — resumes the wait rather than starting the work over. */
|
|
117
113
|
call(name: string, payload?: unknown, opts?: CallOptions): Promise<unknown>;
|
package/dist/client.js
CHANGED
|
@@ -8,21 +8,24 @@ import { DEFAULT_WAIT_TIMEOUT_MS, pollWait, pollWaitByKey } from "./wait.js";
|
|
|
8
8
|
/** API-side handle. Thin wrapper over a TaskStore + SDK-orchestrated wait/call. */
|
|
9
9
|
export class CairnQ {
|
|
10
10
|
_store;
|
|
11
|
-
/** null unless `
|
|
11
|
+
/** null unless `retentionMs` was configured. */
|
|
12
12
|
sweeper;
|
|
13
13
|
constructor(_store, opts = {}) {
|
|
14
14
|
this._store = _store;
|
|
15
15
|
// Installed on the store, not held here: every submit path goes through the
|
|
16
16
|
// store, including TaskContext.submit, which this handle never sees.
|
|
17
17
|
if (opts.maxQueueDepth != null) {
|
|
18
|
-
_store.useBackpressure(
|
|
18
|
+
_store.useBackpressure({
|
|
19
|
+
maxQueueDepth: opts.maxQueueDepth,
|
|
20
|
+
maxQueueWaitMs: opts.maxQueueWaitMs,
|
|
21
|
+
});
|
|
19
22
|
}
|
|
20
23
|
// Retention is the opposite case: it belongs to the handle, because a worker
|
|
21
24
|
// sharing the store must not also be deleting rows behind the API's back.
|
|
22
25
|
// Started here rather than in connect(), which is optional — every other
|
|
23
26
|
// path connects lazily, and retention that silently depends on an optional
|
|
24
27
|
// call is retention that silently does not happen.
|
|
25
|
-
this.sweeper = opts.
|
|
28
|
+
this.sweeper = opts.retentionMs != null ? new RetentionSweeper(_store, opts.retentionMs) : null;
|
|
26
29
|
this.sweeper?.start();
|
|
27
30
|
}
|
|
28
31
|
static sqlite(path, opts = {}) {
|
|
@@ -50,12 +53,25 @@ export class CairnQ {
|
|
|
50
53
|
await this._store.close();
|
|
51
54
|
}
|
|
52
55
|
submit(task, payload, opts = {}) {
|
|
53
|
-
|
|
56
|
+
// Fields picked explicitly rather than spread, so the type is the truth:
|
|
57
|
+
// store-level SubmitInput accepts more (parentId and friends, which
|
|
58
|
+
// TaskContext.submit wires), and a spread would silently keep honoring them.
|
|
59
|
+
return this._store.submit({
|
|
60
|
+
name: taskName(task),
|
|
61
|
+
payload,
|
|
62
|
+
queue: opts.queue,
|
|
63
|
+
key: opts.key,
|
|
64
|
+
conflict: opts.conflict,
|
|
65
|
+
maxAttempts: opts.maxAttempts,
|
|
66
|
+
priority: opts.priority,
|
|
67
|
+
metadata: opts.metadata,
|
|
68
|
+
delayMs: opts.delayMs,
|
|
69
|
+
});
|
|
54
70
|
}
|
|
55
71
|
/** How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
|
|
56
72
|
* The non-blocking read behind `maxQueueDepth`, for a producer that would
|
|
57
|
-
* rather shed load or pick another queue than wait.
|
|
58
|
-
*
|
|
73
|
+
* rather shed load or pick another queue than wait. Bounded at `maxDepth`
|
|
74
|
+
* index entries, so it stays cheap to ask on every enqueue. */
|
|
59
75
|
queueDepth(queue, maxDepth) {
|
|
60
76
|
return this._store.queueDepth(queue, maxDepth);
|
|
61
77
|
}
|
|
@@ -65,15 +81,6 @@ export class CairnQ {
|
|
|
65
81
|
getByKey(key) {
|
|
66
82
|
return this._store.getByKey(key);
|
|
67
83
|
}
|
|
68
|
-
/** The status-only probe wait polls on: id + status, no payload. Public for
|
|
69
|
-
* the same reason it exists — a dashboard or poller that only asks "is it
|
|
70
|
-
* finished yet" should not drag the payload back per ask. */
|
|
71
|
-
getStatus(taskId) {
|
|
72
|
-
return this._store.getStatus(taskId);
|
|
73
|
-
}
|
|
74
|
-
getStatusByKey(key) {
|
|
75
|
-
return this._store.getStatusByKey(key);
|
|
76
|
-
}
|
|
77
84
|
list(input) {
|
|
78
85
|
return this._store.list(input);
|
|
79
86
|
}
|
|
@@ -91,38 +98,15 @@ export class CairnQ {
|
|
|
91
98
|
}
|
|
92
99
|
/** Delete terminal tasks that finished more than `olderThanMs` ago and return
|
|
93
100
|
* their ids. Nothing else in CairnQ removes rows, so a long-lived database
|
|
94
|
-
* needs this on a schedule
|
|
95
|
-
* short; loop until it returns fewer
|
|
101
|
+
* needs this on a schedule — `retentionMs` is this call on a timer. Each call
|
|
102
|
+
* is bounded by `limit` to keep the write short; loop until it returns fewer
|
|
103
|
+
* than `limit`.
|
|
96
104
|
*
|
|
97
105
|
* `queue` / `status` / `name` narrow the sweep — one installation carrying two
|
|
98
106
|
* workloads needs a retention per workload, not one for the whole database. */
|
|
99
107
|
purge(input) {
|
|
100
108
|
return this._store.purge(input);
|
|
101
109
|
}
|
|
102
|
-
/** Task counts per queue, keyed by status and zero-filled across all statuses
|
|
103
|
-
* — `(await stats()).default.queued` is the backlog of a queue. `queue` narrows
|
|
104
|
-
* the aggregate to one queue, which is also what keeps a caller from paying for
|
|
105
|
-
* the other workloads sharing the installation; a named queue is always
|
|
106
|
-
* present, zero-filled if it has no rows.
|
|
107
|
-
*
|
|
108
|
-
* This counts rows, so it costs what it counts — use it for a dashboard, and
|
|
109
|
-
* poll `queueDepth()` instead, which is bounded. */
|
|
110
|
-
stats(queue) {
|
|
111
|
-
return this._store.stats(queue);
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Call `onSignal` when the tasks on `queues` may have changed. Returns an
|
|
115
|
-
* unsubscribe.
|
|
116
|
-
*
|
|
117
|
-
* Notify-accelerated polling, not an event log: a signal means "re-read now",
|
|
118
|
-
* and `stats()` / `list()` / `get()` are where the truth is. On Postgres an
|
|
119
|
-
* idle watch costs nothing and signals land in milliseconds; everywhere else
|
|
120
|
-
* the timer alone still delivers, so the same consumer code is correct either
|
|
121
|
-
* way. See TaskStore.watch for the full contract.
|
|
122
|
-
*/
|
|
123
|
-
watch(opts, onSignal) {
|
|
124
|
-
return this._store.watch(opts, onSignal);
|
|
125
|
-
}
|
|
126
110
|
/** Wait for a task to finish. Resolves with the terminal Task (any status);
|
|
127
111
|
* throws TaskTimeout without stopping the task, so `wait(err.taskId)` picks the
|
|
128
112
|
* same wait back up — from another process, or after a longer deadline. */
|
|
@@ -146,9 +130,9 @@ export class CairnQ {
|
|
|
146
130
|
});
|
|
147
131
|
}
|
|
148
132
|
async call(task, payload, opts = {}) {
|
|
149
|
-
const {
|
|
133
|
+
const { timeoutMs, pollMs, maxPollMs, ...submit } = opts;
|
|
150
134
|
const created = await this.submit(taskName(task), payload, submit);
|
|
151
|
-
const final = await this.wait(created.id, { timeoutMs
|
|
135
|
+
const final = await this.wait(created.id, { timeoutMs, pollMs, maxPollMs });
|
|
152
136
|
if (isSucceeded(final))
|
|
153
137
|
return final.result;
|
|
154
138
|
if (isFailed(final))
|
package/dist/context.d.ts
CHANGED
|
@@ -65,6 +65,19 @@ export declare class TaskContext {
|
|
|
65
65
|
* every payload back on every beat.
|
|
66
66
|
*/
|
|
67
67
|
observeCancel(cancelRequested: boolean): void;
|
|
68
|
+
/**
|
|
69
|
+
* The half of the gate that asks only "is this attempt still mine?".
|
|
70
|
+
*
|
|
71
|
+
* Checked locally, not just via the store's ownership check — after an
|
|
72
|
+
* abandoned (timed-out) attempt the same worker may re-claim this task under
|
|
73
|
+
* the same workerId, and a zombie handler's write would then pass ownership
|
|
74
|
+
* against the NEW attempt.
|
|
75
|
+
*
|
|
76
|
+
* `owned` layers the settled check on top for writes to this task; `submit`
|
|
77
|
+
* takes this half alone, because a handler may legitimately settle a task and
|
|
78
|
+
* then fan out from it.
|
|
79
|
+
*/
|
|
80
|
+
private requireLease;
|
|
68
81
|
private owned;
|
|
69
82
|
progress(value: number | null, message?: string | null): Promise<Task>;
|
|
70
83
|
heartbeat(): Promise<Task>;
|
|
@@ -110,7 +123,16 @@ export declare class TaskContext {
|
|
|
110
123
|
fail(error?: FailReason, opts?: {
|
|
111
124
|
retryable?: boolean;
|
|
112
125
|
}): Promise<Task | null>;
|
|
113
|
-
/**
|
|
126
|
+
/**
|
|
127
|
+
* Submit a child task; parent/root/correlation are wired automatically.
|
|
128
|
+
*
|
|
129
|
+
* Refused once the lease is gone. Not the full `owned` gate — a handler may
|
|
130
|
+
* legitimately settle a task and then fan out from it, so `settled` is no bar
|
|
131
|
+
* — but a context whose lease was lost is an attempt that has been abandoned:
|
|
132
|
+
* it is being retried elsewhere, and every child it creates now will be
|
|
133
|
+
* created again by that retry. Creating work is the one side effect cairnq can
|
|
134
|
+
* actually stop a zombie handler from repeating.
|
|
135
|
+
*/
|
|
114
136
|
submit(name: string, payload?: unknown, opts?: SubmitOptions): Promise<Task>;
|
|
115
137
|
submit<P, R>(task: TaskDef<P, R>, payload?: P, opts?: SubmitOptions): Promise<Task>;
|
|
116
138
|
wait(taskId: string, opts?: {
|
package/dist/context.js
CHANGED
|
@@ -112,17 +112,27 @@ export class TaskContext {
|
|
|
112
112
|
if (cancelRequested)
|
|
113
113
|
this.cancelSeen = true;
|
|
114
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* The half of the gate that asks only "is this attempt still mine?".
|
|
117
|
+
*
|
|
118
|
+
* Checked locally, not just via the store's ownership check — after an
|
|
119
|
+
* abandoned (timed-out) attempt the same worker may re-claim this task under
|
|
120
|
+
* the same workerId, and a zombie handler's write would then pass ownership
|
|
121
|
+
* against the NEW attempt.
|
|
122
|
+
*
|
|
123
|
+
* `owned` layers the settled check on top for writes to this task; `submit`
|
|
124
|
+
* takes this half alone, because a handler may legitimately settle a task and
|
|
125
|
+
* then fan out from it.
|
|
126
|
+
*/
|
|
127
|
+
requireLease() {
|
|
128
|
+
if (this.leaseLost)
|
|
129
|
+
throw new LostLease(this.task.id);
|
|
130
|
+
}
|
|
115
131
|
async owned(write) {
|
|
116
132
|
// One gate for every write through this context, so "may I still write?" is
|
|
117
133
|
// answered in one place rather than at each call site.
|
|
118
134
|
//
|
|
119
|
-
|
|
120
|
-
// locally, not just via the store's ownership check — after an abandoned
|
|
121
|
-
// (timed-out) attempt the same worker may re-claim this task under the same
|
|
122
|
-
// workerId, and a zombie handler's write would then pass ownership against
|
|
123
|
-
// the NEW attempt.
|
|
124
|
-
if (this.leaseLost)
|
|
125
|
-
throw new LostLease(this.task.id);
|
|
135
|
+
this.requireLease();
|
|
126
136
|
// Settled: the task is terminal, so the statement would match no row and come
|
|
127
137
|
// back as a lost lease — telling the handler "another worker took this" when
|
|
128
138
|
// the truth is "you already finished it", and flipping lostLease on the way.
|
|
@@ -242,6 +252,7 @@ export class TaskContext {
|
|
|
242
252
|
return task;
|
|
243
253
|
}
|
|
244
254
|
async submit(task, payload, opts = {}) {
|
|
255
|
+
this.requireLease();
|
|
245
256
|
return this.store.submit({
|
|
246
257
|
name: taskName(task),
|
|
247
258
|
payload,
|
package/dist/errors.d.ts
CHANGED
|
@@ -70,29 +70,6 @@ export declare class TaskCanceled extends CairnQError {
|
|
|
70
70
|
taskId: string;
|
|
71
71
|
constructor(taskId: string);
|
|
72
72
|
}
|
|
73
|
-
/**
|
|
74
|
-
* A heartbeat beat came back later than its own interval allowed.
|
|
75
|
-
*
|
|
76
|
-
* The heartbeat shares the event loop with the handlers whose leases it renews,
|
|
77
|
-
* so a handler that blocks the loop stops the renewal with it: the lease expires,
|
|
78
|
-
* the task is recovered and redelivered, and a second worker starts computing
|
|
79
|
-
* what the first is still computing — one task, billed twice, with no error
|
|
80
|
-
* anywhere. Nothing inside the blocked handler can observe that, which is why it
|
|
81
|
-
* is reported through `onError` alongside the other things the run loop survived.
|
|
82
|
-
*
|
|
83
|
-
* The usual cause is synchronous work in a handler: a tight loop, a large
|
|
84
|
-
* JSON.parse, a `*Sync` filesystem or crypto call. Node has one loop and no way
|
|
85
|
-
* to preempt it — move the work to a worker thread, a child process, or an async
|
|
86
|
-
* API that yields. The other cause is a worker simply oversubscribed for its
|
|
87
|
-
* `leaseMs` — nothing is blocking, there is just more work than turns — which the
|
|
88
|
-
* same report covers, because the lease is at equal risk either way.
|
|
89
|
-
*/
|
|
90
|
-
export declare class EventLoopBlocked extends CairnQError {
|
|
91
|
-
readonly lateMs: number;
|
|
92
|
-
readonly intervalMs: number;
|
|
93
|
-
readonly leaseMs: number;
|
|
94
|
-
constructor(lateMs: number, intervalMs: number, leaseMs: number);
|
|
95
|
-
}
|
|
96
73
|
/** A worker write affected 0 rows: the lease expired and was reclaimed. */
|
|
97
74
|
export declare class LostLease extends CairnQError {
|
|
98
75
|
taskId: string;
|
|
@@ -129,10 +106,27 @@ export declare class UnsupportedBackend extends CairnQError {
|
|
|
129
106
|
export declare class SchemaMismatch extends CairnQError {
|
|
130
107
|
constructor(message: string);
|
|
131
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* The store was closing when this operation asked for it.
|
|
111
|
+
*
|
|
112
|
+
* `close()` waits for the work already accepted — a group commit still holding
|
|
113
|
+
* writes, a transaction with a BEGIN IMMEDIATE open — and turns away everything
|
|
114
|
+
* that arrives after, so that wait cannot be extended indefinitely by a producer
|
|
115
|
+
* that keeps submitting. An operation that lands in that window gets this rather
|
|
116
|
+
* than a driver error about a connection that vanished underneath it.
|
|
117
|
+
*
|
|
118
|
+
* It does not mean the store is finished for good: connecting is lazy, so a
|
|
119
|
+
* store used again after `close()` has returned simply reopens. The Python SDK
|
|
120
|
+
* raises the same named error.
|
|
121
|
+
*/
|
|
122
|
+
export declare class StoreClosed extends CairnQError {
|
|
123
|
+
constructor(message?: string);
|
|
124
|
+
}
|
|
132
125
|
/** A value could not be encoded for a protocol JSON column (non-finite number,
|
|
133
|
-
* BigInt, circular structure,
|
|
134
|
-
* it, and a worker records a handler result
|
|
135
|
-
* `unserializable_result` failure. The Python
|
|
126
|
+
* BigInt, circular structure, an opaque built-in like Map or Set, …). Raised at
|
|
127
|
+
* the boundary — submit rejects with it, and a worker records a handler result
|
|
128
|
+
* that triggers it as a permanent `unserializable_result` failure. The Python
|
|
129
|
+
* SDK raises the same named error. */
|
|
136
130
|
export declare class SerializationError extends CairnQError {
|
|
137
131
|
constructor(message: string);
|
|
138
132
|
}
|