cairnq 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -32
- package/dist/_protocol/sql/postgres/purge.sql +10 -10
- package/dist/_protocol/sql/sqlite/purge.sql +10 -10
- package/dist/backpressure.d.ts +0 -3
- package/dist/backpressure.js +1 -3
- package/dist/client.d.ts +42 -46
- package/dist/client.js +27 -43
- package/dist/context.d.ts +23 -1
- package/dist/context.js +18 -7
- package/dist/errors.d.ts +20 -26
- package/dist/errors.js +23 -35
- package/dist/index.d.ts +4 -10
- package/dist/index.js +3 -6
- package/dist/models.d.ts +14 -1
- package/dist/models.js +17 -5
- package/dist/retention.d.ts +22 -66
- package/dist/retention.js +53 -64
- package/dist/store/base.d.ts +29 -90
- package/dist/store/base.js +136 -123
- package/dist/store/pg-executor.d.ts +11 -0
- package/dist/store/pg-executor.js +11 -0
- package/dist/store/pg-pool.js +14 -0
- package/dist/store/postgres.d.ts +18 -12
- package/dist/store/postgres.js +23 -29
- package/dist/store/sqlite.d.ts +39 -2
- package/dist/store/sqlite.js +108 -29
- package/dist/worker.d.ts +38 -93
- package/dist/worker.js +181 -234
- package/package.json +1 -1
- package/src/backpressure.ts +1 -5
- package/src/client.ts +66 -71
- package/src/context.ts +28 -7
- package/src/errors.ts +25 -36
- package/src/index.ts +4 -27
- package/src/models.ts +17 -5
- package/src/retention.ts +50 -115
- package/src/store/base.ts +131 -166
- package/src/store/pg-executor.ts +11 -0
- package/src/store/pg-pool.ts +14 -0
- package/src/store/postgres.ts +24 -31
- package/src/store/sqlite.ts +108 -27
- package/src/worker.ts +208 -300
- package/dist/_protocol/sql/postgres/stats.sql +0 -23
- package/dist/_protocol/sql/sqlite/stats.sql +0 -23
package/dist/store/base.d.ts
CHANGED
|
@@ -2,10 +2,11 @@ import { type Task, type TaskRef, type TaskStatus } from "../models.js";
|
|
|
2
2
|
import { type BackpressureOptions } from "../backpressure.js";
|
|
3
3
|
/** Encode a value for a protocol JSON column, raising SerializationError on
|
|
4
4
|
* anything JSON cannot represent. Refuses what JSON.stringify would silently
|
|
5
|
-
* mangle
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* mangle: NaN/Infinity anywhere, undefined/function/symbol inside an array, a
|
|
6
|
+
* top-level undefined that disappears entirely, and the objects that would be
|
|
7
|
+
* written as an empty one (see emptiesItselfOut) — either way the twin SDK
|
|
8
|
+
* reads back something other than what the caller meant. The Python SDK rejects
|
|
9
|
+
* the same classes of value; see its dump_json. */
|
|
9
10
|
export declare function dumpJson(value: unknown): string;
|
|
10
11
|
/** Refuse to run against a store whose protocol major this SDK does not speak.
|
|
11
12
|
* The supported major is a protocol fact, not a dialect one — every backend
|
|
@@ -28,7 +29,8 @@ export interface SubmitInput {
|
|
|
28
29
|
parentId?: string | null;
|
|
29
30
|
rootId?: string | null;
|
|
30
31
|
correlationId?: string | null;
|
|
31
|
-
|
|
32
|
+
/** Run no earlier than this many ms from now. */
|
|
33
|
+
delayMs?: number;
|
|
32
34
|
}
|
|
33
35
|
export interface ListInput {
|
|
34
36
|
status?: TaskStatus | null;
|
|
@@ -109,32 +111,6 @@ export declare function statementParams(sql: string): readonly string[];
|
|
|
109
111
|
* them in one place is what stops SQLite and Postgres from drifting apart in
|
|
110
112
|
* behavior; the shared SQL already stops them from drifting in wording.
|
|
111
113
|
*/
|
|
112
|
-
/**
|
|
113
|
-
* Why `watch` is calling back.
|
|
114
|
-
*
|
|
115
|
-
* `queued` / `done` come from the store's push channel and name what moved;
|
|
116
|
-
* `poll` is the timer saying the watch cannot rule out a change. None of them
|
|
117
|
-
* carries state — the row is the truth.
|
|
118
|
-
*/
|
|
119
|
-
export interface WatchSignal {
|
|
120
|
-
reason: "queued" | "done" | "poll";
|
|
121
|
-
/** The queue a task was queued on. Only on `queued`. */
|
|
122
|
-
queue?: string;
|
|
123
|
-
/** The task that reached a terminal status. Only on `done`. */
|
|
124
|
-
taskId?: string;
|
|
125
|
-
}
|
|
126
|
-
export interface WatchOptions {
|
|
127
|
-
/** Restrict `queued` signals to these queues. Unset watches every queue. */
|
|
128
|
-
queues?: string[];
|
|
129
|
-
/**
|
|
130
|
-
* How often to signal in the absence of a push channel — and, where there is
|
|
131
|
-
* one, how long a dropped listener can go unnoticed. The default trades a
|
|
132
|
-
* dashboard's idle query rate against how stale it may look.
|
|
133
|
-
*/
|
|
134
|
-
pollMs?: number;
|
|
135
|
-
}
|
|
136
|
-
/** See WatchOptions.pollMs. */
|
|
137
|
-
export declare const DEFAULT_WATCH_POLL_MS = 2000;
|
|
138
114
|
export declare abstract class TaskStore {
|
|
139
115
|
/** Set by useBackpressure; null means submit is ungated. */
|
|
140
116
|
private gate;
|
|
@@ -167,25 +143,12 @@ export declare abstract class TaskStore {
|
|
|
167
143
|
* `completeIn` reports that. Postgres implements it; SQLite does not.
|
|
168
144
|
*/
|
|
169
145
|
protected txWithSession?<T>(fn: (fetch: Fetch, session: unknown) => Promise<T>): Promise<T>;
|
|
170
|
-
/**
|
|
171
|
-
* Register for this store's push channel, if it has one; returns an
|
|
172
|
-
* unsubscribe. A store without a push channel does not implement this, and
|
|
173
|
-
* `watch` degrades to its timer alone.
|
|
174
|
-
*/
|
|
175
|
-
protected subscribePush?(onSignal: (signal: WatchSignal) => void): () => void;
|
|
176
146
|
/**
|
|
177
147
|
* Tell a store with a push channel which queues this process will wait on, so
|
|
178
148
|
* it can buffer their notifications and ignore everyone else's. Optional: a
|
|
179
149
|
* store without a push channel has nothing to buffer.
|
|
180
150
|
*/
|
|
181
151
|
protected registerWakeable?(queues: string[]): void;
|
|
182
|
-
/**
|
|
183
|
-
* Nudge the push channel back up if it has dropped. Called from `watch`'s
|
|
184
|
-
* timer, which is the only thing keeping a client-side subscriber alive: a
|
|
185
|
-
* process that never claims never calls claimWake, so without this a listener
|
|
186
|
-
* that died once would never come back there.
|
|
187
|
-
*/
|
|
188
|
-
protected warmPush?(): void;
|
|
189
152
|
/**
|
|
190
153
|
* Whether it is worth opening the claim transaction at all — the read-only
|
|
191
154
|
* `claimable_probe`, which every dialect ships.
|
|
@@ -207,13 +170,31 @@ export declare abstract class TaskStore {
|
|
|
207
170
|
/** Resolves when `taskId` may have gone terminal. Plain ref'd sleep —
|
|
208
171
|
* pollWait awaits it directly, so it is what keeps the process alive. */
|
|
209
172
|
taskDoneWake(_taskId: string, timeoutMs: number): Promise<void>;
|
|
173
|
+
/**
|
|
174
|
+
* Whether this store's driver hands JSON columns back as text.
|
|
175
|
+
*
|
|
176
|
+
* Declared by the store rather than sniffed per value, because for a JSON
|
|
177
|
+
* *string* the two wire forms are indistinguishable: the text form of
|
|
178
|
+
* `"hello"` and the decoded form of `"hello"` are both strings, and treating
|
|
179
|
+
* the decoded one as text parses it twice — `"s3://…"` threw, `"42"` came back
|
|
180
|
+
* as the number 42. See rowToTask.
|
|
181
|
+
*
|
|
182
|
+
* True is the SQLite answer and the safe default: its JSON columns are TEXT,
|
|
183
|
+
* always. PostgresStore replaces it with what its driver actually does, which
|
|
184
|
+
* it measures rather than assumes — an injected executor's driver is the
|
|
185
|
+
* application's choice, not cairnq's.
|
|
186
|
+
*/
|
|
187
|
+
protected jsonIsText: boolean;
|
|
188
|
+
/** rowToTask, told this store's wire form. Every row in this class goes
|
|
189
|
+
* through here so no call site has to remember to pass it. */
|
|
190
|
+
private toTask;
|
|
210
191
|
/**
|
|
211
192
|
* An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
|
|
212
193
|
* Each statement's WHERE pins worker_id + a live lease, so 0 rows back means
|
|
213
194
|
* the lease was lost — every such write reports it the same way.
|
|
214
195
|
*/
|
|
215
196
|
private ownedWrite;
|
|
216
|
-
private
|
|
197
|
+
private one;
|
|
217
198
|
private static oneRef;
|
|
218
199
|
/**
|
|
219
200
|
* Bound how deep a queue may get before `submit` blocks. Off unless set.
|
|
@@ -259,54 +240,12 @@ export declare abstract class TaskStore {
|
|
|
259
240
|
* retention expressible at all — see PurgeInput.
|
|
260
241
|
*/
|
|
261
242
|
purge(input?: PurgeInput): Promise<string[]>;
|
|
262
|
-
/**
|
|
263
|
-
* Task counts per queue, keyed by status and zero-filled across all statuses —
|
|
264
|
-
* `(await stats()).default.queued` is the backlog of a queue. A queue appears
|
|
265
|
-
* only while it has rows; terminal tasks keep counting until `purge` removes
|
|
266
|
-
* them.
|
|
267
|
-
*
|
|
268
|
-
* `queue` restricts the aggregate to one queue, which is also what stops the
|
|
269
|
-
* caller paying for every other queue's rows: one installation carrying two
|
|
270
|
-
* workloads is the coordination this project recommends, and the unfiltered
|
|
271
|
-
* form reads the whole table. A named queue is always present in the result,
|
|
272
|
-
* zero-filled if it has no rows at all — asking about a specific queue and
|
|
273
|
-
* getting `undefined` back would make every caller write the same fallback.
|
|
274
|
-
*
|
|
275
|
-
* Filtered or not, this COUNTS, so it costs what it counts: a whole queue,
|
|
276
|
-
* terminal rows included. Right for a dashboard, wrong on an interval — poll
|
|
277
|
-
* `queueDepth`, which is bounded, and keep this for when the real numbers are
|
|
278
|
-
* the point.
|
|
279
|
-
*/
|
|
280
|
-
stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>>;
|
|
281
|
-
/**
|
|
282
|
-
* Call `onSignal` when the tasks on `queues` may have changed — something was
|
|
283
|
-
* queued, or something finished.
|
|
284
|
-
*
|
|
285
|
-
* This is notify-ACCELERATED POLLING, not an event log, and the difference is
|
|
286
|
-
* the whole contract. Where a push channel is available (Postgres LISTEN) an
|
|
287
|
-
* idle watch costs nothing and a signal arrives within milliseconds of the
|
|
288
|
-
* event. Where it is not — a transaction-mode pooler refuses LISTEN, SQLite has
|
|
289
|
-
* no channel at all — the timer alone still delivers `poll` signals, so a
|
|
290
|
-
* consumer that re-reads on every signal is correct in both cases and merely
|
|
291
|
-
* less prompt in one.
|
|
292
|
-
*
|
|
293
|
-
* What it will NOT do is promise that a signal means something happened, or
|
|
294
|
-
* that every event produces its own signal. Treat a signal as "re-read now"
|
|
295
|
-
* and take the truth from `stats()` / `list()` / `get()`, which is where it
|
|
296
|
-
* lives. `reason` is a hint for reading less: a `done` signal names the task,
|
|
297
|
-
* so a dashboard can refresh that row instead of the list.
|
|
298
|
-
*
|
|
299
|
-
* Returns an unsubscribe. The timer is unref'd — watching does not hold a
|
|
300
|
-
* process open.
|
|
301
|
-
*/
|
|
302
|
-
watch(opts: WatchOptions, onSignal: (signal: WatchSignal) => void): () => void;
|
|
303
243
|
/**
|
|
304
244
|
* How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
|
|
305
245
|
*
|
|
306
|
-
* The cheap half of backpressure: bounded at `maxDepth` index entries,
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* load or shape a producer; `QueueDepthGate` builds the blocking form on top.
|
|
246
|
+
* The cheap half of backpressure: bounded at `maxDepth` index entries, so it
|
|
247
|
+
* stays cheap to ask on every enqueue. Use it directly to shed load or shape
|
|
248
|
+
* a producer; `QueueDepthGate` builds the blocking form on top.
|
|
310
249
|
*/
|
|
311
250
|
queueDepth(queue: string, maxDepth: number): Promise<number>;
|
|
312
251
|
/**
|
package/dist/store/base.js
CHANGED
|
@@ -2,6 +2,48 @@ import { newId } from "../ids.js";
|
|
|
2
2
|
import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch, SerializationError, UnsupportedBackend, } from "../errors.js";
|
|
3
3
|
import { isTerminalStatus, rowToRef, rowToTask, STATUSES, } from "../models.js";
|
|
4
4
|
import { QueueDepthGate } from "../backpressure.js";
|
|
5
|
+
/**
|
|
6
|
+
* Whether `JSON.stringify` would write this object without its contents.
|
|
7
|
+
*
|
|
8
|
+
* Every such value serializes to `{}` — or, for a typed array, to an object
|
|
9
|
+
* keyed by index — with everything it actually held silently gone. A handler
|
|
10
|
+
* that returns a `Map` would otherwise record `{}` as the task's result:
|
|
11
|
+
* succeeded, no error, and nothing left to recover the value from. That is the
|
|
12
|
+
* same class of silent mangle as `NaN` becoming `null`, so it is refused the
|
|
13
|
+
* same way.
|
|
14
|
+
*
|
|
15
|
+
* Structural rather than a list of built-in names, because a list is only ever
|
|
16
|
+
* as current as the day it was written. An enumeration of `Map`/`Set`/`Promise`/
|
|
17
|
+
* the typed arrays already missed `Float16Array` (ES2025, and `engines` here is
|
|
18
|
+
* node >=22), `Blob`, `Headers`, `URLSearchParams`, and — unfixably, since its
|
|
19
|
+
* tag is plain `[object Object]` — a class whose state is all private fields.
|
|
20
|
+
* The rule below catches those, and whatever the platform adds next, with
|
|
21
|
+
* nothing to maintain:
|
|
22
|
+
*
|
|
23
|
+
* - **A non-plain object with no own enumerable properties.** Its state lives
|
|
24
|
+
* somewhere `JSON.stringify` cannot reach (internal slots, private fields, a
|
|
25
|
+
* host binding), so `{}` is all that would be written. A *plain* `{}` is
|
|
26
|
+
* deliberately exempt: an empty object is a value, not a loss.
|
|
27
|
+
* - **Any `ArrayBuffer` view.** A typed array writes its indices as string keys
|
|
28
|
+
* and reads back as an object, not an array — non-empty, so the first rule
|
|
29
|
+
* does not see it.
|
|
30
|
+
*
|
|
31
|
+
* A class instance that carries ordinary properties still crosses: those
|
|
32
|
+
* properties are exactly what gets written, and nothing is lost. `Date` never
|
|
33
|
+
* reaches here at all — `toJSON` has already replaced it with its ISO string by
|
|
34
|
+
* the time a replacer is called, which is also why `toJSON` remains the escape
|
|
35
|
+
* hatch for anything that wants to define its own JSON form.
|
|
36
|
+
*/
|
|
37
|
+
function emptiesItselfOut(v) {
|
|
38
|
+
if (ArrayBuffer.isView(v))
|
|
39
|
+
return true;
|
|
40
|
+
const proto = Object.getPrototypeOf(v);
|
|
41
|
+
// Plain and null-prototype objects are the two shapes whose own enumerable
|
|
42
|
+
// properties ARE their contents, so an empty one is honestly empty.
|
|
43
|
+
if (proto === Object.prototype || proto === null)
|
|
44
|
+
return false;
|
|
45
|
+
return Object.keys(v).length === 0;
|
|
46
|
+
}
|
|
5
47
|
const rejectMangled = function (_key, v) {
|
|
6
48
|
if (typeof v === "number" && !Number.isFinite(v)) {
|
|
7
49
|
throw new SerializationError(`non-finite number ${v} is not JSON-serializable`);
|
|
@@ -12,14 +54,19 @@ const rejectMangled = function (_key, v) {
|
|
|
12
54
|
if (Array.isArray(this) && (v === undefined || typeof v === "function" || typeof v === "symbol")) {
|
|
13
55
|
throw new SerializationError(`${typeof v} inside an array is not JSON-serializable`);
|
|
14
56
|
}
|
|
57
|
+
if (v !== null && typeof v === "object" && !Array.isArray(v) && emptiesItselfOut(v)) {
|
|
58
|
+
const name = v.constructor?.name ?? "object";
|
|
59
|
+
throw new SerializationError(`${name} carries nothing JSON.stringify can see and would be written as an empty object`);
|
|
60
|
+
}
|
|
15
61
|
return v;
|
|
16
62
|
};
|
|
17
63
|
/** Encode a value for a protocol JSON column, raising SerializationError on
|
|
18
64
|
* anything JSON cannot represent. Refuses what JSON.stringify would silently
|
|
19
|
-
* mangle
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
65
|
+
* mangle: NaN/Infinity anywhere, undefined/function/symbol inside an array, a
|
|
66
|
+
* top-level undefined that disappears entirely, and the objects that would be
|
|
67
|
+
* written as an empty one (see emptiesItselfOut) — either way the twin SDK
|
|
68
|
+
* reads back something other than what the caller meant. The Python SDK rejects
|
|
69
|
+
* the same classes of value; see its dump_json. */
|
|
23
70
|
export function dumpJson(value) {
|
|
24
71
|
let text;
|
|
25
72
|
try {
|
|
@@ -32,11 +79,36 @@ export function dumpJson(value) {
|
|
|
32
79
|
if (text === undefined) {
|
|
33
80
|
throw new SerializationError(`value of type ${typeof value} is not JSON-serializable`);
|
|
34
81
|
}
|
|
35
|
-
// Every mangled value
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
|
|
82
|
+
// Every mangled value betrays itself in the output, so the strict pass — which
|
|
83
|
+
// forfeits V8's native stringifier — runs only when one of its traces is
|
|
84
|
+
// there: `null` for a mangled scalar, `{}` for an opaque container emptied
|
|
85
|
+
// out, `{"0":` for a typed array rewritten as an index-keyed object. Each is a
|
|
86
|
+
// superset (a payload carrying an empty object, or a literal "0" key, merely
|
|
87
|
+
// pays for a second pass that then finds nothing), and together they are
|
|
88
|
+
// exhaustive over everything rejectMangled rejects. Measured on a null-free
|
|
89
|
+
// payload the three checks cost ~20% of the encode, against ~130% for running
|
|
90
|
+
// the replacer unconditionally.
|
|
91
|
+
//
|
|
92
|
+
// The second pass builds a whole string and throws it away, so a hit costs
|
|
93
|
+
// 2.4x the encode (211us against 87us on a 42KB payload) where a validating
|
|
94
|
+
// WALK would cost 28us. It stays a second stringify anyway, twice over: a walk
|
|
95
|
+
// would have to reproduce seven of JSON.stringify's traversal semantics by
|
|
96
|
+
// hand — toJSON applied before the replacer sees a value, boxed primitives,
|
|
97
|
+
// sparse arrays, own-enumerable ordering, getters called once each, and a
|
|
98
|
+
// cycle guard the native pass does not need here but a walk would (an impure
|
|
99
|
+
// getter can introduce a cycle the first pass did not see, where stringify
|
|
100
|
+
// throws and a walk recurses forever) — and a divergence between the two is
|
|
101
|
+
// either a false rejection or the silent-data-loss bug coming back. It is also
|
|
102
|
+
// the wrong thing to optimise: every dumpJson sits next to a durable write, so
|
|
103
|
+
// 124us is noise beside the transaction it precedes. Revisit only with a
|
|
104
|
+
// profile showing this in double digits of submit latency, and pair any walk
|
|
105
|
+
// with a differential test asserting the two agree over generated values.
|
|
106
|
+
//
|
|
107
|
+
// The `{}` trace cannot be narrowed, either: an emptied-out Map and a
|
|
108
|
+
// legitimate empty object are byte-identical in the output.
|
|
109
|
+
if (text.includes("null") || text.includes("{}") || text.includes('{"0":')) {
|
|
39
110
|
JSON.stringify(value, rejectMangled);
|
|
111
|
+
}
|
|
40
112
|
return text;
|
|
41
113
|
}
|
|
42
114
|
const SUPPORTED_PROTOCOL_MAJOR = 1;
|
|
@@ -140,7 +212,9 @@ export function specialize(sql, params) {
|
|
|
140
212
|
}
|
|
141
213
|
if (!active)
|
|
142
214
|
return sql;
|
|
143
|
-
|
|
215
|
+
// "\u0000" delimits: the names run together with no separator otherwise, and
|
|
216
|
+
// ("a," + "b,SELECT") and ("a,b," + "SELECT") would be one cache entry.
|
|
217
|
+
const key = active + "\u0000" + sql;
|
|
144
218
|
let out = specialCache.get(key);
|
|
145
219
|
if (out === undefined) {
|
|
146
220
|
out = sql.replace(OPTIONAL_FILTER, (whole, name, column) => params[name] != null ? `${column} = :${name}` : whole);
|
|
@@ -168,8 +242,17 @@ export function statementParams(sql) {
|
|
|
168
242
|
}
|
|
169
243
|
return names;
|
|
170
244
|
}
|
|
171
|
-
/**
|
|
172
|
-
|
|
245
|
+
/**
|
|
246
|
+
* The storage seam.
|
|
247
|
+
*
|
|
248
|
+
* A backend supplies three things: how to run one protocol statement, how to run
|
|
249
|
+
* several inside a transaction, and how its dialect binds parameters. Everything
|
|
250
|
+
* above that — the submit conflict branches, the *_by_key lookups, the
|
|
251
|
+
* recover-then-claim sequence, the ownership-checked writes — lives here once,
|
|
252
|
+
* because those are protocol decisions rather than storage decisions. Keeping
|
|
253
|
+
* them in one place is what stops SQLite and Postgres from drifting apart in
|
|
254
|
+
* behavior; the shared SQL already stops them from drifting in wording.
|
|
255
|
+
*/
|
|
173
256
|
export class TaskStore {
|
|
174
257
|
/** Set by useBackpressure; null means submit is ungated. */
|
|
175
258
|
gate = null;
|
|
@@ -207,6 +290,26 @@ export class TaskStore {
|
|
|
207
290
|
return new Promise((resolve) => setTimeout(resolve, timeoutMs));
|
|
208
291
|
}
|
|
209
292
|
// --------------------------------------------------------------- internals
|
|
293
|
+
/**
|
|
294
|
+
* Whether this store's driver hands JSON columns back as text.
|
|
295
|
+
*
|
|
296
|
+
* Declared by the store rather than sniffed per value, because for a JSON
|
|
297
|
+
* *string* the two wire forms are indistinguishable: the text form of
|
|
298
|
+
* `"hello"` and the decoded form of `"hello"` are both strings, and treating
|
|
299
|
+
* the decoded one as text parses it twice — `"s3://…"` threw, `"42"` came back
|
|
300
|
+
* as the number 42. See rowToTask.
|
|
301
|
+
*
|
|
302
|
+
* True is the SQLite answer and the safe default: its JSON columns are TEXT,
|
|
303
|
+
* always. PostgresStore replaces it with what its driver actually does, which
|
|
304
|
+
* it measures rather than assumes — an injected executor's driver is the
|
|
305
|
+
* application's choice, not cairnq's.
|
|
306
|
+
*/
|
|
307
|
+
jsonIsText = true;
|
|
308
|
+
/** rowToTask, told this store's wire form. Every row in this class goes
|
|
309
|
+
* through here so no call site has to remember to pass it. */
|
|
310
|
+
toTask(row) {
|
|
311
|
+
return rowToTask(row, this.jsonIsText);
|
|
312
|
+
}
|
|
210
313
|
/**
|
|
211
314
|
* An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
|
|
212
315
|
* Each statement's WHERE pins worker_id + a live lease, so 0 rows back means
|
|
@@ -216,10 +319,12 @@ export class TaskStore {
|
|
|
216
319
|
const rows = await this.fetch(name, params);
|
|
217
320
|
if (!rows.length)
|
|
218
321
|
throw new LostLease(taskId);
|
|
219
|
-
return
|
|
322
|
+
return this.toTask(rows[0]);
|
|
220
323
|
}
|
|
221
|
-
|
|
222
|
-
|
|
324
|
+
// An instance method, unlike oneRef: mapping a Task reads this store's JSON
|
|
325
|
+
// wire form, and a TaskRef has no JSON column to read.
|
|
326
|
+
one(rows) {
|
|
327
|
+
return rows.length ? this.toTask(rows[0]) : null;
|
|
223
328
|
}
|
|
224
329
|
static oneRef(rows) {
|
|
225
330
|
return rows.length ? rowToRef(rows[0]) : null;
|
|
@@ -246,7 +351,7 @@ export class TaskStore {
|
|
|
246
351
|
metadata: dumpJson(input.metadata ?? {}),
|
|
247
352
|
max_attempts: input.maxAttempts ?? 3,
|
|
248
353
|
priority: input.priority ?? 0,
|
|
249
|
-
delay_ms: input.
|
|
354
|
+
delay_ms: input.delayMs ?? 0,
|
|
250
355
|
parent_id: input.parentId ?? null,
|
|
251
356
|
root_id: input.rootId ?? id,
|
|
252
357
|
correlation_id: input.correlationId ?? null,
|
|
@@ -265,8 +370,8 @@ export class TaskStore {
|
|
|
265
370
|
if (input.maxAttempts != null && input.maxAttempts < 1) {
|
|
266
371
|
throw new Error(`maxAttempts must be >= 1, got ${input.maxAttempts}`);
|
|
267
372
|
}
|
|
268
|
-
if (input.
|
|
269
|
-
throw new Error(`
|
|
373
|
+
if (input.delayMs != null && input.delayMs < 0) {
|
|
374
|
+
throw new Error(`delayMs must be >= 0, got ${input.delayMs}`);
|
|
270
375
|
}
|
|
271
376
|
// After validation and before the first write: bad arguments should fail
|
|
272
377
|
// now, not after waiting out a full queue. Reads the resolved queue, so the
|
|
@@ -274,7 +379,7 @@ export class TaskStore {
|
|
|
274
379
|
if (this.gate)
|
|
275
380
|
await this.gate.acquire(ins.queue);
|
|
276
381
|
if (key === null)
|
|
277
|
-
return
|
|
382
|
+
return this.toTask((await this.fetch("insert_task", ins))[0]);
|
|
278
383
|
// A key makes submit a read-then-write, so it has to be one transaction —
|
|
279
384
|
// opened by taking the key's lock, because on Postgres the transaction alone
|
|
280
385
|
// is not enough: concurrent same-key submits must not both see "no existing
|
|
@@ -292,7 +397,7 @@ export class TaskStore {
|
|
|
292
397
|
if (conflict === "reject")
|
|
293
398
|
throw new AlreadyExists(key);
|
|
294
399
|
if (reusable(conflict, current.status))
|
|
295
|
-
return
|
|
400
|
+
return this.toTask(current);
|
|
296
401
|
// The strategy declined the recorded task, so the key repoints to the
|
|
297
402
|
// fresh one inserted below. Cancel only what is still live: a terminal
|
|
298
403
|
// task has nothing to stop, and cancelling it would rewrite a settled
|
|
@@ -304,14 +409,14 @@ export class TaskStore {
|
|
|
304
409
|
}
|
|
305
410
|
const row = (await fetch("insert_task", ins))[0];
|
|
306
411
|
await fetch("upsert_key", { key, task_id: id });
|
|
307
|
-
return
|
|
412
|
+
return this.toTask(row);
|
|
308
413
|
});
|
|
309
414
|
}
|
|
310
415
|
async get(taskId) {
|
|
311
|
-
return
|
|
416
|
+
return this.one(await this.fetch("get", { id: taskId }));
|
|
312
417
|
}
|
|
313
418
|
async getByKey(key) {
|
|
314
|
-
return
|
|
419
|
+
return this.one(await this.fetch("get_by_key", { key }));
|
|
315
420
|
}
|
|
316
421
|
/** The wait loop's probe: id + status alone, so polling a task with a large
|
|
317
422
|
* payload does not re-read and re-parse that payload on every beat. */
|
|
@@ -341,13 +446,13 @@ export class TaskStore {
|
|
|
341
446
|
limit: input.limit ?? 100,
|
|
342
447
|
offset: input.offset ?? 0,
|
|
343
448
|
});
|
|
344
|
-
return rows.map(
|
|
449
|
+
return rows.map((r) => this.toTask(r));
|
|
345
450
|
}
|
|
346
451
|
async cancel(taskId) {
|
|
347
|
-
return
|
|
452
|
+
return this.one(await this.fetch("cancel", { id: taskId }));
|
|
348
453
|
}
|
|
349
454
|
async retry(taskId, opts = {}) {
|
|
350
|
-
return
|
|
455
|
+
return this.one(await this.fetch("retry", { id: taskId, reset_attempt: opts.resetAttempt ?? false }));
|
|
351
456
|
}
|
|
352
457
|
async cancelByKey(key) {
|
|
353
458
|
return this.byKey("cancel", key, {});
|
|
@@ -367,7 +472,7 @@ export class TaskStore {
|
|
|
367
472
|
const existing = (await fetch("get_key", { key }));
|
|
368
473
|
if (!existing.length)
|
|
369
474
|
return null;
|
|
370
|
-
return
|
|
475
|
+
return this.one(await fetch(name, { id: existing[0].task_id, ...params }));
|
|
371
476
|
});
|
|
372
477
|
}
|
|
373
478
|
/**
|
|
@@ -390,104 +495,12 @@ export class TaskStore {
|
|
|
390
495
|
});
|
|
391
496
|
return rows.map((r) => r.id);
|
|
392
497
|
}
|
|
393
|
-
/**
|
|
394
|
-
* Task counts per queue, keyed by status and zero-filled across all statuses —
|
|
395
|
-
* `(await stats()).default.queued` is the backlog of a queue. A queue appears
|
|
396
|
-
* only while it has rows; terminal tasks keep counting until `purge` removes
|
|
397
|
-
* them.
|
|
398
|
-
*
|
|
399
|
-
* `queue` restricts the aggregate to one queue, which is also what stops the
|
|
400
|
-
* caller paying for every other queue's rows: one installation carrying two
|
|
401
|
-
* workloads is the coordination this project recommends, and the unfiltered
|
|
402
|
-
* form reads the whole table. A named queue is always present in the result,
|
|
403
|
-
* zero-filled if it has no rows at all — asking about a specific queue and
|
|
404
|
-
* getting `undefined` back would make every caller write the same fallback.
|
|
405
|
-
*
|
|
406
|
-
* Filtered or not, this COUNTS, so it costs what it counts: a whole queue,
|
|
407
|
-
* terminal rows included. Right for a dashboard, wrong on an interval — poll
|
|
408
|
-
* `queueDepth`, which is bounded, and keep this for when the real numbers are
|
|
409
|
-
* the point.
|
|
410
|
-
*/
|
|
411
|
-
async stats(queue) {
|
|
412
|
-
const zeros = () => Object.fromEntries(STATUSES.map((s) => [s, 0]));
|
|
413
|
-
const out = {};
|
|
414
|
-
// Seed before the query, not after: a named queue with no rows returns no
|
|
415
|
-
// rows to seed from, and that is exactly the case the promise is about.
|
|
416
|
-
if (queue != null)
|
|
417
|
-
out[queue] = zeros();
|
|
418
|
-
for (const row of await this.fetch("stats", { queue: queue ?? null })) {
|
|
419
|
-
const per = (out[row.queue] ??= zeros());
|
|
420
|
-
per[row.status] = Number(row.count);
|
|
421
|
-
}
|
|
422
|
-
return out;
|
|
423
|
-
}
|
|
424
|
-
/**
|
|
425
|
-
* Call `onSignal` when the tasks on `queues` may have changed — something was
|
|
426
|
-
* queued, or something finished.
|
|
427
|
-
*
|
|
428
|
-
* This is notify-ACCELERATED POLLING, not an event log, and the difference is
|
|
429
|
-
* the whole contract. Where a push channel is available (Postgres LISTEN) an
|
|
430
|
-
* idle watch costs nothing and a signal arrives within milliseconds of the
|
|
431
|
-
* event. Where it is not — a transaction-mode pooler refuses LISTEN, SQLite has
|
|
432
|
-
* no channel at all — the timer alone still delivers `poll` signals, so a
|
|
433
|
-
* consumer that re-reads on every signal is correct in both cases and merely
|
|
434
|
-
* less prompt in one.
|
|
435
|
-
*
|
|
436
|
-
* What it will NOT do is promise that a signal means something happened, or
|
|
437
|
-
* that every event produces its own signal. Treat a signal as "re-read now"
|
|
438
|
-
* and take the truth from `stats()` / `list()` / `get()`, which is where it
|
|
439
|
-
* lives. `reason` is a hint for reading less: a `done` signal names the task,
|
|
440
|
-
* so a dashboard can refresh that row instead of the list.
|
|
441
|
-
*
|
|
442
|
-
* Returns an unsubscribe. The timer is unref'd — watching does not hold a
|
|
443
|
-
* process open.
|
|
444
|
-
*/
|
|
445
|
-
watch(opts, onSignal) {
|
|
446
|
-
const pollMs = Math.max(1, opts.pollMs ?? DEFAULT_WATCH_POLL_MS);
|
|
447
|
-
const queues = opts.queues ?? null;
|
|
448
|
-
let live = true;
|
|
449
|
-
const emit = (signal) => {
|
|
450
|
-
// A signal delivered after unsubscribe would have the consumer re-reading
|
|
451
|
-
// a store it has stopped caring about, possibly a closed one.
|
|
452
|
-
if (live)
|
|
453
|
-
onSignal(signal);
|
|
454
|
-
};
|
|
455
|
-
const unsubscribe = this.subscribePush?.((signal) => {
|
|
456
|
-
// A queued signal names its queue, so a watch scoped to some queues can
|
|
457
|
-
// drop the rest. A done signal names only the task — which queue it was on
|
|
458
|
-
// is not in the notification, so it is never filtered out.
|
|
459
|
-
if (signal.reason === "queued" && queues && signal.queue && !queues.includes(signal.queue)) {
|
|
460
|
-
return;
|
|
461
|
-
}
|
|
462
|
-
emit(signal);
|
|
463
|
-
});
|
|
464
|
-
const timer = setInterval(() => {
|
|
465
|
-
this.warmPush?.();
|
|
466
|
-
// Guarded for the same reason PostgresStore guards its push fan-out: a
|
|
467
|
-
// consumer that throws must not take the timer down with it. Losing the
|
|
468
|
-
// timer would silently retire the fallback that makes watch correct
|
|
469
|
-
// where there is no push channel at all.
|
|
470
|
-
try {
|
|
471
|
-
emit({ reason: "poll" });
|
|
472
|
-
}
|
|
473
|
-
catch {
|
|
474
|
-
// The consumer's problem, not the watch's.
|
|
475
|
-
}
|
|
476
|
-
}, pollMs);
|
|
477
|
-
timer.unref?.();
|
|
478
|
-
return () => {
|
|
479
|
-
live = false;
|
|
480
|
-
unsubscribe?.();
|
|
481
|
-
clearInterval(timer);
|
|
482
|
-
};
|
|
483
|
-
}
|
|
484
498
|
/**
|
|
485
499
|
* How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
|
|
486
500
|
*
|
|
487
|
-
* The cheap half of backpressure: bounded at `maxDepth` index entries,
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
* load or shape a producer; `QueueDepthGate` builds the blocking form on top.
|
|
501
|
+
* The cheap half of backpressure: bounded at `maxDepth` index entries, so it
|
|
502
|
+
* stays cheap to ask on every enqueue. Use it directly to shed load or shape
|
|
503
|
+
* a producer; `QueueDepthGate` builds the blocking form on top.
|
|
491
504
|
*/
|
|
492
505
|
async queueDepth(queue, maxDepth) {
|
|
493
506
|
if (!Number.isInteger(maxDepth) || maxDepth < 0) {
|
|
@@ -580,7 +593,7 @@ export class TaskStore {
|
|
|
580
593
|
name: oneName ? names[0] : null,
|
|
581
594
|
limit,
|
|
582
595
|
});
|
|
583
|
-
return rows.map(
|
|
596
|
+
return rows.map((r) => this.toTask(r));
|
|
584
597
|
});
|
|
585
598
|
});
|
|
586
599
|
}
|
|
@@ -668,7 +681,7 @@ export class TaskStore {
|
|
|
668
681
|
// Rolls back `fn`'s writes along with the settlement that did not land.
|
|
669
682
|
if (!rows.length)
|
|
670
683
|
throw new LostLease(input.taskId);
|
|
671
|
-
return { task:
|
|
684
|
+
return { task: this.toTask(rows[0]), value };
|
|
672
685
|
});
|
|
673
686
|
}
|
|
674
687
|
async fail(input) {
|
|
@@ -44,6 +44,17 @@
|
|
|
44
44
|
* The reference implementation happens not to be affected — `pg` uses the
|
|
45
45
|
* extended protocol by default — which is why this is stated here rather than
|
|
46
46
|
* left to be discovered.
|
|
47
|
+
*
|
|
48
|
+
* A second thing, smaller but just as easy to get wrong: whatever this executor
|
|
49
|
+
* does with json/jsonb, it must do CONSISTENTLY. The store measures the wire
|
|
50
|
+
* form once at connect, with a `select '"cairnq"'::jsonb` probe, and maps every
|
|
51
|
+
* row it later reads according to the answer — so an executor that decodes
|
|
52
|
+
* jsonb for one statement and hands back text for another (or answers the probe
|
|
53
|
+
* differently from how it returns task rows) will have its payloads read the
|
|
54
|
+
* wrong way. Decoding or not are both fine; disagreeing with yourself is not.
|
|
55
|
+
* The probe exists because the two forms cannot be told apart from a value: a
|
|
56
|
+
* JSON string arrives as a string either way, so a guess parses the decoded one
|
|
57
|
+
* twice — `"s3://…"` throws and `"42"` silently becomes the number 42.
|
|
47
58
|
*/
|
|
48
59
|
/** A row as the driver hands it back: column name -> value. */
|
|
49
60
|
export type Row = Record<string, unknown>;
|
|
@@ -44,6 +44,17 @@
|
|
|
44
44
|
* The reference implementation happens not to be affected — `pg` uses the
|
|
45
45
|
* extended protocol by default — which is why this is stated here rather than
|
|
46
46
|
* left to be discovered.
|
|
47
|
+
*
|
|
48
|
+
* A second thing, smaller but just as easy to get wrong: whatever this executor
|
|
49
|
+
* does with json/jsonb, it must do CONSISTENTLY. The store measures the wire
|
|
50
|
+
* form once at connect, with a `select '"cairnq"'::jsonb` probe, and maps every
|
|
51
|
+
* row it later reads according to the answer — so an executor that decodes
|
|
52
|
+
* jsonb for one statement and hands back text for another (or answers the probe
|
|
53
|
+
* differently from how it returns task rows) will have its payloads read the
|
|
54
|
+
* wrong way. Decoding or not are both fine; disagreeing with yourself is not.
|
|
55
|
+
* The probe exists because the two forms cannot be told apart from a value: a
|
|
56
|
+
* JSON string arrives as a string either way, so a guess parses the decoded one
|
|
57
|
+
* twice — `"s3://…"` throws and `"42"` silently becomes the number 42.
|
|
47
58
|
*/
|
|
48
59
|
/**
|
|
49
60
|
* LISTEN will not work on this connection, and retrying cannot change that.
|
package/dist/store/pg-pool.js
CHANGED
|
@@ -150,6 +150,20 @@ export async function createPoolExecutor(dsn, opts = {}) {
|
|
|
150
150
|
// its connection for its whole life, and a pooled one would be a slot the
|
|
151
151
|
// store never gives back. If `opts` ever grows connection-level settings
|
|
152
152
|
// (ssl, application_name), the listener must receive them too.
|
|
153
|
+
//
|
|
154
|
+
// `schema` is deliberately NOT forwarded, and it is the exception worth
|
|
155
|
+
// stating rather than the omission it looks like: a NOTIFY channel name is
|
|
156
|
+
// scoped to the database, never to a schema, so a search_path would change
|
|
157
|
+
// nothing about what arrives here. The consequence is real but bounded —
|
|
158
|
+
// two cairnq installations in different schemas of ONE database hear each
|
|
159
|
+
// other's notifications, and a queue name they happen to share wakes the
|
|
160
|
+
// wrong workers. Correctness is unaffected (a wake is only ever a hint;
|
|
161
|
+
// the claim that follows re-reads the rows and finds none of them), so the
|
|
162
|
+
// cost is one wasted poll per foreign submit. Fixing it properly means
|
|
163
|
+
// putting the schema in the channel name, which is a migration and a
|
|
164
|
+
// mixed-version-fleet hazard: a worker still listening on the old channel
|
|
165
|
+
// would hear nothing and fall back to polling for the length of the
|
|
166
|
+
// rollout.
|
|
153
167
|
const client = new pg.Client({ connectionString: dsn });
|
|
154
168
|
await client.connect(); // failure here is transient: caller retries with backoff
|
|
155
169
|
client.on("notification", (msg) => onNotify(msg.channel, msg.payload));
|