cairnq 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -32
- package/dist/_protocol/sql/postgres/purge.sql +10 -10
- package/dist/_protocol/sql/sqlite/purge.sql +10 -10
- package/dist/backpressure.d.ts +0 -3
- package/dist/backpressure.js +1 -3
- package/dist/client.d.ts +42 -46
- package/dist/client.js +27 -43
- package/dist/context.d.ts +23 -1
- package/dist/context.js +18 -7
- package/dist/errors.d.ts +20 -26
- package/dist/errors.js +23 -35
- package/dist/index.d.ts +4 -10
- package/dist/index.js +3 -6
- package/dist/models.d.ts +14 -1
- package/dist/models.js +17 -5
- package/dist/retention.d.ts +22 -66
- package/dist/retention.js +53 -64
- package/dist/store/base.d.ts +29 -90
- package/dist/store/base.js +136 -123
- package/dist/store/pg-executor.d.ts +11 -0
- package/dist/store/pg-executor.js +11 -0
- package/dist/store/pg-pool.js +14 -0
- package/dist/store/postgres.d.ts +18 -12
- package/dist/store/postgres.js +23 -29
- package/dist/store/sqlite.d.ts +39 -2
- package/dist/store/sqlite.js +108 -29
- package/dist/worker.d.ts +38 -93
- package/dist/worker.js +181 -234
- package/package.json +1 -1
- package/src/backpressure.ts +1 -5
- package/src/client.ts +66 -71
- package/src/context.ts +28 -7
- package/src/errors.ts +25 -36
- package/src/index.ts +4 -27
- package/src/models.ts +17 -5
- package/src/retention.ts +50 -115
- package/src/store/base.ts +131 -166
- package/src/store/pg-executor.ts +11 -0
- package/src/store/pg-pool.ts +14 -0
- package/src/store/postgres.ts +24 -31
- package/src/store/sqlite.ts +108 -27
- package/src/worker.ts +208 -300
- package/dist/_protocol/sql/postgres/stats.sql +0 -23
- package/dist/_protocol/sql/sqlite/stats.sql +0 -23
package/dist/store/postgres.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Fetch, type Params, TaskStore
|
|
1
|
+
import { type Fetch, type Params, TaskStore } from "./base.js";
|
|
2
2
|
import { type PgExecutor, type PgSession } from "./pg-executor.js";
|
|
3
3
|
/**
|
|
4
4
|
* The rewritten SQL and the order its `$n` slots must be filled in.
|
|
@@ -98,9 +98,6 @@ export declare class PostgresStore extends TaskStore {
|
|
|
98
98
|
private readonly wakeableQueues;
|
|
99
99
|
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
100
100
|
private readonly waiters;
|
|
101
|
-
/** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
|
|
102
|
-
* consumes the notification, a subscriber is standing and only observes. */
|
|
103
|
-
private readonly subscribers;
|
|
104
101
|
/**
|
|
105
102
|
* `source` is either a libpq connection string — this store then owns a `pg`
|
|
106
103
|
* pool and requires the optional `pg` package — or a PgExecutor the caller
|
|
@@ -119,8 +116,8 @@ export declare class PostgresStore extends TaskStore {
|
|
|
119
116
|
* Every path here connects lazily, so a store used again after close simply
|
|
120
117
|
* reconnects — and without this it would reconnect permanently degraded to
|
|
121
118
|
* polling, having decided "no push channel" about a connection that no longer
|
|
122
|
-
* exists.
|
|
123
|
-
* for every later one, so the three
|
|
119
|
+
* exists. The fields' initializers give the first state and doConnect calls
|
|
120
|
+
* this for every later one, so the three cannot be re-armed by halves. Only
|
|
124
121
|
* ever safe under doConnect's generation check.
|
|
125
122
|
*/
|
|
126
123
|
private resetListenerState;
|
|
@@ -149,12 +146,25 @@ export declare class PostgresStore extends TaskStore {
|
|
|
149
146
|
*/
|
|
150
147
|
private checkSchema;
|
|
151
148
|
private applyMigrations;
|
|
149
|
+
/**
|
|
150
|
+
* Whether this driver hands json/jsonb back as text, measured rather than
|
|
151
|
+
* assumed.
|
|
152
|
+
*
|
|
153
|
+
* `pg` decodes jsonb by default; asyncpg hands back the text; an application's
|
|
154
|
+
* own driver — or a `pg` with a type parser the application installed — may do
|
|
155
|
+
* either, and an injected executor is exactly that case. The two are
|
|
156
|
+
* indistinguishable from a value once it arrives (see rowToTask), so the
|
|
157
|
+
* answer has to come from the driver itself.
|
|
158
|
+
*
|
|
159
|
+
* A JSON string is the probe because it is the only value whose two forms
|
|
160
|
+
* differ: as text it arrives with its quotes (`"cairnq"`, 8 characters), as a
|
|
161
|
+
* decoded value without them (`cairnq`, 6). One statement, once per connect.
|
|
162
|
+
*/
|
|
163
|
+
private detectJsonWireForm;
|
|
152
164
|
private readProtocolVersion;
|
|
153
165
|
protocolVersion(): Promise<number>;
|
|
154
166
|
protected registerWakeable(queues: string[]): void;
|
|
155
167
|
claimWake(queues: string[], timeoutMs: number): Promise<void>;
|
|
156
|
-
protected subscribePush(onSignal: (signal: WatchSignal) => void): () => void;
|
|
157
|
-
protected warmPush(): void;
|
|
158
168
|
taskDoneWake(taskId: string, timeoutMs: number): Promise<void>;
|
|
159
169
|
/** A promise resolving on notification-or-timeout, deregistering either way.
|
|
160
170
|
* The timer is unref'd: while a listener exists its socket keeps the process
|
|
@@ -167,10 +177,6 @@ export declare class PostgresStore extends TaskStore {
|
|
|
167
177
|
private listenerReady;
|
|
168
178
|
private startListener;
|
|
169
179
|
private onNotification;
|
|
170
|
-
/** Hand a notification to every watch() subscriber. A throwing subscriber is
|
|
171
|
-
* its own problem: it must not cost the others their signal, nor take down the
|
|
172
|
-
* listener connection that delivered it. */
|
|
173
|
-
private publish;
|
|
174
180
|
private dropListener;
|
|
175
181
|
protected fetch(name: string, params: Params): Promise<any[]>;
|
|
176
182
|
protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
|
package/dist/store/postgres.js
CHANGED
|
@@ -125,9 +125,6 @@ export class PostgresStore extends TaskStore {
|
|
|
125
125
|
wakeableQueues = new Set();
|
|
126
126
|
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
127
127
|
waiters = new Map();
|
|
128
|
-
/** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
|
|
129
|
-
* consumes the notification, a subscriber is standing and only observes. */
|
|
130
|
-
subscribers = new Set();
|
|
131
128
|
/**
|
|
132
129
|
* `source` is either a libpq connection string — this store then owns a `pg`
|
|
133
130
|
* pool and requires the optional `pg` package — or a PgExecutor the caller
|
|
@@ -175,8 +172,8 @@ export class PostgresStore extends TaskStore {
|
|
|
175
172
|
* Every path here connects lazily, so a store used again after close simply
|
|
176
173
|
* reconnects — and without this it would reconnect permanently degraded to
|
|
177
174
|
* polling, having decided "no push channel" about a connection that no longer
|
|
178
|
-
* exists.
|
|
179
|
-
* for every later one, so the three
|
|
175
|
+
* exists. The fields' initializers give the first state and doConnect calls
|
|
176
|
+
* this for every later one, so the three cannot be re-armed by halves. Only
|
|
180
177
|
* ever safe under doConnect's generation check.
|
|
181
178
|
*/
|
|
182
179
|
resetListenerState() {
|
|
@@ -194,6 +191,9 @@ export class PostgresStore extends TaskStore {
|
|
|
194
191
|
await this.checkSchema(executor);
|
|
195
192
|
await this.applyMigrations(executor);
|
|
196
193
|
checkProtocolVersion(await this.readProtocolVersion(executor));
|
|
194
|
+
// Before publishing the executor, so no row is ever mapped against an
|
|
195
|
+
// unmeasured wire form.
|
|
196
|
+
this.jsonIsText = await this.detectJsonWireForm(executor);
|
|
197
197
|
}
|
|
198
198
|
catch (e) {
|
|
199
199
|
// Never leak an executor we created; never close one we were handed.
|
|
@@ -286,6 +286,24 @@ export class PostgresStore extends TaskStore {
|
|
|
286
286
|
});
|
|
287
287
|
}
|
|
288
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Whether this driver hands json/jsonb back as text, measured rather than
|
|
291
|
+
* assumed.
|
|
292
|
+
*
|
|
293
|
+
* `pg` decodes jsonb by default; asyncpg hands back the text; an application's
|
|
294
|
+
* own driver — or a `pg` with a type parser the application installed — may do
|
|
295
|
+
* either, and an injected executor is exactly that case. The two are
|
|
296
|
+
* indistinguishable from a value once it arrives (see rowToTask), so the
|
|
297
|
+
* answer has to come from the driver itself.
|
|
298
|
+
*
|
|
299
|
+
* A JSON string is the probe because it is the only value whose two forms
|
|
300
|
+
* differ: as text it arrives with its quotes (`"cairnq"`, 8 characters), as a
|
|
301
|
+
* decoded value without them (`cairnq`, 6). One statement, once per connect.
|
|
302
|
+
*/
|
|
303
|
+
async detectJsonWireForm(s) {
|
|
304
|
+
const rows = await s.query(`select '"cairnq"'::jsonb as probe`, []);
|
|
305
|
+
return rows[0]?.probe === '"cairnq"';
|
|
306
|
+
}
|
|
289
307
|
// Takes an explicit session: during doConnect the executor is not published
|
|
290
308
|
// yet, so this cannot go through fetch(). The statement binds nothing.
|
|
291
309
|
async readProtocolVersion(s) {
|
|
@@ -322,15 +340,6 @@ export class PostgresStore extends TaskStore {
|
|
|
322
340
|
await this.wakeOn("queued", remaining);
|
|
323
341
|
}
|
|
324
342
|
}
|
|
325
|
-
subscribePush(onSignal) {
|
|
326
|
-
this.subscribers.add(onSignal);
|
|
327
|
-
this.listenerReady(); // an API-side watcher is often the only thing asking
|
|
328
|
-
return () => void this.subscribers.delete(onSignal);
|
|
329
|
-
}
|
|
330
|
-
warmPush() {
|
|
331
|
-
if (this.subscribers.size > 0)
|
|
332
|
-
this.listenerReady();
|
|
333
|
-
}
|
|
334
343
|
taskDoneWake(taskId, timeoutMs) {
|
|
335
344
|
if (!this.listenerReady())
|
|
336
345
|
return super.taskDoneWake(taskId, timeoutMs);
|
|
@@ -421,11 +430,9 @@ export class PostgresStore extends TaskStore {
|
|
|
421
430
|
if (this.wakeableQueues.has(payload))
|
|
422
431
|
this.pendingQueues.add(payload);
|
|
423
432
|
key = "queued";
|
|
424
|
-
this.publish({ reason: "queued", queue: payload });
|
|
425
433
|
}
|
|
426
434
|
else if (channel === DONE_CHANNEL && payload) {
|
|
427
435
|
key = `done:${payload}`;
|
|
428
|
-
this.publish({ reason: "done", taskId: payload });
|
|
429
436
|
}
|
|
430
437
|
else {
|
|
431
438
|
return;
|
|
@@ -436,19 +443,6 @@ export class PostgresStore extends TaskStore {
|
|
|
436
443
|
for (const w of set)
|
|
437
444
|
w();
|
|
438
445
|
}
|
|
439
|
-
/** Hand a notification to every watch() subscriber. A throwing subscriber is
|
|
440
|
-
* its own problem: it must not cost the others their signal, nor take down the
|
|
441
|
-
* listener connection that delivered it. */
|
|
442
|
-
publish(signal) {
|
|
443
|
-
for (const s of [...this.subscribers]) {
|
|
444
|
-
try {
|
|
445
|
-
s(signal);
|
|
446
|
-
}
|
|
447
|
-
catch {
|
|
448
|
-
// Deliberately swallowed — see above.
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
446
|
dropListener() {
|
|
453
447
|
const stop = this.listener;
|
|
454
448
|
this.listener = null;
|
package/dist/store/sqlite.d.ts
CHANGED
|
@@ -46,13 +46,48 @@ export declare class SQLiteStore extends TaskStore {
|
|
|
46
46
|
private readonly writes;
|
|
47
47
|
/** Writes waiting to be group-committed — see flush(). */
|
|
48
48
|
private pending;
|
|
49
|
-
/**
|
|
50
|
-
|
|
49
|
+
/**
|
|
50
|
+
* The flusher draining `pending`, or null when none is running — both the
|
|
51
|
+
* "is one already going" guard and the handle `close()` waits on, which are
|
|
52
|
+
* the same question. The Python twin keeps two fields only because its
|
|
53
|
+
* _do_close hands the flusher off before awaiting it.
|
|
54
|
+
*/
|
|
55
|
+
private flusher;
|
|
56
|
+
/** Set for the duration of close(), so the drain it waits on is finite. */
|
|
57
|
+
private closing;
|
|
58
|
+
/** The close in progress, so a second caller waits for it rather than
|
|
59
|
+
* returning while the connection is still open. */
|
|
60
|
+
private closed;
|
|
61
|
+
/** Whether a connection was ever opened — see ensure() on in-memory reopen. */
|
|
62
|
+
private everOpened;
|
|
51
63
|
constructor(path: string, opts?: {
|
|
52
64
|
busyTimeoutMs?: number;
|
|
53
65
|
});
|
|
54
66
|
connect(): Promise<void>;
|
|
67
|
+
/**
|
|
68
|
+
* Close the connection, once everything already accepted has landed.
|
|
69
|
+
*
|
|
70
|
+
* Two things can be in flight, and cutting off either loses a write that a
|
|
71
|
+
* caller is still awaiting: a group commit holding a batch (`flush`), and a
|
|
72
|
+
* transaction with a BEGIN IMMEDIATE open on this same connection (a keyed
|
|
73
|
+
* submit, a claim). The first is awaited directly. The second is waited out by
|
|
74
|
+
* queuing an empty operation behind it on the file lock — the lock is what
|
|
75
|
+
* serializes them in the first place, so anything already queued there runs
|
|
76
|
+
* before this does.
|
|
77
|
+
*
|
|
78
|
+
* Neither wait can be stretched indefinitely: `closing` turns away everything
|
|
79
|
+
* that arrives from now on, so the queue this drains is the one that existed
|
|
80
|
+
* when close() was called. Without that barrier a write landing between the
|
|
81
|
+
* flusher finishing and the connection closing would start a fresh flusher
|
|
82
|
+
* against a `db` already null — the TypeError that used to surface here.
|
|
83
|
+
*
|
|
84
|
+
* Closing does not retire the store. Connecting is lazy, so a store used again
|
|
85
|
+
* afterwards reopens; `closing` is cleared for exactly that reason. The one
|
|
86
|
+
* case where reopening cannot mean what the caller wants is an in-memory
|
|
87
|
+
* database, whose contents live in the connection — see ensure().
|
|
88
|
+
*/
|
|
55
89
|
close(): Promise<void>;
|
|
90
|
+
private doClose;
|
|
56
91
|
private ensure;
|
|
57
92
|
private applyMigrations;
|
|
58
93
|
private readProtocolVersion;
|
|
@@ -136,6 +171,8 @@ export declare class SQLiteStore extends TaskStore {
|
|
|
136
171
|
* and one that arrives after sees the flag down and starts a new flusher.
|
|
137
172
|
*/
|
|
138
173
|
private scheduleFlush;
|
|
174
|
+
/** The flusher's body; see scheduleFlush for why it is a separate method. */
|
|
175
|
+
private drain;
|
|
139
176
|
protected fetch(name: string, params: Params): Promise<any[]>;
|
|
140
177
|
protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
|
|
141
178
|
}
|
package/dist/store/sqlite.js
CHANGED
|
@@ -3,6 +3,7 @@ import { dirname, resolve } from "node:path";
|
|
|
3
3
|
import { createRequire } from "node:module";
|
|
4
4
|
import { nowMs } from "../ids.js";
|
|
5
5
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
6
|
+
import { StoreClosed } from "../errors.js";
|
|
6
7
|
import { checkProtocolVersion, COMMENT, specialize, statementParams, TaskStore, } from "./base.js";
|
|
7
8
|
// `better-sqlite3` is an optional dependency, matching `pg` on the Postgres side:
|
|
8
9
|
// a Postgres-only deployment should not have to build a native module it never
|
|
@@ -213,8 +214,20 @@ export class SQLiteStore extends TaskStore {
|
|
|
213
214
|
writes;
|
|
214
215
|
/** Writes waiting to be group-committed — see flush(). */
|
|
215
216
|
pending = [];
|
|
216
|
-
/**
|
|
217
|
-
|
|
217
|
+
/**
|
|
218
|
+
* The flusher draining `pending`, or null when none is running — both the
|
|
219
|
+
* "is one already going" guard and the handle `close()` waits on, which are
|
|
220
|
+
* the same question. The Python twin keeps two fields only because its
|
|
221
|
+
* _do_close hands the flusher off before awaiting it.
|
|
222
|
+
*/
|
|
223
|
+
flusher = null;
|
|
224
|
+
/** Set for the duration of close(), so the drain it waits on is finite. */
|
|
225
|
+
closing = false;
|
|
226
|
+
/** The close in progress, so a second caller waits for it rather than
|
|
227
|
+
* returning while the connection is still open. */
|
|
228
|
+
closed = null;
|
|
229
|
+
/** Whether a connection was ever opened — see ensure() on in-memory reopen. */
|
|
230
|
+
everOpened = false;
|
|
218
231
|
constructor(path, opts = {}) {
|
|
219
232
|
super();
|
|
220
233
|
this.path = path;
|
|
@@ -233,17 +246,81 @@ export class SQLiteStore extends TaskStore {
|
|
|
233
246
|
async connect() {
|
|
234
247
|
this.ensure();
|
|
235
248
|
}
|
|
249
|
+
/**
|
|
250
|
+
* Close the connection, once everything already accepted has landed.
|
|
251
|
+
*
|
|
252
|
+
* Two things can be in flight, and cutting off either loses a write that a
|
|
253
|
+
* caller is still awaiting: a group commit holding a batch (`flush`), and a
|
|
254
|
+
* transaction with a BEGIN IMMEDIATE open on this same connection (a keyed
|
|
255
|
+
* submit, a claim). The first is awaited directly. The second is waited out by
|
|
256
|
+
* queuing an empty operation behind it on the file lock — the lock is what
|
|
257
|
+
* serializes them in the first place, so anything already queued there runs
|
|
258
|
+
* before this does.
|
|
259
|
+
*
|
|
260
|
+
* Neither wait can be stretched indefinitely: `closing` turns away everything
|
|
261
|
+
* that arrives from now on, so the queue this drains is the one that existed
|
|
262
|
+
* when close() was called. Without that barrier a write landing between the
|
|
263
|
+
* flusher finishing and the connection closing would start a fresh flusher
|
|
264
|
+
* against a `db` already null — the TypeError that used to surface here.
|
|
265
|
+
*
|
|
266
|
+
* Closing does not retire the store. Connecting is lazy, so a store used again
|
|
267
|
+
* afterwards reopens; `closing` is cleared for exactly that reason. The one
|
|
268
|
+
* case where reopening cannot mean what the caller wants is an in-memory
|
|
269
|
+
* database, whose contents live in the connection — see ensure().
|
|
270
|
+
*/
|
|
236
271
|
async close() {
|
|
237
|
-
|
|
238
|
-
this.
|
|
239
|
-
|
|
240
|
-
|
|
272
|
+
return (this.closed ??= this.doClose().finally(() => {
|
|
273
|
+
this.closed = null;
|
|
274
|
+
}));
|
|
275
|
+
}
|
|
276
|
+
async doClose() {
|
|
277
|
+
this.closing = true;
|
|
278
|
+
try {
|
|
279
|
+
// Errors belong to the writes that carry them, and the flusher has already
|
|
280
|
+
// delivered each one to its own waiter; close() only needs it finished.
|
|
281
|
+
await this.flusher?.catch(() => { });
|
|
282
|
+
// Runs behind every operation currently on the lock, including a
|
|
283
|
+
// transaction mid-await. Reads are queued there too, so this also waits
|
|
284
|
+
// out a statement that would otherwise land on a closed connection.
|
|
285
|
+
await this.enqueue(() => { }).catch(() => { });
|
|
286
|
+
if (this.db) {
|
|
287
|
+
this.db.close();
|
|
288
|
+
this.db = null;
|
|
289
|
+
this.stmts.clear();
|
|
290
|
+
}
|
|
291
|
+
// An in-memory database's lock guards nothing once its connection is gone:
|
|
292
|
+
// the key is minted per store (see lockKey), so no other store can be
|
|
293
|
+
// queued behind it. Left in place, a process that opens many short-lived
|
|
294
|
+
// in-memory stores — a test suite, a per-request scratch queue — grows
|
|
295
|
+
// fileLocks by one settled promise per store, forever. A file's key is NOT
|
|
296
|
+
// dropped: other stores may still be serializing on it.
|
|
297
|
+
if (this.lockKey.startsWith("memory#"))
|
|
298
|
+
fileLocks.delete(this.lockKey);
|
|
299
|
+
}
|
|
300
|
+
finally {
|
|
301
|
+
this.closing = false;
|
|
241
302
|
}
|
|
242
303
|
}
|
|
243
304
|
ensure() {
|
|
305
|
+
// A store mid-close is about to drop the connection this would hand back.
|
|
306
|
+
// Refusing keeps close()'s drain finite, and gives the caller a typed error
|
|
307
|
+
// instead of whatever the driver says about a connection pulled out from
|
|
308
|
+
// under it.
|
|
309
|
+
if (this.closing)
|
|
310
|
+
throw new StoreClosed();
|
|
244
311
|
if (this.db)
|
|
245
312
|
return this.db;
|
|
246
313
|
const memory = isMemory(this.path);
|
|
314
|
+
// Reopening is how a store used again after close() carries on, and for a
|
|
315
|
+
// file that is exactly right — the tasks are in the file. An in-memory
|
|
316
|
+
// database lives in the connection, so the same path would hand back an
|
|
317
|
+
// EMPTY store: every task the caller submitted gone, no error, and a `get`
|
|
318
|
+
// that answers null as if the id had never existed. Say so instead.
|
|
319
|
+
if (memory && this.everOpened) {
|
|
320
|
+
throw new StoreClosed(`in-memory database ${JSON.stringify(this.path)} was closed — its contents ` +
|
|
321
|
+
`went with the connection, so reopening would silently start from empty. ` +
|
|
322
|
+
`Use a file path if the store needs to outlive a close.`);
|
|
323
|
+
}
|
|
247
324
|
if (!memory)
|
|
248
325
|
mkdirSync(dirname(this.path), { recursive: true });
|
|
249
326
|
const db = new (loadSqlite())(this.path);
|
|
@@ -277,6 +354,7 @@ export class SQLiteStore extends TaskStore {
|
|
|
277
354
|
this.stmts.set(sql, db.prepare(sql));
|
|
278
355
|
}
|
|
279
356
|
this.db = db;
|
|
357
|
+
this.everOpened = true;
|
|
280
358
|
checkProtocolVersion(this.readProtocolVersion());
|
|
281
359
|
return db;
|
|
282
360
|
}
|
|
@@ -547,33 +625,34 @@ export class SQLiteStore extends TaskStore {
|
|
|
547
625
|
* and one that arrives after sees the flag down and starts a new flusher.
|
|
548
626
|
*/
|
|
549
627
|
scheduleFlush(db) {
|
|
550
|
-
if (this.
|
|
628
|
+
if (this.flusher)
|
|
551
629
|
return;
|
|
552
|
-
|
|
553
|
-
|
|
630
|
+
// Assigned before the `.finally` can possibly run — a finally callback is a
|
|
631
|
+
// microtask at the earliest — so the field is never cleared before it is
|
|
632
|
+
// set, however little `drain` does before returning.
|
|
633
|
+
this.flusher = this.drain(db).finally(() => {
|
|
634
|
+
this.flusher = null;
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
/** The flusher's body; see scheduleFlush for why it is a separate method. */
|
|
638
|
+
async drain(db) {
|
|
639
|
+
while (this.pending.length) {
|
|
554
640
|
try {
|
|
555
|
-
|
|
556
|
-
try {
|
|
557
|
-
await this.withLock(() => this.flush(db));
|
|
558
|
-
}
|
|
559
|
-
catch (err) {
|
|
560
|
-
// flush only throws on a lost write lock, and only after putting its
|
|
561
|
-
// batch back — so reaching here means withLock spent the whole budget
|
|
562
|
-
// and those writes are still queued with nobody else coming for them.
|
|
563
|
-
// Anything that arrived behind them is failed with the same error
|
|
564
|
-
// rather than left hanging: this store cannot write at all right now,
|
|
565
|
-
// which is what a lone write would have been told too.
|
|
566
|
-
const stranded = this.pending;
|
|
567
|
-
this.pending = [];
|
|
568
|
-
for (const w of stranded)
|
|
569
|
-
w.reject(err);
|
|
570
|
-
}
|
|
571
|
-
}
|
|
641
|
+
await this.withLock(() => this.flush(db));
|
|
572
642
|
}
|
|
573
|
-
|
|
574
|
-
|
|
643
|
+
catch (err) {
|
|
644
|
+
// flush only throws on a lost write lock, and only after putting its
|
|
645
|
+
// batch back — so reaching here means withLock spent the whole budget
|
|
646
|
+
// and those writes are still queued with nobody else coming for them.
|
|
647
|
+
// Anything that arrived behind them is failed with the same error
|
|
648
|
+
// rather than left hanging: this store cannot write at all right now,
|
|
649
|
+
// which is what a lone write would have been told too.
|
|
650
|
+
const stranded = this.pending;
|
|
651
|
+
this.pending = [];
|
|
652
|
+
for (const w of stranded)
|
|
653
|
+
w.reject(err);
|
|
575
654
|
}
|
|
576
|
-
}
|
|
655
|
+
}
|
|
577
656
|
}
|
|
578
657
|
async fetch(name, params) {
|
|
579
658
|
const db = this.ensure();
|
package/dist/worker.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { BackpressureOptions } from "./backpressure.js";
|
|
2
1
|
import { TaskContext } from "./context.js";
|
|
3
2
|
import type { PgExecutor } from "./store/pg-executor.js";
|
|
4
3
|
import type { TaskStore } from "./store/base.js";
|
|
@@ -24,18 +23,15 @@ export type ErrorPhase = "claim" | "execute";
|
|
|
24
23
|
* children through TaskContext.submit is a producer, and in a worker process
|
|
25
24
|
* there is usually no CairnQ handle to have configured the store.
|
|
26
25
|
*/
|
|
27
|
-
export interface WorkerOptions
|
|
26
|
+
export interface WorkerOptions {
|
|
28
27
|
/**
|
|
29
28
|
* Handler calls allowed to run at once. A batch call counts as one, however
|
|
30
29
|
* many tasks it carries — size it for how much work you want in flight, not
|
|
31
|
-
* for how many tasks that comes to.
|
|
32
|
-
* bounds memory, which task counts never did.
|
|
30
|
+
* for how many tasks that comes to.
|
|
33
31
|
*/
|
|
34
32
|
concurrency?: number;
|
|
35
33
|
leaseMs?: number;
|
|
36
|
-
heartbeatIntervalMs?: number;
|
|
37
34
|
pollIntervalMs?: number;
|
|
38
|
-
claimBatch?: number;
|
|
39
35
|
/** Base delay before re-running a failed attempt; doubles per attempt. 0 disables. */
|
|
40
36
|
retryBackoffMs?: number;
|
|
41
37
|
/** Ceiling for the doubling. */
|
|
@@ -49,39 +45,21 @@ export interface WorkerOptions extends Partial<BackpressureOptions> {
|
|
|
49
45
|
* a retryable `handler_timeout` failure. Unset disables the ceiling.
|
|
50
46
|
*/
|
|
51
47
|
maxRunMs?: number;
|
|
52
|
-
/**
|
|
53
|
-
* Resident payload bytes allowed across running handlers, independent of
|
|
54
|
-
* their count.
|
|
55
|
-
*
|
|
56
|
-
* `concurrency` bounds tasks, not memory, so a worker sized for small payloads
|
|
57
|
-
* holds concurrency * largest-payload bytes the moment a batch of big ones
|
|
58
|
-
* arrives — for payloads that carry media inline, that is the difference
|
|
59
|
-
* between megabytes and gigabytes resident. Once the budget is spent the
|
|
60
|
-
* worker stops claiming until running handlers give it back.
|
|
61
|
-
*
|
|
62
|
-
* The bound is on tasks already executing: it is read between claims, never
|
|
63
|
-
* during one, and a claim commits to its rows before any size is known. One
|
|
64
|
-
* poll can therefore overshoot by up to `claimBatch` rows per registered name
|
|
65
|
-
* (or one whole `batch`, whichever is larger). Lower `claimBatch`, or the batch
|
|
66
|
-
* sizes, to tighten that. A single payload larger than the entire budget still
|
|
67
|
-
* runs — alone, rather than deadlocking the worker.
|
|
68
|
-
*
|
|
69
|
-
* Costs one JSON serialization per task to measure, so it is only computed
|
|
70
|
-
* when set. Unset disables the budget.
|
|
71
|
-
*/
|
|
72
|
-
maxInFlightBytes?: number;
|
|
73
48
|
/**
|
|
74
49
|
* Call ceilings that several names can draw from, by name — `{ gpu: 1 }`.
|
|
75
50
|
* A handler joins one with `task(name, { resource: "gpu" }, fn)`.
|
|
76
51
|
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
* is declared here, once, and at capacity 1 it is mutual exclusion across the
|
|
82
|
-
* names that join it.
|
|
52
|
+
* The limit belongs to the scarce thing rather than to any one name — a GPU,
|
|
53
|
+
* an index that tolerates a single writer — so it is declared here, once, and
|
|
54
|
+
* at capacity 1 it is mutual exclusion across the names that join it. A name
|
|
55
|
+
* that only needs to cap itself declares a resource of its own.
|
|
83
56
|
*/
|
|
84
57
|
resources?: Record<string, number>;
|
|
58
|
+
/** Queued tasks a queue may hold before `submit` blocks — a number for every
|
|
59
|
+
* queue, or a per-queue record that leaves unnamed queues unbounded. */
|
|
60
|
+
maxQueueDepth?: number | Record<string, number>;
|
|
61
|
+
/** How long a blocked submit waits before raising QueueFull. Default 600_000. */
|
|
62
|
+
maxQueueWaitMs?: number;
|
|
85
63
|
/**
|
|
86
64
|
* Called for errors the worker survived — a claim that threw, a store write
|
|
87
65
|
* that failed while finalizing a task. Without it these are silent: the run
|
|
@@ -99,22 +77,7 @@ export declare class Worker {
|
|
|
99
77
|
private readonly opts;
|
|
100
78
|
private readonly handlers;
|
|
101
79
|
private readonly ownId;
|
|
102
|
-
|
|
103
|
-
private inFlightBytes;
|
|
104
|
-
/** Calls in flight, for the names that cap their own concurrency. */
|
|
105
|
-
private readonly callsInFlight;
|
|
106
|
-
/**
|
|
107
|
-
* Calls holding units of each declared resource. A resource is the same shape
|
|
108
|
-
* of budget as a name's own `concurrency` — a ceiling on calls — differing
|
|
109
|
-
* only in who draws from it: several names rather than one. That is what
|
|
110
|
-
* expresses "these handlers share one GPU" without inventing a queue per
|
|
111
|
-
* resource.
|
|
112
|
-
*/
|
|
113
|
-
private readonly resourceCalls;
|
|
114
|
-
/** Rotates which source is offered the free budget first — see loop(). */
|
|
115
|
-
private claimCursor;
|
|
116
|
-
/** Invalidated by task(); see schedule(). */
|
|
117
|
-
private scheduleCache;
|
|
80
|
+
private readonly planner;
|
|
118
81
|
/**
|
|
119
82
|
* Retry backoff, resolved once. Both settlement paths read these — the
|
|
120
83
|
* worker's own `safeFail` and the TaskContext it hands a handler — so
|
|
@@ -151,22 +114,16 @@ export declare class Worker {
|
|
|
151
114
|
* batched — one embedding call over 256 texts rather than 256 calls — and size
|
|
152
115
|
* it by what the downstream API wants, not by the queue.
|
|
153
116
|
*
|
|
154
|
-
* `concurrency` caps the calls this name may run at once, under the worker's
|
|
155
|
-
* own. Use it to keep one expensive name from taking the whole worker.
|
|
156
|
-
*
|
|
157
117
|
* `resource` draws each call from a ceiling declared in
|
|
158
118
|
* `WorkerOptions.resources` and shared with every other name that names it —
|
|
159
119
|
* at capacity 1, mutual exclusion across those names.
|
|
160
120
|
*/
|
|
161
121
|
task(name: string | TaskDef, opts: {
|
|
162
|
-
batch
|
|
163
|
-
concurrency?: number;
|
|
122
|
+
batch?: number;
|
|
164
123
|
resource?: string;
|
|
165
124
|
}, handler: BatchHandler): this;
|
|
166
|
-
/**
|
|
167
|
-
* still takes `(ctx, payload)`. */
|
|
125
|
+
/** A shared resource without batching: the handler still takes `(ctx, payload)`. */
|
|
168
126
|
task(name: string | TaskDef, opts: {
|
|
169
|
-
concurrency?: number;
|
|
170
127
|
resource?: string;
|
|
171
128
|
}, handler: Handler): this;
|
|
172
129
|
stop(): void;
|
|
@@ -192,37 +149,6 @@ export declare class Worker {
|
|
|
192
149
|
*/
|
|
193
150
|
private deliveries;
|
|
194
151
|
private context;
|
|
195
|
-
/**
|
|
196
|
-
* How this poll's claim is split into per-name quotas, plus the union of names
|
|
197
|
-
* the probe spans.
|
|
198
|
-
*
|
|
199
|
-
* Cached and invalidated by `task()`, rather than rebuilt each poll: handlers
|
|
200
|
-
* may be registered after run() started, but only there, and this otherwise
|
|
201
|
-
* allocates a source per name on every tick for a worker's whole lifetime.
|
|
202
|
-
*
|
|
203
|
-
* A name that limits itself — by `batch`, by its own `concurrency`, or by a
|
|
204
|
-
* `resource` — needs a quota the shared draw cannot express, so it gets a
|
|
205
|
-
* source of its own; every other name shares one, where a task is a call.
|
|
206
|
-
*
|
|
207
|
-
* A resource is deliberately *not* one source spanning its names: `batch` is
|
|
208
|
-
* per name, and a single source carries one batch size, so two members that
|
|
209
|
-
* batch differently could not share a draw. Keeping a source per name and
|
|
210
|
-
* letting several of them draw down one shared ceiling composes with batching
|
|
211
|
-
* instead of excluding it.
|
|
212
|
-
*/
|
|
213
|
-
private schedule;
|
|
214
|
-
/**
|
|
215
|
-
* A source's own call ceiling for one poll, or undefined when only the
|
|
216
|
-
* worker-wide budget applies.
|
|
217
|
-
*
|
|
218
|
-
* Three independent ceilings, whichever binds first: the name's own concurrency
|
|
219
|
-
* less what it is already running; its resource's capacity less what is running
|
|
220
|
-
* *and* what earlier draws in this same poll already took (`taken`); and
|
|
221
|
-
* `claimBatch`. The last is a ceiling on *rows* per poll, so it converts at this
|
|
222
|
-
* source's batch size — and never below one call, or a `claimBatch` under some
|
|
223
|
-
* name's batch would stall that name outright.
|
|
224
|
-
*/
|
|
225
|
-
private sourceCalls;
|
|
226
152
|
private loop;
|
|
227
153
|
/** Blocking-style entry point for a standalone worker process: run until
|
|
228
154
|
* SIGINT/SIGTERM, then close the store. Use this at a script's top level;
|
|
@@ -230,7 +156,23 @@ export declare class Worker {
|
|
|
230
156
|
serve(opts?: {
|
|
231
157
|
concurrency?: number;
|
|
232
158
|
}): Promise<void>;
|
|
233
|
-
/**
|
|
159
|
+
/**
|
|
160
|
+
* Run the worker in the same process for the duration of fn (deployment mode A).
|
|
161
|
+
*
|
|
162
|
+
* `run()` can fail before `fn` has even started — a database that will not
|
|
163
|
+
* open, a migration that will not apply, a protocol version this SDK does not
|
|
164
|
+
* speak — so its rejection is captured from the first tick rather than only at
|
|
165
|
+
* the `await` in the teardown. Left uncaught until then it is an unhandled
|
|
166
|
+
* rejection, which Node terminates the process for by default: the caller
|
|
167
|
+
* never saw the error, and `fn` never finished. (The same mistake `runCall`
|
|
168
|
+
* documents having made.)
|
|
169
|
+
*
|
|
170
|
+
* If `fn` returned, the worker's failure is what the caller hears: a value
|
|
171
|
+
* produced with no worker running is not a success worth reporting quietly. If
|
|
172
|
+
* `fn` threw, its own error wins — it is the caller's code, and replacing it
|
|
173
|
+
* would hide what they were actually doing — with the worker's failure
|
|
174
|
+
* attached as `cause` so the root of it stays reachable.
|
|
175
|
+
*/
|
|
234
176
|
background<T>(fn: () => Promise<T>, opts?: {
|
|
235
177
|
concurrency?: number;
|
|
236
178
|
}): Promise<T>;
|
|
@@ -307,10 +249,13 @@ export declare class Worker {
|
|
|
307
249
|
* set.
|
|
308
250
|
*
|
|
309
251
|
* On timeout the attempt is abandoned: every context it covers is flagged
|
|
310
|
-
* lease-lost first (ctx.signal aborts
|
|
311
|
-
*
|
|
312
|
-
* TaskContext.owned
|
|
313
|
-
*
|
|
252
|
+
* lease-lost first (ctx.signal aborts; a handler that keeps running can never
|
|
253
|
+
* write to its task again, nor settle anything behind the worker's back — see
|
|
254
|
+
* TaskContext.owned — and cannot create child tasks either, which would
|
|
255
|
+
* otherwise be created a second time by the retry), then the still-pending
|
|
256
|
+
* promise is left to settle on its own, its outcome discarded. What a zombie
|
|
257
|
+
* does OUTSIDE cairnq is its own business: `ctx.signal` is how a handler is
|
|
258
|
+
* told to stop, and only the handler can act on it. The caller records the handler_timeout failure;
|
|
314
259
|
* lease recovery is NOT involved, so redelivery is immediate.
|
|
315
260
|
*/
|
|
316
261
|
private attempt;
|