cairnq 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -32
- package/dist/_protocol/sql/postgres/purge.sql +10 -10
- package/dist/_protocol/sql/sqlite/purge.sql +10 -10
- package/dist/backpressure.d.ts +0 -3
- package/dist/backpressure.js +1 -3
- package/dist/client.d.ts +42 -46
- package/dist/client.js +27 -43
- package/dist/context.d.ts +23 -1
- package/dist/context.js +18 -7
- package/dist/errors.d.ts +20 -26
- package/dist/errors.js +23 -35
- package/dist/index.d.ts +4 -10
- package/dist/index.js +3 -6
- package/dist/models.d.ts +14 -1
- package/dist/models.js +17 -5
- package/dist/retention.d.ts +22 -66
- package/dist/retention.js +53 -64
- package/dist/store/base.d.ts +29 -90
- package/dist/store/base.js +136 -123
- package/dist/store/pg-executor.d.ts +11 -0
- package/dist/store/pg-executor.js +11 -0
- package/dist/store/pg-pool.js +14 -0
- package/dist/store/postgres.d.ts +18 -12
- package/dist/store/postgres.js +23 -29
- package/dist/store/sqlite.d.ts +39 -2
- package/dist/store/sqlite.js +108 -29
- package/dist/worker.d.ts +38 -93
- package/dist/worker.js +181 -234
- package/package.json +1 -1
- package/src/backpressure.ts +1 -5
- package/src/client.ts +66 -71
- package/src/context.ts +28 -7
- package/src/errors.ts +25 -36
- package/src/index.ts +4 -27
- package/src/models.ts +17 -5
- package/src/retention.ts +50 -115
- package/src/store/base.ts +131 -166
- package/src/store/pg-executor.ts +11 -0
- package/src/store/pg-pool.ts +14 -0
- package/src/store/postgres.ts +24 -31
- package/src/store/sqlite.ts +108 -27
- package/src/worker.ts +208 -300
- package/dist/_protocol/sql/postgres/stats.sql +0 -23
- package/dist/_protocol/sql/sqlite/stats.sql +0 -23
package/dist/worker.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DEFAULT_RETRY_BACKOFF_MAX_MS, DEFAULT_RETRY_BACKOFF_MS, failDelayMs } from "./backoff.js";
|
|
2
2
|
import { TaskContext } from "./context.js";
|
|
3
|
-
import { asEnvelope, errorEnvelope,
|
|
3
|
+
import { asEnvelope, errorEnvelope, exceptionEnvelope, LostLease, SerializationError, } from "./errors.js";
|
|
4
4
|
import { newId } from "./ids.js";
|
|
5
5
|
import { SQLiteStore } from "./store/sqlite.js";
|
|
6
6
|
import { PostgresStore } from "./store/postgres.js";
|
|
@@ -29,43 +29,122 @@ function timeoutEnvelope(name, maxRunMs) {
|
|
|
29
29
|
}
|
|
30
30
|
const TIMED_OUT = Symbol("cairnq.timedOut");
|
|
31
31
|
/**
|
|
32
|
-
*
|
|
32
|
+
* The worker's claim-scheduling state, in one object: which names draw together
|
|
33
|
+
* and at what batch size, which shared resource each draw spends, how many
|
|
34
|
+
* calls each resource still has room for, and whose turn it is.
|
|
33
35
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* sees before parsing — capturing its length there would make this free, at the
|
|
38
|
-
* cost of carrying a non-protocol field on Task in both SDKs. Left for when the
|
|
39
|
-
* measurement shows up in a profile.
|
|
36
|
+
* A name that limits itself — by `batch`, or by a `resource` — needs a quota
|
|
37
|
+
* the shared draw cannot express, so it gets a source of its own; every other
|
|
38
|
+
* name shares one, where a task is a call.
|
|
40
39
|
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
40
|
+
* A resource is deliberately *not* one source spanning its names: `batch` is
|
|
41
|
+
* per name, and a single source carries one batch size, so two members that
|
|
42
|
+
* batch differently could not share a draw. Keeping a source per name and
|
|
43
|
+
* letting several of them draw down one shared ceiling composes with batching
|
|
44
|
+
* instead of excluding it.
|
|
43
45
|
*/
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
class ClaimPlanner {
|
|
47
|
+
capacities;
|
|
48
|
+
sources = [];
|
|
49
|
+
/** The union of names the claim probe spans. */
|
|
50
|
+
names = [];
|
|
51
|
+
/** Calls holding units of each declared resource. */
|
|
52
|
+
inFlight = new Map();
|
|
53
|
+
/** Rotates which source is offered the free budget first — see plan(). */
|
|
54
|
+
cursor = 0;
|
|
55
|
+
constructor(capacities) {
|
|
56
|
+
this.capacities = capacities;
|
|
47
57
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
58
|
+
/** Rebuild the sources from the registrations. Called from task(), which is
|
|
59
|
+
* the only place the set changes, so a poll never pays for the rebuild. */
|
|
60
|
+
rebuild(handlers) {
|
|
61
|
+
const sources = [];
|
|
62
|
+
const shared = [];
|
|
63
|
+
for (const [name, reg] of handlers) {
|
|
64
|
+
if (reg.batch != null || reg.resource != null) {
|
|
65
|
+
sources.push({ names: [name], batch: reg.batch ?? 1, resource: reg.resource });
|
|
66
|
+
}
|
|
67
|
+
else
|
|
68
|
+
shared.push(name);
|
|
69
|
+
}
|
|
70
|
+
if (shared.length)
|
|
71
|
+
sources.push({ names: shared, batch: 1 });
|
|
72
|
+
this.sources = sources;
|
|
73
|
+
this.names = [...handlers.keys()];
|
|
74
|
+
}
|
|
75
|
+
get empty() {
|
|
76
|
+
return this.sources.length === 0;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Draw up to `free` handler calls from one claim transaction.
|
|
80
|
+
*
|
|
81
|
+
* Sources are served in rotating order — without rotating, the first source
|
|
82
|
+
* would take every free slot and the rest would starve behind its backlog.
|
|
83
|
+
* Each draw sizes itself from what the previous ones took, so a source with
|
|
84
|
+
* nothing queued costs nothing; that feedback is why this runs inside the
|
|
85
|
+
* claim transaction rather than dividing the budget up front. It must await
|
|
86
|
+
* nothing but `claim` — the store holds the write lock while it runs.
|
|
87
|
+
*
|
|
88
|
+
* `split` is the worker's deliveries(): rows become handler calls there, so
|
|
89
|
+
* spending the budget against its result is the only way the two cannot
|
|
90
|
+
* disagree.
|
|
91
|
+
*/
|
|
92
|
+
async plan(free, claim, split) {
|
|
93
|
+
const cursor = this.cursor % this.sources.length;
|
|
94
|
+
const order = [...this.sources.slice(cursor), ...this.sources.slice(0, cursor)];
|
|
95
|
+
this.cursor = (cursor + 1) % this.sources.length;
|
|
96
|
+
const drawn = [];
|
|
97
|
+
let left = free;
|
|
98
|
+
// Resource units this poll has already drawn. `inFlight` only moves when a
|
|
99
|
+
// call is dispatched, which happens after this whole plan returns — so
|
|
100
|
+
// without a local tally two sources sharing a resource would each see its
|
|
101
|
+
// full ceiling and together overshoot it.
|
|
102
|
+
const taken = new Map();
|
|
103
|
+
for (const src of order) {
|
|
104
|
+
if (left <= 0)
|
|
105
|
+
break;
|
|
106
|
+
const quota = Math.min(this.headroom(src, taken) ?? left, left);
|
|
107
|
+
if (quota <= 0)
|
|
108
|
+
continue;
|
|
109
|
+
const rows = await claim(src.names, src.batch * quota);
|
|
110
|
+
if (!rows.length)
|
|
111
|
+
continue;
|
|
112
|
+
const calls = split(rows);
|
|
113
|
+
drawn.push({ src, calls });
|
|
114
|
+
left -= calls.length;
|
|
115
|
+
if (src.resource != null) {
|
|
116
|
+
taken.set(src.resource, (taken.get(src.resource) ?? 0) + calls.length);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return drawn;
|
|
120
|
+
}
|
|
121
|
+
/** A source's own call ceiling for one poll, or undefined when only the
|
|
122
|
+
* worker-wide budget applies: its resource's capacity less what is running
|
|
123
|
+
* and what earlier draws in this same poll already took. */
|
|
124
|
+
headroom(src, taken) {
|
|
125
|
+
if (src.resource == null)
|
|
126
|
+
return undefined;
|
|
127
|
+
return Math.max(0, this.capacities[src.resource] -
|
|
128
|
+
(this.inFlight.get(src.resource) ?? 0) -
|
|
129
|
+
(taken.get(src.resource) ?? 0));
|
|
130
|
+
}
|
|
131
|
+
/** Charge one dispatched call to its resource, for the span of the call. */
|
|
132
|
+
acquire(resource) {
|
|
133
|
+
if (resource == null)
|
|
134
|
+
return;
|
|
135
|
+
this.inFlight.set(resource, (this.inFlight.get(resource) ?? 0) + 1);
|
|
136
|
+
}
|
|
137
|
+
/** Give the unit back. Deleting at zero keeps the map to the resources
|
|
138
|
+
* actually in flight, so an idle worker holds no entries at all. */
|
|
139
|
+
release(resource) {
|
|
140
|
+
if (resource == null)
|
|
141
|
+
return;
|
|
142
|
+
const rest = (this.inFlight.get(resource) ?? 1) - 1;
|
|
143
|
+
if (rest > 0)
|
|
144
|
+
this.inFlight.set(resource, rest);
|
|
145
|
+
else
|
|
146
|
+
this.inFlight.delete(resource);
|
|
53
147
|
}
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Give back one call's unit of a counted budget. Deleting at zero is what keeps
|
|
57
|
-
* the map to the keys actually in flight, so an idle worker holds no entries at
|
|
58
|
-
* all — and both budgets (a name's own concurrency, a resource's capacity)
|
|
59
|
-
* settle the same way, from one place.
|
|
60
|
-
*/
|
|
61
|
-
function release(counts, key) {
|
|
62
|
-
if (key == null)
|
|
63
|
-
return;
|
|
64
|
-
const rest = (counts.get(key) ?? 1) - 1;
|
|
65
|
-
if (rest > 0)
|
|
66
|
-
counts.set(key, rest);
|
|
67
|
-
else
|
|
68
|
-
counts.delete(key);
|
|
69
148
|
}
|
|
70
149
|
export class Worker {
|
|
71
150
|
store;
|
|
@@ -73,22 +152,7 @@ export class Worker {
|
|
|
73
152
|
opts;
|
|
74
153
|
handlers = new Map();
|
|
75
154
|
ownId = newId("worker");
|
|
76
|
-
|
|
77
|
-
inFlightBytes = 0;
|
|
78
|
-
/** Calls in flight, for the names that cap their own concurrency. */
|
|
79
|
-
callsInFlight = new Map();
|
|
80
|
-
/**
|
|
81
|
-
* Calls holding units of each declared resource. A resource is the same shape
|
|
82
|
-
* of budget as a name's own `concurrency` — a ceiling on calls — differing
|
|
83
|
-
* only in who draws from it: several names rather than one. That is what
|
|
84
|
-
* expresses "these handlers share one GPU" without inventing a queue per
|
|
85
|
-
* resource.
|
|
86
|
-
*/
|
|
87
|
-
resourceCalls = new Map();
|
|
88
|
-
/** Rotates which source is offered the free budget first — see loop(). */
|
|
89
|
-
claimCursor = 0;
|
|
90
|
-
/** Invalidated by task(); see schedule(). */
|
|
91
|
-
scheduleCache = null;
|
|
155
|
+
planner;
|
|
92
156
|
/**
|
|
93
157
|
* Retry backoff, resolved once. Both settlement paths read these — the
|
|
94
158
|
* worker's own `safeFail` and the TaskContext it hands a handler — so
|
|
@@ -111,19 +175,18 @@ export class Worker {
|
|
|
111
175
|
if (opts.maxRunMs != null && opts.maxRunMs <= 0) {
|
|
112
176
|
throw new Error(`maxRunMs must be > 0, got ${opts.maxRunMs}`);
|
|
113
177
|
}
|
|
114
|
-
// 0 would make the budget permanently spent, so the worker would claim
|
|
115
|
-
// nothing and look hung. Rejected here, as the Python SDK does.
|
|
116
|
-
if (opts.maxInFlightBytes != null && opts.maxInFlightBytes <= 0) {
|
|
117
|
-
throw new Error(`maxInFlightBytes must be > 0, got ${opts.maxInFlightBytes}`);
|
|
118
|
-
}
|
|
119
178
|
for (const [resource, capacity] of Object.entries(opts.resources ?? {})) {
|
|
120
179
|
if (!Number.isInteger(capacity) || capacity < 1) {
|
|
121
180
|
throw new Error(`resources[${JSON.stringify(resource)}] must be an integer >= 1, got ${capacity}`);
|
|
122
181
|
}
|
|
123
182
|
}
|
|
124
183
|
if (opts.maxQueueDepth != null) {
|
|
125
|
-
store.useBackpressure(
|
|
184
|
+
store.useBackpressure({
|
|
185
|
+
maxQueueDepth: opts.maxQueueDepth,
|
|
186
|
+
maxQueueWaitMs: opts.maxQueueWaitMs,
|
|
187
|
+
});
|
|
126
188
|
}
|
|
189
|
+
this.planner = new ClaimPlanner(opts.resources ?? {});
|
|
127
190
|
this.backoffMs = opts.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
|
|
128
191
|
this.backoffMaxMs = opts.retryBackoffMaxMs ?? DEFAULT_RETRY_BACKOFF_MAX_MS;
|
|
129
192
|
}
|
|
@@ -148,11 +211,10 @@ export class Worker {
|
|
|
148
211
|
return this.ownId;
|
|
149
212
|
}
|
|
150
213
|
task(arg, second, third) {
|
|
151
|
-
// Option form: (name | def, { batch?,
|
|
152
|
-
//
|
|
153
|
-
//
|
|
214
|
+
// Option form: (name | def, { batch?, resource? }, handler). Peel the
|
|
215
|
+
// options off and fall through, so name resolution and registration stay
|
|
216
|
+
// single-sited.
|
|
154
217
|
let batch;
|
|
155
|
-
let concurrency;
|
|
156
218
|
let resource;
|
|
157
219
|
let handler = second;
|
|
158
220
|
if (second != null && typeof second === "object") {
|
|
@@ -162,12 +224,6 @@ export class Worker {
|
|
|
162
224
|
}
|
|
163
225
|
batch = second.batch;
|
|
164
226
|
}
|
|
165
|
-
if (second.concurrency != null) {
|
|
166
|
-
if (!Number.isInteger(second.concurrency) || second.concurrency < 1) {
|
|
167
|
-
throw new Error(`concurrency must be an integer >= 1, got ${second.concurrency}`);
|
|
168
|
-
}
|
|
169
|
-
concurrency = second.concurrency;
|
|
170
|
-
}
|
|
171
227
|
resource = second.resource;
|
|
172
228
|
handler = third;
|
|
173
229
|
}
|
|
@@ -197,8 +253,8 @@ export class Worker {
|
|
|
197
253
|
throw new Error(`task ${JSON.stringify(name)} declares resource ${JSON.stringify(resource)}, ` +
|
|
198
254
|
`which is not in WorkerOptions.resources; declared: ${known}`);
|
|
199
255
|
}
|
|
200
|
-
this.handlers.set(name, { fn, batch,
|
|
201
|
-
this.
|
|
256
|
+
this.handlers.set(name, { fn, batch, resource });
|
|
257
|
+
this.planner.rebuild(this.handlers);
|
|
202
258
|
return this;
|
|
203
259
|
}
|
|
204
260
|
stop() {
|
|
@@ -231,7 +287,7 @@ export class Worker {
|
|
|
231
287
|
const leaseMs = this.opts.leaseMs ?? 30_000;
|
|
232
288
|
await this.store.connect();
|
|
233
289
|
// The calls in flight — `concurrency` counts these, so the set's size is the
|
|
234
|
-
// budget. How many tasks they carry is
|
|
290
|
+
// budget. How many tasks they carry is the batch sizes' business.
|
|
235
291
|
const running = new Set();
|
|
236
292
|
try {
|
|
237
293
|
await this.loop(concurrency, leaseMs, running);
|
|
@@ -281,139 +337,29 @@ export class Worker {
|
|
|
281
337
|
retryBackoffMaxMs: this.backoffMaxMs,
|
|
282
338
|
});
|
|
283
339
|
}
|
|
284
|
-
/**
|
|
285
|
-
* How this poll's claim is split into per-name quotas, plus the union of names
|
|
286
|
-
* the probe spans.
|
|
287
|
-
*
|
|
288
|
-
* Cached and invalidated by `task()`, rather than rebuilt each poll: handlers
|
|
289
|
-
* may be registered after run() started, but only there, and this otherwise
|
|
290
|
-
* allocates a source per name on every tick for a worker's whole lifetime.
|
|
291
|
-
*
|
|
292
|
-
* A name that limits itself — by `batch`, by its own `concurrency`, or by a
|
|
293
|
-
* `resource` — needs a quota the shared draw cannot express, so it gets a
|
|
294
|
-
* source of its own; every other name shares one, where a task is a call.
|
|
295
|
-
*
|
|
296
|
-
* A resource is deliberately *not* one source spanning its names: `batch` is
|
|
297
|
-
* per name, and a single source carries one batch size, so two members that
|
|
298
|
-
* batch differently could not share a draw. Keeping a source per name and
|
|
299
|
-
* letting several of them draw down one shared ceiling composes with batching
|
|
300
|
-
* instead of excluding it.
|
|
301
|
-
*/
|
|
302
|
-
schedule() {
|
|
303
|
-
if (this.scheduleCache)
|
|
304
|
-
return this.scheduleCache;
|
|
305
|
-
const sources = [];
|
|
306
|
-
const shared = [];
|
|
307
|
-
for (const [name, reg] of this.handlers) {
|
|
308
|
-
if (reg.batch != null || reg.concurrency != null || reg.resource != null) {
|
|
309
|
-
sources.push({
|
|
310
|
-
key: reg.concurrency == null ? undefined : name,
|
|
311
|
-
names: [name],
|
|
312
|
-
batch: reg.batch ?? 1,
|
|
313
|
-
concurrency: reg.concurrency,
|
|
314
|
-
resource: reg.resource,
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
else
|
|
318
|
-
shared.push(name);
|
|
319
|
-
}
|
|
320
|
-
if (shared.length)
|
|
321
|
-
sources.push({ names: shared, batch: 1 });
|
|
322
|
-
return (this.scheduleCache = { sources, names: [...this.handlers.keys()] });
|
|
323
|
-
}
|
|
324
|
-
/**
|
|
325
|
-
* A source's own call ceiling for one poll, or undefined when only the
|
|
326
|
-
* worker-wide budget applies.
|
|
327
|
-
*
|
|
328
|
-
* Three independent ceilings, whichever binds first: the name's own concurrency
|
|
329
|
-
* less what it is already running; its resource's capacity less what is running
|
|
330
|
-
* *and* what earlier draws in this same poll already took (`taken`); and
|
|
331
|
-
* `claimBatch`. The last is a ceiling on *rows* per poll, so it converts at this
|
|
332
|
-
* source's batch size — and never below one call, or a `claimBatch` under some
|
|
333
|
-
* name's batch would stall that name outright.
|
|
334
|
-
*/
|
|
335
|
-
sourceCalls(src, taken) {
|
|
336
|
-
const rows = this.opts.claimBatch;
|
|
337
|
-
const byRows = rows == null ? undefined : Math.max(1, Math.floor(rows / src.batch));
|
|
338
|
-
const byName = src.concurrency == null
|
|
339
|
-
? undefined
|
|
340
|
-
: Math.max(0, src.concurrency - (this.callsInFlight.get(src.key) ?? 0));
|
|
341
|
-
const byResource = src.resource == null
|
|
342
|
-
? undefined
|
|
343
|
-
: Math.max(0, this.opts.resources[src.resource] -
|
|
344
|
-
(this.resourceCalls.get(src.resource) ?? 0) -
|
|
345
|
-
(taken.get(src.resource) ?? 0));
|
|
346
|
-
const limits = [byRows, byName, byResource].filter((n) => n != null);
|
|
347
|
-
return limits.length ? Math.min(...limits) : undefined;
|
|
348
|
-
}
|
|
349
340
|
async loop(concurrency, leaseMs, running) {
|
|
350
341
|
const pollMs = this.opts.pollIntervalMs ?? 500;
|
|
351
|
-
const byteBudget = this.opts.maxInFlightBytes;
|
|
352
342
|
while (!this.stopped) {
|
|
353
343
|
// `concurrency` counts calls, so the calls in flight *are* the running
|
|
354
344
|
// promises — a batch holding 256 tasks is one of them.
|
|
355
345
|
const free = concurrency - running.size;
|
|
356
|
-
|
|
357
|
-
// resident payload bytes. The byte arm is guarded on running.size because
|
|
358
|
-
// it must never be the reason we race an empty set — Promise.race([]) is
|
|
359
|
-
// pending forever, past even stop(). With nothing running, nothing is
|
|
360
|
-
// resident, so the budget cannot be the thing holding us back anyway.
|
|
361
|
-
const overBudget = byteBudget != null && this.inFlightBytes >= byteBudget;
|
|
362
|
-
if (running.size > 0 && (free <= 0 || overBudget)) {
|
|
346
|
+
if (free <= 0) {
|
|
363
347
|
// Wait for a slot rather than spinning. runCall() never rejects, so
|
|
364
348
|
// racing these is safe.
|
|
365
349
|
await Promise.race([...running]);
|
|
366
350
|
continue;
|
|
367
351
|
}
|
|
368
|
-
|
|
369
|
-
if (!sources.length) {
|
|
352
|
+
if (this.planner.empty) {
|
|
370
353
|
await this.idle(pollMs);
|
|
371
354
|
continue;
|
|
372
355
|
}
|
|
373
|
-
|
|
374
|
-
// rotating it the first source would take every free slot and the rest
|
|
375
|
-
// would starve behind its backlog.
|
|
376
|
-
const cursor = this.claimCursor % sources.length;
|
|
377
|
-
const order = [...sources.slice(cursor), ...sources.slice(0, cursor)];
|
|
378
|
-
this.claimCursor = (cursor + 1) % sources.length;
|
|
379
|
-
let claimed;
|
|
356
|
+
let drawn;
|
|
380
357
|
try {
|
|
381
|
-
|
|
358
|
+
drawn = await this.store.claimSession(
|
|
382
359
|
// Only what this worker can run. Queues do not partition work by task
|
|
383
360
|
// name, so another worker's tasks would otherwise be claimed here and
|
|
384
361
|
// failed for want of a handler.
|
|
385
|
-
{ queues: this.queues, workerId: this.ownId, leaseMs, names },
|
|
386
|
-
const drawn = [];
|
|
387
|
-
let left = free;
|
|
388
|
-
// Resource units this poll has already drawn. resourceCalls only
|
|
389
|
-
// moves when a call is dispatched, which happens after this whole
|
|
390
|
-
// plan returns — so without a local tally two sources sharing a
|
|
391
|
-
// resource would each see its full ceiling and together overshoot
|
|
392
|
-
// it. Same shape as `left`, one budget down.
|
|
393
|
-
const taken = new Map();
|
|
394
|
-
for (const src of order) {
|
|
395
|
-
if (left <= 0)
|
|
396
|
-
break;
|
|
397
|
-
const quota = Math.min(this.sourceCalls(src, taken) ?? left, left);
|
|
398
|
-
if (quota <= 0)
|
|
399
|
-
continue;
|
|
400
|
-
const rows = await claim(src.names, src.batch * quota);
|
|
401
|
-
if (!rows.length)
|
|
402
|
-
continue;
|
|
403
|
-
// deliveries() is what actually turns rows into handler calls, so
|
|
404
|
-
// spending the budget against its result is the only way the two
|
|
405
|
-
// cannot disagree. A source with nothing queued costs nothing,
|
|
406
|
-
// which is why the budget is spent here, draw by draw, rather than
|
|
407
|
-
// divided up before the claim.
|
|
408
|
-
const calls = this.deliveries(rows);
|
|
409
|
-
drawn.push({ src, calls });
|
|
410
|
-
left -= calls.length;
|
|
411
|
-
if (src.resource != null) {
|
|
412
|
-
taken.set(src.resource, (taken.get(src.resource) ?? 0) + calls.length);
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
return drawn;
|
|
416
|
-
});
|
|
362
|
+
{ queues: this.queues, workerId: this.ownId, leaseMs, names: this.planner.names }, (claim) => this.planner.plan(free, claim, (rows) => this.deliveries(rows)));
|
|
417
363
|
}
|
|
418
364
|
catch (err) {
|
|
419
365
|
// A claim can fail transiently (lock contention, a dropped connection).
|
|
@@ -422,31 +368,21 @@ export class Worker {
|
|
|
422
368
|
await this.sleepOrStop(CLAIM_ERROR_BACKOFF_MS);
|
|
423
369
|
continue;
|
|
424
370
|
}
|
|
425
|
-
if (!
|
|
371
|
+
if (!drawn?.length) {
|
|
426
372
|
await this.idle(pollMs);
|
|
427
373
|
continue;
|
|
428
374
|
}
|
|
429
|
-
for (const { src, calls } of
|
|
375
|
+
for (const { src, calls } of drawn) {
|
|
430
376
|
for (const [reg, group] of calls) {
|
|
431
377
|
// Charged before the handler starts and refunded when it settles, so
|
|
432
|
-
// the
|
|
433
|
-
// payloads stay pinned in memory.
|
|
434
|
-
const bytes = byteBudget == null ? 0 : group.reduce((sum, t) => sum + payloadBytes(t), 0);
|
|
435
|
-
this.inFlightBytes += bytes;
|
|
436
|
-
const key = src.key;
|
|
378
|
+
// the resource covers exactly the span the call holds its slot.
|
|
437
379
|
const resource = src.resource;
|
|
438
|
-
|
|
439
|
-
this.callsInFlight.set(key, (this.callsInFlight.get(key) ?? 0) + 1);
|
|
440
|
-
if (resource != null) {
|
|
441
|
-
this.resourceCalls.set(resource, (this.resourceCalls.get(resource) ?? 0) + 1);
|
|
442
|
-
}
|
|
380
|
+
this.planner.acquire(resource);
|
|
443
381
|
// An unregistered name has nothing to run, so it never starts a handler
|
|
444
382
|
// or a heartbeat; everything else is one call, batched or not.
|
|
445
383
|
const call = reg == null ? this.failNoHandler(group[0], leaseMs) : this.runCall(reg, group, leaseMs);
|
|
446
384
|
const p = call.finally(() => {
|
|
447
|
-
this.
|
|
448
|
-
release(this.callsInFlight, key);
|
|
449
|
-
release(this.resourceCalls, resource);
|
|
385
|
+
this.planner.release(resource);
|
|
450
386
|
running.delete(p);
|
|
451
387
|
});
|
|
452
388
|
running.add(p);
|
|
@@ -471,17 +407,50 @@ export class Worker {
|
|
|
471
407
|
await this.closeIfOwned();
|
|
472
408
|
}
|
|
473
409
|
}
|
|
474
|
-
/**
|
|
410
|
+
/**
|
|
411
|
+
* Run the worker in the same process for the duration of fn (deployment mode A).
|
|
412
|
+
*
|
|
413
|
+
* `run()` can fail before `fn` has even started — a database that will not
|
|
414
|
+
* open, a migration that will not apply, a protocol version this SDK does not
|
|
415
|
+
* speak — so its rejection is captured from the first tick rather than only at
|
|
416
|
+
* the `await` in the teardown. Left uncaught until then it is an unhandled
|
|
417
|
+
* rejection, which Node terminates the process for by default: the caller
|
|
418
|
+
* never saw the error, and `fn` never finished. (The same mistake `runCall`
|
|
419
|
+
* documents having made.)
|
|
420
|
+
*
|
|
421
|
+
* If `fn` returned, the worker's failure is what the caller hears: a value
|
|
422
|
+
* produced with no worker running is not a success worth reporting quietly. If
|
|
423
|
+
* `fn` threw, its own error wins — it is the caller's code, and replacing it
|
|
424
|
+
* would hide what they were actually doing — with the worker's failure
|
|
425
|
+
* attached as `cause` so the root of it stays reachable.
|
|
426
|
+
*/
|
|
475
427
|
async background(fn, opts = {}) {
|
|
476
428
|
const runner = this.run(opts);
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
429
|
+
// Attached now, not at the await in the teardown: that is what keeps an
|
|
430
|
+
// early rejection from being an unhandled one. Carried as a value rather
|
|
431
|
+
// than assigned to a captured variable, so the outcome flows out of
|
|
432
|
+
// `shutdown` and needs no cast to be read.
|
|
433
|
+
const settled = runner.then(() => null, (err) => ({ err }));
|
|
434
|
+
const shutdown = async () => {
|
|
481
435
|
this.stop();
|
|
482
|
-
await
|
|
436
|
+
const failure = await settled;
|
|
483
437
|
await this.closeIfOwned();
|
|
438
|
+
return failure;
|
|
439
|
+
};
|
|
440
|
+
let value;
|
|
441
|
+
try {
|
|
442
|
+
value = await fn();
|
|
443
|
+
}
|
|
444
|
+
catch (err) {
|
|
445
|
+
const failure = await shutdown();
|
|
446
|
+
if (failure && err instanceof Error && err.cause === undefined)
|
|
447
|
+
err.cause = failure.err;
|
|
448
|
+
throw err;
|
|
484
449
|
}
|
|
450
|
+
const failure = await shutdown();
|
|
451
|
+
if (failure)
|
|
452
|
+
throw failure.err;
|
|
453
|
+
return value;
|
|
485
454
|
}
|
|
486
455
|
/**
|
|
487
456
|
* Record a claimed task this worker cannot run. Reachable only if a name is
|
|
@@ -661,10 +630,13 @@ export class Worker {
|
|
|
661
630
|
* set.
|
|
662
631
|
*
|
|
663
632
|
* On timeout the attempt is abandoned: every context it covers is flagged
|
|
664
|
-
* lease-lost first (ctx.signal aborts
|
|
665
|
-
*
|
|
666
|
-
* TaskContext.owned
|
|
667
|
-
*
|
|
633
|
+
* lease-lost first (ctx.signal aborts; a handler that keeps running can never
|
|
634
|
+
* write to its task again, nor settle anything behind the worker's back — see
|
|
635
|
+
* TaskContext.owned — and cannot create child tasks either, which would
|
|
636
|
+
* otherwise be created a second time by the retry), then the still-pending
|
|
637
|
+
* promise is left to settle on its own, its outcome discarded. What a zombie
|
|
638
|
+
* does OUTSIDE cairnq is its own business: `ctx.signal` is how a handler is
|
|
639
|
+
* told to stop, and only the handler can act on it. The caller records the handler_timeout failure;
|
|
668
640
|
* lease recovery is NOT involved, so redelivery is immediate.
|
|
669
641
|
*/
|
|
670
642
|
async attempt(invoke, ctxs) {
|
|
@@ -705,27 +677,7 @@ export class Worker {
|
|
|
705
677
|
let active = true;
|
|
706
678
|
let wake = null;
|
|
707
679
|
// lease/3 gives two beats of slack; the floor only matters for sub-150ms leases.
|
|
708
|
-
const interval =
|
|
709
|
-
// When this heartbeat last ran. A loop cannot report its own absence — a
|
|
710
|
-
// handler that blocks for its whole attempt never lets the timer fire at all
|
|
711
|
-
// — so the check lives outside the loop and cancel() runs it too.
|
|
712
|
-
let lastBeatAt = Date.now();
|
|
713
|
-
/** Report a heartbeat that has not run for more than two intervals: a whole
|
|
714
|
-
* beat missed, which at lease/3 means the next such block loses the lease
|
|
715
|
-
* outright. Fires while the lease still holds — after it expires the only
|
|
716
|
-
* evidence is a task that ran twice, in two workers' logs, with no error in
|
|
717
|
-
* either. */
|
|
718
|
-
const checkBeat = () => {
|
|
719
|
-
const now = Date.now();
|
|
720
|
-
const lateMs = now - lastBeatAt - interval;
|
|
721
|
-
if (lateMs > interval) {
|
|
722
|
-
this.report(new EventLoopBlocked(lateMs, interval, leaseMs), {
|
|
723
|
-
phase: "execute",
|
|
724
|
-
taskId: ctxs[0].taskId,
|
|
725
|
-
});
|
|
726
|
-
}
|
|
727
|
-
lastBeatAt = now;
|
|
728
|
-
};
|
|
680
|
+
const interval = Math.max(50, Math.floor(leaseMs / 3));
|
|
729
681
|
const done = (async () => {
|
|
730
682
|
while (active) {
|
|
731
683
|
// Cancellable sleep: cancel() resolves this immediately and clears the
|
|
@@ -740,7 +692,6 @@ export class Worker {
|
|
|
740
692
|
wake = null;
|
|
741
693
|
if (!active)
|
|
742
694
|
break;
|
|
743
|
-
checkBeat();
|
|
744
695
|
const live = ctxs.filter((c) => !c.settled && !c.lostLease);
|
|
745
696
|
if (!live.length)
|
|
746
697
|
break;
|
|
@@ -770,10 +721,6 @@ export class Worker {
|
|
|
770
721
|
active = false;
|
|
771
722
|
if (wake)
|
|
772
723
|
wake();
|
|
773
|
-
// The attempt is over, so this is the last chance to notice that the
|
|
774
|
-
// heartbeat never got one — the whole-attempt block, which is also the
|
|
775
|
-
// one where the lease is already gone.
|
|
776
|
-
checkBeat();
|
|
777
724
|
},
|
|
778
725
|
done,
|
|
779
726
|
};
|
package/package.json
CHANGED
package/src/backpressure.ts
CHANGED
|
@@ -30,8 +30,6 @@ export interface BackpressureOptions {
|
|
|
30
30
|
maxQueueDepth: QueueDepthLimit;
|
|
31
31
|
/** How long a blocked submit waits before raising QueueFull. Default 600_000. */
|
|
32
32
|
maxQueueWaitMs?: number;
|
|
33
|
-
/** First backoff between depth probes; doubles to a 5s ceiling. Default 250. */
|
|
34
|
-
queuePollIntervalMs?: number;
|
|
35
33
|
}
|
|
36
34
|
|
|
37
35
|
/**
|
|
@@ -60,7 +58,6 @@ export class QueueDepthGate {
|
|
|
60
58
|
private readonly probing = new Map<string, Promise<void>>();
|
|
61
59
|
private readonly limits: QueueDepthLimit;
|
|
62
60
|
private readonly maxWaitMs: number;
|
|
63
|
-
private readonly initialProbeMs: number;
|
|
64
61
|
|
|
65
62
|
constructor(
|
|
66
63
|
private readonly store: TaskStore,
|
|
@@ -68,7 +65,6 @@ export class QueueDepthGate {
|
|
|
68
65
|
) {
|
|
69
66
|
this.limits = opts.maxQueueDepth;
|
|
70
67
|
this.maxWaitMs = opts.maxQueueWaitMs ?? DEFAULT_MAX_WAIT_MS;
|
|
71
|
-
this.initialProbeMs = opts.queuePollIntervalMs ?? INITIAL_PROBE_INTERVAL_MS;
|
|
72
68
|
if (typeof this.limits === "number") this.validate("*", this.limits);
|
|
73
69
|
else for (const [q, v] of Object.entries(this.limits)) this.validate(q, v);
|
|
74
70
|
}
|
|
@@ -98,7 +94,7 @@ export class QueueDepthGate {
|
|
|
98
94
|
if (limit == null) return;
|
|
99
95
|
|
|
100
96
|
const startedAt = Date.now();
|
|
101
|
-
let waitMs =
|
|
97
|
+
let waitMs = INITIAL_PROBE_INTERVAL_MS;
|
|
102
98
|
for (;;) {
|
|
103
99
|
const left = this.headroom.get(queue) ?? 0;
|
|
104
100
|
if (left > 0) {
|