@blamejs/core 0.14.17 → 0.14.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/README.md +2 -2
- package/lib/agent-orchestrator.js +10 -4
- package/lib/ai-prompt.js +1 -1
- package/lib/app-shutdown.js +28 -0
- package/lib/archive-read.js +215 -16
- package/lib/breach-deadline.js +166 -1
- package/lib/cloud-events.js +3 -1
- package/lib/codepoint-class.js +21 -0
- package/lib/db-schema.js +120 -3
- package/lib/db.js +10 -3
- package/lib/error-page.js +88 -8
- package/lib/external-db.js +164 -13
- package/lib/guard-email.js +36 -3
- package/lib/mail-send-deliver.js +15 -5
- package/lib/mail-sieve.js +2 -1
- package/lib/middleware/ai-act-disclosure.js +88 -19
- package/lib/middleware/asyncapi-serve.js +56 -4
- package/lib/middleware/attach-user.js +45 -10
- package/lib/middleware/body-parser.js +70 -14
- package/lib/middleware/csp-report.js +30 -2
- package/lib/middleware/deny-response.js +22 -7
- package/lib/middleware/openapi-serve.js +56 -4
- package/lib/middleware/scim-server.js +7 -4
- package/lib/queue-local.js +148 -38
- package/lib/queue.js +41 -11
- package/lib/render.js +21 -3
- package/lib/safe-buffer.js +55 -0
- package/lib/static.js +46 -17
- package/lib/uri-template.js +3 -1
- package/package.json +1 -1
- package/sbom.cdx.json +6 -6
package/lib/queue-local.js
CHANGED
|
@@ -28,6 +28,18 @@
|
|
|
28
28
|
* before INSERT; lease unseals the leased rows before returning to
|
|
29
29
|
* the caller. cluster-storage's RETURNING clause hands back sealed
|
|
30
30
|
* blobs which we run through cryptoField.unsealRow explicitly.
|
|
31
|
+
*
|
|
32
|
+
* Bring-your-own database: the local backend defaults to the
|
|
33
|
+
* framework's main DB (single-node) / external-db (cluster) via
|
|
34
|
+
* cluster-storage, and to the table "_blamejs_jobs". An operator can
|
|
35
|
+
* point the backend at their own store, table, and schema through the
|
|
36
|
+
* protocol config — see create(config). The physical table reference
|
|
37
|
+
* is composed through b.safeSql identifier quoting (never raw string
|
|
38
|
+
* interpolation), so an operator-supplied table/schema cannot smuggle
|
|
39
|
+
* SQL through the identifier slot (SQL identifier injection, CWE-89).
|
|
40
|
+
* Sealing stays keyed off the logical column map registered for
|
|
41
|
+
* "_blamejs_jobs", so payload + lastError remain sealed regardless of
|
|
42
|
+
* which physical table the rows land in.
|
|
31
43
|
*/
|
|
32
44
|
var cluster = require("./cluster");
|
|
33
45
|
var clusterStorage = require("./cluster-storage");
|
|
@@ -37,11 +49,22 @@ var cryptoField = require("./crypto-field");
|
|
|
37
49
|
var lazyRequire = require("./lazy-require");
|
|
38
50
|
var numericBounds = require("./numeric-bounds");
|
|
39
51
|
var safeJson = require("./safe-json");
|
|
52
|
+
var safeSql = require("./safe-sql");
|
|
40
53
|
var scheduler = require("./scheduler");
|
|
41
54
|
var { QueueError } = require("./framework-error");
|
|
42
55
|
|
|
43
56
|
var _err = QueueError.factory;
|
|
44
57
|
|
|
58
|
+
// Logical table name the field-crypto schema is keyed on. This is the
|
|
59
|
+
// COLUMN→seal map registered in db.js's FRAMEWORK_SCHEMA, NOT the
|
|
60
|
+
// physical table the SQL writes to. An operator who points the backend
|
|
61
|
+
// at their own table still seals payload + lastError through this map,
|
|
62
|
+
// so a bring-your-own table inherits the same at-rest protection.
|
|
63
|
+
var SEAL_TABLE = "_blamejs_jobs";
|
|
64
|
+
|
|
65
|
+
// Default physical table for the local backend.
|
|
66
|
+
var DEFAULT_TABLE = "_blamejs_jobs";
|
|
67
|
+
|
|
45
68
|
// vault is lazy-required because some flows (sealed lastError) only
|
|
46
69
|
// touch it on retry-with-error paths, and the import order
|
|
47
70
|
// (queue-local → vault → db → audit → cluster) tolerates the late bind.
|
|
@@ -84,7 +107,7 @@ function _shapeLeasedRow(raw) {
|
|
|
84
107
|
// raw is a row coming back from RETURNING — payload is sealed if
|
|
85
108
|
// present. Run through cryptoField's unseal pipeline so the caller
|
|
86
109
|
// gets cleartext.
|
|
87
|
-
var unsealed = cryptoField.unsealRow(
|
|
110
|
+
var unsealed = cryptoField.unsealRow(SEAL_TABLE, raw);
|
|
88
111
|
return {
|
|
89
112
|
jobId: unsealed._id,
|
|
90
113
|
queueName: unsealed.queueName,
|
|
@@ -102,9 +125,78 @@ function _shapeLeasedRow(raw) {
|
|
|
102
125
|
};
|
|
103
126
|
}
|
|
104
127
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
128
|
+
// Validate a bring-your-own store handle at config time. It must expose
|
|
129
|
+
// the execute / executeOne / executeAll trio that cluster-storage does,
|
|
130
|
+
// since every method here dispatches through that surface.
|
|
131
|
+
var _REQUIRED_STORE_METHODS = ["execute", "executeOne", "executeAll"];
|
|
132
|
+
function _resolveStore(handle) {
|
|
133
|
+
if (handle === undefined || handle === null) return clusterStorage;
|
|
134
|
+
if (typeof handle !== "object") {
|
|
135
|
+
throw _err("INVALID_DB_HANDLE",
|
|
136
|
+
"queue local config.db must be a storage handle exposing execute/executeOne/executeAll, got " +
|
|
137
|
+
typeof handle, true);
|
|
138
|
+
}
|
|
139
|
+
for (var i = 0; i < _REQUIRED_STORE_METHODS.length; i++) {
|
|
140
|
+
var m = _REQUIRED_STORE_METHODS[i];
|
|
141
|
+
if (typeof handle[m] !== "function") {
|
|
142
|
+
throw _err("INVALID_DB_HANDLE",
|
|
143
|
+
"queue local config.db is missing required method '" + m + "()'", true);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return handle;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Compose the physical table reference from config.table + config.schema,
|
|
150
|
+
// quoting each identifier through b.safeSql so an operator-supplied name
|
|
151
|
+
// cannot interpolate SQL through the identifier slot (CWE-89). Returns
|
|
152
|
+
// the bare default name unquoted when no custom table/schema is given so
|
|
153
|
+
// the framework's cluster-mode table rewrite (resolveTables) still fires
|
|
154
|
+
// on the default jobs table; any custom name is fully validated + quoted.
|
|
155
|
+
function _resolveTableRef(config) {
|
|
156
|
+
var table = config.table !== undefined && config.table !== null
|
|
157
|
+
? config.table : DEFAULT_TABLE;
|
|
158
|
+
if (typeof table !== "string") {
|
|
159
|
+
throw _err("INVALID_TABLE",
|
|
160
|
+
"queue local config.table must be a string identifier, got " + typeof table, true);
|
|
161
|
+
}
|
|
162
|
+
var schema = config.schema;
|
|
163
|
+
if (schema !== undefined && schema !== null && typeof schema !== "string") {
|
|
164
|
+
throw _err("INVALID_SCHEMA",
|
|
165
|
+
"queue local config.schema must be a string identifier, got " + typeof schema, true);
|
|
166
|
+
}
|
|
167
|
+
var usingDefault = (table === DEFAULT_TABLE) &&
|
|
168
|
+
(schema === undefined || schema === null);
|
|
169
|
+
if (usingDefault) {
|
|
170
|
+
// Byte-identical default SQL — unquoted bare name so cluster-mode
|
|
171
|
+
// resolveTables continues to recognize and rewrite the jobs table.
|
|
172
|
+
return DEFAULT_TABLE;
|
|
173
|
+
}
|
|
174
|
+
// Any custom table/schema is validated + dialect-quoted. validateIdentifier
|
|
175
|
+
// / quoteQualified THROW (SafeSqlError) on a bad identifier; surface that
|
|
176
|
+
// as the queue's config-time error so the operator catches the typo at
|
|
177
|
+
// boot rather than on first enqueue.
|
|
178
|
+
try {
|
|
179
|
+
if (schema !== undefined && schema !== null && schema !== "") {
|
|
180
|
+
return safeSql.quoteQualified([schema, table]);
|
|
181
|
+
}
|
|
182
|
+
return safeSql.quoteIdentifier(table);
|
|
183
|
+
} catch (e) {
|
|
184
|
+
throw _err("INVALID_TABLE",
|
|
185
|
+
"queue local table/schema failed identifier validation: " + e.message, true);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function create(config) {
|
|
190
|
+
config = config || {};
|
|
191
|
+
// Bring-your-own store + table. Defaults preserve the prior behavior
|
|
192
|
+
// exactly: cluster-storage dispatch to the framework's main DB
|
|
193
|
+
// (single-node) / external-db (cluster), table "_blamejs_jobs".
|
|
194
|
+
var store = _resolveStore(config.db);
|
|
195
|
+
// qTable holds the physical table reference already validated +
|
|
196
|
+
// dialect-quoted by _resolveTableRef (via safeSql.quoteIdentifier /
|
|
197
|
+
// quoteQualified, or the framework's bare default name). The `q` prefix
|
|
198
|
+
// marks it as a safe-to-interpolate identifier so it is never re-quoted.
|
|
199
|
+
var qTable = _resolveTableRef(config);
|
|
108
200
|
|
|
109
201
|
async function enqueue(queueName, payload, opts) {
|
|
110
202
|
cluster.requireLeader();
|
|
@@ -171,11 +263,11 @@ function create(_config) {
|
|
|
171
263
|
flowChildName: flowChildName,
|
|
172
264
|
dependsOn: dependsOn,
|
|
173
265
|
};
|
|
174
|
-
var sealed = cryptoField.sealRow(
|
|
266
|
+
var sealed = cryptoField.sealRow(SEAL_TABLE, row);
|
|
175
267
|
var values = JOB_COLS.map(function (c) { return c in sealed ? sealed[c] : null; });
|
|
176
268
|
|
|
177
|
-
await
|
|
178
|
-
"INSERT INTO
|
|
269
|
+
await store.execute(
|
|
270
|
+
"INSERT INTO " + qTable + " (" + _quotedList(JOB_COLS) + ") " +
|
|
179
271
|
"VALUES (" + _placeholders(JOB_COLS) + ")",
|
|
180
272
|
values
|
|
181
273
|
);
|
|
@@ -201,16 +293,16 @@ function create(_config) {
|
|
|
201
293
|
// can't be picked twice). RETURNING hands back the leased columns
|
|
202
294
|
// so we don't need a separate SELECT after the UPDATE.
|
|
203
295
|
var sql =
|
|
204
|
-
"UPDATE
|
|
296
|
+
"UPDATE " + qTable + " " +
|
|
205
297
|
"SET status = 'inflight', leasedAt = ?, leaseExpiresAt = ?, attempts = attempts + 1 " +
|
|
206
298
|
"WHERE _id IN (" +
|
|
207
|
-
" SELECT _id FROM
|
|
299
|
+
" SELECT _id FROM " + qTable + " " +
|
|
208
300
|
" WHERE queueName = ? AND status = 'pending' AND availableAt <= ? " +
|
|
209
301
|
" ORDER BY priority DESC, availableAt ASC, enqueuedAt ASC " +
|
|
210
302
|
" LIMIT ?" +
|
|
211
303
|
") " +
|
|
212
304
|
"RETURNING " + _quotedList(LEASE_RETURN_COLS);
|
|
213
|
-
var result = await
|
|
305
|
+
var result = await store.execute(
|
|
214
306
|
sql,
|
|
215
307
|
[nowMs, leaseExpiresAt, queueName, nowMs, maxRows]
|
|
216
308
|
);
|
|
@@ -232,8 +324,8 @@ function create(_config) {
|
|
|
232
324
|
"extendLease: additionalMs must be a positive number", true);
|
|
233
325
|
}
|
|
234
326
|
var newExpiry = Date.now() + additionalMs;
|
|
235
|
-
var result = await
|
|
236
|
-
"UPDATE
|
|
327
|
+
var result = await store.execute(
|
|
328
|
+
"UPDATE " + qTable + " SET leaseExpiresAt = ? " +
|
|
237
329
|
"WHERE _id = ? AND status = 'inflight'",
|
|
238
330
|
[newExpiry, jobId]
|
|
239
331
|
);
|
|
@@ -247,16 +339,16 @@ function create(_config) {
|
|
|
247
339
|
// the status flip. Single SELECT + UPDATE pair under the same
|
|
248
340
|
// jobId — race-free under SQLite (single-writer); cluster-storage
|
|
249
341
|
// dispatches both calls to the same backend.
|
|
250
|
-
var rowRes = await
|
|
342
|
+
var rowRes = await store.execute(
|
|
251
343
|
"SELECT _id, queueName, payload, repeatCron, repeatTimezone, " +
|
|
252
344
|
" flowId, flowChildName, priority, classification, traceId " +
|
|
253
|
-
"FROM
|
|
345
|
+
"FROM " + qTable + " WHERE _id = ?",
|
|
254
346
|
[jobId]
|
|
255
347
|
);
|
|
256
348
|
var row = (rowRes && rowRes.rows && rowRes.rows[0]) || null;
|
|
257
349
|
|
|
258
|
-
await
|
|
259
|
-
"UPDATE
|
|
350
|
+
await store.execute(
|
|
351
|
+
"UPDATE " + qTable + " SET status = 'done', finishedAt = ?, leaseExpiresAt = NULL " +
|
|
260
352
|
"WHERE _id = ? AND status = 'inflight'",
|
|
261
353
|
[nowMs, jobId]
|
|
262
354
|
);
|
|
@@ -266,7 +358,7 @@ function create(_config) {
|
|
|
266
358
|
// re-enqueue — operators investigate before the cron resumes.
|
|
267
359
|
if (row && row.repeatCron) {
|
|
268
360
|
try {
|
|
269
|
-
var unsealedRow = cryptoField.unsealRow(
|
|
361
|
+
var unsealedRow = cryptoField.unsealRow(SEAL_TABLE, row);
|
|
270
362
|
var cron = scheduler.parseCron(unsealedRow.repeatCron);
|
|
271
363
|
var nextMs = scheduler.nextCronFire(cron, new Date(nowMs), unsealedRow.repeatTimezone || null);
|
|
272
364
|
await enqueue(unsealedRow.queueName,
|
|
@@ -296,8 +388,8 @@ function create(_config) {
|
|
|
296
388
|
}
|
|
297
389
|
|
|
298
390
|
async function _maybeReleaseFlowChildren(flowId, completedJobId, completedChildName, nowMs) {
|
|
299
|
-
var siblingsRes = await
|
|
300
|
-
"SELECT _id, dependsOn, flowChildName, status, availableAt FROM
|
|
391
|
+
var siblingsRes = await store.execute(
|
|
392
|
+
"SELECT _id, dependsOn, flowChildName, status, availableAt FROM " + qTable + " " +
|
|
301
393
|
"WHERE flowId = ? AND status = 'pending' AND availableAt > ?",
|
|
302
394
|
[flowId, nowMs]
|
|
303
395
|
);
|
|
@@ -317,16 +409,16 @@ function create(_config) {
|
|
|
317
409
|
// Quick path: just-completed job matches by id or child name.
|
|
318
410
|
if (dep === completedJobId || (completedChildName && dep === completedChildName)) continue;
|
|
319
411
|
// Otherwise SELECT to confirm done.
|
|
320
|
-
var depRes = await
|
|
321
|
-
"SELECT 1 FROM
|
|
412
|
+
var depRes = await store.execute(
|
|
413
|
+
"SELECT 1 FROM " + qTable + " WHERE flowId = ? AND status = 'done' AND " +
|
|
322
414
|
" (_id = ? OR flowChildName = ?) LIMIT 1",
|
|
323
415
|
[flowId, dep, dep]
|
|
324
416
|
);
|
|
325
417
|
if (!depRes || !depRes.rows || depRes.rows.length === 0) { allDone = false; break; }
|
|
326
418
|
}
|
|
327
419
|
if (allDone) {
|
|
328
|
-
await
|
|
329
|
-
"UPDATE
|
|
420
|
+
await store.execute(
|
|
421
|
+
"UPDATE " + qTable + " SET availableAt = ? WHERE _id = ?",
|
|
330
422
|
[nowMs, sib._id]
|
|
331
423
|
);
|
|
332
424
|
}
|
|
@@ -345,8 +437,8 @@ function create(_config) {
|
|
|
345
437
|
// status / availableAt / finishedAt updates per branch — same
|
|
346
438
|
// semantics as the previous SELECT-then-UPDATE-in-transaction
|
|
347
439
|
// path, but no cross-dialect transaction primitive needed.
|
|
348
|
-
await
|
|
349
|
-
"UPDATE
|
|
440
|
+
await store.execute(
|
|
441
|
+
"UPDATE " + qTable + " SET " +
|
|
350
442
|
" status = CASE WHEN attempts < maxAttempts THEN 'pending' ELSE 'failed' END, " +
|
|
351
443
|
" lastError = ?, " +
|
|
352
444
|
" leaseExpiresAt = NULL, " +
|
|
@@ -360,8 +452,8 @@ function create(_config) {
|
|
|
360
452
|
|
|
361
453
|
async function sweepExpired() {
|
|
362
454
|
cluster.requireLeader();
|
|
363
|
-
var result = await
|
|
364
|
-
"UPDATE
|
|
455
|
+
var result = await store.execute(
|
|
456
|
+
"UPDATE " + qTable + " SET status = 'pending', leaseExpiresAt = NULL " +
|
|
365
457
|
"WHERE status = 'inflight' AND leaseExpiresAt < ?",
|
|
366
458
|
[Date.now()]
|
|
367
459
|
);
|
|
@@ -369,8 +461,8 @@ function create(_config) {
|
|
|
369
461
|
}
|
|
370
462
|
|
|
371
463
|
async function size(queueName) {
|
|
372
|
-
var row = await
|
|
373
|
-
"SELECT COUNT(*) AS n FROM
|
|
464
|
+
var row = await store.executeOne(
|
|
465
|
+
"SELECT COUNT(*) AS n FROM " + qTable + " " +
|
|
374
466
|
"WHERE queueName = ? AND (status = 'pending' OR status = 'inflight')",
|
|
375
467
|
[queueName]
|
|
376
468
|
);
|
|
@@ -395,16 +487,16 @@ function create(_config) {
|
|
|
395
487
|
}
|
|
396
488
|
limit = opts.limit;
|
|
397
489
|
}
|
|
398
|
-
var rows = await
|
|
490
|
+
var rows = await store.executeAll(
|
|
399
491
|
"SELECT _id, queueName, payload, status, enqueuedAt, finishedAt, " +
|
|
400
492
|
" attempts, maxAttempts, lastError, traceId, classification " +
|
|
401
|
-
"FROM
|
|
493
|
+
"FROM " + qTable + " " +
|
|
402
494
|
"WHERE queueName = ? AND status = 'failed' " +
|
|
403
495
|
"ORDER BY finishedAt DESC LIMIT ?",
|
|
404
496
|
[queueName, limit]
|
|
405
497
|
);
|
|
406
498
|
return rows.map(function (row) {
|
|
407
|
-
var unsealed = cryptoField.unsealRow(
|
|
499
|
+
var unsealed = cryptoField.unsealRow(SEAL_TABLE, row);
|
|
408
500
|
return {
|
|
409
501
|
jobId: row._id,
|
|
410
502
|
queueName: row.queueName,
|
|
@@ -424,8 +516,8 @@ function create(_config) {
|
|
|
424
516
|
async function dlqRetry(jobId) {
|
|
425
517
|
cluster.requireLeader();
|
|
426
518
|
var nowMs = Date.now();
|
|
427
|
-
var result = await
|
|
428
|
-
"UPDATE
|
|
519
|
+
var result = await store.execute(
|
|
520
|
+
"UPDATE " + qTable + " SET " +
|
|
429
521
|
" status = 'pending', " +
|
|
430
522
|
" attempts = 0, " +
|
|
431
523
|
" availableAt = ?, " +
|
|
@@ -440,8 +532,8 @@ function create(_config) {
|
|
|
440
532
|
}
|
|
441
533
|
|
|
442
534
|
async function dlqSize(queueName) {
|
|
443
|
-
var row = await
|
|
444
|
-
"SELECT COUNT(*) AS n FROM
|
|
535
|
+
var row = await store.executeOne(
|
|
536
|
+
"SELECT COUNT(*) AS n FROM " + qTable + " " +
|
|
445
537
|
"WHERE queueName = ? AND status = 'failed'",
|
|
446
538
|
[queueName]
|
|
447
539
|
);
|
|
@@ -450,13 +542,30 @@ function create(_config) {
|
|
|
450
542
|
|
|
451
543
|
async function purge(queueName) {
|
|
452
544
|
cluster.requireLeader();
|
|
453
|
-
var result = await
|
|
454
|
-
"DELETE FROM
|
|
545
|
+
var result = await store.execute(
|
|
546
|
+
"DELETE FROM " + qTable + " WHERE queueName = ?",
|
|
455
547
|
[queueName]
|
|
456
548
|
);
|
|
457
549
|
return result.rowCount || 0;
|
|
458
550
|
}
|
|
459
551
|
|
|
552
|
+
// patchFlowDeps — the second pass of enqueueFlow. Writes the resolved
|
|
553
|
+
// dependsOn jobIds and parks availableAt at MAX_SAFE_INTEGER for a flow
|
|
554
|
+
// child that has dependencies. Lives on the backend (not in queue.js)
|
|
555
|
+
// so it targets THIS backend's configured store + table — a
|
|
556
|
+
// bring-your-own table receives the flow graph the same way the
|
|
557
|
+
// first-pass enqueue did, instead of the dispatcher writing to the
|
|
558
|
+
// default jobs table behind the backend's back. depIds is serialized
|
|
559
|
+
// to JSON for the dependsOn column.
|
|
560
|
+
async function patchFlowDeps(jobId, depIds) {
|
|
561
|
+
cluster.requireLeader();
|
|
562
|
+
var result = await store.execute(
|
|
563
|
+
"UPDATE " + qTable + " SET dependsOn = ?, availableAt = ? WHERE _id = ?",
|
|
564
|
+
[JSON.stringify(depIds), FLOW_BLOCKED_AVAILABLE_AT, jobId]
|
|
565
|
+
);
|
|
566
|
+
return (result.rowCount || 0) > 0;
|
|
567
|
+
}
|
|
568
|
+
|
|
460
569
|
return {
|
|
461
570
|
protocol: "local",
|
|
462
571
|
enqueue: enqueue,
|
|
@@ -470,6 +579,7 @@ function create(_config) {
|
|
|
470
579
|
dlqList: dlqList,
|
|
471
580
|
dlqRetry: dlqRetry,
|
|
472
581
|
dlqSize: dlqSize,
|
|
582
|
+
patchFlowDeps: patchFlowDeps,
|
|
473
583
|
};
|
|
474
584
|
}
|
|
475
585
|
|
package/lib/queue.js
CHANGED
|
@@ -13,7 +13,9 @@
|
|
|
13
13
|
* backend declares a `protocol` plus protocol-specific options. The
|
|
14
14
|
* built-in `local` protocol is SQLite-backed (rows live in the
|
|
15
15
|
* framework's main DB so persistence survives crashes / restarts
|
|
16
|
-
* without external infrastructure)
|
|
16
|
+
* without external infrastructure), and can be pointed at an
|
|
17
|
+
* operator's own database handle, table, and schema via the `local`
|
|
18
|
+
* config (`db` / `table` / `schema`). `redis` and `sqs` ship; `amqp`
|
|
17
19
|
* and `nats` are listed as deferred and surface a clear error if
|
|
18
20
|
* selected.
|
|
19
21
|
*
|
|
@@ -54,7 +56,6 @@
|
|
|
54
56
|
* Durable, pluggable job queue with priority-aware leasing, retry + deterministic backoff, graceful shutdown, parent/child flows, and a dead-letter surface for jobs that exhaust their retries.
|
|
55
57
|
*/
|
|
56
58
|
var C = require("./constants");
|
|
57
|
-
var clusterStorage = require("./cluster-storage");
|
|
58
59
|
var bCrypto = require("./crypto");
|
|
59
60
|
var lazyRequire = require("./lazy-require");
|
|
60
61
|
var { boot } = require("./log");
|
|
@@ -107,13 +108,29 @@ var sweepTimer = null;
|
|
|
107
108
|
* Throws when `opts.backends` is missing — operators catch the typo
|
|
108
109
|
* at boot rather than discovering it on first enqueue.
|
|
109
110
|
*
|
|
111
|
+
* The `local` protocol defaults to the framework's own database (the
|
|
112
|
+
* main SQLite in single-node mode, the operator-supplied external DB in
|
|
113
|
+
* cluster mode) and the `_blamejs_jobs` table. An operator who wants the
|
|
114
|
+
* queue rows to live in their own database, table, or schema supplies
|
|
115
|
+
* `db` / `table` / `schema` in the `local` backend config. The `db`
|
|
116
|
+
* handle must expose the same `execute` / `executeOne` / `executeAll`
|
|
117
|
+
* surface as `b.clusterStorage`; `table` / `schema` are validated as SQL
|
|
118
|
+
* identifiers and quoted through `b.safeSql` (an identifier that isn't a
|
|
119
|
+
* safe name is refused at `init` time, not interpolated into SQL).
|
|
120
|
+
* Sealed columns (`payload`, `lastError`) stay sealed regardless of
|
|
121
|
+
* where the rows land.
|
|
122
|
+
*
|
|
110
123
|
* @opts
|
|
111
124
|
* backends: {
|
|
112
125
|
* [name: string]: {
|
|
113
126
|
* protocol: "local" | "redis" | "sqs",
|
|
114
127
|
* breaker?: { ... }, // see b.retry.CircuitBreaker opts
|
|
115
128
|
* retry?: { ... }, // see b.retry.withRetry opts
|
|
116
|
-
* //
|
|
129
|
+
* // local protocol — bring-your-own database (all optional):
|
|
130
|
+
* db?: object, // store handle (execute/executeOne/executeAll); default cluster-storage
|
|
131
|
+
* table?: string, // table name (validated + quoted); default "_blamejs_jobs"
|
|
132
|
+
* schema?: string, // schema/namespace qualifier (validated + quoted)
|
|
133
|
+
* // ...other protocol-specific opts (e.g. redis url, sqs queueUrl)
|
|
117
134
|
* },
|
|
118
135
|
* },
|
|
119
136
|
* defaultBackend?: string, // name to use when enqueue/consume omit { backend }
|
|
@@ -122,11 +139,12 @@ var sweepTimer = null;
|
|
|
122
139
|
* b.queue.init({
|
|
123
140
|
* backends: {
|
|
124
141
|
* primary: { protocol: "local" },
|
|
142
|
+
* app: { protocol: "local", table: "app_jobs", schema: "work" },
|
|
125
143
|
* },
|
|
126
144
|
* defaultBackend: "primary",
|
|
127
145
|
* });
|
|
128
146
|
* b.queue.listBackends();
|
|
129
|
-
* // → [{ name: "primary", protocol: "local", breakerState: "closed" }]
|
|
147
|
+
* // → [{ name: "primary", protocol: "local", breakerState: "closed" }, ...]
|
|
130
148
|
*/
|
|
131
149
|
function init(opts) {
|
|
132
150
|
if (initialized) return;
|
|
@@ -183,6 +201,7 @@ function init(opts) {
|
|
|
183
201
|
dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
|
|
184
202
|
dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
|
|
185
203
|
dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
|
|
204
|
+
patchFlowDeps: raw.patchFlowDeps ? wrapWithRetry(raw.patchFlowDeps) : null,
|
|
186
205
|
};
|
|
187
206
|
});
|
|
188
207
|
|
|
@@ -897,6 +916,17 @@ function enqueueFlow(spec) {
|
|
|
897
916
|
|
|
898
917
|
var flowId = "flow-" + bCrypto.generateToken(C.BYTES.bytes(8));
|
|
899
918
|
|
|
919
|
+
// Resolve the backend up front so the second-pass dependsOn patch
|
|
920
|
+
// targets the SAME backend (and its configured store + table) that the
|
|
921
|
+
// first-pass enqueue wrote to. A backend pointed at a bring-your-own
|
|
922
|
+
// table must receive the flow graph through its own writer, not a
|
|
923
|
+
// dispatcher-level write to the default jobs table.
|
|
924
|
+
var flowBackend = _backendFor(spec);
|
|
925
|
+
if (typeof flowBackend.patchFlowDeps !== "function") {
|
|
926
|
+
return Promise.reject(_err("FLOW_UNSUPPORTED",
|
|
927
|
+
"queue backend '" + flowBackend.name + "' does not support enqueueFlow", true));
|
|
928
|
+
}
|
|
929
|
+
|
|
900
930
|
return observability.tap("queue.enqueueFlow",
|
|
901
931
|
{ queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
|
|
902
932
|
async function () {
|
|
@@ -910,6 +940,7 @@ function enqueueFlow(spec) {
|
|
|
910
940
|
var ch = spec.children[p];
|
|
911
941
|
// Hold off setting dependsOn until we know all sibling jobIds.
|
|
912
942
|
var enqOpts = {
|
|
943
|
+
backend: flowBackend.name,
|
|
913
944
|
flowId: flowId,
|
|
914
945
|
flowChildName: ch.name,
|
|
915
946
|
priority: ch.priority || 0,
|
|
@@ -917,9 +948,9 @@ function enqueueFlow(spec) {
|
|
|
917
948
|
traceId: ch.traceId || null,
|
|
918
949
|
maxAttempts: ch.maxAttempts,
|
|
919
950
|
// dependsOn intentionally omitted on first pass — will be patched
|
|
920
|
-
// in via
|
|
921
|
-
//
|
|
922
|
-
// children get
|
|
951
|
+
// in via the backend's patchFlowDeps after all jobIds are known.
|
|
952
|
+
// Root children (no deps) are immediately leaseable; deps-bearing
|
|
953
|
+
// children get parked at MAX_SAFE_INTEGER via the second pass.
|
|
923
954
|
};
|
|
924
955
|
var result = await enqueue(spec.queueName, ch.payload, enqOpts);
|
|
925
956
|
nameToJobId[ch.name] = result.jobId;
|
|
@@ -927,14 +958,13 @@ function enqueueFlow(spec) {
|
|
|
927
958
|
}
|
|
928
959
|
// Second pass: write dependsOn (translated to jobIds) for children
|
|
929
960
|
// that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
|
|
961
|
+
// Routed through the backend's writer so it lands in the backend's
|
|
962
|
+
// configured store + table (bring-your-own DB safe).
|
|
930
963
|
for (var q = 0; q < jobs.length; q++) {
|
|
931
964
|
var j = jobs[q];
|
|
932
965
|
if (j.dependsOn.length === 0) continue;
|
|
933
966
|
var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
|
|
934
|
-
await
|
|
935
|
-
"UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
|
|
936
|
-
[JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
|
|
937
|
-
);
|
|
967
|
+
await flowBackend.patchFlowDeps(j.jobId, depIds);
|
|
938
968
|
}
|
|
939
969
|
_emit("system.queue.flow.enqueue", {
|
|
940
970
|
metadata: {
|
package/lib/render.js
CHANGED
|
@@ -82,17 +82,35 @@ var DEFAULT_DYNAMIC_CACHE_CONTROL = "private, no-cache, must-revalidate";
|
|
|
82
82
|
* without losing Content-Type. Returns `undefined` — the response
|
|
83
83
|
* is fully written by the time the call returns.
|
|
84
84
|
*
|
|
85
|
+
* `opts.replacer` is forwarded to `JSON.stringify` (ECMA-262 §25.5.2,
|
|
86
|
+
* the second argument) so handlers can serialize values that have no
|
|
87
|
+
* native JSON form — `BigInt` (which otherwise throws), `Date` in a
|
|
88
|
+
* custom shape, `Map` / `Set`, or a redaction filter over secret-
|
|
89
|
+
* shaped keys — without pre-walking the body. Accepts the same
|
|
90
|
+
* function or property-name array `JSON.stringify` does; a non-
|
|
91
|
+
* function / non-array value is a config typo and throws.
|
|
92
|
+
*
|
|
85
93
|
* @opts
|
|
86
|
-
* status:
|
|
87
|
-
* headers:
|
|
94
|
+
* status: 200, // numeric HTTP status (200/201/202/4xx/5xx)
|
|
95
|
+
* headers: {}, // merged over defaults; later wins
|
|
96
|
+
* replacer: function|string[], // JSON.stringify replacer (BigInt/Date/redaction)
|
|
88
97
|
*
|
|
89
98
|
* @example
|
|
90
99
|
* b.render.json(res, { ok: true, id: 42 }, { status: 201 });
|
|
91
100
|
* // → response: 201, application/json, body `{"ok":true,"id":42}`
|
|
101
|
+
*
|
|
102
|
+
* b.render.json(res, { total: 9007199254740993n }, {
|
|
103
|
+
* replacer: function (k, v) { return typeof v === "bigint" ? v.toString() : v; },
|
|
104
|
+
* });
|
|
105
|
+
* // → body `{"total":"9007199254740993"}`
|
|
92
106
|
*/
|
|
93
107
|
function json(res, body, opts) {
|
|
94
108
|
opts = opts || {};
|
|
95
|
-
|
|
109
|
+
if (opts.replacer !== undefined && opts.replacer !== null &&
|
|
110
|
+
typeof opts.replacer !== "function" && !Array.isArray(opts.replacer)) {
|
|
111
|
+
throw new TypeError("render.json: opts.replacer must be a function or an array of keys");
|
|
112
|
+
}
|
|
113
|
+
var encoded = JSON.stringify(body, opts.replacer);
|
|
96
114
|
var headers = _mergedHeaders({
|
|
97
115
|
"Content-Type": "application/json; charset=utf-8",
|
|
98
116
|
"Content-Length": Buffer.byteLength(encoded, "utf8"),
|
package/lib/safe-buffer.js
CHANGED
|
@@ -321,6 +321,60 @@ function boundedChunkCollector(opts) {
|
|
|
321
321
|
};
|
|
322
322
|
}
|
|
323
323
|
|
|
324
|
+
/**
|
|
325
|
+
* @primitive b.safeBuffer.collectStream
|
|
326
|
+
* @signature b.safeBuffer.collectStream(stream, opts)
|
|
327
|
+
* @since 0.14.18
|
|
328
|
+
* @related b.safeBuffer.boundedChunkCollector, b.safeBuffer.toBuffer
|
|
329
|
+
*
|
|
330
|
+
* Read a Node Readable (an `http.IncomingMessage` request body, a file
|
|
331
|
+
* stream, an upstream response) fully into one Buffer with the byte cap
|
|
332
|
+
* enforced at every chunk — the streaming sibling of
|
|
333
|
+
* `boundedChunkCollector`. `boundedChunkCollector` is a push-based
|
|
334
|
+
* collector object; `collectStream` is the pump around it, so callers
|
|
335
|
+
* compose the stream case instead of reaching for a `(stream, opts)`
|
|
336
|
+
* overload that does not exist.
|
|
337
|
+
*
|
|
338
|
+
* Resolves with the concatenated Buffer when the stream ends. Rejects
|
|
339
|
+
* (and destroys the stream) the moment a chunk would overflow
|
|
340
|
+
* `maxBytes`, so a hostile sender cannot force unbounded buffering. A
|
|
341
|
+
* bad `maxBytes` (missing / non-finite / `Infinity`) rejects rather than
|
|
342
|
+
* throwing synchronously.
|
|
343
|
+
*
|
|
344
|
+
* @opts
|
|
345
|
+
* maxBytes: number, // REQUIRED positive finite int; total byte cap
|
|
346
|
+
* errorClass: Function, // caller Error subclass for the too-large reject
|
|
347
|
+
* sizeCode: string, // default "buffer/too-large"
|
|
348
|
+
* sizeMessage: string, // override the too-large message
|
|
349
|
+
*
|
|
350
|
+
* @example
|
|
351
|
+
* var body = await b.safeBuffer.collectStream(req, { maxBytes: 65536 });
|
|
352
|
+
* var json = b.safeJson.parse(body.toString("utf8"));
|
|
353
|
+
* // → the parsed request body, never more than 64 KiB buffered
|
|
354
|
+
*/
|
|
355
|
+
function collectStream(stream, opts) {
|
|
356
|
+
return new Promise(function (resolve, reject) {
|
|
357
|
+
var collector;
|
|
358
|
+
try { collector = boundedChunkCollector(opts || {}); }
|
|
359
|
+
catch (e) { reject(e); return; }
|
|
360
|
+
var done = false;
|
|
361
|
+
function fail(e) {
|
|
362
|
+
if (done) return;
|
|
363
|
+
done = true;
|
|
364
|
+
try { if (stream && typeof stream.destroy === "function") stream.destroy(); }
|
|
365
|
+
catch (_e) { /* socket already closed */ }
|
|
366
|
+
reject(e);
|
|
367
|
+
}
|
|
368
|
+
stream.on("data", function (chunk) {
|
|
369
|
+
if (done) return;
|
|
370
|
+
try { collector.push(chunk); }
|
|
371
|
+
catch (e) { fail(e); }
|
|
372
|
+
});
|
|
373
|
+
stream.on("end", function () { if (!done) { done = true; resolve(collector.result()); } });
|
|
374
|
+
stream.on("error", fail);
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
|
|
324
378
|
/**
|
|
325
379
|
* @primitive b.safeBuffer.secureZero
|
|
326
380
|
* @signature b.safeBuffer.secureZero(buf)
|
|
@@ -552,6 +606,7 @@ module.exports = {
|
|
|
552
606
|
normalizeText: normalizeText,
|
|
553
607
|
toBuffer: toBuffer,
|
|
554
608
|
boundedChunkCollector: boundedChunkCollector,
|
|
609
|
+
collectStream: collectStream,
|
|
555
610
|
secureZero: secureZero,
|
|
556
611
|
isHex: isHex,
|
|
557
612
|
hasCrlf: hasCrlf,
|