@blamejs/core 0.7.1 → 0.7.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +423 -389
- package/README.md +150 -149
- package/bin/blamejs.js +0 -0
- package/index.js +308 -282
- package/lib/api-key.js +660 -672
- package/lib/api-snapshot.js +338 -338
- package/lib/app-shutdown.js +385 -385
- package/lib/app.js +365 -365
- package/lib/archive.js +250 -250
- package/lib/atomic-file.js +544 -544
- package/lib/audit-chain.js +177 -177
- package/lib/audit-sign.js +344 -344
- package/lib/audit-tools.js +677 -677
- package/lib/audit.js +766 -766
- package/lib/auth/jwt.js +311 -311
- package/lib/auth/lockout.js +436 -436
- package/lib/auth/oauth.js +721 -721
- package/lib/auth/passkey.js +181 -181
- package/lib/auth/password.js +594 -594
- package/lib/backup/bundle.js +217 -217
- package/lib/backup/crypto.js +176 -176
- package/lib/backup/index.js +515 -515
- package/lib/backup/manifest.js +282 -282
- package/lib/break-glass.js +1338 -1338
- package/lib/bundler.js +441 -441
- package/lib/cache-redis.js +256 -266
- package/lib/cache.js +1206 -1211
- package/lib/canonical-json.js +115 -115
- package/lib/chain-writer.js +234 -234
- package/lib/cli-helpers.js +206 -206
- package/lib/cli.js +2334 -2334
- package/lib/cluster-provider-db.js +317 -317
- package/lib/cluster-storage.js +226 -226
- package/lib/cluster.js +703 -703
- package/lib/codepoint-class.js +196 -0
- package/lib/config-drift.js +301 -301
- package/lib/consent.js +222 -222
- package/lib/constants.js +191 -191
- package/lib/cookies.js +315 -315
- package/lib/credential-hash.js +322 -322
- package/lib/crypto.js +266 -266
- package/lib/csv.js +275 -286
- package/lib/db-declare-row-policy.js +267 -267
- package/lib/db-declare-view.js +420 -421
- package/lib/db-query.js +406 -406
- package/lib/db-schema.js +319 -319
- package/lib/db.js +1288 -1288
- package/lib/deprecate.js +222 -222
- package/lib/dev.js +335 -335
- package/lib/dual-control.js +473 -473
- package/lib/error-page.js +420 -420
- package/lib/external-db-migrate.js +441 -441
- package/lib/external-db.js +1061 -1061
- package/lib/file-type.js +273 -273
- package/lib/file-upload.js +1136 -0
- package/lib/forms.js +422 -422
- package/lib/framework-error.js +293 -202
- package/lib/framework-schema.js +717 -717
- package/lib/gate-contract.js +971 -0
- package/lib/guard-all.js +405 -0
- package/lib/guard-archive.js +739 -0
- package/lib/guard-csv.js +816 -0
- package/lib/guard-email.js +744 -0
- package/lib/guard-filename.js +724 -0
- package/lib/guard-html.js +976 -0
- package/lib/guard-json.js +729 -0
- package/lib/guard-markdown.js +586 -0
- package/lib/guard-svg.js +976 -0
- package/lib/guard-xml.js +405 -0
- package/lib/guard-yaml.js +529 -0
- package/lib/handlers.js +350 -350
- package/lib/http-client-cookie-jar.js +508 -508
- package/lib/http-client.js +1195 -1195
- package/lib/i18n.js +878 -878
- package/lib/jobs.js +185 -185
- package/lib/log-stream-cloudwatch.js +369 -369
- package/lib/log-stream-local.js +146 -146
- package/lib/log-stream-otlp-grpc.js +410 -410
- package/lib/log-stream-otlp.js +286 -286
- package/lib/log-stream-syslog.js +302 -302
- package/lib/log-stream-webhook.js +199 -199
- package/lib/log-stream.js +330 -330
- package/lib/log.js +500 -500
- package/lib/mail-bounce.js +528 -528
- package/lib/mail-dkim.js +369 -362
- package/lib/mail.js +981 -962
- package/lib/metrics.js +683 -683
- package/lib/middleware/api-encrypt.js +936 -573
- package/lib/middleware/attach-user.js +157 -157
- package/lib/middleware/body-parser.js +1170 -1091
- package/lib/middleware/bot-guard.js +178 -178
- package/lib/middleware/compression.js +452 -452
- package/lib/middleware/cors.js +314 -314
- package/lib/middleware/csp-nonce.js +348 -348
- package/lib/middleware/csrf-protect.js +316 -316
- package/lib/middleware/db-role-for.js +264 -269
- package/lib/middleware/health.js +392 -392
- package/lib/middleware/index.js +79 -79
- package/lib/middleware/rate-limit.js +358 -358
- package/lib/middleware/request-id.js +61 -61
- package/lib/middleware/request-log.js +168 -168
- package/lib/middleware/require-auth.js +104 -104
- package/lib/middleware/security-headers.js +116 -116
- package/lib/middleware/sse.js +166 -166
- package/lib/migrations.js +383 -383
- package/lib/mtls-ca.js +518 -518
- package/lib/mtls-engine-default.js +481 -481
- package/lib/network-dns.js +632 -632
- package/lib/network-heartbeat.js +290 -290
- package/lib/network-nts.js +574 -574
- package/lib/network-proxy.js +265 -265
- package/lib/network-tls.js +328 -328
- package/lib/network.js +233 -233
- package/lib/notify.js +612 -614
- package/lib/ntp-check.js +229 -229
- package/lib/numeric-bounds.js +111 -91
- package/lib/object-store/azure-blob-bucket-ops.js +349 -349
- package/lib/object-store/azure-blob.js +488 -451
- package/lib/object-store/gcs-bucket-ops.js +351 -351
- package/lib/object-store/gcs.js +519 -479
- package/lib/object-store/http-put.js +153 -153
- package/lib/object-store/index.js +197 -197
- package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
- package/lib/object-store/sigv4.js +903 -855
- package/lib/observability.js +151 -151
- package/lib/otel-export.js +269 -269
- package/lib/pagination.js +464 -464
- package/lib/parsers/index.js +80 -80
- package/lib/parsers/safe-env.js +642 -642
- package/lib/parsers/safe-ini.js +292 -292
- package/lib/parsers/safe-toml.js +784 -784
- package/lib/parsers/safe-xml.js +390 -390
- package/lib/parsers/safe-yaml.js +1015 -1015
- package/lib/permissions.js +708 -708
- package/lib/pqc-agent.js +87 -87
- package/lib/pqc-gate.js +279 -279
- package/lib/protobuf-encoder.js +190 -190
- package/lib/protocol-dispatcher.js +161 -161
- package/lib/pubsub-redis.js +167 -177
- package/lib/pubsub.js +429 -429
- package/lib/queue-local.js +476 -476
- package/lib/queue-redis.js +745 -752
- package/lib/queue-sqs.js +319 -319
- package/lib/queue.js +695 -695
- package/lib/redis-client.js +519 -489
- package/lib/request-helpers.js +340 -336
- package/lib/restore-bundle.js +237 -237
- package/lib/restore-rollback.js +259 -259
- package/lib/restore.js +409 -409
- package/lib/retry.js +376 -376
- package/lib/router.js +748 -748
- package/lib/safe-async.js +735 -735
- package/lib/safe-buffer.js +237 -237
- package/lib/safe-json.js +541 -541
- package/lib/safe-schema.js +1266 -1266
- package/lib/safe-url.js +159 -159
- package/lib/scheduler.js +706 -706
- package/lib/security-assert.js +373 -373
- package/lib/seeders.js +618 -630
- package/lib/session.js +478 -478
- package/lib/slug.js +269 -269
- package/lib/ssrf-guard.js +401 -401
- package/lib/static.js +879 -114
- package/lib/storage.js +471 -471
- package/lib/subject.js +281 -281
- package/lib/template.js +791 -791
- package/lib/testing.js +798 -798
- package/lib/time.js +310 -310
- package/lib/totp.js +302 -302
- package/lib/tracing.js +494 -494
- package/lib/uuid.js +132 -132
- package/lib/validate-opts.js +340 -270
- package/lib/vault/index.js +308 -308
- package/lib/vault/rotate.js +784 -784
- package/lib/vault/wrap.js +296 -296
- package/lib/vendor/noble-ciphers.cjs +9 -9
- package/lib/webhook.js +595 -598
- package/lib/websocket.js +1048 -1048
- package/package.json +77 -77
- package/sbom.cyclonedx.json +7 -7
package/lib/queue.js
CHANGED
|
@@ -1,695 +1,695 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Queue dispatcher — pluggable job queue with retry, lease semantics, and
|
|
4
|
-
* graceful shutdown.
|
|
5
|
-
*
|
|
6
|
-
* Same dispatcher pattern as object-store: backends are configured per-name,
|
|
7
|
-
* each with a protocol + protocol-specific options. The built-in 'local'
|
|
8
|
-
* protocol is SQLite-backed (baked into the framework's main DB).
|
|
9
|
-
* External protocols (redis, sqs, amqp, nats) are listed as deferred and
|
|
10
|
-
* surface a clear error when selected.
|
|
11
|
-
*
|
|
12
|
-
* Public API:
|
|
13
|
-
* queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
|
|
14
|
-
* queue.enqueue(queueName, payload, opts?)
|
|
15
|
-
* → { jobId, queueName, ... }
|
|
16
|
-
* queue.consume(queueName, handler, opts?)
|
|
17
|
-
* → consumer handle (with cancel())
|
|
18
|
-
* queue.size(queueName, opts?) → number (pending + inflight)
|
|
19
|
-
* queue.purge(queueName, opts?) → number deleted
|
|
20
|
-
* queue.shutdown(opts?) → drain handlers gracefully
|
|
21
|
-
* queue.listBackends() → [{ name, protocol }]
|
|
22
|
-
*
|
|
23
|
-
* Job lifecycle:
|
|
24
|
-
* enqueued (status='pending')
|
|
25
|
-
* ↓ availableAt reached + consumer leases
|
|
26
|
-
* inflight (status='inflight', lease expires after leaseDurationMs)
|
|
27
|
-
* ↓ handler returns ↓ handler throws
|
|
28
|
-
* done (status='done') if attempts < maxAttempts:
|
|
29
|
-
* pending (with backoff)
|
|
30
|
-
* else:
|
|
31
|
-
* failed (status='failed')
|
|
32
|
-
*/
|
|
33
|
-
var C = require("./constants");
|
|
34
|
-
var clusterStorage = require("./cluster-storage");
|
|
35
|
-
var crypto = require("./crypto");
|
|
36
|
-
var lazyRequire = require("./lazy-require");
|
|
37
|
-
var { boot } = require("./log");
|
|
38
|
-
var numericChecks = require("./numeric-checks");
|
|
39
|
-
var observability = require("./observability");
|
|
40
|
-
var protocolDispatcher = require("./protocol-dispatcher");
|
|
41
|
-
var localProto = require("./queue-local");
|
|
42
|
-
var redisProto = require("./queue-redis");
|
|
43
|
-
var sqsProto = require("./queue-sqs");
|
|
44
|
-
var retryHelper = require("./retry");
|
|
45
|
-
var safeAsync = require("./safe-async");
|
|
46
|
-
var { QueueError } = require("./framework-error");
|
|
47
|
-
|
|
48
|
-
var log = boot("queue");
|
|
49
|
-
|
|
50
|
-
var dispatcher = protocolDispatcher.create({
|
|
51
|
-
name: "queue",
|
|
52
|
-
errorClass: QueueError,
|
|
53
|
-
protocols: { "local": localProto, "redis": redisProto, "sqs": sqsProto },
|
|
54
|
-
deferred: {
|
|
55
|
-
"amqp": { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
|
|
56
|
-
"nats": { description: "NATS JetStream" },
|
|
57
|
-
},
|
|
58
|
-
fallbackProtocol: "local",
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
var _err = QueueError.factory;
|
|
62
|
-
|
|
63
|
-
var audit = lazyRequire(function () { return require("./audit"); });
|
|
64
|
-
|
|
65
|
-
var initialized = false;
|
|
66
|
-
var backends = {};
|
|
67
|
-
var defaultBackend = null;
|
|
68
|
-
var consumers = []; // [{ queueName, backendName, cancel(), running, inFlight: Set }]
|
|
69
|
-
var sweepTimer = null;
|
|
70
|
-
|
|
71
|
-
function init(opts) {
|
|
72
|
-
if (initialized) return;
|
|
73
|
-
if (!opts || !opts.backends) {
|
|
74
|
-
throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
backends = {};
|
|
78
|
-
// IIFE per-iteration so each backend's wrappers close over its own
|
|
79
|
-
// raw / breaker / cfg. With `var` (function-scoped) those bindings
|
|
80
|
-
// would otherwise be shared across iterations and every wrapper
|
|
81
|
-
// would end up using the LAST backend's breaker + retry config —
|
|
82
|
-
// multi-backend isolation broken without a single error surfacing.
|
|
83
|
-
Object.keys(opts.backends).forEach(function (name) {
|
|
84
|
-
var cfg = opts.backends[name];
|
|
85
|
-
var proto = dispatcher.resolve(cfg.protocol);
|
|
86
|
-
var raw = proto.create(cfg);
|
|
87
|
-
var breaker = new retryHelper.CircuitBreaker(
|
|
88
|
-
"queue:" + name,
|
|
89
|
-
cfg.breaker
|
|
90
|
-
);
|
|
91
|
-
|
|
92
|
-
// Wrap mutating ops with retry + breaker (idempotent only — enqueue and
|
|
93
|
-
// complete are safe to retry; lease isn't because partial-double-lease
|
|
94
|
-
// is dangerous, so it goes through the breaker but not retry).
|
|
95
|
-
function wrapWithRetry(fn) {
|
|
96
|
-
return function () {
|
|
97
|
-
var args = Array.prototype.slice.call(arguments);
|
|
98
|
-
return retryHelper.withRetry(function () {
|
|
99
|
-
return breaker.wrap(function () { return fn.apply(raw, args); });
|
|
100
|
-
}, cfg.retry);
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
function wrapBreakerOnly(fn) {
|
|
104
|
-
return function () {
|
|
105
|
-
var args = Array.prototype.slice.call(arguments);
|
|
106
|
-
return breaker.wrap(function () { return fn.apply(raw, args); });
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
backends[name] = {
|
|
111
|
-
name: name,
|
|
112
|
-
protocol: cfg.protocol,
|
|
113
|
-
breaker: breaker,
|
|
114
|
-
raw: raw,
|
|
115
|
-
enqueue: wrapWithRetry(raw.enqueue),
|
|
116
|
-
lease: wrapBreakerOnly(raw.lease),
|
|
117
|
-
extendLease: raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
|
|
118
|
-
complete: wrapWithRetry(raw.complete),
|
|
119
|
-
fail: wrapWithRetry(raw.fail),
|
|
120
|
-
sweepExpired: raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
|
|
121
|
-
size: wrapWithRetry(raw.size),
|
|
122
|
-
purge: wrapWithRetry(raw.purge),
|
|
123
|
-
dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
|
|
124
|
-
dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
|
|
125
|
-
dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
|
|
126
|
-
};
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
|
|
130
|
-
|
|
131
|
-
// Sweep expired leases periodically (every 30s) so crashed-handler jobs
|
|
132
|
-
// get re-pended.
|
|
133
|
-
sweepTimer = safeAsync.repeating(function () {
|
|
134
|
-
Object.keys(backends).forEach(function (n) {
|
|
135
|
-
if (backends[n].sweepExpired) {
|
|
136
|
-
backends[n].sweepExpired().catch(function () { /* best effort */ });
|
|
137
|
-
}
|
|
138
|
-
});
|
|
139
|
-
}, C.TIME.seconds(30), { name: "queue-sweep" });
|
|
140
|
-
|
|
141
|
-
initialized = true;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function _backendFor(opts) {
|
|
145
|
-
opts = opts || {};
|
|
146
|
-
var name = opts.backend || defaultBackend;
|
|
147
|
-
var b = backends[name];
|
|
148
|
-
if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
|
|
149
|
-
return b;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
// ---- Public API ----
|
|
153
|
-
|
|
154
|
-
function enqueue(queueName, payload, opts) {
|
|
155
|
-
_requireInit();
|
|
156
|
-
if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
|
|
157
|
-
opts = opts || {};
|
|
158
|
-
var b = _backendFor(opts);
|
|
159
|
-
return observability.tap("queue.enqueue",
|
|
160
|
-
{ queueName: queueName, backend: b.name },
|
|
161
|
-
function () {
|
|
162
|
-
return b.enqueue(queueName, payload, opts).then(function (result) {
|
|
163
|
-
_emit("system.queue.enqueue", {
|
|
164
|
-
metadata: {
|
|
165
|
-
queue: queueName,
|
|
166
|
-
backend: b.name,
|
|
167
|
-
jobId: result.jobId,
|
|
168
|
-
classification: result.classification,
|
|
169
|
-
traceId: opts.traceId,
|
|
170
|
-
delaySeconds: opts.delaySeconds || 0,
|
|
171
|
-
},
|
|
172
|
-
});
|
|
173
|
-
return result;
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
function consume(queueName, handler, opts) {
|
|
180
|
-
_requireInit();
|
|
181
|
-
if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
|
|
182
|
-
if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
|
|
183
|
-
opts = opts || {};
|
|
184
|
-
var b = _backendFor(opts);
|
|
185
|
-
var concurrency = opts.concurrency || 1;
|
|
186
|
-
var leaseDurationMs = opts.leaseDurationMs || C.TIME.seconds(30);
|
|
187
|
-
var pollIntervalMs = opts.pollIntervalMs || C.TIME.seconds(1);
|
|
188
|
-
var fastPollMs = opts.fastPollMs || 50;
|
|
189
|
-
|
|
190
|
-
// Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
|
|
191
|
-
// start within any rolling perSeconds window. Token-bucket-style
|
|
192
|
-
// accounting keeps it cheap (just a sliding deque of timestamps).
|
|
193
|
-
var rateLimit = null;
|
|
194
|
-
if (opts.rateLimit) {
|
|
195
|
-
var rlMax = opts.rateLimit.max;
|
|
196
|
-
var rlPer = opts.rateLimit.perSeconds;
|
|
197
|
-
// NaN, Infinity, 0, negatives, fractional max all produce undefined
|
|
198
|
-
// throttling math (NaN deque comparisons, perma-locked queues,
|
|
199
|
-
// perma-open windows). Reject at config time.
|
|
200
|
-
if (!numericChecks.isPositiveInt(rlMax) || !numericChecks.isPositiveFinite(rlPer)) {
|
|
201
|
-
throw _err("BAD_RATE_LIMIT",
|
|
202
|
-
"consume({ rateLimit }): expected { max: positive integer, perSeconds: positive finite number }, got " +
|
|
203
|
-
JSON.stringify(opts.rateLimit), true);
|
|
204
|
-
}
|
|
205
|
-
rateLimit = {
|
|
206
|
-
max: rlMax,
|
|
207
|
-
windowMs: C.TIME.seconds(rlPer),
|
|
208
|
-
timestamps: [],
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
function _rateLimitWaitMs() {
|
|
212
|
-
if (!rateLimit) return 0;
|
|
213
|
-
var now = Date.now();
|
|
214
|
-
var cutoff = now - rateLimit.windowMs;
|
|
215
|
-
while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
|
|
216
|
-
rateLimit.timestamps.shift();
|
|
217
|
-
}
|
|
218
|
-
if (rateLimit.timestamps.length < rateLimit.max) return 0;
|
|
219
|
-
return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
|
|
220
|
-
}
|
|
221
|
-
function _rateLimitConsume() {
|
|
222
|
-
if (rateLimit) rateLimit.timestamps.push(Date.now());
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
// Progress audit-emit rate-limit — protect the audit chain from a
|
|
226
|
-
// chatty handler that calls progress() every loop iteration.
|
|
227
|
-
var PROGRESS_MIN_INTERVAL_MS = 250;
|
|
228
|
-
|
|
229
|
-
// Each consumer has its own AbortController so cancel() unblocks any
|
|
230
|
-
// in-flight poll-sleep immediately rather than waiting up to
|
|
231
|
-
// pollIntervalMs (default 1s) for the next while-loop iteration.
|
|
232
|
-
var abortCtrl = new AbortController();
|
|
233
|
-
var state = {
|
|
234
|
-
queueName: queueName,
|
|
235
|
-
backendName: b.name,
|
|
236
|
-
cancelled: false,
|
|
237
|
-
inFlight: new Set(),
|
|
238
|
-
abortCtrl: abortCtrl,
|
|
239
|
-
cancel: function () {
|
|
240
|
-
state.cancelled = true;
|
|
241
|
-
try { abortCtrl.abort(); }
|
|
242
|
-
catch (e) { log.debug("cancel-cleanup-failed", { op: "abortCtrl.abort", error: e.message }); }
|
|
243
|
-
},
|
|
244
|
-
};
|
|
245
|
-
consumers.push(state);
|
|
246
|
-
|
|
247
|
-
(async function loop() {
|
|
248
|
-
// Helper — sleep with cancellation. On abort, returns instead of
|
|
249
|
-
// rejecting so the next while-iteration sees `state.cancelled` and
|
|
250
|
-
// exits cleanly.
|
|
251
|
-
async function _pollSleep(ms) {
|
|
252
|
-
try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
|
|
253
|
-
catch (_e) { /* aborted — loop condition will catch it */ }
|
|
254
|
-
}
|
|
255
|
-
while (!state.cancelled) {
|
|
256
|
-
// Don't lease more than (concurrency - inFlight) at a time
|
|
257
|
-
var slots = concurrency - state.inFlight.size;
|
|
258
|
-
if (slots <= 0) {
|
|
259
|
-
await _pollSleep(fastPollMs);
|
|
260
|
-
continue;
|
|
261
|
-
}
|
|
262
|
-
// If rate-limited and we'd exceed the budget, sleep until the
|
|
263
|
-
// next slot opens. We lease at most `max - currentTokens` jobs to
|
|
264
|
-
// stay under the cap.
|
|
265
|
-
if (rateLimit) {
|
|
266
|
-
var wait = _rateLimitWaitMs();
|
|
267
|
-
if (wait > 0) {
|
|
268
|
-
await _pollSleep(Math.min(wait, pollIntervalMs));
|
|
269
|
-
continue;
|
|
270
|
-
}
|
|
271
|
-
var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
|
|
272
|
-
if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
|
|
273
|
-
}
|
|
274
|
-
var jobs;
|
|
275
|
-
try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
|
|
276
|
-
catch {
|
|
277
|
-
// Backend down (breaker open, etc.) — back off
|
|
278
|
-
await _pollSleep(pollIntervalMs);
|
|
279
|
-
continue;
|
|
280
|
-
}
|
|
281
|
-
if (!jobs || jobs.length === 0) {
|
|
282
|
-
await _pollSleep(pollIntervalMs);
|
|
283
|
-
continue;
|
|
284
|
-
}
|
|
285
|
-
for (var i = 0; i < jobs.length; i++) {
|
|
286
|
-
observability.event("queue.lease", 1, { queueName: queueName });
|
|
287
|
-
(function (job) {
|
|
288
|
-
state.inFlight.add(job.jobId);
|
|
289
|
-
_emit("system.queue.consume.start", {
|
|
290
|
-
metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
|
|
291
|
-
});
|
|
292
|
-
// Consume a rate-limit slot at handler-start so the budget
|
|
293
|
-
// tracks invocation rate, not lease rate (a single lease that
|
|
294
|
-
// splits work across many sub-units doesn't double-count).
|
|
295
|
-
_rateLimitConsume();
|
|
296
|
-
|
|
297
|
-
// Handler context — second arg to handler. Carries
|
|
298
|
-
// ctx.extendLease(ms) for long-running handlers and
|
|
299
|
-
// ctx.progress(0..100) for surfacing job progress to the
|
|
300
|
-
// audit chain (rate-limited so chatty handlers don't drown it).
|
|
301
|
-
var lastProgressEmitAt = 0;
|
|
302
|
-
var lastProgressValue = -1;
|
|
303
|
-
var ctx = {
|
|
304
|
-
extendLease: function (additionalMs) {
|
|
305
|
-
if (typeof b.extendLease !== "function") {
|
|
306
|
-
throw _err("EXTEND_LEASE_UNSUPPORTED",
|
|
307
|
-
"queue backend '" + b.name + "' does not support extendLease",
|
|
308
|
-
true);
|
|
309
|
-
}
|
|
310
|
-
return b.extendLease(job.jobId, additionalMs).then(function (ok) {
|
|
311
|
-
if (ok) {
|
|
312
|
-
_emit("system.queue.lease.extended", {
|
|
313
|
-
metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
return ok;
|
|
317
|
-
});
|
|
318
|
-
},
|
|
319
|
-
progress: function (pct) {
|
|
320
|
-
if (typeof pct !== "number" || !isFinite(pct)) return;
|
|
321
|
-
var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
|
|
322
|
-
var now = Date.now();
|
|
323
|
-
// Always emit 0 and 100 (start/done markers); throttle the rest.
|
|
324
|
-
var isMarker = clamped === 0 || clamped === 100;
|
|
325
|
-
if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
|
|
326
|
-
if (clamped === lastProgressValue && !isMarker) return;
|
|
327
|
-
lastProgressEmitAt = now;
|
|
328
|
-
lastProgressValue = clamped;
|
|
329
|
-
observability.event("queue.progress", clamped, { queueName: queueName });
|
|
330
|
-
_emit("system.queue.progress", {
|
|
331
|
-
metadata: {
|
|
332
|
-
queue: queueName, backend: b.name, jobId: job.jobId,
|
|
333
|
-
attempt: job.attempts, traceId: job.traceId,
|
|
334
|
-
percent: clamped,
|
|
335
|
-
},
|
|
336
|
-
});
|
|
337
|
-
},
|
|
338
|
-
};
|
|
339
|
-
observability.tap("queue.consume",
|
|
340
|
-
{ queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
|
|
341
|
-
function () {
|
|
342
|
-
return Promise.resolve()
|
|
343
|
-
.then(function () { return handler(job, ctx); })
|
|
344
|
-
.then(function () {
|
|
345
|
-
return b.complete(job.jobId).then(function () {
|
|
346
|
-
_emit("system.queue.consume.success", {
|
|
347
|
-
metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
|
|
348
|
-
});
|
|
349
|
-
observability.event("queue.complete", 1, { queueName: queueName });
|
|
350
|
-
});
|
|
351
|
-
}, function (err) {
|
|
352
|
-
var msg = (err && err.message) || String(err);
|
|
353
|
-
var willRetry = job.attempts < job.maxAttempts;
|
|
354
|
-
return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
|
|
355
|
-
.then(function () {
|
|
356
|
-
observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
|
|
357
|
-
_emit("system.queue.consume.failure", {
|
|
358
|
-
metadata: {
|
|
359
|
-
queue: queueName, backend: b.name, jobId: job.jobId,
|
|
360
|
-
attempt: job.attempts, traceId: job.traceId,
|
|
361
|
-
maxAttempts: job.maxAttempts, willRetry: willRetry,
|
|
362
|
-
},
|
|
363
|
-
reason: msg,
|
|
364
|
-
outcome: "failure",
|
|
365
|
-
});
|
|
366
|
-
// DLQ-write event when the job has exhausted its retries.
|
|
367
|
-
// Operators wire this to their alerting / dashboards
|
|
368
|
-
// — failed-after-retries is "needs human review" not
|
|
369
|
-
// "in the normal flow." Audit chain captures the
|
|
370
|
-
// final state for forensics.
|
|
371
|
-
if (!willRetry) {
|
|
372
|
-
_emit("system.queue.dlq.write", {
|
|
373
|
-
metadata: {
|
|
374
|
-
queue: queueName, backend: b.name, jobId: job.jobId,
|
|
375
|
-
attempts: job.attempts, traceId: job.traceId,
|
|
376
|
-
},
|
|
377
|
-
reason: msg,
|
|
378
|
-
outcome: "failure",
|
|
379
|
-
});
|
|
380
|
-
}
|
|
381
|
-
});
|
|
382
|
-
})
|
|
383
|
-
.catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
|
|
384
|
-
.then(function () { state.inFlight.delete(job.jobId); });
|
|
385
|
-
}
|
|
386
|
-
);
|
|
387
|
-
})(jobs[i]);
|
|
388
|
-
}
|
|
389
|
-
await _pollSleep(fastPollMs);
|
|
390
|
-
}
|
|
391
|
-
})();
|
|
392
|
-
|
|
393
|
-
return state;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
var _QUEUE_BACKOFF_OPTS = {
|
|
397
|
-
baseDelayMs: C.TIME.seconds(1),
|
|
398
|
-
maxDelayMs: C.TIME.minutes(5),
|
|
399
|
-
jitterFactor: 0, // queue uses deterministic backoff (no jitter) for predictability
|
|
400
|
-
};
|
|
401
|
-
function _backoffDelay(attempt) {
|
|
402
|
-
return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
function size(queueName, opts) {
|
|
406
|
-
_requireInit();
|
|
407
|
-
return _backendFor(opts).size(queueName);
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
function purge(queueName, opts) {
|
|
411
|
-
_requireInit();
|
|
412
|
-
var b = _backendFor(opts);
|
|
413
|
-
return b.purge(queueName).then(function (n) {
|
|
414
|
-
_emit("system.queue.purge", {
|
|
415
|
-
metadata: { queue: queueName, backend: b.name, deleted: n },
|
|
416
|
-
});
|
|
417
|
-
return n;
|
|
418
|
-
});
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
// ---- Dead-letter queue ----
|
|
422
|
-
//
|
|
423
|
-
// `dlqList` returns failed-after-retries jobs for operator review;
|
|
424
|
-
// `dlqRetry` resets a single job back to 'pending' so it gets picked
|
|
425
|
-
// up by consumers again (operator-driven — never automatic).
|
|
426
|
-
|
|
427
|
-
function dlqList(queueName, opts) {
|
|
428
|
-
_requireInit();
|
|
429
|
-
var b = _backendFor(opts);
|
|
430
|
-
if (typeof b.dlqList !== "function") {
|
|
431
|
-
return Promise.reject(_err("DLQ_UNSUPPORTED",
|
|
432
|
-
"queue backend '" + b.name + "' does not support dlqList", true));
|
|
433
|
-
}
|
|
434
|
-
return b.dlqList(queueName, opts);
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
function dlqRetry(jobId, opts) {
|
|
438
|
-
_requireInit();
|
|
439
|
-
var b = _backendFor(opts);
|
|
440
|
-
if (typeof b.dlqRetry !== "function") {
|
|
441
|
-
return Promise.reject(_err("DLQ_UNSUPPORTED",
|
|
442
|
-
"queue backend '" + b.name + "' does not support dlqRetry", true));
|
|
443
|
-
}
|
|
444
|
-
return b.dlqRetry(jobId).then(function (ok) {
|
|
445
|
-
if (ok) {
|
|
446
|
-
_emit("system.queue.dlq.retry", {
|
|
447
|
-
metadata: { jobId: jobId, backend: b.name },
|
|
448
|
-
});
|
|
449
|
-
}
|
|
450
|
-
return ok;
|
|
451
|
-
});
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
function dlqSize(queueName, opts) {
|
|
455
|
-
_requireInit();
|
|
456
|
-
var b = _backendFor(opts);
|
|
457
|
-
if (typeof b.dlqSize !== "function") {
|
|
458
|
-
return Promise.reject(_err("DLQ_UNSUPPORTED",
|
|
459
|
-
"queue backend '" + b.name + "' does not support dlqSize", true));
|
|
460
|
-
}
|
|
461
|
-
return b.dlqSize(queueName);
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
async function shutdown(opts) {
|
|
465
|
-
if (!initialized) return;
|
|
466
|
-
opts = opts || {};
|
|
467
|
-
var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
|
|
468
|
-
// Signal all consumers to stop
|
|
469
|
-
consumers.forEach(function (c) { c.cancel(); });
|
|
470
|
-
// Wait for in-flight handlers to complete
|
|
471
|
-
var deadline = Date.now() + timeoutMs;
|
|
472
|
-
while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
|
|
473
|
-
if (Date.now() > deadline) break;
|
|
474
|
-
await safeAsync.sleep(50);
|
|
475
|
-
}
|
|
476
|
-
consumers = [];
|
|
477
|
-
if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
function listBackends() {
|
|
481
|
-
_requireInit();
|
|
482
|
-
return Object.keys(backends).map(function (name) {
|
|
483
|
-
return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
|
|
484
|
-
});
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
function _emit(action, info) {
|
|
488
|
-
audit().safeEmit({ action: action, ...(info || {}) });
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
function _requireInit() {
|
|
492
|
-
if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
function _resetForTest() {
|
|
496
|
-
if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
|
|
497
|
-
consumers.forEach(function (c) { c.cancel(); });
|
|
498
|
-
consumers = [];
|
|
499
|
-
backends = {};
|
|
500
|
-
defaultBackend = null;
|
|
501
|
-
initialized = false;
|
|
502
|
-
audit.reset();
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
// enqueueFlow — atomic registration of a parent-child job graph.
|
|
506
|
-
//
|
|
507
|
-
// await b.queue.enqueueFlow({
|
|
508
|
-
// queueName: "ingest",
|
|
509
|
-
// children: [
|
|
510
|
-
// { name: "fetch", payload: { url } },
|
|
511
|
-
// { name: "transform", payload: { ... }, dependsOn: ["fetch"] },
|
|
512
|
-
// { name: "publish", payload: { ... }, dependsOn: ["transform"] },
|
|
513
|
-
// ],
|
|
514
|
-
// });
|
|
515
|
-
//
|
|
516
|
-
// Cycle detection runs at registration (throws at call site). Each child enters
|
|
517
|
-
// the queue with availableAt = MAX_SAFE_INTEGER until parent completion
|
|
518
|
-
// bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
|
|
519
|
-
function enqueueFlow(spec) {
|
|
520
|
-
_requireInit();
|
|
521
|
-
if (!spec || typeof spec !== "object") {
|
|
522
|
-
return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
|
|
523
|
-
}
|
|
524
|
-
if (typeof spec.queueName !== "string" || !spec.queueName) {
|
|
525
|
-
return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
|
|
526
|
-
}
|
|
527
|
-
if (!Array.isArray(spec.children) || spec.children.length === 0) {
|
|
528
|
-
return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
|
|
529
|
-
}
|
|
530
|
-
// Validate each child's shape.
|
|
531
|
-
var byName = {};
|
|
532
|
-
for (var i = 0; i < spec.children.length; i++) {
|
|
533
|
-
var c = spec.children[i];
|
|
534
|
-
if (!c || typeof c !== "object") {
|
|
535
|
-
return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
|
|
536
|
-
}
|
|
537
|
-
if (typeof c.name !== "string" || !c.name) {
|
|
538
|
-
return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
|
|
539
|
-
}
|
|
540
|
-
if (byName[c.name]) {
|
|
541
|
-
return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
|
|
542
|
-
}
|
|
543
|
-
byName[c.name] = c;
|
|
544
|
-
if (c.dependsOn !== undefined) {
|
|
545
|
-
if (!Array.isArray(c.dependsOn)) {
|
|
546
|
-
return Promise.reject(_err("BAD_FLOW",
|
|
547
|
-
"children[" + i + "].dependsOn must be an array of names", true));
|
|
548
|
-
}
|
|
549
|
-
for (var di = 0; di < c.dependsOn.length; di++) {
|
|
550
|
-
if (typeof c.dependsOn[di] !== "string") {
|
|
551
|
-
return Promise.reject(_err("BAD_FLOW",
|
|
552
|
-
"children[" + i + "].dependsOn[" + di + "] must be a string name", true));
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
// Cycle detection — depth-first traversal with visited set.
|
|
558
|
-
function _visit(name, stack) {
|
|
559
|
-
if (stack.indexOf(name) !== -1) {
|
|
560
|
-
throw _err("FLOW_CYCLE", "flow cycle detected: " +
|
|
561
|
-
stack.concat([name]).join(" → "), true);
|
|
562
|
-
}
|
|
563
|
-
var child = byName[name];
|
|
564
|
-
if (!child || !child.dependsOn) return;
|
|
565
|
-
var nextStack = stack.concat([name]);
|
|
566
|
-
for (var k = 0; k < child.dependsOn.length; k++) {
|
|
567
|
-
var dep = child.dependsOn[k];
|
|
568
|
-
if (!byName[dep]) {
|
|
569
|
-
throw _err("FLOW_UNKNOWN_DEP",
|
|
570
|
-
"child '" + name + "' dependsOn unknown name '" + dep + "'", true);
|
|
571
|
-
}
|
|
572
|
-
_visit(dep, nextStack);
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
try {
|
|
576
|
-
var names = Object.keys(byName);
|
|
577
|
-
for (var n = 0; n < names.length; n++) _visit(names[n], []);
|
|
578
|
-
} catch (e) {
|
|
579
|
-
return Promise.reject(e);
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
var flowId = "flow-" + crypto.generateToken(C.BYTES.bytes(8));
|
|
583
|
-
|
|
584
|
-
return observability.tap("queue.enqueueFlow",
|
|
585
|
-
{ queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
|
|
586
|
-
async function () {
|
|
587
|
-
var jobs = [];
|
|
588
|
-
// Two-pass insert: first pass enqueues all children with their
|
|
589
|
-
// names attached so the second pass can write dependsOn jobIds
|
|
590
|
-
// resolved by name. Children with deps land at MAX_SAFE_INTEGER
|
|
591
|
-
// availableAt automatically (see queue-local enqueue logic).
|
|
592
|
-
var nameToJobId = {};
|
|
593
|
-
for (var p = 0; p < spec.children.length; p++) {
|
|
594
|
-
var ch = spec.children[p];
|
|
595
|
-
// Hold off setting dependsOn until we know all sibling jobIds.
|
|
596
|
-
var enqOpts = {
|
|
597
|
-
flowId: flowId,
|
|
598
|
-
flowChildName: ch.name,
|
|
599
|
-
priority: ch.priority || 0,
|
|
600
|
-
classification: ch.classification || null,
|
|
601
|
-
traceId: ch.traceId || null,
|
|
602
|
-
maxAttempts: ch.maxAttempts,
|
|
603
|
-
// dependsOn intentionally omitted on first pass — will be patched
|
|
604
|
-
// in via direct UPDATE after all jobIds are known. This means
|
|
605
|
-
// root children (no deps) are immediately leaseable; deps-bearing
|
|
606
|
-
// children get patched to MAX_SAFE_INTEGER via second pass.
|
|
607
|
-
};
|
|
608
|
-
var result = await enqueue(spec.queueName, ch.payload, enqOpts);
|
|
609
|
-
nameToJobId[ch.name] = result.jobId;
|
|
610
|
-
jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
|
|
611
|
-
}
|
|
612
|
-
// Second pass: write dependsOn (translated to jobIds) for children
|
|
613
|
-
// that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
|
|
614
|
-
for (var q = 0; q < jobs.length; q++) {
|
|
615
|
-
var j = jobs[q];
|
|
616
|
-
if (j.dependsOn.length === 0) continue;
|
|
617
|
-
var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
|
|
618
|
-
await clusterStorage.execute(
|
|
619
|
-
"UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
|
|
620
|
-
[JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
|
|
621
|
-
);
|
|
622
|
-
}
|
|
623
|
-
_emit("system.queue.flow.enqueue", {
|
|
624
|
-
metadata: {
|
|
625
|
-
queue: spec.queueName,
|
|
626
|
-
flowId: flowId,
|
|
627
|
-
childCount: spec.children.length,
|
|
628
|
-
},
|
|
629
|
-
});
|
|
630
|
-
return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
|
|
631
|
-
}
|
|
632
|
-
);
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
// bootFromEnv — env-driven init mirroring b.network.bootFromEnv and
|
|
636
|
-
// b.logStream.bootFromEnv. Reads the BLAMEJS_QUEUE_* env vars and
|
|
637
|
-
// calls queue.init({ backends }) accordingly. Operators get a working
|
|
638
|
-
// queue backend without writing build-app code.
|
|
639
|
-
//
|
|
640
|
-
// BLAMEJS_QUEUE_PROTOCOL local | redis (default: local)
|
|
641
|
-
// BLAMEJS_QUEUE_REDIS_URL redis://host:port/db (required when protocol=redis)
|
|
642
|
-
// BLAMEJS_QUEUE_REDIS_PASSWORD auth password
|
|
643
|
-
// BLAMEJS_QUEUE_REDIS_USERNAME ACL username (optional)
|
|
644
|
-
// BLAMEJS_QUEUE_REDIS_TLS "1"/"true" forces TLS (else inferred from rediss://)
|
|
645
|
-
// BLAMEJS_QUEUE_REDIS_KEY_PREFIX key prefix (default "blamejs:queue")
|
|
646
|
-
function bootFromEnv(opts) {
|
|
647
|
-
opts = opts || {};
|
|
648
|
-
var env = opts.env || process.env;
|
|
649
|
-
if (initialized) return;
|
|
650
|
-
var protocol = env.BLAMEJS_QUEUE_PROTOCOL || "local";
|
|
651
|
-
var backendCfg;
|
|
652
|
-
if (protocol === "local") {
|
|
653
|
-
backendCfg = { protocol: "local" };
|
|
654
|
-
} else if (protocol === "redis") {
|
|
655
|
-
var url = env.BLAMEJS_QUEUE_REDIS_URL;
|
|
656
|
-
if (!url) {
|
|
657
|
-
throw _err("INVALID_CONFIG",
|
|
658
|
-
"queue.bootFromEnv: BLAMEJS_QUEUE_REDIS_URL is required when BLAMEJS_QUEUE_PROTOCOL=redis",
|
|
659
|
-
true);
|
|
660
|
-
}
|
|
661
|
-
var tlsRaw = env.BLAMEJS_QUEUE_REDIS_TLS;
|
|
662
|
-
var tls = tlsRaw === "1" || tlsRaw === "true";
|
|
663
|
-
backendCfg = {
|
|
664
|
-
protocol: "redis",
|
|
665
|
-
url: url,
|
|
666
|
-
password: env.BLAMEJS_QUEUE_REDIS_PASSWORD || null,
|
|
667
|
-
username: env.BLAMEJS_QUEUE_REDIS_USERNAME || null,
|
|
668
|
-
tls: tlsRaw !== undefined ? tls : undefined, // undefined → inferred from rediss://
|
|
669
|
-
keyPrefix: env.BLAMEJS_QUEUE_REDIS_KEY_PREFIX || undefined,
|
|
670
|
-
};
|
|
671
|
-
} else {
|
|
672
|
-
throw _err("INVALID_CONFIG",
|
|
673
|
-
"queue.bootFromEnv: BLAMEJS_QUEUE_PROTOCOL must be 'local' or 'redis', got '" + protocol + "'",
|
|
674
|
-
true);
|
|
675
|
-
}
|
|
676
|
-
init({ backends: { default: backendCfg }, defaultBackend: "default" });
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
module.exports = {
|
|
680
|
-
init: init,
|
|
681
|
-
bootFromEnv: bootFromEnv,
|
|
682
|
-
enqueue: enqueue,
|
|
683
|
-
enqueueFlow: enqueueFlow,
|
|
684
|
-
consume: consume,
|
|
685
|
-
size: size,
|
|
686
|
-
purge: purge,
|
|
687
|
-
shutdown: shutdown,
|
|
688
|
-
listBackends: listBackends,
|
|
689
|
-
dlqList: dlqList,
|
|
690
|
-
dlqRetry: dlqRetry,
|
|
691
|
-
dlqSize: dlqSize,
|
|
692
|
-
PROTOCOLS: dispatcher.protocols,
|
|
693
|
-
DEFERRED_PROTOCOLS: dispatcher.deferred,
|
|
694
|
-
_resetForTest: _resetForTest,
|
|
695
|
-
};
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Queue dispatcher — pluggable job queue with retry, lease semantics, and
|
|
4
|
+
* graceful shutdown.
|
|
5
|
+
*
|
|
6
|
+
* Same dispatcher pattern as object-store: backends are configured per-name,
|
|
7
|
+
* each with a protocol + protocol-specific options. The built-in 'local'
|
|
8
|
+
* protocol is SQLite-backed (baked into the framework's main DB).
|
|
9
|
+
* External protocols (redis, sqs, amqp, nats) are listed as deferred and
|
|
10
|
+
* surface a clear error when selected.
|
|
11
|
+
*
|
|
12
|
+
* Public API:
|
|
13
|
+
* queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
|
|
14
|
+
* queue.enqueue(queueName, payload, opts?)
|
|
15
|
+
* → { jobId, queueName, ... }
|
|
16
|
+
* queue.consume(queueName, handler, opts?)
|
|
17
|
+
* → consumer handle (with cancel())
|
|
18
|
+
* queue.size(queueName, opts?) → number (pending + inflight)
|
|
19
|
+
* queue.purge(queueName, opts?) → number deleted
|
|
20
|
+
* queue.shutdown(opts?) → drain handlers gracefully
|
|
21
|
+
* queue.listBackends() → [{ name, protocol }]
|
|
22
|
+
*
|
|
23
|
+
* Job lifecycle:
|
|
24
|
+
* enqueued (status='pending')
|
|
25
|
+
* ↓ availableAt reached + consumer leases
|
|
26
|
+
* inflight (status='inflight', lease expires after leaseDurationMs)
|
|
27
|
+
* ↓ handler returns ↓ handler throws
|
|
28
|
+
* done (status='done') if attempts < maxAttempts:
|
|
29
|
+
* pending (with backoff)
|
|
30
|
+
* else:
|
|
31
|
+
* failed (status='failed')
|
|
32
|
+
*/
|
|
33
|
+
var C = require("./constants");
|
|
34
|
+
var clusterStorage = require("./cluster-storage");
|
|
35
|
+
var crypto = require("./crypto");
|
|
36
|
+
var lazyRequire = require("./lazy-require");
|
|
37
|
+
var { boot } = require("./log");
|
|
38
|
+
var numericChecks = require("./numeric-checks");
|
|
39
|
+
var observability = require("./observability");
|
|
40
|
+
var protocolDispatcher = require("./protocol-dispatcher");
|
|
41
|
+
var localProto = require("./queue-local");
|
|
42
|
+
var redisProto = require("./queue-redis");
|
|
43
|
+
var sqsProto = require("./queue-sqs");
|
|
44
|
+
var retryHelper = require("./retry");
|
|
45
|
+
var safeAsync = require("./safe-async");
|
|
46
|
+
var { QueueError } = require("./framework-error");
|
|
47
|
+
|
|
48
|
+
var log = boot("queue");
|
|
49
|
+
|
|
50
|
+
var dispatcher = protocolDispatcher.create({
|
|
51
|
+
name: "queue",
|
|
52
|
+
errorClass: QueueError,
|
|
53
|
+
protocols: { "local": localProto, "redis": redisProto, "sqs": sqsProto },
|
|
54
|
+
deferred: {
|
|
55
|
+
"amqp": { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
|
|
56
|
+
"nats": { description: "NATS JetStream" },
|
|
57
|
+
},
|
|
58
|
+
fallbackProtocol: "local",
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
var _err = QueueError.factory;
|
|
62
|
+
|
|
63
|
+
var audit = lazyRequire(function () { return require("./audit"); });
|
|
64
|
+
|
|
65
|
+
var initialized = false;
|
|
66
|
+
var backends = {};
|
|
67
|
+
var defaultBackend = null;
|
|
68
|
+
var consumers = []; // [{ queueName, backendName, cancel(), running, inFlight: Set }]
|
|
69
|
+
var sweepTimer = null;
|
|
70
|
+
|
|
71
|
+
function init(opts) {
|
|
72
|
+
if (initialized) return;
|
|
73
|
+
if (!opts || !opts.backends) {
|
|
74
|
+
throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
backends = {};
|
|
78
|
+
// IIFE per-iteration so each backend's wrappers close over its own
|
|
79
|
+
// raw / breaker / cfg. With `var` (function-scoped) those bindings
|
|
80
|
+
// would otherwise be shared across iterations and every wrapper
|
|
81
|
+
// would end up using the LAST backend's breaker + retry config —
|
|
82
|
+
// multi-backend isolation broken without a single error surfacing.
|
|
83
|
+
Object.keys(opts.backends).forEach(function (name) {
|
|
84
|
+
var cfg = opts.backends[name];
|
|
85
|
+
var proto = dispatcher.resolve(cfg.protocol);
|
|
86
|
+
var raw = proto.create(cfg);
|
|
87
|
+
var breaker = new retryHelper.CircuitBreaker(
|
|
88
|
+
"queue:" + name,
|
|
89
|
+
cfg.breaker
|
|
90
|
+
);
|
|
91
|
+
|
|
92
|
+
// Wrap mutating ops with retry + breaker (idempotent only — enqueue and
|
|
93
|
+
// complete are safe to retry; lease isn't because partial-double-lease
|
|
94
|
+
// is dangerous, so it goes through the breaker but not retry).
|
|
95
|
+
function wrapWithRetry(fn) {
|
|
96
|
+
return function () {
|
|
97
|
+
var args = Array.prototype.slice.call(arguments);
|
|
98
|
+
return retryHelper.withRetry(function () {
|
|
99
|
+
return breaker.wrap(function () { return fn.apply(raw, args); });
|
|
100
|
+
}, cfg.retry);
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
function wrapBreakerOnly(fn) {
|
|
104
|
+
return function () {
|
|
105
|
+
var args = Array.prototype.slice.call(arguments);
|
|
106
|
+
return breaker.wrap(function () { return fn.apply(raw, args); });
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
backends[name] = {
|
|
111
|
+
name: name,
|
|
112
|
+
protocol: cfg.protocol,
|
|
113
|
+
breaker: breaker,
|
|
114
|
+
raw: raw,
|
|
115
|
+
enqueue: wrapWithRetry(raw.enqueue),
|
|
116
|
+
lease: wrapBreakerOnly(raw.lease),
|
|
117
|
+
extendLease: raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
|
|
118
|
+
complete: wrapWithRetry(raw.complete),
|
|
119
|
+
fail: wrapWithRetry(raw.fail),
|
|
120
|
+
sweepExpired: raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
|
|
121
|
+
size: wrapWithRetry(raw.size),
|
|
122
|
+
purge: wrapWithRetry(raw.purge),
|
|
123
|
+
dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
|
|
124
|
+
dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
|
|
125
|
+
dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
|
|
126
|
+
};
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
|
|
130
|
+
|
|
131
|
+
// Sweep expired leases periodically (every 30s) so crashed-handler jobs
|
|
132
|
+
// get re-pended.
|
|
133
|
+
sweepTimer = safeAsync.repeating(function () {
|
|
134
|
+
Object.keys(backends).forEach(function (n) {
|
|
135
|
+
if (backends[n].sweepExpired) {
|
|
136
|
+
backends[n].sweepExpired().catch(function () { /* best effort */ });
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
}, C.TIME.seconds(30), { name: "queue-sweep" });
|
|
140
|
+
|
|
141
|
+
initialized = true;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function _backendFor(opts) {
|
|
145
|
+
opts = opts || {};
|
|
146
|
+
var name = opts.backend || defaultBackend;
|
|
147
|
+
var b = backends[name];
|
|
148
|
+
if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
|
|
149
|
+
return b;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ---- Public API ----
|
|
153
|
+
|
|
154
|
+
function enqueue(queueName, payload, opts) {
|
|
155
|
+
_requireInit();
|
|
156
|
+
if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
|
|
157
|
+
opts = opts || {};
|
|
158
|
+
var b = _backendFor(opts);
|
|
159
|
+
return observability.tap("queue.enqueue",
|
|
160
|
+
{ queueName: queueName, backend: b.name },
|
|
161
|
+
function () {
|
|
162
|
+
return b.enqueue(queueName, payload, opts).then(function (result) {
|
|
163
|
+
_emit("system.queue.enqueue", {
|
|
164
|
+
metadata: {
|
|
165
|
+
queue: queueName,
|
|
166
|
+
backend: b.name,
|
|
167
|
+
jobId: result.jobId,
|
|
168
|
+
classification: result.classification,
|
|
169
|
+
traceId: opts.traceId,
|
|
170
|
+
delaySeconds: opts.delaySeconds || 0,
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
return result;
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function consume(queueName, handler, opts) {
|
|
180
|
+
_requireInit();
|
|
181
|
+
if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
|
|
182
|
+
if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
|
|
183
|
+
opts = opts || {};
|
|
184
|
+
var b = _backendFor(opts);
|
|
185
|
+
var concurrency = opts.concurrency || 1;
|
|
186
|
+
var leaseDurationMs = opts.leaseDurationMs || C.TIME.seconds(30);
|
|
187
|
+
var pollIntervalMs = opts.pollIntervalMs || C.TIME.seconds(1);
|
|
188
|
+
var fastPollMs = opts.fastPollMs || 50;
|
|
189
|
+
|
|
190
|
+
// Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
|
|
191
|
+
// start within any rolling perSeconds window. Token-bucket-style
|
|
192
|
+
// accounting keeps it cheap (just a sliding deque of timestamps).
|
|
193
|
+
var rateLimit = null;
|
|
194
|
+
if (opts.rateLimit) {
|
|
195
|
+
var rlMax = opts.rateLimit.max;
|
|
196
|
+
var rlPer = opts.rateLimit.perSeconds;
|
|
197
|
+
// NaN, Infinity, 0, negatives, fractional max all produce undefined
|
|
198
|
+
// throttling math (NaN deque comparisons, perma-locked queues,
|
|
199
|
+
// perma-open windows). Reject at config time.
|
|
200
|
+
if (!numericChecks.isPositiveInt(rlMax) || !numericChecks.isPositiveFinite(rlPer)) {
|
|
201
|
+
throw _err("BAD_RATE_LIMIT",
|
|
202
|
+
"consume({ rateLimit }): expected { max: positive integer, perSeconds: positive finite number }, got " +
|
|
203
|
+
JSON.stringify(opts.rateLimit), true);
|
|
204
|
+
}
|
|
205
|
+
rateLimit = {
|
|
206
|
+
max: rlMax,
|
|
207
|
+
windowMs: C.TIME.seconds(rlPer),
|
|
208
|
+
timestamps: [],
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
function _rateLimitWaitMs() {
|
|
212
|
+
if (!rateLimit) return 0;
|
|
213
|
+
var now = Date.now();
|
|
214
|
+
var cutoff = now - rateLimit.windowMs;
|
|
215
|
+
while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
|
|
216
|
+
rateLimit.timestamps.shift();
|
|
217
|
+
}
|
|
218
|
+
if (rateLimit.timestamps.length < rateLimit.max) return 0;
|
|
219
|
+
return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
|
|
220
|
+
}
|
|
221
|
+
function _rateLimitConsume() {
|
|
222
|
+
if (rateLimit) rateLimit.timestamps.push(Date.now());
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// Progress audit-emit rate-limit — protect the audit chain from a
|
|
226
|
+
// chatty handler that calls progress() every loop iteration.
|
|
227
|
+
var PROGRESS_MIN_INTERVAL_MS = 250;
|
|
228
|
+
|
|
229
|
+
// Each consumer has its own AbortController so cancel() unblocks any
|
|
230
|
+
// in-flight poll-sleep immediately rather than waiting up to
|
|
231
|
+
// pollIntervalMs (default 1s) for the next while-loop iteration.
|
|
232
|
+
var abortCtrl = new AbortController();
|
|
233
|
+
var state = {
|
|
234
|
+
queueName: queueName,
|
|
235
|
+
backendName: b.name,
|
|
236
|
+
cancelled: false,
|
|
237
|
+
inFlight: new Set(),
|
|
238
|
+
abortCtrl: abortCtrl,
|
|
239
|
+
cancel: function () {
|
|
240
|
+
state.cancelled = true;
|
|
241
|
+
try { abortCtrl.abort(); }
|
|
242
|
+
catch (e) { log.debug("cancel-cleanup-failed", { op: "abortCtrl.abort", error: e.message }); }
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
consumers.push(state);
|
|
246
|
+
|
|
247
|
+
(async function loop() {
|
|
248
|
+
// Helper — sleep with cancellation. On abort, returns instead of
|
|
249
|
+
// rejecting so the next while-iteration sees `state.cancelled` and
|
|
250
|
+
// exits cleanly.
|
|
251
|
+
async function _pollSleep(ms) {
|
|
252
|
+
try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
|
|
253
|
+
catch (_e) { /* aborted — loop condition will catch it */ }
|
|
254
|
+
}
|
|
255
|
+
while (!state.cancelled) {
|
|
256
|
+
// Don't lease more than (concurrency - inFlight) at a time
|
|
257
|
+
var slots = concurrency - state.inFlight.size;
|
|
258
|
+
if (slots <= 0) {
|
|
259
|
+
await _pollSleep(fastPollMs);
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
// If rate-limited and we'd exceed the budget, sleep until the
|
|
263
|
+
// next slot opens. We lease at most `max - currentTokens` jobs to
|
|
264
|
+
// stay under the cap.
|
|
265
|
+
if (rateLimit) {
|
|
266
|
+
var wait = _rateLimitWaitMs();
|
|
267
|
+
if (wait > 0) {
|
|
268
|
+
await _pollSleep(Math.min(wait, pollIntervalMs));
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
|
|
272
|
+
if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
|
|
273
|
+
}
|
|
274
|
+
var jobs;
|
|
275
|
+
try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
|
|
276
|
+
catch {
|
|
277
|
+
// Backend down (breaker open, etc.) — back off
|
|
278
|
+
await _pollSleep(pollIntervalMs);
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (!jobs || jobs.length === 0) {
|
|
282
|
+
await _pollSleep(pollIntervalMs);
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
for (var i = 0; i < jobs.length; i++) {
|
|
286
|
+
observability.event("queue.lease", 1, { queueName: queueName });
|
|
287
|
+
(function (job) {
|
|
288
|
+
state.inFlight.add(job.jobId);
|
|
289
|
+
_emit("system.queue.consume.start", {
|
|
290
|
+
metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
|
|
291
|
+
});
|
|
292
|
+
// Consume a rate-limit slot at handler-start so the budget
|
|
293
|
+
// tracks invocation rate, not lease rate (a single lease that
|
|
294
|
+
// splits work across many sub-units doesn't double-count).
|
|
295
|
+
_rateLimitConsume();
|
|
296
|
+
|
|
297
|
+
// Handler context — second arg to handler. Carries
|
|
298
|
+
// ctx.extendLease(ms) for long-running handlers and
|
|
299
|
+
// ctx.progress(0..100) for surfacing job progress to the
|
|
300
|
+
// audit chain (rate-limited so chatty handlers don't drown it).
|
|
301
|
+
var lastProgressEmitAt = 0;
|
|
302
|
+
var lastProgressValue = -1;
|
|
303
|
+
var ctx = {
|
|
304
|
+
extendLease: function (additionalMs) {
|
|
305
|
+
if (typeof b.extendLease !== "function") {
|
|
306
|
+
throw _err("EXTEND_LEASE_UNSUPPORTED",
|
|
307
|
+
"queue backend '" + b.name + "' does not support extendLease",
|
|
308
|
+
true);
|
|
309
|
+
}
|
|
310
|
+
return b.extendLease(job.jobId, additionalMs).then(function (ok) {
|
|
311
|
+
if (ok) {
|
|
312
|
+
_emit("system.queue.lease.extended", {
|
|
313
|
+
metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
return ok;
|
|
317
|
+
});
|
|
318
|
+
},
|
|
319
|
+
progress: function (pct) {
|
|
320
|
+
if (typeof pct !== "number" || !isFinite(pct)) return;
|
|
321
|
+
var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
|
|
322
|
+
var now = Date.now();
|
|
323
|
+
// Always emit 0 and 100 (start/done markers); throttle the rest.
|
|
324
|
+
var isMarker = clamped === 0 || clamped === 100;
|
|
325
|
+
if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
|
|
326
|
+
if (clamped === lastProgressValue && !isMarker) return;
|
|
327
|
+
lastProgressEmitAt = now;
|
|
328
|
+
lastProgressValue = clamped;
|
|
329
|
+
observability.event("queue.progress", clamped, { queueName: queueName });
|
|
330
|
+
_emit("system.queue.progress", {
|
|
331
|
+
metadata: {
|
|
332
|
+
queue: queueName, backend: b.name, jobId: job.jobId,
|
|
333
|
+
attempt: job.attempts, traceId: job.traceId,
|
|
334
|
+
percent: clamped,
|
|
335
|
+
},
|
|
336
|
+
});
|
|
337
|
+
},
|
|
338
|
+
};
|
|
339
|
+
observability.tap("queue.consume",
|
|
340
|
+
{ queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
|
|
341
|
+
function () {
|
|
342
|
+
return Promise.resolve()
|
|
343
|
+
.then(function () { return handler(job, ctx); })
|
|
344
|
+
.then(function () {
|
|
345
|
+
return b.complete(job.jobId).then(function () {
|
|
346
|
+
_emit("system.queue.consume.success", {
|
|
347
|
+
metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
|
|
348
|
+
});
|
|
349
|
+
observability.event("queue.complete", 1, { queueName: queueName });
|
|
350
|
+
});
|
|
351
|
+
}, function (err) {
|
|
352
|
+
var msg = (err && err.message) || String(err);
|
|
353
|
+
var willRetry = job.attempts < job.maxAttempts;
|
|
354
|
+
return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
|
|
355
|
+
.then(function () {
|
|
356
|
+
observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
|
|
357
|
+
_emit("system.queue.consume.failure", {
|
|
358
|
+
metadata: {
|
|
359
|
+
queue: queueName, backend: b.name, jobId: job.jobId,
|
|
360
|
+
attempt: job.attempts, traceId: job.traceId,
|
|
361
|
+
maxAttempts: job.maxAttempts, willRetry: willRetry,
|
|
362
|
+
},
|
|
363
|
+
reason: msg,
|
|
364
|
+
outcome: "failure",
|
|
365
|
+
});
|
|
366
|
+
// DLQ-write event when the job has exhausted its retries.
|
|
367
|
+
// Operators wire this to their alerting / dashboards
|
|
368
|
+
// — failed-after-retries is "needs human review" not
|
|
369
|
+
// "in the normal flow." Audit chain captures the
|
|
370
|
+
// final state for forensics.
|
|
371
|
+
if (!willRetry) {
|
|
372
|
+
_emit("system.queue.dlq.write", {
|
|
373
|
+
metadata: {
|
|
374
|
+
queue: queueName, backend: b.name, jobId: job.jobId,
|
|
375
|
+
attempts: job.attempts, traceId: job.traceId,
|
|
376
|
+
},
|
|
377
|
+
reason: msg,
|
|
378
|
+
outcome: "failure",
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
});
|
|
382
|
+
})
|
|
383
|
+
.catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
|
|
384
|
+
.then(function () { state.inFlight.delete(job.jobId); });
|
|
385
|
+
}
|
|
386
|
+
);
|
|
387
|
+
})(jobs[i]);
|
|
388
|
+
}
|
|
389
|
+
await _pollSleep(fastPollMs);
|
|
390
|
+
}
|
|
391
|
+
})();
|
|
392
|
+
|
|
393
|
+
return state;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
var _QUEUE_BACKOFF_OPTS = {
|
|
397
|
+
baseDelayMs: C.TIME.seconds(1),
|
|
398
|
+
maxDelayMs: C.TIME.minutes(5),
|
|
399
|
+
jitterFactor: 0, // queue uses deterministic backoff (no jitter) for predictability
|
|
400
|
+
};
|
|
401
|
+
function _backoffDelay(attempt) {
|
|
402
|
+
return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function size(queueName, opts) {
|
|
406
|
+
_requireInit();
|
|
407
|
+
return _backendFor(opts).size(queueName);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function purge(queueName, opts) {
|
|
411
|
+
_requireInit();
|
|
412
|
+
var b = _backendFor(opts);
|
|
413
|
+
return b.purge(queueName).then(function (n) {
|
|
414
|
+
_emit("system.queue.purge", {
|
|
415
|
+
metadata: { queue: queueName, backend: b.name, deleted: n },
|
|
416
|
+
});
|
|
417
|
+
return n;
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// ---- Dead-letter queue ----
|
|
422
|
+
//
|
|
423
|
+
// `dlqList` returns failed-after-retries jobs for operator review;
|
|
424
|
+
// `dlqRetry` resets a single job back to 'pending' so it gets picked
|
|
425
|
+
// up by consumers again (operator-driven — never automatic).
|
|
426
|
+
|
|
427
|
+
function dlqList(queueName, opts) {
|
|
428
|
+
_requireInit();
|
|
429
|
+
var b = _backendFor(opts);
|
|
430
|
+
if (typeof b.dlqList !== "function") {
|
|
431
|
+
return Promise.reject(_err("DLQ_UNSUPPORTED",
|
|
432
|
+
"queue backend '" + b.name + "' does not support dlqList", true));
|
|
433
|
+
}
|
|
434
|
+
return b.dlqList(queueName, opts);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function dlqRetry(jobId, opts) {
|
|
438
|
+
_requireInit();
|
|
439
|
+
var b = _backendFor(opts);
|
|
440
|
+
if (typeof b.dlqRetry !== "function") {
|
|
441
|
+
return Promise.reject(_err("DLQ_UNSUPPORTED",
|
|
442
|
+
"queue backend '" + b.name + "' does not support dlqRetry", true));
|
|
443
|
+
}
|
|
444
|
+
return b.dlqRetry(jobId).then(function (ok) {
|
|
445
|
+
if (ok) {
|
|
446
|
+
_emit("system.queue.dlq.retry", {
|
|
447
|
+
metadata: { jobId: jobId, backend: b.name },
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
return ok;
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function dlqSize(queueName, opts) {
|
|
455
|
+
_requireInit();
|
|
456
|
+
var b = _backendFor(opts);
|
|
457
|
+
if (typeof b.dlqSize !== "function") {
|
|
458
|
+
return Promise.reject(_err("DLQ_UNSUPPORTED",
|
|
459
|
+
"queue backend '" + b.name + "' does not support dlqSize", true));
|
|
460
|
+
}
|
|
461
|
+
return b.dlqSize(queueName);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
async function shutdown(opts) {
|
|
465
|
+
if (!initialized) return;
|
|
466
|
+
opts = opts || {};
|
|
467
|
+
var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
|
|
468
|
+
// Signal all consumers to stop
|
|
469
|
+
consumers.forEach(function (c) { c.cancel(); });
|
|
470
|
+
// Wait for in-flight handlers to complete
|
|
471
|
+
var deadline = Date.now() + timeoutMs;
|
|
472
|
+
while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
|
|
473
|
+
if (Date.now() > deadline) break;
|
|
474
|
+
await safeAsync.sleep(50);
|
|
475
|
+
}
|
|
476
|
+
consumers = [];
|
|
477
|
+
if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function listBackends() {
|
|
481
|
+
_requireInit();
|
|
482
|
+
return Object.keys(backends).map(function (name) {
|
|
483
|
+
return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
function _emit(action, info) {
|
|
488
|
+
audit().safeEmit({ action: action, ...(info || {}) });
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
function _requireInit() {
|
|
492
|
+
if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
function _resetForTest() {
|
|
496
|
+
if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
|
|
497
|
+
consumers.forEach(function (c) { c.cancel(); });
|
|
498
|
+
consumers = [];
|
|
499
|
+
backends = {};
|
|
500
|
+
defaultBackend = null;
|
|
501
|
+
initialized = false;
|
|
502
|
+
audit.reset();
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// enqueueFlow — atomic registration of a parent-child job graph.
|
|
506
|
+
//
|
|
507
|
+
// await b.queue.enqueueFlow({
|
|
508
|
+
// queueName: "ingest",
|
|
509
|
+
// children: [
|
|
510
|
+
// { name: "fetch", payload: { url } },
|
|
511
|
+
// { name: "transform", payload: { ... }, dependsOn: ["fetch"] },
|
|
512
|
+
// { name: "publish", payload: { ... }, dependsOn: ["transform"] },
|
|
513
|
+
// ],
|
|
514
|
+
// });
|
|
515
|
+
//
|
|
516
|
+
// Cycle detection runs at registration (throws at call site). Each child enters
|
|
517
|
+
// the queue with availableAt = MAX_SAFE_INTEGER until parent completion
|
|
518
|
+
// bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
|
|
519
|
+
function enqueueFlow(spec) {
|
|
520
|
+
_requireInit();
|
|
521
|
+
if (!spec || typeof spec !== "object") {
|
|
522
|
+
return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
|
|
523
|
+
}
|
|
524
|
+
if (typeof spec.queueName !== "string" || !spec.queueName) {
|
|
525
|
+
return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
|
|
526
|
+
}
|
|
527
|
+
if (!Array.isArray(spec.children) || spec.children.length === 0) {
|
|
528
|
+
return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
|
|
529
|
+
}
|
|
530
|
+
// Validate each child's shape.
|
|
531
|
+
var byName = {};
|
|
532
|
+
for (var i = 0; i < spec.children.length; i++) {
|
|
533
|
+
var c = spec.children[i];
|
|
534
|
+
if (!c || typeof c !== "object") {
|
|
535
|
+
return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
|
|
536
|
+
}
|
|
537
|
+
if (typeof c.name !== "string" || !c.name) {
|
|
538
|
+
return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
|
|
539
|
+
}
|
|
540
|
+
if (byName[c.name]) {
|
|
541
|
+
return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
|
|
542
|
+
}
|
|
543
|
+
byName[c.name] = c;
|
|
544
|
+
if (c.dependsOn !== undefined) {
|
|
545
|
+
if (!Array.isArray(c.dependsOn)) {
|
|
546
|
+
return Promise.reject(_err("BAD_FLOW",
|
|
547
|
+
"children[" + i + "].dependsOn must be an array of names", true));
|
|
548
|
+
}
|
|
549
|
+
for (var di = 0; di < c.dependsOn.length; di++) {
|
|
550
|
+
if (typeof c.dependsOn[di] !== "string") {
|
|
551
|
+
return Promise.reject(_err("BAD_FLOW",
|
|
552
|
+
"children[" + i + "].dependsOn[" + di + "] must be a string name", true));
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
// Cycle detection — depth-first traversal with visited set.
|
|
558
|
+
function _visit(name, stack) {
|
|
559
|
+
if (stack.indexOf(name) !== -1) {
|
|
560
|
+
throw _err("FLOW_CYCLE", "flow cycle detected: " +
|
|
561
|
+
stack.concat([name]).join(" → "), true);
|
|
562
|
+
}
|
|
563
|
+
var child = byName[name];
|
|
564
|
+
if (!child || !child.dependsOn) return;
|
|
565
|
+
var nextStack = stack.concat([name]);
|
|
566
|
+
for (var k = 0; k < child.dependsOn.length; k++) {
|
|
567
|
+
var dep = child.dependsOn[k];
|
|
568
|
+
if (!byName[dep]) {
|
|
569
|
+
throw _err("FLOW_UNKNOWN_DEP",
|
|
570
|
+
"child '" + name + "' dependsOn unknown name '" + dep + "'", true);
|
|
571
|
+
}
|
|
572
|
+
_visit(dep, nextStack);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
try {
|
|
576
|
+
var names = Object.keys(byName);
|
|
577
|
+
for (var n = 0; n < names.length; n++) _visit(names[n], []);
|
|
578
|
+
} catch (e) {
|
|
579
|
+
return Promise.reject(e);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
var flowId = "flow-" + crypto.generateToken(C.BYTES.bytes(8));
|
|
583
|
+
|
|
584
|
+
return observability.tap("queue.enqueueFlow",
|
|
585
|
+
{ queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
|
|
586
|
+
async function () {
|
|
587
|
+
var jobs = [];
|
|
588
|
+
// Two-pass insert: first pass enqueues all children with their
|
|
589
|
+
// names attached so the second pass can write dependsOn jobIds
|
|
590
|
+
// resolved by name. Children with deps land at MAX_SAFE_INTEGER
|
|
591
|
+
// availableAt automatically (see queue-local enqueue logic).
|
|
592
|
+
var nameToJobId = {};
|
|
593
|
+
for (var p = 0; p < spec.children.length; p++) {
|
|
594
|
+
var ch = spec.children[p];
|
|
595
|
+
// Hold off setting dependsOn until we know all sibling jobIds.
|
|
596
|
+
var enqOpts = {
|
|
597
|
+
flowId: flowId,
|
|
598
|
+
flowChildName: ch.name,
|
|
599
|
+
priority: ch.priority || 0,
|
|
600
|
+
classification: ch.classification || null,
|
|
601
|
+
traceId: ch.traceId || null,
|
|
602
|
+
maxAttempts: ch.maxAttempts,
|
|
603
|
+
// dependsOn intentionally omitted on first pass — will be patched
|
|
604
|
+
// in via direct UPDATE after all jobIds are known. This means
|
|
605
|
+
// root children (no deps) are immediately leaseable; deps-bearing
|
|
606
|
+
// children get patched to MAX_SAFE_INTEGER via second pass.
|
|
607
|
+
};
|
|
608
|
+
var result = await enqueue(spec.queueName, ch.payload, enqOpts);
|
|
609
|
+
nameToJobId[ch.name] = result.jobId;
|
|
610
|
+
jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
|
|
611
|
+
}
|
|
612
|
+
// Second pass: write dependsOn (translated to jobIds) for children
|
|
613
|
+
// that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
|
|
614
|
+
for (var q = 0; q < jobs.length; q++) {
|
|
615
|
+
var j = jobs[q];
|
|
616
|
+
if (j.dependsOn.length === 0) continue;
|
|
617
|
+
var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
|
|
618
|
+
await clusterStorage.execute(
|
|
619
|
+
"UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
|
|
620
|
+
[JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
_emit("system.queue.flow.enqueue", {
|
|
624
|
+
metadata: {
|
|
625
|
+
queue: spec.queueName,
|
|
626
|
+
flowId: flowId,
|
|
627
|
+
childCount: spec.children.length,
|
|
628
|
+
},
|
|
629
|
+
});
|
|
630
|
+
return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
|
|
631
|
+
}
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// bootFromEnv — env-driven init mirroring b.network.bootFromEnv and
|
|
636
|
+
// b.logStream.bootFromEnv. Reads the BLAMEJS_QUEUE_* env vars and
|
|
637
|
+
// calls queue.init({ backends }) accordingly. Operators get a working
|
|
638
|
+
// queue backend without writing build-app code.
|
|
639
|
+
//
|
|
640
|
+
// BLAMEJS_QUEUE_PROTOCOL local | redis (default: local)
|
|
641
|
+
// BLAMEJS_QUEUE_REDIS_URL redis://host:port/db (required when protocol=redis)
|
|
642
|
+
// BLAMEJS_QUEUE_REDIS_PASSWORD auth password
|
|
643
|
+
// BLAMEJS_QUEUE_REDIS_USERNAME ACL username (optional)
|
|
644
|
+
// BLAMEJS_QUEUE_REDIS_TLS "1"/"true" forces TLS (else inferred from rediss://)
|
|
645
|
+
// BLAMEJS_QUEUE_REDIS_KEY_PREFIX key prefix (default "blamejs:queue")
|
|
646
|
+
function bootFromEnv(opts) {
|
|
647
|
+
opts = opts || {};
|
|
648
|
+
var env = opts.env || process.env;
|
|
649
|
+
if (initialized) return;
|
|
650
|
+
var protocol = env.BLAMEJS_QUEUE_PROTOCOL || "local";
|
|
651
|
+
var backendCfg;
|
|
652
|
+
if (protocol === "local") {
|
|
653
|
+
backendCfg = { protocol: "local" };
|
|
654
|
+
} else if (protocol === "redis") {
|
|
655
|
+
var url = env.BLAMEJS_QUEUE_REDIS_URL;
|
|
656
|
+
if (!url) {
|
|
657
|
+
throw _err("INVALID_CONFIG",
|
|
658
|
+
"queue.bootFromEnv: BLAMEJS_QUEUE_REDIS_URL is required when BLAMEJS_QUEUE_PROTOCOL=redis",
|
|
659
|
+
true);
|
|
660
|
+
}
|
|
661
|
+
var tlsRaw = env.BLAMEJS_QUEUE_REDIS_TLS;
|
|
662
|
+
var tls = tlsRaw === "1" || tlsRaw === "true";
|
|
663
|
+
backendCfg = {
|
|
664
|
+
protocol: "redis",
|
|
665
|
+
url: url,
|
|
666
|
+
password: env.BLAMEJS_QUEUE_REDIS_PASSWORD || null,
|
|
667
|
+
username: env.BLAMEJS_QUEUE_REDIS_USERNAME || null,
|
|
668
|
+
tls: tlsRaw !== undefined ? tls : undefined, // undefined → inferred from rediss://
|
|
669
|
+
keyPrefix: env.BLAMEJS_QUEUE_REDIS_KEY_PREFIX || undefined,
|
|
670
|
+
};
|
|
671
|
+
} else {
|
|
672
|
+
throw _err("INVALID_CONFIG",
|
|
673
|
+
"queue.bootFromEnv: BLAMEJS_QUEUE_PROTOCOL must be 'local' or 'redis', got '" + protocol + "'",
|
|
674
|
+
true);
|
|
675
|
+
}
|
|
676
|
+
init({ backends: { default: backendCfg }, defaultBackend: "default" });
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
module.exports = {
|
|
680
|
+
init: init,
|
|
681
|
+
bootFromEnv: bootFromEnv,
|
|
682
|
+
enqueue: enqueue,
|
|
683
|
+
enqueueFlow: enqueueFlow,
|
|
684
|
+
consume: consume,
|
|
685
|
+
size: size,
|
|
686
|
+
purge: purge,
|
|
687
|
+
shutdown: shutdown,
|
|
688
|
+
listBackends: listBackends,
|
|
689
|
+
dlqList: dlqList,
|
|
690
|
+
dlqRetry: dlqRetry,
|
|
691
|
+
dlqSize: dlqSize,
|
|
692
|
+
PROTOCOLS: dispatcher.protocols,
|
|
693
|
+
DEFERRED_PROTOCOLS: dispatcher.deferred,
|
|
694
|
+
_resetForTest: _resetForTest,
|
|
695
|
+
};
|