@blamejs/core 0.7.18 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/CHANGELOG.md +425 -423
  2. package/README.md +150 -150
  3. package/bin/blamejs.js +0 -0
  4. package/index.js +310 -308
  5. package/lib/api-key.js +660 -660
  6. package/lib/api-snapshot.js +338 -338
  7. package/lib/app-shutdown.js +385 -385
  8. package/lib/app.js +365 -365
  9. package/lib/archive.js +250 -250
  10. package/lib/atomic-file.js +544 -544
  11. package/lib/audit-chain.js +177 -177
  12. package/lib/audit-sign.js +344 -344
  13. package/lib/audit-tools.js +677 -677
  14. package/lib/audit.js +766 -766
  15. package/lib/auth/jwt-external.js +365 -0
  16. package/lib/auth/jwt.js +337 -311
  17. package/lib/auth/lockout.js +436 -436
  18. package/lib/auth/oauth.js +721 -721
  19. package/lib/auth/passkey.js +181 -181
  20. package/lib/auth/password.js +628 -594
  21. package/lib/backup/bundle.js +217 -217
  22. package/lib/backup/crypto.js +176 -176
  23. package/lib/backup/index.js +515 -515
  24. package/lib/backup/manifest.js +282 -282
  25. package/lib/break-glass.js +1338 -1338
  26. package/lib/bundler.js +441 -441
  27. package/lib/cache-redis.js +256 -256
  28. package/lib/cache.js +1206 -1206
  29. package/lib/canonical-json.js +115 -115
  30. package/lib/chain-writer.js +234 -234
  31. package/lib/cli-helpers.js +206 -206
  32. package/lib/cli.js +2334 -2334
  33. package/lib/cluster-provider-db.js +317 -317
  34. package/lib/cluster-storage.js +226 -226
  35. package/lib/cluster.js +703 -703
  36. package/lib/config-drift.js +301 -301
  37. package/lib/consent.js +222 -222
  38. package/lib/constants.js +191 -191
  39. package/lib/cookies.js +315 -315
  40. package/lib/credential-hash.js +322 -322
  41. package/lib/crypto.js +266 -266
  42. package/lib/csv.js +275 -275
  43. package/lib/db-declare-row-policy.js +267 -267
  44. package/lib/db-declare-view.js +420 -420
  45. package/lib/db-query.js +406 -406
  46. package/lib/db-schema.js +319 -319
  47. package/lib/db.js +1288 -1288
  48. package/lib/deprecate.js +222 -222
  49. package/lib/dev.js +335 -335
  50. package/lib/dual-control.js +473 -473
  51. package/lib/error-page.js +420 -420
  52. package/lib/external-db-migrate.js +441 -441
  53. package/lib/external-db.js +1061 -1061
  54. package/lib/file-type.js +273 -273
  55. package/lib/forms.js +422 -422
  56. package/lib/framework-error.js +293 -293
  57. package/lib/framework-schema.js +717 -717
  58. package/lib/handlers.js +350 -350
  59. package/lib/http-client-cookie-jar.js +508 -508
  60. package/lib/http-client.js +1195 -1195
  61. package/lib/i18n.js +878 -878
  62. package/lib/jobs.js +185 -185
  63. package/lib/log-stream-cloudwatch.js +369 -369
  64. package/lib/log-stream-local.js +146 -146
  65. package/lib/log-stream-otlp-grpc.js +410 -410
  66. package/lib/log-stream-otlp.js +286 -286
  67. package/lib/log-stream-syslog.js +302 -302
  68. package/lib/log-stream-webhook.js +199 -199
  69. package/lib/log-stream.js +330 -330
  70. package/lib/log.js +500 -500
  71. package/lib/mail-bounce.js +528 -528
  72. package/lib/mail-dkim.js +369 -369
  73. package/lib/mail.js +981 -981
  74. package/lib/metrics.js +683 -683
  75. package/lib/middleware/api-encrypt.js +936 -936
  76. package/lib/middleware/attach-user.js +157 -157
  77. package/lib/middleware/bearer-auth.js +152 -0
  78. package/lib/middleware/body-parser.js +1170 -1170
  79. package/lib/middleware/bot-guard.js +178 -178
  80. package/lib/middleware/compression.js +452 -452
  81. package/lib/middleware/cors.js +314 -314
  82. package/lib/middleware/csp-nonce.js +348 -348
  83. package/lib/middleware/csrf-protect.js +316 -316
  84. package/lib/middleware/db-role-for.js +264 -264
  85. package/lib/middleware/health.js +392 -392
  86. package/lib/middleware/index.js +82 -79
  87. package/lib/middleware/rate-limit.js +358 -358
  88. package/lib/middleware/request-id.js +61 -61
  89. package/lib/middleware/request-log.js +168 -168
  90. package/lib/middleware/require-auth.js +104 -104
  91. package/lib/middleware/security-headers.js +116 -116
  92. package/lib/middleware/sse.js +166 -166
  93. package/lib/migrations.js +383 -383
  94. package/lib/mtls-ca.js +518 -518
  95. package/lib/mtls-engine-default.js +481 -481
  96. package/lib/network-dns.js +632 -632
  97. package/lib/network-heartbeat.js +290 -290
  98. package/lib/network-nts.js +574 -574
  99. package/lib/network-proxy.js +265 -265
  100. package/lib/network-tls.js +328 -328
  101. package/lib/network.js +233 -233
  102. package/lib/notify.js +612 -612
  103. package/lib/ntp-check.js +229 -229
  104. package/lib/numeric-bounds.js +111 -111
  105. package/lib/object-store/azure-blob-bucket-ops.js +349 -349
  106. package/lib/object-store/azure-blob.js +488 -488
  107. package/lib/object-store/gcs-bucket-ops.js +351 -351
  108. package/lib/object-store/gcs.js +519 -519
  109. package/lib/object-store/http-put.js +153 -153
  110. package/lib/object-store/index.js +197 -197
  111. package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
  112. package/lib/object-store/sigv4.js +903 -903
  113. package/lib/observability.js +151 -151
  114. package/lib/otel-export.js +269 -269
  115. package/lib/pagination.js +464 -464
  116. package/lib/parsers/index.js +80 -80
  117. package/lib/parsers/safe-env.js +642 -642
  118. package/lib/parsers/safe-ini.js +292 -292
  119. package/lib/parsers/safe-toml.js +784 -784
  120. package/lib/parsers/safe-xml.js +390 -390
  121. package/lib/parsers/safe-yaml.js +1015 -1015
  122. package/lib/permissions.js +708 -708
  123. package/lib/pqc-agent.js +87 -87
  124. package/lib/pqc-gate.js +279 -279
  125. package/lib/protobuf-encoder.js +190 -190
  126. package/lib/protocol-dispatcher.js +161 -161
  127. package/lib/pubsub-redis.js +167 -167
  128. package/lib/pubsub.js +429 -429
  129. package/lib/queue-local.js +476 -476
  130. package/lib/queue-redis.js +745 -745
  131. package/lib/queue-sqs.js +319 -319
  132. package/lib/queue.js +695 -695
  133. package/lib/redis-client.js +519 -519
  134. package/lib/request-helpers.js +340 -340
  135. package/lib/restore-bundle.js +237 -237
  136. package/lib/restore-rollback.js +259 -259
  137. package/lib/restore.js +409 -409
  138. package/lib/retry.js +376 -376
  139. package/lib/router.js +748 -748
  140. package/lib/safe-async.js +735 -735
  141. package/lib/safe-buffer.js +237 -237
  142. package/lib/safe-json.js +541 -541
  143. package/lib/safe-schema.js +1266 -1266
  144. package/lib/safe-url.js +159 -159
  145. package/lib/scheduler.js +706 -706
  146. package/lib/security-assert.js +373 -373
  147. package/lib/seeders.js +618 -618
  148. package/lib/session.js +535 -478
  149. package/lib/slug.js +269 -269
  150. package/lib/ssrf-guard.js +401 -401
  151. package/lib/storage.js +471 -471
  152. package/lib/subject.js +281 -281
  153. package/lib/template.js +791 -791
  154. package/lib/testing.js +798 -798
  155. package/lib/time.js +310 -310
  156. package/lib/totp.js +302 -302
  157. package/lib/tracing.js +494 -494
  158. package/lib/uuid.js +132 -132
  159. package/lib/validate-opts.js +340 -340
  160. package/lib/vault/index.js +308 -308
  161. package/lib/vault/rotate.js +784 -784
  162. package/lib/vault/wrap.js +296 -296
  163. package/lib/vendor/noble-ciphers.cjs +9 -9
  164. package/lib/webhook.js +595 -595
  165. package/lib/websocket.js +1048 -1048
  166. package/package.json +77 -77
  167. package/sbom.cyclonedx.json +7 -7
package/lib/queue.js CHANGED
@@ -1,695 +1,695 @@
1
- "use strict";
2
- /**
3
- * Queue dispatcher — pluggable job queue with retry, lease semantics, and
4
- * graceful shutdown.
5
- *
6
- * Same dispatcher pattern as object-store: backends are configured per-name,
7
- * each with a protocol + protocol-specific options. The built-in 'local'
8
- * protocol is SQLite-backed (baked into the framework's main DB).
9
- * External protocols (redis, sqs, amqp, nats) are listed as deferred and
10
- * surface a clear error when selected.
11
- *
12
- * Public API:
13
- * queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
14
- * queue.enqueue(queueName, payload, opts?)
15
- * → { jobId, queueName, ... }
16
- * queue.consume(queueName, handler, opts?)
17
- * → consumer handle (with cancel())
18
- * queue.size(queueName, opts?) → number (pending + inflight)
19
- * queue.purge(queueName, opts?) → number deleted
20
- * queue.shutdown(opts?) → drain handlers gracefully
21
- * queue.listBackends() → [{ name, protocol }]
22
- *
23
- * Job lifecycle:
24
- * enqueued (status='pending')
25
- * ↓ availableAt reached + consumer leases
26
- * inflight (status='inflight', lease expires after leaseDurationMs)
27
- * ↓ handler returns ↓ handler throws
28
- * done (status='done') if attempts < maxAttempts:
29
- * pending (with backoff)
30
- * else:
31
- * failed (status='failed')
32
- */
33
- var C = require("./constants");
34
- var clusterStorage = require("./cluster-storage");
35
- var crypto = require("./crypto");
36
- var lazyRequire = require("./lazy-require");
37
- var { boot } = require("./log");
38
- var numericChecks = require("./numeric-checks");
39
- var observability = require("./observability");
40
- var protocolDispatcher = require("./protocol-dispatcher");
41
- var localProto = require("./queue-local");
42
- var redisProto = require("./queue-redis");
43
- var sqsProto = require("./queue-sqs");
44
- var retryHelper = require("./retry");
45
- var safeAsync = require("./safe-async");
46
- var { QueueError } = require("./framework-error");
47
-
48
- var log = boot("queue");
49
-
50
- var dispatcher = protocolDispatcher.create({
51
- name: "queue",
52
- errorClass: QueueError,
53
- protocols: { "local": localProto, "redis": redisProto, "sqs": sqsProto },
54
- deferred: {
55
- "amqp": { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
56
- "nats": { description: "NATS JetStream" },
57
- },
58
- fallbackProtocol: "local",
59
- });
60
-
61
- var _err = QueueError.factory;
62
-
63
- var audit = lazyRequire(function () { return require("./audit"); });
64
-
65
- var initialized = false;
66
- var backends = {};
67
- var defaultBackend = null;
68
- var consumers = []; // [{ queueName, backendName, cancel(), running, inFlight: Set }]
69
- var sweepTimer = null;
70
-
71
- function init(opts) {
72
- if (initialized) return;
73
- if (!opts || !opts.backends) {
74
- throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
75
- }
76
-
77
- backends = {};
78
- // IIFE per-iteration so each backend's wrappers close over its own
79
- // raw / breaker / cfg. With `var` (function-scoped) those bindings
80
- // would otherwise be shared across iterations and every wrapper
81
- // would end up using the LAST backend's breaker + retry config —
82
- // multi-backend isolation broken without a single error surfacing.
83
- Object.keys(opts.backends).forEach(function (name) {
84
- var cfg = opts.backends[name];
85
- var proto = dispatcher.resolve(cfg.protocol);
86
- var raw = proto.create(cfg);
87
- var breaker = new retryHelper.CircuitBreaker(
88
- "queue:" + name,
89
- cfg.breaker
90
- );
91
-
92
- // Wrap mutating ops with retry + breaker (idempotent only — enqueue and
93
- // complete are safe to retry; lease isn't because partial-double-lease
94
- // is dangerous, so it goes through the breaker but not retry).
95
- function wrapWithRetry(fn) {
96
- return function () {
97
- var args = Array.prototype.slice.call(arguments);
98
- return retryHelper.withRetry(function () {
99
- return breaker.wrap(function () { return fn.apply(raw, args); });
100
- }, cfg.retry);
101
- };
102
- }
103
- function wrapBreakerOnly(fn) {
104
- return function () {
105
- var args = Array.prototype.slice.call(arguments);
106
- return breaker.wrap(function () { return fn.apply(raw, args); });
107
- };
108
- }
109
-
110
- backends[name] = {
111
- name: name,
112
- protocol: cfg.protocol,
113
- breaker: breaker,
114
- raw: raw,
115
- enqueue: wrapWithRetry(raw.enqueue),
116
- lease: wrapBreakerOnly(raw.lease),
117
- extendLease: raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
118
- complete: wrapWithRetry(raw.complete),
119
- fail: wrapWithRetry(raw.fail),
120
- sweepExpired: raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
121
- size: wrapWithRetry(raw.size),
122
- purge: wrapWithRetry(raw.purge),
123
- dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
124
- dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
125
- dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
126
- };
127
- });
128
-
129
- defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
130
-
131
- // Sweep expired leases periodically (every 30s) so crashed-handler jobs
132
- // get re-pended.
133
- sweepTimer = safeAsync.repeating(function () {
134
- Object.keys(backends).forEach(function (n) {
135
- if (backends[n].sweepExpired) {
136
- backends[n].sweepExpired().catch(function () { /* best effort */ });
137
- }
138
- });
139
- }, C.TIME.seconds(30), { name: "queue-sweep" });
140
-
141
- initialized = true;
142
- }
143
-
144
- function _backendFor(opts) {
145
- opts = opts || {};
146
- var name = opts.backend || defaultBackend;
147
- var b = backends[name];
148
- if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
149
- return b;
150
- }
151
-
152
- // ---- Public API ----
153
-
154
- function enqueue(queueName, payload, opts) {
155
- _requireInit();
156
- if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
157
- opts = opts || {};
158
- var b = _backendFor(opts);
159
- return observability.tap("queue.enqueue",
160
- { queueName: queueName, backend: b.name },
161
- function () {
162
- return b.enqueue(queueName, payload, opts).then(function (result) {
163
- _emit("system.queue.enqueue", {
164
- metadata: {
165
- queue: queueName,
166
- backend: b.name,
167
- jobId: result.jobId,
168
- classification: result.classification,
169
- traceId: opts.traceId,
170
- delaySeconds: opts.delaySeconds || 0,
171
- },
172
- });
173
- return result;
174
- });
175
- }
176
- );
177
- }
178
-
179
- function consume(queueName, handler, opts) {
180
- _requireInit();
181
- if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
182
- if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
183
- opts = opts || {};
184
- var b = _backendFor(opts);
185
- var concurrency = opts.concurrency || 1;
186
- var leaseDurationMs = opts.leaseDurationMs || C.TIME.seconds(30);
187
- var pollIntervalMs = opts.pollIntervalMs || C.TIME.seconds(1);
188
- var fastPollMs = opts.fastPollMs || 50;
189
-
190
- // Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
191
- // start within any rolling perSeconds window. Token-bucket-style
192
- // accounting keeps it cheap (just a sliding deque of timestamps).
193
- var rateLimit = null;
194
- if (opts.rateLimit) {
195
- var rlMax = opts.rateLimit.max;
196
- var rlPer = opts.rateLimit.perSeconds;
197
- // NaN, Infinity, 0, negatives, fractional max all produce undefined
198
- // throttling math (NaN deque comparisons, perma-locked queues,
199
- // perma-open windows). Reject at config time.
200
- if (!numericChecks.isPositiveInt(rlMax) || !numericChecks.isPositiveFinite(rlPer)) {
201
- throw _err("BAD_RATE_LIMIT",
202
- "consume({ rateLimit }): expected { max: positive integer, perSeconds: positive finite number }, got " +
203
- JSON.stringify(opts.rateLimit), true);
204
- }
205
- rateLimit = {
206
- max: rlMax,
207
- windowMs: C.TIME.seconds(rlPer),
208
- timestamps: [],
209
- };
210
- }
211
- function _rateLimitWaitMs() {
212
- if (!rateLimit) return 0;
213
- var now = Date.now();
214
- var cutoff = now - rateLimit.windowMs;
215
- while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
216
- rateLimit.timestamps.shift();
217
- }
218
- if (rateLimit.timestamps.length < rateLimit.max) return 0;
219
- return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
220
- }
221
- function _rateLimitConsume() {
222
- if (rateLimit) rateLimit.timestamps.push(Date.now());
223
- }
224
-
225
- // Progress audit-emit rate-limit — protect the audit chain from a
226
- // chatty handler that calls progress() every loop iteration.
227
- var PROGRESS_MIN_INTERVAL_MS = 250;
228
-
229
- // Each consumer has its own AbortController so cancel() unblocks any
230
- // in-flight poll-sleep immediately rather than waiting up to
231
- // pollIntervalMs (default 1s) for the next while-loop iteration.
232
- var abortCtrl = new AbortController();
233
- var state = {
234
- queueName: queueName,
235
- backendName: b.name,
236
- cancelled: false,
237
- inFlight: new Set(),
238
- abortCtrl: abortCtrl,
239
- cancel: function () {
240
- state.cancelled = true;
241
- try { abortCtrl.abort(); }
242
- catch (e) { log.debug("cancel-cleanup-failed", { op: "abortCtrl.abort", error: e.message }); }
243
- },
244
- };
245
- consumers.push(state);
246
-
247
- (async function loop() {
248
- // Helper — sleep with cancellation. On abort, returns instead of
249
- // rejecting so the next while-iteration sees `state.cancelled` and
250
- // exits cleanly.
251
- async function _pollSleep(ms) {
252
- try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
253
- catch (_e) { /* aborted — loop condition will catch it */ }
254
- }
255
- while (!state.cancelled) {
256
- // Don't lease more than (concurrency - inFlight) at a time
257
- var slots = concurrency - state.inFlight.size;
258
- if (slots <= 0) {
259
- await _pollSleep(fastPollMs);
260
- continue;
261
- }
262
- // If rate-limited and we'd exceed the budget, sleep until the
263
- // next slot opens. We lease at most `max - currentTokens` jobs to
264
- // stay under the cap.
265
- if (rateLimit) {
266
- var wait = _rateLimitWaitMs();
267
- if (wait > 0) {
268
- await _pollSleep(Math.min(wait, pollIntervalMs));
269
- continue;
270
- }
271
- var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
272
- if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
273
- }
274
- var jobs;
275
- try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
276
- catch {
277
- // Backend down (breaker open, etc.) — back off
278
- await _pollSleep(pollIntervalMs);
279
- continue;
280
- }
281
- if (!jobs || jobs.length === 0) {
282
- await _pollSleep(pollIntervalMs);
283
- continue;
284
- }
285
- for (var i = 0; i < jobs.length; i++) {
286
- observability.event("queue.lease", 1, { queueName: queueName });
287
- (function (job) {
288
- state.inFlight.add(job.jobId);
289
- _emit("system.queue.consume.start", {
290
- metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
291
- });
292
- // Consume a rate-limit slot at handler-start so the budget
293
- // tracks invocation rate, not lease rate (a single lease that
294
- // splits work across many sub-units doesn't double-count).
295
- _rateLimitConsume();
296
-
297
- // Handler context — second arg to handler. Carries
298
- // ctx.extendLease(ms) for long-running handlers and
299
- // ctx.progress(0..100) for surfacing job progress to the
300
- // audit chain (rate-limited so chatty handlers don't drown it).
301
- var lastProgressEmitAt = 0;
302
- var lastProgressValue = -1;
303
- var ctx = {
304
- extendLease: function (additionalMs) {
305
- if (typeof b.extendLease !== "function") {
306
- throw _err("EXTEND_LEASE_UNSUPPORTED",
307
- "queue backend '" + b.name + "' does not support extendLease",
308
- true);
309
- }
310
- return b.extendLease(job.jobId, additionalMs).then(function (ok) {
311
- if (ok) {
312
- _emit("system.queue.lease.extended", {
313
- metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
314
- });
315
- }
316
- return ok;
317
- });
318
- },
319
- progress: function (pct) {
320
- if (typeof pct !== "number" || !isFinite(pct)) return;
321
- var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
322
- var now = Date.now();
323
- // Always emit 0 and 100 (start/done markers); throttle the rest.
324
- var isMarker = clamped === 0 || clamped === 100;
325
- if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
326
- if (clamped === lastProgressValue && !isMarker) return;
327
- lastProgressEmitAt = now;
328
- lastProgressValue = clamped;
329
- observability.event("queue.progress", clamped, { queueName: queueName });
330
- _emit("system.queue.progress", {
331
- metadata: {
332
- queue: queueName, backend: b.name, jobId: job.jobId,
333
- attempt: job.attempts, traceId: job.traceId,
334
- percent: clamped,
335
- },
336
- });
337
- },
338
- };
339
- observability.tap("queue.consume",
340
- { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
341
- function () {
342
- return Promise.resolve()
343
- .then(function () { return handler(job, ctx); })
344
- .then(function () {
345
- return b.complete(job.jobId).then(function () {
346
- _emit("system.queue.consume.success", {
347
- metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
348
- });
349
- observability.event("queue.complete", 1, { queueName: queueName });
350
- });
351
- }, function (err) {
352
- var msg = (err && err.message) || String(err);
353
- var willRetry = job.attempts < job.maxAttempts;
354
- return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
355
- .then(function () {
356
- observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
357
- _emit("system.queue.consume.failure", {
358
- metadata: {
359
- queue: queueName, backend: b.name, jobId: job.jobId,
360
- attempt: job.attempts, traceId: job.traceId,
361
- maxAttempts: job.maxAttempts, willRetry: willRetry,
362
- },
363
- reason: msg,
364
- outcome: "failure",
365
- });
366
- // DLQ-write event when the job has exhausted its retries.
367
- // Operators wire this to their alerting / dashboards
368
- // — failed-after-retries is "needs human review" not
369
- // "in the normal flow." Audit chain captures the
370
- // final state for forensics.
371
- if (!willRetry) {
372
- _emit("system.queue.dlq.write", {
373
- metadata: {
374
- queue: queueName, backend: b.name, jobId: job.jobId,
375
- attempts: job.attempts, traceId: job.traceId,
376
- },
377
- reason: msg,
378
- outcome: "failure",
379
- });
380
- }
381
- });
382
- })
383
- .catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
384
- .then(function () { state.inFlight.delete(job.jobId); });
385
- }
386
- );
387
- })(jobs[i]);
388
- }
389
- await _pollSleep(fastPollMs);
390
- }
391
- })();
392
-
393
- return state;
394
- }
395
-
396
- var _QUEUE_BACKOFF_OPTS = {
397
- baseDelayMs: C.TIME.seconds(1),
398
- maxDelayMs: C.TIME.minutes(5),
399
- jitterFactor: 0, // queue uses deterministic backoff (no jitter) for predictability
400
- };
401
- function _backoffDelay(attempt) {
402
- return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
403
- }
404
-
405
- function size(queueName, opts) {
406
- _requireInit();
407
- return _backendFor(opts).size(queueName);
408
- }
409
-
410
- function purge(queueName, opts) {
411
- _requireInit();
412
- var b = _backendFor(opts);
413
- return b.purge(queueName).then(function (n) {
414
- _emit("system.queue.purge", {
415
- metadata: { queue: queueName, backend: b.name, deleted: n },
416
- });
417
- return n;
418
- });
419
- }
420
-
421
- // ---- Dead-letter queue ----
422
- //
423
- // `dlqList` returns failed-after-retries jobs for operator review;
424
- // `dlqRetry` resets a single job back to 'pending' so it gets picked
425
- // up by consumers again (operator-driven — never automatic).
426
-
427
- function dlqList(queueName, opts) {
428
- _requireInit();
429
- var b = _backendFor(opts);
430
- if (typeof b.dlqList !== "function") {
431
- return Promise.reject(_err("DLQ_UNSUPPORTED",
432
- "queue backend '" + b.name + "' does not support dlqList", true));
433
- }
434
- return b.dlqList(queueName, opts);
435
- }
436
-
437
- function dlqRetry(jobId, opts) {
438
- _requireInit();
439
- var b = _backendFor(opts);
440
- if (typeof b.dlqRetry !== "function") {
441
- return Promise.reject(_err("DLQ_UNSUPPORTED",
442
- "queue backend '" + b.name + "' does not support dlqRetry", true));
443
- }
444
- return b.dlqRetry(jobId).then(function (ok) {
445
- if (ok) {
446
- _emit("system.queue.dlq.retry", {
447
- metadata: { jobId: jobId, backend: b.name },
448
- });
449
- }
450
- return ok;
451
- });
452
- }
453
-
454
- function dlqSize(queueName, opts) {
455
- _requireInit();
456
- var b = _backendFor(opts);
457
- if (typeof b.dlqSize !== "function") {
458
- return Promise.reject(_err("DLQ_UNSUPPORTED",
459
- "queue backend '" + b.name + "' does not support dlqSize", true));
460
- }
461
- return b.dlqSize(queueName);
462
- }
463
-
464
- async function shutdown(opts) {
465
- if (!initialized) return;
466
- opts = opts || {};
467
- var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
468
- // Signal all consumers to stop
469
- consumers.forEach(function (c) { c.cancel(); });
470
- // Wait for in-flight handlers to complete
471
- var deadline = Date.now() + timeoutMs;
472
- while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
473
- if (Date.now() > deadline) break;
474
- await safeAsync.sleep(50);
475
- }
476
- consumers = [];
477
- if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
478
- }
479
-
480
- function listBackends() {
481
- _requireInit();
482
- return Object.keys(backends).map(function (name) {
483
- return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
484
- });
485
- }
486
-
487
- function _emit(action, info) {
488
- audit().safeEmit({ action: action, ...(info || {}) });
489
- }
490
-
491
- function _requireInit() {
492
- if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
493
- }
494
-
495
- function _resetForTest() {
496
- if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
497
- consumers.forEach(function (c) { c.cancel(); });
498
- consumers = [];
499
- backends = {};
500
- defaultBackend = null;
501
- initialized = false;
502
- audit.reset();
503
- }
504
-
505
- // enqueueFlow — atomic registration of a parent-child job graph.
506
- //
507
- // await b.queue.enqueueFlow({
508
- // queueName: "ingest",
509
- // children: [
510
- // { name: "fetch", payload: { url } },
511
- // { name: "transform", payload: { ... }, dependsOn: ["fetch"] },
512
- // { name: "publish", payload: { ... }, dependsOn: ["transform"] },
513
- // ],
514
- // });
515
- //
516
- // Cycle detection runs at registration (throws at call site). Each child enters
517
- // the queue with availableAt = MAX_SAFE_INTEGER until parent completion
518
- // bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
519
- function enqueueFlow(spec) {
520
- _requireInit();
521
- if (!spec || typeof spec !== "object") {
522
- return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
523
- }
524
- if (typeof spec.queueName !== "string" || !spec.queueName) {
525
- return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
526
- }
527
- if (!Array.isArray(spec.children) || spec.children.length === 0) {
528
- return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
529
- }
530
- // Validate each child's shape.
531
- var byName = {};
532
- for (var i = 0; i < spec.children.length; i++) {
533
- var c = spec.children[i];
534
- if (!c || typeof c !== "object") {
535
- return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
536
- }
537
- if (typeof c.name !== "string" || !c.name) {
538
- return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
539
- }
540
- if (byName[c.name]) {
541
- return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
542
- }
543
- byName[c.name] = c;
544
- if (c.dependsOn !== undefined) {
545
- if (!Array.isArray(c.dependsOn)) {
546
- return Promise.reject(_err("BAD_FLOW",
547
- "children[" + i + "].dependsOn must be an array of names", true));
548
- }
549
- for (var di = 0; di < c.dependsOn.length; di++) {
550
- if (typeof c.dependsOn[di] !== "string") {
551
- return Promise.reject(_err("BAD_FLOW",
552
- "children[" + i + "].dependsOn[" + di + "] must be a string name", true));
553
- }
554
- }
555
- }
556
- }
557
- // Cycle detection — depth-first traversal with visited set.
558
- function _visit(name, stack) {
559
- if (stack.indexOf(name) !== -1) {
560
- throw _err("FLOW_CYCLE", "flow cycle detected: " +
561
- stack.concat([name]).join(" → "), true);
562
- }
563
- var child = byName[name];
564
- if (!child || !child.dependsOn) return;
565
- var nextStack = stack.concat([name]);
566
- for (var k = 0; k < child.dependsOn.length; k++) {
567
- var dep = child.dependsOn[k];
568
- if (!byName[dep]) {
569
- throw _err("FLOW_UNKNOWN_DEP",
570
- "child '" + name + "' dependsOn unknown name '" + dep + "'", true);
571
- }
572
- _visit(dep, nextStack);
573
- }
574
- }
575
- try {
576
- var names = Object.keys(byName);
577
- for (var n = 0; n < names.length; n++) _visit(names[n], []);
578
- } catch (e) {
579
- return Promise.reject(e);
580
- }
581
-
582
- var flowId = "flow-" + crypto.generateToken(C.BYTES.bytes(8));
583
-
584
- return observability.tap("queue.enqueueFlow",
585
- { queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
586
- async function () {
587
- var jobs = [];
588
- // Two-pass insert: first pass enqueues all children with their
589
- // names attached so the second pass can write dependsOn jobIds
590
- // resolved by name. Children with deps land at MAX_SAFE_INTEGER
591
- // availableAt automatically (see queue-local enqueue logic).
592
- var nameToJobId = {};
593
- for (var p = 0; p < spec.children.length; p++) {
594
- var ch = spec.children[p];
595
- // Hold off setting dependsOn until we know all sibling jobIds.
596
- var enqOpts = {
597
- flowId: flowId,
598
- flowChildName: ch.name,
599
- priority: ch.priority || 0,
600
- classification: ch.classification || null,
601
- traceId: ch.traceId || null,
602
- maxAttempts: ch.maxAttempts,
603
- // dependsOn intentionally omitted on first pass — will be patched
604
- // in via direct UPDATE after all jobIds are known. This means
605
- // root children (no deps) are immediately leaseable; deps-bearing
606
- // children get patched to MAX_SAFE_INTEGER via second pass.
607
- };
608
- var result = await enqueue(spec.queueName, ch.payload, enqOpts);
609
- nameToJobId[ch.name] = result.jobId;
610
- jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
611
- }
612
- // Second pass: write dependsOn (translated to jobIds) for children
613
- // that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
614
- for (var q = 0; q < jobs.length; q++) {
615
- var j = jobs[q];
616
- if (j.dependsOn.length === 0) continue;
617
- var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
618
- await clusterStorage.execute(
619
- "UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
620
- [JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
621
- );
622
- }
623
- _emit("system.queue.flow.enqueue", {
624
- metadata: {
625
- queue: spec.queueName,
626
- flowId: flowId,
627
- childCount: spec.children.length,
628
- },
629
- });
630
- return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
631
- }
632
- );
633
- }
634
-
635
- // bootFromEnv — env-driven init mirroring b.network.bootFromEnv and
636
- // b.logStream.bootFromEnv. Reads the BLAMEJS_QUEUE_* env vars and
637
- // calls queue.init({ backends }) accordingly. Operators get a working
638
- // queue backend without writing build-app code.
639
- //
640
- // BLAMEJS_QUEUE_PROTOCOL local | redis (default: local)
641
- // BLAMEJS_QUEUE_REDIS_URL redis://host:port/db (required when protocol=redis)
642
- // BLAMEJS_QUEUE_REDIS_PASSWORD auth password
643
- // BLAMEJS_QUEUE_REDIS_USERNAME ACL username (optional)
644
- // BLAMEJS_QUEUE_REDIS_TLS "1"/"true" forces TLS (else inferred from rediss://)
645
- // BLAMEJS_QUEUE_REDIS_KEY_PREFIX key prefix (default "blamejs:queue")
646
- function bootFromEnv(opts) {
647
- opts = opts || {};
648
- var env = opts.env || process.env;
649
- if (initialized) return;
650
- var protocol = env.BLAMEJS_QUEUE_PROTOCOL || "local";
651
- var backendCfg;
652
- if (protocol === "local") {
653
- backendCfg = { protocol: "local" };
654
- } else if (protocol === "redis") {
655
- var url = env.BLAMEJS_QUEUE_REDIS_URL;
656
- if (!url) {
657
- throw _err("INVALID_CONFIG",
658
- "queue.bootFromEnv: BLAMEJS_QUEUE_REDIS_URL is required when BLAMEJS_QUEUE_PROTOCOL=redis",
659
- true);
660
- }
661
- var tlsRaw = env.BLAMEJS_QUEUE_REDIS_TLS;
662
- var tls = tlsRaw === "1" || tlsRaw === "true";
663
- backendCfg = {
664
- protocol: "redis",
665
- url: url,
666
- password: env.BLAMEJS_QUEUE_REDIS_PASSWORD || null,
667
- username: env.BLAMEJS_QUEUE_REDIS_USERNAME || null,
668
- tls: tlsRaw !== undefined ? tls : undefined, // undefined → inferred from rediss://
669
- keyPrefix: env.BLAMEJS_QUEUE_REDIS_KEY_PREFIX || undefined,
670
- };
671
- } else {
672
- throw _err("INVALID_CONFIG",
673
- "queue.bootFromEnv: BLAMEJS_QUEUE_PROTOCOL must be 'local' or 'redis', got '" + protocol + "'",
674
- true);
675
- }
676
- init({ backends: { default: backendCfg }, defaultBackend: "default" });
677
- }
678
-
679
- module.exports = {
680
- init: init,
681
- bootFromEnv: bootFromEnv,
682
- enqueue: enqueue,
683
- enqueueFlow: enqueueFlow,
684
- consume: consume,
685
- size: size,
686
- purge: purge,
687
- shutdown: shutdown,
688
- listBackends: listBackends,
689
- dlqList: dlqList,
690
- dlqRetry: dlqRetry,
691
- dlqSize: dlqSize,
692
- PROTOCOLS: dispatcher.protocols,
693
- DEFERRED_PROTOCOLS: dispatcher.deferred,
694
- _resetForTest: _resetForTest,
695
- };
1
+ "use strict";
2
+ /**
3
+ * Queue dispatcher — pluggable job queue with retry, lease semantics, and
4
+ * graceful shutdown.
5
+ *
6
+ * Same dispatcher pattern as object-store: backends are configured per-name,
7
+ * each with a protocol + protocol-specific options. The built-in 'local'
8
+ * protocol is SQLite-backed (baked into the framework's main DB).
9
+ * External protocols (redis, sqs, amqp, nats) are listed as deferred and
10
+ * surface a clear error when selected.
11
+ *
12
+ * Public API:
13
+ * queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
14
+ * queue.enqueue(queueName, payload, opts?)
15
+ * → { jobId, queueName, ... }
16
+ * queue.consume(queueName, handler, opts?)
17
+ * → consumer handle (with cancel())
18
+ * queue.size(queueName, opts?) → number (pending + inflight)
19
+ * queue.purge(queueName, opts?) → number deleted
20
+ * queue.shutdown(opts?) → drain handlers gracefully
21
+ * queue.listBackends() → [{ name, protocol }]
22
+ *
23
+ * Job lifecycle:
24
+ * enqueued (status='pending')
25
+ * ↓ availableAt reached + consumer leases
26
+ * inflight (status='inflight', lease expires after leaseDurationMs)
27
+ * ↓ handler returns ↓ handler throws
28
+ * done (status='done') if attempts < maxAttempts:
29
+ * pending (with backoff)
30
+ * else:
31
+ * failed (status='failed')
32
+ */
33
+ var C = require("./constants");
34
+ var clusterStorage = require("./cluster-storage");
35
+ var crypto = require("./crypto");
36
+ var lazyRequire = require("./lazy-require");
37
+ var { boot } = require("./log");
38
+ var numericChecks = require("./numeric-checks");
39
+ var observability = require("./observability");
40
+ var protocolDispatcher = require("./protocol-dispatcher");
41
+ var localProto = require("./queue-local");
42
+ var redisProto = require("./queue-redis");
43
+ var sqsProto = require("./queue-sqs");
44
+ var retryHelper = require("./retry");
45
+ var safeAsync = require("./safe-async");
46
+ var { QueueError } = require("./framework-error");
47
+
48
+ var log = boot("queue");
49
+
50
+ var dispatcher = protocolDispatcher.create({
51
+ name: "queue",
52
+ errorClass: QueueError,
53
+ protocols: { "local": localProto, "redis": redisProto, "sqs": sqsProto },
54
+ deferred: {
55
+ "amqp": { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
56
+ "nats": { description: "NATS JetStream" },
57
+ },
58
+ fallbackProtocol: "local",
59
+ });
60
+
61
+ var _err = QueueError.factory;
62
+
63
+ var audit = lazyRequire(function () { return require("./audit"); });
64
+
65
+ var initialized = false;
66
+ var backends = {};
67
+ var defaultBackend = null;
68
+ var consumers = []; // [{ queueName, backendName, cancel(), running, inFlight: Set }]
69
+ var sweepTimer = null;
70
+
71
+ function init(opts) {
72
+ if (initialized) return;
73
+ if (!opts || !opts.backends) {
74
+ throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
75
+ }
76
+
77
+ backends = {};
78
+ // IIFE per-iteration so each backend's wrappers close over its own
79
+ // raw / breaker / cfg. With `var` (function-scoped) those bindings
80
+ // would otherwise be shared across iterations and every wrapper
81
+ // would end up using the LAST backend's breaker + retry config —
82
+ // multi-backend isolation broken without a single error surfacing.
83
+ Object.keys(opts.backends).forEach(function (name) {
84
+ var cfg = opts.backends[name];
85
+ var proto = dispatcher.resolve(cfg.protocol);
86
+ var raw = proto.create(cfg);
87
+ var breaker = new retryHelper.CircuitBreaker(
88
+ "queue:" + name,
89
+ cfg.breaker
90
+ );
91
+
92
+ // Wrap mutating ops with retry + breaker (idempotent only — enqueue and
93
+ // complete are safe to retry; lease isn't because partial-double-lease
94
+ // is dangerous, so it goes through the breaker but not retry).
95
+ function wrapWithRetry(fn) {
96
+ return function () {
97
+ var args = Array.prototype.slice.call(arguments);
98
+ return retryHelper.withRetry(function () {
99
+ return breaker.wrap(function () { return fn.apply(raw, args); });
100
+ }, cfg.retry);
101
+ };
102
+ }
103
+ function wrapBreakerOnly(fn) {
104
+ return function () {
105
+ var args = Array.prototype.slice.call(arguments);
106
+ return breaker.wrap(function () { return fn.apply(raw, args); });
107
+ };
108
+ }
109
+
110
+ backends[name] = {
111
+ name: name,
112
+ protocol: cfg.protocol,
113
+ breaker: breaker,
114
+ raw: raw,
115
+ enqueue: wrapWithRetry(raw.enqueue),
116
+ lease: wrapBreakerOnly(raw.lease),
117
+ extendLease: raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
118
+ complete: wrapWithRetry(raw.complete),
119
+ fail: wrapWithRetry(raw.fail),
120
+ sweepExpired: raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
121
+ size: wrapWithRetry(raw.size),
122
+ purge: wrapWithRetry(raw.purge),
123
+ dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
124
+ dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
125
+ dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
126
+ };
127
+ });
128
+
129
+ defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
130
+
131
+ // Sweep expired leases periodically (every 30s) so crashed-handler jobs
132
+ // get re-pended.
133
+ sweepTimer = safeAsync.repeating(function () {
134
+ Object.keys(backends).forEach(function (n) {
135
+ if (backends[n].sweepExpired) {
136
+ backends[n].sweepExpired().catch(function () { /* best effort */ });
137
+ }
138
+ });
139
+ }, C.TIME.seconds(30), { name: "queue-sweep" });
140
+
141
+ initialized = true;
142
+ }
143
+
144
+ function _backendFor(opts) {
145
+ opts = opts || {};
146
+ var name = opts.backend || defaultBackend;
147
+ var b = backends[name];
148
+ if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
149
+ return b;
150
+ }
151
+
152
+ // ---- Public API ----
153
+
154
+ function enqueue(queueName, payload, opts) {
155
+ _requireInit();
156
+ if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
157
+ opts = opts || {};
158
+ var b = _backendFor(opts);
159
+ return observability.tap("queue.enqueue",
160
+ { queueName: queueName, backend: b.name },
161
+ function () {
162
+ return b.enqueue(queueName, payload, opts).then(function (result) {
163
+ _emit("system.queue.enqueue", {
164
+ metadata: {
165
+ queue: queueName,
166
+ backend: b.name,
167
+ jobId: result.jobId,
168
+ classification: result.classification,
169
+ traceId: opts.traceId,
170
+ delaySeconds: opts.delaySeconds || 0,
171
+ },
172
+ });
173
+ return result;
174
+ });
175
+ }
176
+ );
177
+ }
178
+
179
+ function consume(queueName, handler, opts) {
180
+ _requireInit();
181
+ if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
182
+ if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
183
+ opts = opts || {};
184
+ var b = _backendFor(opts);
185
+ var concurrency = opts.concurrency || 1;
186
+ var leaseDurationMs = opts.leaseDurationMs || C.TIME.seconds(30);
187
+ var pollIntervalMs = opts.pollIntervalMs || C.TIME.seconds(1);
188
+ var fastPollMs = opts.fastPollMs || 50;
189
+
190
+ // Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
191
+ // start within any rolling perSeconds window. Token-bucket-style
192
+ // accounting keeps it cheap (just a sliding deque of timestamps).
193
+ var rateLimit = null;
194
+ if (opts.rateLimit) {
195
+ var rlMax = opts.rateLimit.max;
196
+ var rlPer = opts.rateLimit.perSeconds;
197
+ // NaN, Infinity, 0, negatives, fractional max all produce undefined
198
+ // throttling math (NaN deque comparisons, perma-locked queues,
199
+ // perma-open windows). Reject at config time.
200
+ if (!numericChecks.isPositiveInt(rlMax) || !numericChecks.isPositiveFinite(rlPer)) {
201
+ throw _err("BAD_RATE_LIMIT",
202
+ "consume({ rateLimit }): expected { max: positive integer, perSeconds: positive finite number }, got " +
203
+ JSON.stringify(opts.rateLimit), true);
204
+ }
205
+ rateLimit = {
206
+ max: rlMax,
207
+ windowMs: C.TIME.seconds(rlPer),
208
+ timestamps: [],
209
+ };
210
+ }
211
+ function _rateLimitWaitMs() {
212
+ if (!rateLimit) return 0;
213
+ var now = Date.now();
214
+ var cutoff = now - rateLimit.windowMs;
215
+ while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
216
+ rateLimit.timestamps.shift();
217
+ }
218
+ if (rateLimit.timestamps.length < rateLimit.max) return 0;
219
+ return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
220
+ }
221
+ function _rateLimitConsume() {
222
+ if (rateLimit) rateLimit.timestamps.push(Date.now());
223
+ }
224
+
225
+ // Progress audit-emit rate-limit — protect the audit chain from a
226
+ // chatty handler that calls progress() every loop iteration.
227
+ var PROGRESS_MIN_INTERVAL_MS = 250;
228
+
229
+ // Each consumer has its own AbortController so cancel() unblocks any
230
+ // in-flight poll-sleep immediately rather than waiting up to
231
+ // pollIntervalMs (default 1s) for the next while-loop iteration.
232
+ var abortCtrl = new AbortController();
233
+ var state = {
234
+ queueName: queueName,
235
+ backendName: b.name,
236
+ cancelled: false,
237
+ inFlight: new Set(),
238
+ abortCtrl: abortCtrl,
239
+ cancel: function () {
240
+ state.cancelled = true;
241
+ try { abortCtrl.abort(); }
242
+ catch (e) { log.debug("cancel-cleanup-failed", { op: "abortCtrl.abort", error: e.message }); }
243
+ },
244
+ };
245
+ consumers.push(state);
246
+
247
+ (async function loop() {
248
+ // Helper — sleep with cancellation. On abort, returns instead of
249
+ // rejecting so the next while-iteration sees `state.cancelled` and
250
+ // exits cleanly.
251
+ async function _pollSleep(ms) {
252
+ try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
253
+ catch (_e) { /* aborted — loop condition will catch it */ }
254
+ }
255
+ while (!state.cancelled) {
256
+ // Don't lease more than (concurrency - inFlight) at a time
257
+ var slots = concurrency - state.inFlight.size;
258
+ if (slots <= 0) {
259
+ await _pollSleep(fastPollMs);
260
+ continue;
261
+ }
262
+ // If rate-limited and we'd exceed the budget, sleep until the
263
+ // next slot opens. We lease at most `max - currentTokens` jobs to
264
+ // stay under the cap.
265
+ if (rateLimit) {
266
+ var wait = _rateLimitWaitMs();
267
+ if (wait > 0) {
268
+ await _pollSleep(Math.min(wait, pollIntervalMs));
269
+ continue;
270
+ }
271
+ var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
272
+ if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
273
+ }
274
+ var jobs;
275
+ try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
276
+ catch {
277
+ // Backend down (breaker open, etc.) — back off
278
+ await _pollSleep(pollIntervalMs);
279
+ continue;
280
+ }
281
+ if (!jobs || jobs.length === 0) {
282
+ await _pollSleep(pollIntervalMs);
283
+ continue;
284
+ }
285
+ for (var i = 0; i < jobs.length; i++) {
286
+ observability.event("queue.lease", 1, { queueName: queueName });
287
+ (function (job) {
288
+ state.inFlight.add(job.jobId);
289
+ _emit("system.queue.consume.start", {
290
+ metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
291
+ });
292
+ // Consume a rate-limit slot at handler-start so the budget
293
+ // tracks invocation rate, not lease rate (a single lease that
294
+ // splits work across many sub-units doesn't double-count).
295
+ _rateLimitConsume();
296
+
297
+ // Handler context — second arg to handler. Carries
298
+ // ctx.extendLease(ms) for long-running handlers and
299
+ // ctx.progress(0..100) for surfacing job progress to the
300
+ // audit chain (rate-limited so chatty handlers don't drown it).
301
+ var lastProgressEmitAt = 0;
302
+ var lastProgressValue = -1;
303
+ var ctx = {
304
+ extendLease: function (additionalMs) {
305
+ if (typeof b.extendLease !== "function") {
306
+ throw _err("EXTEND_LEASE_UNSUPPORTED",
307
+ "queue backend '" + b.name + "' does not support extendLease",
308
+ true);
309
+ }
310
+ return b.extendLease(job.jobId, additionalMs).then(function (ok) {
311
+ if (ok) {
312
+ _emit("system.queue.lease.extended", {
313
+ metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
314
+ });
315
+ }
316
+ return ok;
317
+ });
318
+ },
319
+ progress: function (pct) {
320
+ if (typeof pct !== "number" || !isFinite(pct)) return;
321
+ var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
322
+ var now = Date.now();
323
+ // Always emit 0 and 100 (start/done markers); throttle the rest.
324
+ var isMarker = clamped === 0 || clamped === 100;
325
+ if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
326
+ if (clamped === lastProgressValue && !isMarker) return;
327
+ lastProgressEmitAt = now;
328
+ lastProgressValue = clamped;
329
+ observability.event("queue.progress", clamped, { queueName: queueName });
330
+ _emit("system.queue.progress", {
331
+ metadata: {
332
+ queue: queueName, backend: b.name, jobId: job.jobId,
333
+ attempt: job.attempts, traceId: job.traceId,
334
+ percent: clamped,
335
+ },
336
+ });
337
+ },
338
+ };
339
+ observability.tap("queue.consume",
340
+ { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
341
+ function () {
342
+ return Promise.resolve()
343
+ .then(function () { return handler(job, ctx); })
344
+ .then(function () {
345
+ return b.complete(job.jobId).then(function () {
346
+ _emit("system.queue.consume.success", {
347
+ metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
348
+ });
349
+ observability.event("queue.complete", 1, { queueName: queueName });
350
+ });
351
+ }, function (err) {
352
+ var msg = (err && err.message) || String(err);
353
+ var willRetry = job.attempts < job.maxAttempts;
354
+ return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
355
+ .then(function () {
356
+ observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
357
+ _emit("system.queue.consume.failure", {
358
+ metadata: {
359
+ queue: queueName, backend: b.name, jobId: job.jobId,
360
+ attempt: job.attempts, traceId: job.traceId,
361
+ maxAttempts: job.maxAttempts, willRetry: willRetry,
362
+ },
363
+ reason: msg,
364
+ outcome: "failure",
365
+ });
366
+ // DLQ-write event when the job has exhausted its retries.
367
+ // Operators wire this to their alerting / dashboards
368
+ // — failed-after-retries is "needs human review" not
369
+ // "in the normal flow." Audit chain captures the
370
+ // final state for forensics.
371
+ if (!willRetry) {
372
+ _emit("system.queue.dlq.write", {
373
+ metadata: {
374
+ queue: queueName, backend: b.name, jobId: job.jobId,
375
+ attempts: job.attempts, traceId: job.traceId,
376
+ },
377
+ reason: msg,
378
+ outcome: "failure",
379
+ });
380
+ }
381
+ });
382
+ })
383
+ .catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
384
+ .then(function () { state.inFlight.delete(job.jobId); });
385
+ }
386
+ );
387
+ })(jobs[i]);
388
+ }
389
+ await _pollSleep(fastPollMs);
390
+ }
391
+ })();
392
+
393
+ return state;
394
+ }
395
+
396
+ var _QUEUE_BACKOFF_OPTS = {
397
+ baseDelayMs: C.TIME.seconds(1),
398
+ maxDelayMs: C.TIME.minutes(5),
399
+ jitterFactor: 0, // queue uses deterministic backoff (no jitter) for predictability
400
+ };
401
+ function _backoffDelay(attempt) {
402
+ return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
403
+ }
404
+
405
+ function size(queueName, opts) {
406
+ _requireInit();
407
+ return _backendFor(opts).size(queueName);
408
+ }
409
+
410
+ function purge(queueName, opts) {
411
+ _requireInit();
412
+ var b = _backendFor(opts);
413
+ return b.purge(queueName).then(function (n) {
414
+ _emit("system.queue.purge", {
415
+ metadata: { queue: queueName, backend: b.name, deleted: n },
416
+ });
417
+ return n;
418
+ });
419
+ }
420
+
421
+ // ---- Dead-letter queue ----
422
+ //
423
+ // `dlqList` returns failed-after-retries jobs for operator review;
424
+ // `dlqRetry` resets a single job back to 'pending' so it gets picked
425
+ // up by consumers again (operator-driven — never automatic).
426
+
427
+ function dlqList(queueName, opts) {
428
+ _requireInit();
429
+ var b = _backendFor(opts);
430
+ if (typeof b.dlqList !== "function") {
431
+ return Promise.reject(_err("DLQ_UNSUPPORTED",
432
+ "queue backend '" + b.name + "' does not support dlqList", true));
433
+ }
434
+ return b.dlqList(queueName, opts);
435
+ }
436
+
437
+ function dlqRetry(jobId, opts) {
438
+ _requireInit();
439
+ var b = _backendFor(opts);
440
+ if (typeof b.dlqRetry !== "function") {
441
+ return Promise.reject(_err("DLQ_UNSUPPORTED",
442
+ "queue backend '" + b.name + "' does not support dlqRetry", true));
443
+ }
444
+ return b.dlqRetry(jobId).then(function (ok) {
445
+ if (ok) {
446
+ _emit("system.queue.dlq.retry", {
447
+ metadata: { jobId: jobId, backend: b.name },
448
+ });
449
+ }
450
+ return ok;
451
+ });
452
+ }
453
+
454
+ function dlqSize(queueName, opts) {
455
+ _requireInit();
456
+ var b = _backendFor(opts);
457
+ if (typeof b.dlqSize !== "function") {
458
+ return Promise.reject(_err("DLQ_UNSUPPORTED",
459
+ "queue backend '" + b.name + "' does not support dlqSize", true));
460
+ }
461
+ return b.dlqSize(queueName);
462
+ }
463
+
464
+ async function shutdown(opts) {
465
+ if (!initialized) return;
466
+ opts = opts || {};
467
+ var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
468
+ // Signal all consumers to stop
469
+ consumers.forEach(function (c) { c.cancel(); });
470
+ // Wait for in-flight handlers to complete
471
+ var deadline = Date.now() + timeoutMs;
472
+ while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
473
+ if (Date.now() > deadline) break;
474
+ await safeAsync.sleep(50);
475
+ }
476
+ consumers = [];
477
+ if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
478
+ }
479
+
480
+ function listBackends() {
481
+ _requireInit();
482
+ return Object.keys(backends).map(function (name) {
483
+ return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
484
+ });
485
+ }
486
+
487
+ function _emit(action, info) {
488
+ audit().safeEmit({ action: action, ...(info || {}) });
489
+ }
490
+
491
+ function _requireInit() {
492
+ if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
493
+ }
494
+
495
+ function _resetForTest() {
496
+ if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
497
+ consumers.forEach(function (c) { c.cancel(); });
498
+ consumers = [];
499
+ backends = {};
500
+ defaultBackend = null;
501
+ initialized = false;
502
+ audit.reset();
503
+ }
504
+
505
+ // enqueueFlow — atomic registration of a parent-child job graph.
506
+ //
507
+ // await b.queue.enqueueFlow({
508
+ // queueName: "ingest",
509
+ // children: [
510
+ // { name: "fetch", payload: { url } },
511
+ // { name: "transform", payload: { ... }, dependsOn: ["fetch"] },
512
+ // { name: "publish", payload: { ... }, dependsOn: ["transform"] },
513
+ // ],
514
+ // });
515
+ //
516
+ // Cycle detection runs at registration (throws at call site). Each child enters
517
+ // the queue with availableAt = MAX_SAFE_INTEGER until parent completion
518
+ // bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
519
+ function enqueueFlow(spec) {
520
+ _requireInit();
521
+ if (!spec || typeof spec !== "object") {
522
+ return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
523
+ }
524
+ if (typeof spec.queueName !== "string" || !spec.queueName) {
525
+ return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
526
+ }
527
+ if (!Array.isArray(spec.children) || spec.children.length === 0) {
528
+ return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
529
+ }
530
+ // Validate each child's shape.
531
+ var byName = {};
532
+ for (var i = 0; i < spec.children.length; i++) {
533
+ var c = spec.children[i];
534
+ if (!c || typeof c !== "object") {
535
+ return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
536
+ }
537
+ if (typeof c.name !== "string" || !c.name) {
538
+ return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
539
+ }
540
+ if (byName[c.name]) {
541
+ return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
542
+ }
543
+ byName[c.name] = c;
544
+ if (c.dependsOn !== undefined) {
545
+ if (!Array.isArray(c.dependsOn)) {
546
+ return Promise.reject(_err("BAD_FLOW",
547
+ "children[" + i + "].dependsOn must be an array of names", true));
548
+ }
549
+ for (var di = 0; di < c.dependsOn.length; di++) {
550
+ if (typeof c.dependsOn[di] !== "string") {
551
+ return Promise.reject(_err("BAD_FLOW",
552
+ "children[" + i + "].dependsOn[" + di + "] must be a string name", true));
553
+ }
554
+ }
555
+ }
556
+ }
557
+ // Cycle detection — depth-first traversal with visited set.
558
+ function _visit(name, stack) {
559
+ if (stack.indexOf(name) !== -1) {
560
+ throw _err("FLOW_CYCLE", "flow cycle detected: " +
561
+ stack.concat([name]).join(" → "), true);
562
+ }
563
+ var child = byName[name];
564
+ if (!child || !child.dependsOn) return;
565
+ var nextStack = stack.concat([name]);
566
+ for (var k = 0; k < child.dependsOn.length; k++) {
567
+ var dep = child.dependsOn[k];
568
+ if (!byName[dep]) {
569
+ throw _err("FLOW_UNKNOWN_DEP",
570
+ "child '" + name + "' dependsOn unknown name '" + dep + "'", true);
571
+ }
572
+ _visit(dep, nextStack);
573
+ }
574
+ }
575
+ try {
576
+ var names = Object.keys(byName);
577
+ for (var n = 0; n < names.length; n++) _visit(names[n], []);
578
+ } catch (e) {
579
+ return Promise.reject(e);
580
+ }
581
+
582
+ var flowId = "flow-" + crypto.generateToken(C.BYTES.bytes(8));
583
+
584
+ return observability.tap("queue.enqueueFlow",
585
+ { queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
586
+ async function () {
587
+ var jobs = [];
588
+ // Two-pass insert: first pass enqueues all children with their
589
+ // names attached so the second pass can write dependsOn jobIds
590
+ // resolved by name. Children with deps land at MAX_SAFE_INTEGER
591
+ // availableAt automatically (see queue-local enqueue logic).
592
+ var nameToJobId = {};
593
+ for (var p = 0; p < spec.children.length; p++) {
594
+ var ch = spec.children[p];
595
+ // Hold off setting dependsOn until we know all sibling jobIds.
596
+ var enqOpts = {
597
+ flowId: flowId,
598
+ flowChildName: ch.name,
599
+ priority: ch.priority || 0,
600
+ classification: ch.classification || null,
601
+ traceId: ch.traceId || null,
602
+ maxAttempts: ch.maxAttempts,
603
+ // dependsOn intentionally omitted on first pass — will be patched
604
+ // in via direct UPDATE after all jobIds are known. This means
605
+ // root children (no deps) are immediately leaseable; deps-bearing
606
+ // children get patched to MAX_SAFE_INTEGER via second pass.
607
+ };
608
+ var result = await enqueue(spec.queueName, ch.payload, enqOpts);
609
+ nameToJobId[ch.name] = result.jobId;
610
+ jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
611
+ }
612
+ // Second pass: write dependsOn (translated to jobIds) for children
613
+ // that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
614
+ for (var q = 0; q < jobs.length; q++) {
615
+ var j = jobs[q];
616
+ if (j.dependsOn.length === 0) continue;
617
+ var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
618
+ await clusterStorage.execute(
619
+ "UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
620
+ [JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
621
+ );
622
+ }
623
+ _emit("system.queue.flow.enqueue", {
624
+ metadata: {
625
+ queue: spec.queueName,
626
+ flowId: flowId,
627
+ childCount: spec.children.length,
628
+ },
629
+ });
630
+ return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
631
+ }
632
+ );
633
+ }
634
+
635
+ // bootFromEnv — env-driven init mirroring b.network.bootFromEnv and
636
+ // b.logStream.bootFromEnv. Reads the BLAMEJS_QUEUE_* env vars and
637
+ // calls queue.init({ backends }) accordingly. Operators get a working
638
+ // queue backend without writing build-app code.
639
+ //
640
+ // BLAMEJS_QUEUE_PROTOCOL local | redis (default: local)
641
+ // BLAMEJS_QUEUE_REDIS_URL redis://host:port/db (required when protocol=redis)
642
+ // BLAMEJS_QUEUE_REDIS_PASSWORD auth password
643
+ // BLAMEJS_QUEUE_REDIS_USERNAME ACL username (optional)
644
+ // BLAMEJS_QUEUE_REDIS_TLS "1"/"true" forces TLS (else inferred from rediss://)
645
+ // BLAMEJS_QUEUE_REDIS_KEY_PREFIX key prefix (default "blamejs:queue")
646
+ function bootFromEnv(opts) {
647
+ opts = opts || {};
648
+ var env = opts.env || process.env;
649
+ if (initialized) return;
650
+ var protocol = env.BLAMEJS_QUEUE_PROTOCOL || "local";
651
+ var backendCfg;
652
+ if (protocol === "local") {
653
+ backendCfg = { protocol: "local" };
654
+ } else if (protocol === "redis") {
655
+ var url = env.BLAMEJS_QUEUE_REDIS_URL;
656
+ if (!url) {
657
+ throw _err("INVALID_CONFIG",
658
+ "queue.bootFromEnv: BLAMEJS_QUEUE_REDIS_URL is required when BLAMEJS_QUEUE_PROTOCOL=redis",
659
+ true);
660
+ }
661
+ var tlsRaw = env.BLAMEJS_QUEUE_REDIS_TLS;
662
+ var tls = tlsRaw === "1" || tlsRaw === "true";
663
+ backendCfg = {
664
+ protocol: "redis",
665
+ url: url,
666
+ password: env.BLAMEJS_QUEUE_REDIS_PASSWORD || null,
667
+ username: env.BLAMEJS_QUEUE_REDIS_USERNAME || null,
668
+ tls: tlsRaw !== undefined ? tls : undefined, // undefined → inferred from rediss://
669
+ keyPrefix: env.BLAMEJS_QUEUE_REDIS_KEY_PREFIX || undefined,
670
+ };
671
+ } else {
672
+ throw _err("INVALID_CONFIG",
673
+ "queue.bootFromEnv: BLAMEJS_QUEUE_PROTOCOL must be 'local' or 'redis', got '" + protocol + "'",
674
+ true);
675
+ }
676
+ init({ backends: { default: backendCfg }, defaultBackend: "default" });
677
+ }
678
+
679
+ module.exports = {
680
+ init: init,
681
+ bootFromEnv: bootFromEnv,
682
+ enqueue: enqueue,
683
+ enqueueFlow: enqueueFlow,
684
+ consume: consume,
685
+ size: size,
686
+ purge: purge,
687
+ shutdown: shutdown,
688
+ listBackends: listBackends,
689
+ dlqList: dlqList,
690
+ dlqRetry: dlqRetry,
691
+ dlqSize: dlqSize,
692
+ PROTOCOLS: dispatcher.protocols,
693
+ DEFERRED_PROTOCOLS: dispatcher.deferred,
694
+ _resetForTest: _resetForTest,
695
+ };