@blamejs/core 0.7.1 → 0.7.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/CHANGELOG.md +423 -389
  2. package/README.md +150 -149
  3. package/bin/blamejs.js +0 -0
  4. package/index.js +308 -282
  5. package/lib/api-key.js +660 -672
  6. package/lib/api-snapshot.js +338 -338
  7. package/lib/app-shutdown.js +385 -385
  8. package/lib/app.js +365 -365
  9. package/lib/archive.js +250 -250
  10. package/lib/atomic-file.js +544 -544
  11. package/lib/audit-chain.js +177 -177
  12. package/lib/audit-sign.js +344 -344
  13. package/lib/audit-tools.js +677 -677
  14. package/lib/audit.js +766 -766
  15. package/lib/auth/jwt.js +311 -311
  16. package/lib/auth/lockout.js +436 -436
  17. package/lib/auth/oauth.js +721 -721
  18. package/lib/auth/passkey.js +181 -181
  19. package/lib/auth/password.js +594 -594
  20. package/lib/backup/bundle.js +217 -217
  21. package/lib/backup/crypto.js +176 -176
  22. package/lib/backup/index.js +515 -515
  23. package/lib/backup/manifest.js +282 -282
  24. package/lib/break-glass.js +1338 -1338
  25. package/lib/bundler.js +441 -441
  26. package/lib/cache-redis.js +256 -266
  27. package/lib/cache.js +1206 -1211
  28. package/lib/canonical-json.js +115 -115
  29. package/lib/chain-writer.js +234 -234
  30. package/lib/cli-helpers.js +206 -206
  31. package/lib/cli.js +2334 -2334
  32. package/lib/cluster-provider-db.js +317 -317
  33. package/lib/cluster-storage.js +226 -226
  34. package/lib/cluster.js +703 -703
  35. package/lib/codepoint-class.js +196 -0
  36. package/lib/config-drift.js +301 -301
  37. package/lib/consent.js +222 -222
  38. package/lib/constants.js +191 -191
  39. package/lib/cookies.js +315 -315
  40. package/lib/credential-hash.js +322 -322
  41. package/lib/crypto.js +266 -266
  42. package/lib/csv.js +275 -286
  43. package/lib/db-declare-row-policy.js +267 -267
  44. package/lib/db-declare-view.js +420 -421
  45. package/lib/db-query.js +406 -406
  46. package/lib/db-schema.js +319 -319
  47. package/lib/db.js +1288 -1288
  48. package/lib/deprecate.js +222 -222
  49. package/lib/dev.js +335 -335
  50. package/lib/dual-control.js +473 -473
  51. package/lib/error-page.js +420 -420
  52. package/lib/external-db-migrate.js +441 -441
  53. package/lib/external-db.js +1061 -1061
  54. package/lib/file-type.js +273 -273
  55. package/lib/file-upload.js +1136 -0
  56. package/lib/forms.js +422 -422
  57. package/lib/framework-error.js +293 -202
  58. package/lib/framework-schema.js +717 -717
  59. package/lib/gate-contract.js +971 -0
  60. package/lib/guard-all.js +405 -0
  61. package/lib/guard-archive.js +739 -0
  62. package/lib/guard-csv.js +816 -0
  63. package/lib/guard-email.js +744 -0
  64. package/lib/guard-filename.js +724 -0
  65. package/lib/guard-html.js +976 -0
  66. package/lib/guard-json.js +729 -0
  67. package/lib/guard-markdown.js +586 -0
  68. package/lib/guard-svg.js +976 -0
  69. package/lib/guard-xml.js +405 -0
  70. package/lib/guard-yaml.js +529 -0
  71. package/lib/handlers.js +350 -350
  72. package/lib/http-client-cookie-jar.js +508 -508
  73. package/lib/http-client.js +1195 -1195
  74. package/lib/i18n.js +878 -878
  75. package/lib/jobs.js +185 -185
  76. package/lib/log-stream-cloudwatch.js +369 -369
  77. package/lib/log-stream-local.js +146 -146
  78. package/lib/log-stream-otlp-grpc.js +410 -410
  79. package/lib/log-stream-otlp.js +286 -286
  80. package/lib/log-stream-syslog.js +302 -302
  81. package/lib/log-stream-webhook.js +199 -199
  82. package/lib/log-stream.js +330 -330
  83. package/lib/log.js +500 -500
  84. package/lib/mail-bounce.js +528 -528
  85. package/lib/mail-dkim.js +369 -362
  86. package/lib/mail.js +981 -962
  87. package/lib/metrics.js +683 -683
  88. package/lib/middleware/api-encrypt.js +936 -573
  89. package/lib/middleware/attach-user.js +157 -157
  90. package/lib/middleware/body-parser.js +1170 -1091
  91. package/lib/middleware/bot-guard.js +178 -178
  92. package/lib/middleware/compression.js +452 -452
  93. package/lib/middleware/cors.js +314 -314
  94. package/lib/middleware/csp-nonce.js +348 -348
  95. package/lib/middleware/csrf-protect.js +316 -316
  96. package/lib/middleware/db-role-for.js +264 -269
  97. package/lib/middleware/health.js +392 -392
  98. package/lib/middleware/index.js +79 -79
  99. package/lib/middleware/rate-limit.js +358 -358
  100. package/lib/middleware/request-id.js +61 -61
  101. package/lib/middleware/request-log.js +168 -168
  102. package/lib/middleware/require-auth.js +104 -104
  103. package/lib/middleware/security-headers.js +116 -116
  104. package/lib/middleware/sse.js +166 -166
  105. package/lib/migrations.js +383 -383
  106. package/lib/mtls-ca.js +518 -518
  107. package/lib/mtls-engine-default.js +481 -481
  108. package/lib/network-dns.js +632 -632
  109. package/lib/network-heartbeat.js +290 -290
  110. package/lib/network-nts.js +574 -574
  111. package/lib/network-proxy.js +265 -265
  112. package/lib/network-tls.js +328 -328
  113. package/lib/network.js +233 -233
  114. package/lib/notify.js +612 -614
  115. package/lib/ntp-check.js +229 -229
  116. package/lib/numeric-bounds.js +111 -91
  117. package/lib/object-store/azure-blob-bucket-ops.js +349 -349
  118. package/lib/object-store/azure-blob.js +488 -451
  119. package/lib/object-store/gcs-bucket-ops.js +351 -351
  120. package/lib/object-store/gcs.js +519 -479
  121. package/lib/object-store/http-put.js +153 -153
  122. package/lib/object-store/index.js +197 -197
  123. package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
  124. package/lib/object-store/sigv4.js +903 -855
  125. package/lib/observability.js +151 -151
  126. package/lib/otel-export.js +269 -269
  127. package/lib/pagination.js +464 -464
  128. package/lib/parsers/index.js +80 -80
  129. package/lib/parsers/safe-env.js +642 -642
  130. package/lib/parsers/safe-ini.js +292 -292
  131. package/lib/parsers/safe-toml.js +784 -784
  132. package/lib/parsers/safe-xml.js +390 -390
  133. package/lib/parsers/safe-yaml.js +1015 -1015
  134. package/lib/permissions.js +708 -708
  135. package/lib/pqc-agent.js +87 -87
  136. package/lib/pqc-gate.js +279 -279
  137. package/lib/protobuf-encoder.js +190 -190
  138. package/lib/protocol-dispatcher.js +161 -161
  139. package/lib/pubsub-redis.js +167 -177
  140. package/lib/pubsub.js +429 -429
  141. package/lib/queue-local.js +476 -476
  142. package/lib/queue-redis.js +745 -752
  143. package/lib/queue-sqs.js +319 -319
  144. package/lib/queue.js +695 -695
  145. package/lib/redis-client.js +519 -489
  146. package/lib/request-helpers.js +340 -336
  147. package/lib/restore-bundle.js +237 -237
  148. package/lib/restore-rollback.js +259 -259
  149. package/lib/restore.js +409 -409
  150. package/lib/retry.js +376 -376
  151. package/lib/router.js +748 -748
  152. package/lib/safe-async.js +735 -735
  153. package/lib/safe-buffer.js +237 -237
  154. package/lib/safe-json.js +541 -541
  155. package/lib/safe-schema.js +1266 -1266
  156. package/lib/safe-url.js +159 -159
  157. package/lib/scheduler.js +706 -706
  158. package/lib/security-assert.js +373 -373
  159. package/lib/seeders.js +618 -630
  160. package/lib/session.js +478 -478
  161. package/lib/slug.js +269 -269
  162. package/lib/ssrf-guard.js +401 -401
  163. package/lib/static.js +879 -114
  164. package/lib/storage.js +471 -471
  165. package/lib/subject.js +281 -281
  166. package/lib/template.js +791 -791
  167. package/lib/testing.js +798 -798
  168. package/lib/time.js +310 -310
  169. package/lib/totp.js +302 -302
  170. package/lib/tracing.js +494 -494
  171. package/lib/uuid.js +132 -132
  172. package/lib/validate-opts.js +340 -270
  173. package/lib/vault/index.js +308 -308
  174. package/lib/vault/rotate.js +784 -784
  175. package/lib/vault/wrap.js +296 -296
  176. package/lib/vendor/noble-ciphers.cjs +9 -9
  177. package/lib/webhook.js +595 -598
  178. package/lib/websocket.js +1048 -1048
  179. package/package.json +77 -77
  180. package/sbom.cyclonedx.json +7 -7
package/lib/queue.js CHANGED
@@ -1,695 +1,695 @@
1
- "use strict";
2
- /**
3
- * Queue dispatcher — pluggable job queue with retry, lease semantics, and
4
- * graceful shutdown.
5
- *
6
- * Same dispatcher pattern as object-store: backends are configured per-name,
7
- * each with a protocol + protocol-specific options. The built-in 'local'
8
- * protocol is SQLite-backed (baked into the framework's main DB).
9
- * External protocols (redis, sqs, amqp, nats) are listed as deferred and
10
- * surface a clear error when selected.
11
- *
12
- * Public API:
13
- * queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
14
- * queue.enqueue(queueName, payload, opts?)
15
- * → { jobId, queueName, ... }
16
- * queue.consume(queueName, handler, opts?)
17
- * → consumer handle (with cancel())
18
- * queue.size(queueName, opts?) → number (pending + inflight)
19
- * queue.purge(queueName, opts?) → number deleted
20
- * queue.shutdown(opts?) → drain handlers gracefully
21
- * queue.listBackends() → [{ name, protocol }]
22
- *
23
- * Job lifecycle:
24
- * enqueued (status='pending')
25
- * ↓ availableAt reached + consumer leases
26
- * inflight (status='inflight', lease expires after leaseDurationMs)
27
- * ↓ handler returns ↓ handler throws
28
- * done (status='done') if attempts < maxAttempts:
29
- * pending (with backoff)
30
- * else:
31
- * failed (status='failed')
32
- */
33
- var C = require("./constants");
34
- var clusterStorage = require("./cluster-storage");
35
- var crypto = require("./crypto");
36
- var lazyRequire = require("./lazy-require");
37
- var { boot } = require("./log");
38
- var numericChecks = require("./numeric-checks");
39
- var observability = require("./observability");
40
- var protocolDispatcher = require("./protocol-dispatcher");
41
- var localProto = require("./queue-local");
42
- var redisProto = require("./queue-redis");
43
- var sqsProto = require("./queue-sqs");
44
- var retryHelper = require("./retry");
45
- var safeAsync = require("./safe-async");
46
- var { QueueError } = require("./framework-error");
47
-
48
- var log = boot("queue");
49
-
50
- var dispatcher = protocolDispatcher.create({
51
- name: "queue",
52
- errorClass: QueueError,
53
- protocols: { "local": localProto, "redis": redisProto, "sqs": sqsProto },
54
- deferred: {
55
- "amqp": { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
56
- "nats": { description: "NATS JetStream" },
57
- },
58
- fallbackProtocol: "local",
59
- });
60
-
61
- var _err = QueueError.factory;
62
-
63
- var audit = lazyRequire(function () { return require("./audit"); });
64
-
65
- var initialized = false;
66
- var backends = {};
67
- var defaultBackend = null;
68
- var consumers = []; // [{ queueName, backendName, cancel(), running, inFlight: Set }]
69
- var sweepTimer = null;
70
-
71
- function init(opts) {
72
- if (initialized) return;
73
- if (!opts || !opts.backends) {
74
- throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
75
- }
76
-
77
- backends = {};
78
- // IIFE per-iteration so each backend's wrappers close over its own
79
- // raw / breaker / cfg. With `var` (function-scoped) those bindings
80
- // would otherwise be shared across iterations and every wrapper
81
- // would end up using the LAST backend's breaker + retry config —
82
- // multi-backend isolation broken without a single error surfacing.
83
- Object.keys(opts.backends).forEach(function (name) {
84
- var cfg = opts.backends[name];
85
- var proto = dispatcher.resolve(cfg.protocol);
86
- var raw = proto.create(cfg);
87
- var breaker = new retryHelper.CircuitBreaker(
88
- "queue:" + name,
89
- cfg.breaker
90
- );
91
-
92
- // Wrap mutating ops with retry + breaker (idempotent only — enqueue and
93
- // complete are safe to retry; lease isn't because partial-double-lease
94
- // is dangerous, so it goes through the breaker but not retry).
95
- function wrapWithRetry(fn) {
96
- return function () {
97
- var args = Array.prototype.slice.call(arguments);
98
- return retryHelper.withRetry(function () {
99
- return breaker.wrap(function () { return fn.apply(raw, args); });
100
- }, cfg.retry);
101
- };
102
- }
103
- function wrapBreakerOnly(fn) {
104
- return function () {
105
- var args = Array.prototype.slice.call(arguments);
106
- return breaker.wrap(function () { return fn.apply(raw, args); });
107
- };
108
- }
109
-
110
- backends[name] = {
111
- name: name,
112
- protocol: cfg.protocol,
113
- breaker: breaker,
114
- raw: raw,
115
- enqueue: wrapWithRetry(raw.enqueue),
116
- lease: wrapBreakerOnly(raw.lease),
117
- extendLease: raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
118
- complete: wrapWithRetry(raw.complete),
119
- fail: wrapWithRetry(raw.fail),
120
- sweepExpired: raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
121
- size: wrapWithRetry(raw.size),
122
- purge: wrapWithRetry(raw.purge),
123
- dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
124
- dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
125
- dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
126
- };
127
- });
128
-
129
- defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
130
-
131
- // Sweep expired leases periodically (every 30s) so crashed-handler jobs
132
- // get re-pended.
133
- sweepTimer = safeAsync.repeating(function () {
134
- Object.keys(backends).forEach(function (n) {
135
- if (backends[n].sweepExpired) {
136
- backends[n].sweepExpired().catch(function () { /* best effort */ });
137
- }
138
- });
139
- }, C.TIME.seconds(30), { name: "queue-sweep" });
140
-
141
- initialized = true;
142
- }
143
-
144
- function _backendFor(opts) {
145
- opts = opts || {};
146
- var name = opts.backend || defaultBackend;
147
- var b = backends[name];
148
- if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
149
- return b;
150
- }
151
-
152
- // ---- Public API ----
153
-
154
- function enqueue(queueName, payload, opts) {
155
- _requireInit();
156
- if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
157
- opts = opts || {};
158
- var b = _backendFor(opts);
159
- return observability.tap("queue.enqueue",
160
- { queueName: queueName, backend: b.name },
161
- function () {
162
- return b.enqueue(queueName, payload, opts).then(function (result) {
163
- _emit("system.queue.enqueue", {
164
- metadata: {
165
- queue: queueName,
166
- backend: b.name,
167
- jobId: result.jobId,
168
- classification: result.classification,
169
- traceId: opts.traceId,
170
- delaySeconds: opts.delaySeconds || 0,
171
- },
172
- });
173
- return result;
174
- });
175
- }
176
- );
177
- }
178
-
179
- function consume(queueName, handler, opts) {
180
- _requireInit();
181
- if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
182
- if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
183
- opts = opts || {};
184
- var b = _backendFor(opts);
185
- var concurrency = opts.concurrency || 1;
186
- var leaseDurationMs = opts.leaseDurationMs || C.TIME.seconds(30);
187
- var pollIntervalMs = opts.pollIntervalMs || C.TIME.seconds(1);
188
- var fastPollMs = opts.fastPollMs || 50;
189
-
190
- // Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
191
- // start within any rolling perSeconds window. Token-bucket-style
192
- // accounting keeps it cheap (just a sliding deque of timestamps).
193
- var rateLimit = null;
194
- if (opts.rateLimit) {
195
- var rlMax = opts.rateLimit.max;
196
- var rlPer = opts.rateLimit.perSeconds;
197
- // NaN, Infinity, 0, negatives, fractional max all produce undefined
198
- // throttling math (NaN deque comparisons, perma-locked queues,
199
- // perma-open windows). Reject at config time.
200
- if (!numericChecks.isPositiveInt(rlMax) || !numericChecks.isPositiveFinite(rlPer)) {
201
- throw _err("BAD_RATE_LIMIT",
202
- "consume({ rateLimit }): expected { max: positive integer, perSeconds: positive finite number }, got " +
203
- JSON.stringify(opts.rateLimit), true);
204
- }
205
- rateLimit = {
206
- max: rlMax,
207
- windowMs: C.TIME.seconds(rlPer),
208
- timestamps: [],
209
- };
210
- }
211
- function _rateLimitWaitMs() {
212
- if (!rateLimit) return 0;
213
- var now = Date.now();
214
- var cutoff = now - rateLimit.windowMs;
215
- while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
216
- rateLimit.timestamps.shift();
217
- }
218
- if (rateLimit.timestamps.length < rateLimit.max) return 0;
219
- return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
220
- }
221
- function _rateLimitConsume() {
222
- if (rateLimit) rateLimit.timestamps.push(Date.now());
223
- }
224
-
225
- // Progress audit-emit rate-limit — protect the audit chain from a
226
- // chatty handler that calls progress() every loop iteration.
227
- var PROGRESS_MIN_INTERVAL_MS = 250;
228
-
229
- // Each consumer has its own AbortController so cancel() unblocks any
230
- // in-flight poll-sleep immediately rather than waiting up to
231
- // pollIntervalMs (default 1s) for the next while-loop iteration.
232
- var abortCtrl = new AbortController();
233
- var state = {
234
- queueName: queueName,
235
- backendName: b.name,
236
- cancelled: false,
237
- inFlight: new Set(),
238
- abortCtrl: abortCtrl,
239
- cancel: function () {
240
- state.cancelled = true;
241
- try { abortCtrl.abort(); }
242
- catch (e) { log.debug("cancel-cleanup-failed", { op: "abortCtrl.abort", error: e.message }); }
243
- },
244
- };
245
- consumers.push(state);
246
-
247
- (async function loop() {
248
- // Helper — sleep with cancellation. On abort, returns instead of
249
- // rejecting so the next while-iteration sees `state.cancelled` and
250
- // exits cleanly.
251
- async function _pollSleep(ms) {
252
- try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
253
- catch (_e) { /* aborted — loop condition will catch it */ }
254
- }
255
- while (!state.cancelled) {
256
- // Don't lease more than (concurrency - inFlight) at a time
257
- var slots = concurrency - state.inFlight.size;
258
- if (slots <= 0) {
259
- await _pollSleep(fastPollMs);
260
- continue;
261
- }
262
- // If rate-limited and we'd exceed the budget, sleep until the
263
- // next slot opens. We lease at most `max - currentTokens` jobs to
264
- // stay under the cap.
265
- if (rateLimit) {
266
- var wait = _rateLimitWaitMs();
267
- if (wait > 0) {
268
- await _pollSleep(Math.min(wait, pollIntervalMs));
269
- continue;
270
- }
271
- var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
272
- if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
273
- }
274
- var jobs;
275
- try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
276
- catch {
277
- // Backend down (breaker open, etc.) — back off
278
- await _pollSleep(pollIntervalMs);
279
- continue;
280
- }
281
- if (!jobs || jobs.length === 0) {
282
- await _pollSleep(pollIntervalMs);
283
- continue;
284
- }
285
- for (var i = 0; i < jobs.length; i++) {
286
- observability.event("queue.lease", 1, { queueName: queueName });
287
- (function (job) {
288
- state.inFlight.add(job.jobId);
289
- _emit("system.queue.consume.start", {
290
- metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
291
- });
292
- // Consume a rate-limit slot at handler-start so the budget
293
- // tracks invocation rate, not lease rate (a single lease that
294
- // splits work across many sub-units doesn't double-count).
295
- _rateLimitConsume();
296
-
297
- // Handler context — second arg to handler. Carries
298
- // ctx.extendLease(ms) for long-running handlers and
299
- // ctx.progress(0..100) for surfacing job progress to the
300
- // audit chain (rate-limited so chatty handlers don't drown it).
301
- var lastProgressEmitAt = 0;
302
- var lastProgressValue = -1;
303
- var ctx = {
304
- extendLease: function (additionalMs) {
305
- if (typeof b.extendLease !== "function") {
306
- throw _err("EXTEND_LEASE_UNSUPPORTED",
307
- "queue backend '" + b.name + "' does not support extendLease",
308
- true);
309
- }
310
- return b.extendLease(job.jobId, additionalMs).then(function (ok) {
311
- if (ok) {
312
- _emit("system.queue.lease.extended", {
313
- metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
314
- });
315
- }
316
- return ok;
317
- });
318
- },
319
- progress: function (pct) {
320
- if (typeof pct !== "number" || !isFinite(pct)) return;
321
- var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
322
- var now = Date.now();
323
- // Always emit 0 and 100 (start/done markers); throttle the rest.
324
- var isMarker = clamped === 0 || clamped === 100;
325
- if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
326
- if (clamped === lastProgressValue && !isMarker) return;
327
- lastProgressEmitAt = now;
328
- lastProgressValue = clamped;
329
- observability.event("queue.progress", clamped, { queueName: queueName });
330
- _emit("system.queue.progress", {
331
- metadata: {
332
- queue: queueName, backend: b.name, jobId: job.jobId,
333
- attempt: job.attempts, traceId: job.traceId,
334
- percent: clamped,
335
- },
336
- });
337
- },
338
- };
339
- observability.tap("queue.consume",
340
- { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
341
- function () {
342
- return Promise.resolve()
343
- .then(function () { return handler(job, ctx); })
344
- .then(function () {
345
- return b.complete(job.jobId).then(function () {
346
- _emit("system.queue.consume.success", {
347
- metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
348
- });
349
- observability.event("queue.complete", 1, { queueName: queueName });
350
- });
351
- }, function (err) {
352
- var msg = (err && err.message) || String(err);
353
- var willRetry = job.attempts < job.maxAttempts;
354
- return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
355
- .then(function () {
356
- observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
357
- _emit("system.queue.consume.failure", {
358
- metadata: {
359
- queue: queueName, backend: b.name, jobId: job.jobId,
360
- attempt: job.attempts, traceId: job.traceId,
361
- maxAttempts: job.maxAttempts, willRetry: willRetry,
362
- },
363
- reason: msg,
364
- outcome: "failure",
365
- });
366
- // DLQ-write event when the job has exhausted its retries.
367
- // Operators wire this to their alerting / dashboards
368
- // — failed-after-retries is "needs human review" not
369
- // "in the normal flow." Audit chain captures the
370
- // final state for forensics.
371
- if (!willRetry) {
372
- _emit("system.queue.dlq.write", {
373
- metadata: {
374
- queue: queueName, backend: b.name, jobId: job.jobId,
375
- attempts: job.attempts, traceId: job.traceId,
376
- },
377
- reason: msg,
378
- outcome: "failure",
379
- });
380
- }
381
- });
382
- })
383
- .catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
384
- .then(function () { state.inFlight.delete(job.jobId); });
385
- }
386
- );
387
- })(jobs[i]);
388
- }
389
- await _pollSleep(fastPollMs);
390
- }
391
- })();
392
-
393
- return state;
394
- }
395
-
396
- var _QUEUE_BACKOFF_OPTS = {
397
- baseDelayMs: C.TIME.seconds(1),
398
- maxDelayMs: C.TIME.minutes(5),
399
- jitterFactor: 0, // queue uses deterministic backoff (no jitter) for predictability
400
- };
401
- function _backoffDelay(attempt) {
402
- return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
403
- }
404
-
405
- function size(queueName, opts) {
406
- _requireInit();
407
- return _backendFor(opts).size(queueName);
408
- }
409
-
410
- function purge(queueName, opts) {
411
- _requireInit();
412
- var b = _backendFor(opts);
413
- return b.purge(queueName).then(function (n) {
414
- _emit("system.queue.purge", {
415
- metadata: { queue: queueName, backend: b.name, deleted: n },
416
- });
417
- return n;
418
- });
419
- }
420
-
421
- // ---- Dead-letter queue ----
422
- //
423
- // `dlqList` returns failed-after-retries jobs for operator review;
424
- // `dlqRetry` resets a single job back to 'pending' so it gets picked
425
- // up by consumers again (operator-driven — never automatic).
426
-
427
- function dlqList(queueName, opts) {
428
- _requireInit();
429
- var b = _backendFor(opts);
430
- if (typeof b.dlqList !== "function") {
431
- return Promise.reject(_err("DLQ_UNSUPPORTED",
432
- "queue backend '" + b.name + "' does not support dlqList", true));
433
- }
434
- return b.dlqList(queueName, opts);
435
- }
436
-
437
- function dlqRetry(jobId, opts) {
438
- _requireInit();
439
- var b = _backendFor(opts);
440
- if (typeof b.dlqRetry !== "function") {
441
- return Promise.reject(_err("DLQ_UNSUPPORTED",
442
- "queue backend '" + b.name + "' does not support dlqRetry", true));
443
- }
444
- return b.dlqRetry(jobId).then(function (ok) {
445
- if (ok) {
446
- _emit("system.queue.dlq.retry", {
447
- metadata: { jobId: jobId, backend: b.name },
448
- });
449
- }
450
- return ok;
451
- });
452
- }
453
-
454
- function dlqSize(queueName, opts) {
455
- _requireInit();
456
- var b = _backendFor(opts);
457
- if (typeof b.dlqSize !== "function") {
458
- return Promise.reject(_err("DLQ_UNSUPPORTED",
459
- "queue backend '" + b.name + "' does not support dlqSize", true));
460
- }
461
- return b.dlqSize(queueName);
462
- }
463
-
464
- async function shutdown(opts) {
465
- if (!initialized) return;
466
- opts = opts || {};
467
- var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
468
- // Signal all consumers to stop
469
- consumers.forEach(function (c) { c.cancel(); });
470
- // Wait for in-flight handlers to complete
471
- var deadline = Date.now() + timeoutMs;
472
- while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
473
- if (Date.now() > deadline) break;
474
- await safeAsync.sleep(50);
475
- }
476
- consumers = [];
477
- if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
478
- }
479
-
480
- function listBackends() {
481
- _requireInit();
482
- return Object.keys(backends).map(function (name) {
483
- return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
484
- });
485
- }
486
-
487
- function _emit(action, info) {
488
- audit().safeEmit({ action: action, ...(info || {}) });
489
- }
490
-
491
- function _requireInit() {
492
- if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
493
- }
494
-
495
- function _resetForTest() {
496
- if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
497
- consumers.forEach(function (c) { c.cancel(); });
498
- consumers = [];
499
- backends = {};
500
- defaultBackend = null;
501
- initialized = false;
502
- audit.reset();
503
- }
504
-
505
- // enqueueFlow — atomic registration of a parent-child job graph.
506
- //
507
- // await b.queue.enqueueFlow({
508
- // queueName: "ingest",
509
- // children: [
510
- // { name: "fetch", payload: { url } },
511
- // { name: "transform", payload: { ... }, dependsOn: ["fetch"] },
512
- // { name: "publish", payload: { ... }, dependsOn: ["transform"] },
513
- // ],
514
- // });
515
- //
516
- // Cycle detection runs at registration (throws at call site). Each child enters
517
- // the queue with availableAt = MAX_SAFE_INTEGER until parent completion
518
- // bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
519
- function enqueueFlow(spec) {
520
- _requireInit();
521
- if (!spec || typeof spec !== "object") {
522
- return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
523
- }
524
- if (typeof spec.queueName !== "string" || !spec.queueName) {
525
- return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
526
- }
527
- if (!Array.isArray(spec.children) || spec.children.length === 0) {
528
- return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
529
- }
530
- // Validate each child's shape.
531
- var byName = {};
532
- for (var i = 0; i < spec.children.length; i++) {
533
- var c = spec.children[i];
534
- if (!c || typeof c !== "object") {
535
- return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
536
- }
537
- if (typeof c.name !== "string" || !c.name) {
538
- return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
539
- }
540
- if (byName[c.name]) {
541
- return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
542
- }
543
- byName[c.name] = c;
544
- if (c.dependsOn !== undefined) {
545
- if (!Array.isArray(c.dependsOn)) {
546
- return Promise.reject(_err("BAD_FLOW",
547
- "children[" + i + "].dependsOn must be an array of names", true));
548
- }
549
- for (var di = 0; di < c.dependsOn.length; di++) {
550
- if (typeof c.dependsOn[di] !== "string") {
551
- return Promise.reject(_err("BAD_FLOW",
552
- "children[" + i + "].dependsOn[" + di + "] must be a string name", true));
553
- }
554
- }
555
- }
556
- }
557
- // Cycle detection — depth-first traversal with visited set.
558
- function _visit(name, stack) {
559
- if (stack.indexOf(name) !== -1) {
560
- throw _err("FLOW_CYCLE", "flow cycle detected: " +
561
- stack.concat([name]).join(" → "), true);
562
- }
563
- var child = byName[name];
564
- if (!child || !child.dependsOn) return;
565
- var nextStack = stack.concat([name]);
566
- for (var k = 0; k < child.dependsOn.length; k++) {
567
- var dep = child.dependsOn[k];
568
- if (!byName[dep]) {
569
- throw _err("FLOW_UNKNOWN_DEP",
570
- "child '" + name + "' dependsOn unknown name '" + dep + "'", true);
571
- }
572
- _visit(dep, nextStack);
573
- }
574
- }
575
- try {
576
- var names = Object.keys(byName);
577
- for (var n = 0; n < names.length; n++) _visit(names[n], []);
578
- } catch (e) {
579
- return Promise.reject(e);
580
- }
581
-
582
- var flowId = "flow-" + crypto.generateToken(C.BYTES.bytes(8));
583
-
584
- return observability.tap("queue.enqueueFlow",
585
- { queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
586
- async function () {
587
- var jobs = [];
588
- // Two-pass insert: first pass enqueues all children with their
589
- // names attached so the second pass can write dependsOn jobIds
590
- // resolved by name. Children with deps land at MAX_SAFE_INTEGER
591
- // availableAt automatically (see queue-local enqueue logic).
592
- var nameToJobId = {};
593
- for (var p = 0; p < spec.children.length; p++) {
594
- var ch = spec.children[p];
595
- // Hold off setting dependsOn until we know all sibling jobIds.
596
- var enqOpts = {
597
- flowId: flowId,
598
- flowChildName: ch.name,
599
- priority: ch.priority || 0,
600
- classification: ch.classification || null,
601
- traceId: ch.traceId || null,
602
- maxAttempts: ch.maxAttempts,
603
- // dependsOn intentionally omitted on first pass — will be patched
604
- // in via direct UPDATE after all jobIds are known. This means
605
- // root children (no deps) are immediately leaseable; deps-bearing
606
- // children get patched to MAX_SAFE_INTEGER via second pass.
607
- };
608
- var result = await enqueue(spec.queueName, ch.payload, enqOpts);
609
- nameToJobId[ch.name] = result.jobId;
610
- jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
611
- }
612
- // Second pass: write dependsOn (translated to jobIds) for children
613
- // that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
614
- for (var q = 0; q < jobs.length; q++) {
615
- var j = jobs[q];
616
- if (j.dependsOn.length === 0) continue;
617
- var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
618
- await clusterStorage.execute(
619
- "UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
620
- [JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
621
- );
622
- }
623
- _emit("system.queue.flow.enqueue", {
624
- metadata: {
625
- queue: spec.queueName,
626
- flowId: flowId,
627
- childCount: spec.children.length,
628
- },
629
- });
630
- return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
631
- }
632
- );
633
- }
634
-
635
- // bootFromEnv — env-driven init mirroring b.network.bootFromEnv and
636
- // b.logStream.bootFromEnv. Reads the BLAMEJS_QUEUE_* env vars and
637
- // calls queue.init({ backends }) accordingly. Operators get a working
638
- // queue backend without writing build-app code.
639
- //
640
- // BLAMEJS_QUEUE_PROTOCOL local | redis (default: local)
641
- // BLAMEJS_QUEUE_REDIS_URL redis://host:port/db (required when protocol=redis)
642
- // BLAMEJS_QUEUE_REDIS_PASSWORD auth password
643
- // BLAMEJS_QUEUE_REDIS_USERNAME ACL username (optional)
644
- // BLAMEJS_QUEUE_REDIS_TLS "1"/"true" forces TLS (else inferred from rediss://)
645
- // BLAMEJS_QUEUE_REDIS_KEY_PREFIX key prefix (default "blamejs:queue")
646
- function bootFromEnv(opts) {
647
- opts = opts || {};
648
- var env = opts.env || process.env;
649
- if (initialized) return;
650
- var protocol = env.BLAMEJS_QUEUE_PROTOCOL || "local";
651
- var backendCfg;
652
- if (protocol === "local") {
653
- backendCfg = { protocol: "local" };
654
- } else if (protocol === "redis") {
655
- var url = env.BLAMEJS_QUEUE_REDIS_URL;
656
- if (!url) {
657
- throw _err("INVALID_CONFIG",
658
- "queue.bootFromEnv: BLAMEJS_QUEUE_REDIS_URL is required when BLAMEJS_QUEUE_PROTOCOL=redis",
659
- true);
660
- }
661
- var tlsRaw = env.BLAMEJS_QUEUE_REDIS_TLS;
662
- var tls = tlsRaw === "1" || tlsRaw === "true";
663
- backendCfg = {
664
- protocol: "redis",
665
- url: url,
666
- password: env.BLAMEJS_QUEUE_REDIS_PASSWORD || null,
667
- username: env.BLAMEJS_QUEUE_REDIS_USERNAME || null,
668
- tls: tlsRaw !== undefined ? tls : undefined, // undefined → inferred from rediss://
669
- keyPrefix: env.BLAMEJS_QUEUE_REDIS_KEY_PREFIX || undefined,
670
- };
671
- } else {
672
- throw _err("INVALID_CONFIG",
673
- "queue.bootFromEnv: BLAMEJS_QUEUE_PROTOCOL must be 'local' or 'redis', got '" + protocol + "'",
674
- true);
675
- }
676
- init({ backends: { default: backendCfg }, defaultBackend: "default" });
677
- }
678
-
679
- module.exports = {
680
- init: init,
681
- bootFromEnv: bootFromEnv,
682
- enqueue: enqueue,
683
- enqueueFlow: enqueueFlow,
684
- consume: consume,
685
- size: size,
686
- purge: purge,
687
- shutdown: shutdown,
688
- listBackends: listBackends,
689
- dlqList: dlqList,
690
- dlqRetry: dlqRetry,
691
- dlqSize: dlqSize,
692
- PROTOCOLS: dispatcher.protocols,
693
- DEFERRED_PROTOCOLS: dispatcher.deferred,
694
- _resetForTest: _resetForTest,
695
- };
1
+ "use strict";
2
+ /**
3
+ * Queue dispatcher — pluggable job queue with retry, lease semantics, and
4
+ * graceful shutdown.
5
+ *
6
+ * Same dispatcher pattern as object-store: backends are configured per-name,
7
+ * each with a protocol + protocol-specific options. The built-in 'local'
8
+ * protocol is SQLite-backed (baked into the framework's main DB).
9
+ * External protocols (redis, sqs, amqp, nats) are listed as deferred and
10
+ * surface a clear error when selected.
11
+ *
12
+ * Public API:
13
+ * queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
14
+ * queue.enqueue(queueName, payload, opts?)
15
+ * → { jobId, queueName, ... }
16
+ * queue.consume(queueName, handler, opts?)
17
+ * → consumer handle (with cancel())
18
+ * queue.size(queueName, opts?) → number (pending + inflight)
19
+ * queue.purge(queueName, opts?) → number deleted
20
+ * queue.shutdown(opts?) → drain handlers gracefully
21
+ * queue.listBackends() → [{ name, protocol }]
22
+ *
23
+ * Job lifecycle:
24
+ * enqueued (status='pending')
25
+ * ↓ availableAt reached + consumer leases
26
+ * inflight (status='inflight', lease expires after leaseDurationMs)
27
+ * ↓ handler returns ↓ handler throws
28
+ * done (status='done') if attempts < maxAttempts:
29
+ * pending (with backoff)
30
+ * else:
31
+ * failed (status='failed')
32
+ */
33
+ var C = require("./constants");
34
+ var clusterStorage = require("./cluster-storage");
35
+ var crypto = require("./crypto");
36
+ var lazyRequire = require("./lazy-require");
37
+ var { boot } = require("./log");
38
+ var numericChecks = require("./numeric-checks");
39
+ var observability = require("./observability");
40
+ var protocolDispatcher = require("./protocol-dispatcher");
41
+ var localProto = require("./queue-local");
42
+ var redisProto = require("./queue-redis");
43
+ var sqsProto = require("./queue-sqs");
44
+ var retryHelper = require("./retry");
45
+ var safeAsync = require("./safe-async");
46
+ var { QueueError } = require("./framework-error");
47
+
48
+ var log = boot("queue");
49
+
50
+ var dispatcher = protocolDispatcher.create({
51
+ name: "queue",
52
+ errorClass: QueueError,
53
+ protocols: { "local": localProto, "redis": redisProto, "sqs": sqsProto },
54
+ deferred: {
55
+ "amqp": { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
56
+ "nats": { description: "NATS JetStream" },
57
+ },
58
+ fallbackProtocol: "local",
59
+ });
60
+
61
+ var _err = QueueError.factory;
62
+
63
+ var audit = lazyRequire(function () { return require("./audit"); });
64
+
65
+ var initialized = false;
66
+ var backends = {};
67
+ var defaultBackend = null;
68
+ var consumers = []; // [{ queueName, backendName, cancel(), running, inFlight: Set }]
69
+ var sweepTimer = null;
70
+
71
+ function init(opts) {
72
+ if (initialized) return;
73
+ if (!opts || !opts.backends) {
74
+ throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
75
+ }
76
+
77
+ backends = {};
78
+ // IIFE per-iteration so each backend's wrappers close over its own
79
+ // raw / breaker / cfg. With `var` (function-scoped) those bindings
80
+ // would otherwise be shared across iterations and every wrapper
81
+ // would end up using the LAST backend's breaker + retry config —
82
+ // multi-backend isolation broken without a single error surfacing.
83
+ Object.keys(opts.backends).forEach(function (name) {
84
+ var cfg = opts.backends[name];
85
+ var proto = dispatcher.resolve(cfg.protocol);
86
+ var raw = proto.create(cfg);
87
+ var breaker = new retryHelper.CircuitBreaker(
88
+ "queue:" + name,
89
+ cfg.breaker
90
+ );
91
+
92
+ // Wrap mutating ops with retry + breaker (idempotent only — enqueue and
93
+ // complete are safe to retry; lease isn't because partial-double-lease
94
+ // is dangerous, so it goes through the breaker but not retry).
95
+ function wrapWithRetry(fn) {
96
+ return function () {
97
+ var args = Array.prototype.slice.call(arguments);
98
+ return retryHelper.withRetry(function () {
99
+ return breaker.wrap(function () { return fn.apply(raw, args); });
100
+ }, cfg.retry);
101
+ };
102
+ }
103
+ function wrapBreakerOnly(fn) {
104
+ return function () {
105
+ var args = Array.prototype.slice.call(arguments);
106
+ return breaker.wrap(function () { return fn.apply(raw, args); });
107
+ };
108
+ }
109
+
110
+ backends[name] = {
111
+ name: name,
112
+ protocol: cfg.protocol,
113
+ breaker: breaker,
114
+ raw: raw,
115
+ enqueue: wrapWithRetry(raw.enqueue),
116
+ lease: wrapBreakerOnly(raw.lease),
117
+ extendLease: raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
118
+ complete: wrapWithRetry(raw.complete),
119
+ fail: wrapWithRetry(raw.fail),
120
+ sweepExpired: raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
121
+ size: wrapWithRetry(raw.size),
122
+ purge: wrapWithRetry(raw.purge),
123
+ dlqList: raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
124
+ dlqRetry: raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
125
+ dlqSize: raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
126
+ };
127
+ });
128
+
129
+ defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
130
+
131
+ // Sweep expired leases periodically (every 30s) so crashed-handler jobs
132
+ // get re-pended.
133
+ sweepTimer = safeAsync.repeating(function () {
134
+ Object.keys(backends).forEach(function (n) {
135
+ if (backends[n].sweepExpired) {
136
+ backends[n].sweepExpired().catch(function () { /* best effort */ });
137
+ }
138
+ });
139
+ }, C.TIME.seconds(30), { name: "queue-sweep" });
140
+
141
+ initialized = true;
142
+ }
143
+
144
+ function _backendFor(opts) {
145
+ opts = opts || {};
146
+ var name = opts.backend || defaultBackend;
147
+ var b = backends[name];
148
+ if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
149
+ return b;
150
+ }
151
+
152
+ // ---- Public API ----
153
+
154
+ function enqueue(queueName, payload, opts) {
155
+ _requireInit();
156
+ if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
157
+ opts = opts || {};
158
+ var b = _backendFor(opts);
159
+ return observability.tap("queue.enqueue",
160
+ { queueName: queueName, backend: b.name },
161
+ function () {
162
+ return b.enqueue(queueName, payload, opts).then(function (result) {
163
+ _emit("system.queue.enqueue", {
164
+ metadata: {
165
+ queue: queueName,
166
+ backend: b.name,
167
+ jobId: result.jobId,
168
+ classification: result.classification,
169
+ traceId: opts.traceId,
170
+ delaySeconds: opts.delaySeconds || 0,
171
+ },
172
+ });
173
+ return result;
174
+ });
175
+ }
176
+ );
177
+ }
178
+
179
+ function consume(queueName, handler, opts) {
180
+ _requireInit();
181
+ if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
182
+ if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
183
+ opts = opts || {};
184
+ var b = _backendFor(opts);
185
+ var concurrency = opts.concurrency || 1;
186
+ var leaseDurationMs = opts.leaseDurationMs || C.TIME.seconds(30);
187
+ var pollIntervalMs = opts.pollIntervalMs || C.TIME.seconds(1);
188
+ var fastPollMs = opts.fastPollMs || 50;
189
+
190
+ // Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
191
+ // start within any rolling perSeconds window. Token-bucket-style
192
+ // accounting keeps it cheap (just a sliding deque of timestamps).
193
+ var rateLimit = null;
194
+ if (opts.rateLimit) {
195
+ var rlMax = opts.rateLimit.max;
196
+ var rlPer = opts.rateLimit.perSeconds;
197
+ // NaN, Infinity, 0, negatives, fractional max all produce undefined
198
+ // throttling math (NaN deque comparisons, perma-locked queues,
199
+ // perma-open windows). Reject at config time.
200
+ if (!numericChecks.isPositiveInt(rlMax) || !numericChecks.isPositiveFinite(rlPer)) {
201
+ throw _err("BAD_RATE_LIMIT",
202
+ "consume({ rateLimit }): expected { max: positive integer, perSeconds: positive finite number }, got " +
203
+ JSON.stringify(opts.rateLimit), true);
204
+ }
205
+ rateLimit = {
206
+ max: rlMax,
207
+ windowMs: C.TIME.seconds(rlPer),
208
+ timestamps: [],
209
+ };
210
+ }
211
+ function _rateLimitWaitMs() {
212
+ if (!rateLimit) return 0;
213
+ var now = Date.now();
214
+ var cutoff = now - rateLimit.windowMs;
215
+ while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
216
+ rateLimit.timestamps.shift();
217
+ }
218
+ if (rateLimit.timestamps.length < rateLimit.max) return 0;
219
+ return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
220
+ }
221
+ function _rateLimitConsume() {
222
+ if (rateLimit) rateLimit.timestamps.push(Date.now());
223
+ }
224
+
225
+ // Progress audit-emit rate-limit — protect the audit chain from a
226
+ // chatty handler that calls progress() every loop iteration.
227
+ var PROGRESS_MIN_INTERVAL_MS = 250;
228
+
229
+ // Each consumer has its own AbortController so cancel() unblocks any
230
+ // in-flight poll-sleep immediately rather than waiting up to
231
+ // pollIntervalMs (default 1s) for the next while-loop iteration.
232
+ var abortCtrl = new AbortController();
233
+ var state = {
234
+ queueName: queueName,
235
+ backendName: b.name,
236
+ cancelled: false,
237
+ inFlight: new Set(),
238
+ abortCtrl: abortCtrl,
239
+ cancel: function () {
240
+ state.cancelled = true;
241
+ try { abortCtrl.abort(); }
242
+ catch (e) { log.debug("cancel-cleanup-failed", { op: "abortCtrl.abort", error: e.message }); }
243
+ },
244
+ };
245
+ consumers.push(state);
246
+
247
+ (async function loop() {
248
+ // Helper — sleep with cancellation. On abort, returns instead of
249
+ // rejecting so the next while-iteration sees `state.cancelled` and
250
+ // exits cleanly.
251
+ async function _pollSleep(ms) {
252
+ try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
253
+ catch (_e) { /* aborted — loop condition will catch it */ }
254
+ }
255
+ while (!state.cancelled) {
256
+ // Don't lease more than (concurrency - inFlight) at a time
257
+ var slots = concurrency - state.inFlight.size;
258
+ if (slots <= 0) {
259
+ await _pollSleep(fastPollMs);
260
+ continue;
261
+ }
262
+ // If rate-limited and we'd exceed the budget, sleep until the
263
+ // next slot opens. We lease at most `max - currentTokens` jobs to
264
+ // stay under the cap.
265
+ if (rateLimit) {
266
+ var wait = _rateLimitWaitMs();
267
+ if (wait > 0) {
268
+ await _pollSleep(Math.min(wait, pollIntervalMs));
269
+ continue;
270
+ }
271
+ var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
272
+ if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
273
+ }
274
+ var jobs;
275
+ try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
276
+ catch {
277
+ // Backend down (breaker open, etc.) — back off
278
+ await _pollSleep(pollIntervalMs);
279
+ continue;
280
+ }
281
+ if (!jobs || jobs.length === 0) {
282
+ await _pollSleep(pollIntervalMs);
283
+ continue;
284
+ }
285
+ for (var i = 0; i < jobs.length; i++) {
286
+ observability.event("queue.lease", 1, { queueName: queueName });
287
+ (function (job) {
288
+ state.inFlight.add(job.jobId);
289
+ _emit("system.queue.consume.start", {
290
+ metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
291
+ });
292
+ // Consume a rate-limit slot at handler-start so the budget
293
+ // tracks invocation rate, not lease rate (a single lease that
294
+ // splits work across many sub-units doesn't double-count).
295
+ _rateLimitConsume();
296
+
297
+ // Handler context — second arg to handler. Carries
298
+ // ctx.extendLease(ms) for long-running handlers and
299
+ // ctx.progress(0..100) for surfacing job progress to the
300
+ // audit chain (rate-limited so chatty handlers don't drown it).
301
+ var lastProgressEmitAt = 0;
302
+ var lastProgressValue = -1;
303
+ var ctx = {
304
+ extendLease: function (additionalMs) {
305
+ if (typeof b.extendLease !== "function") {
306
+ throw _err("EXTEND_LEASE_UNSUPPORTED",
307
+ "queue backend '" + b.name + "' does not support extendLease",
308
+ true);
309
+ }
310
+ return b.extendLease(job.jobId, additionalMs).then(function (ok) {
311
+ if (ok) {
312
+ _emit("system.queue.lease.extended", {
313
+ metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
314
+ });
315
+ }
316
+ return ok;
317
+ });
318
+ },
319
+ progress: function (pct) {
320
+ if (typeof pct !== "number" || !isFinite(pct)) return;
321
+ var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
322
+ var now = Date.now();
323
+ // Always emit 0 and 100 (start/done markers); throttle the rest.
324
+ var isMarker = clamped === 0 || clamped === 100;
325
+ if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
326
+ if (clamped === lastProgressValue && !isMarker) return;
327
+ lastProgressEmitAt = now;
328
+ lastProgressValue = clamped;
329
+ observability.event("queue.progress", clamped, { queueName: queueName });
330
+ _emit("system.queue.progress", {
331
+ metadata: {
332
+ queue: queueName, backend: b.name, jobId: job.jobId,
333
+ attempt: job.attempts, traceId: job.traceId,
334
+ percent: clamped,
335
+ },
336
+ });
337
+ },
338
+ };
339
+ observability.tap("queue.consume",
340
+ { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
341
+ function () {
342
+ return Promise.resolve()
343
+ .then(function () { return handler(job, ctx); })
344
+ .then(function () {
345
+ return b.complete(job.jobId).then(function () {
346
+ _emit("system.queue.consume.success", {
347
+ metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
348
+ });
349
+ observability.event("queue.complete", 1, { queueName: queueName });
350
+ });
351
+ }, function (err) {
352
+ var msg = (err && err.message) || String(err);
353
+ var willRetry = job.attempts < job.maxAttempts;
354
+ return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
355
+ .then(function () {
356
+ observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
357
+ _emit("system.queue.consume.failure", {
358
+ metadata: {
359
+ queue: queueName, backend: b.name, jobId: job.jobId,
360
+ attempt: job.attempts, traceId: job.traceId,
361
+ maxAttempts: job.maxAttempts, willRetry: willRetry,
362
+ },
363
+ reason: msg,
364
+ outcome: "failure",
365
+ });
366
+ // DLQ-write event when the job has exhausted its retries.
367
+ // Operators wire this to their alerting / dashboards
368
+ // — failed-after-retries is "needs human review" not
369
+ // "in the normal flow." Audit chain captures the
370
+ // final state for forensics.
371
+ if (!willRetry) {
372
+ _emit("system.queue.dlq.write", {
373
+ metadata: {
374
+ queue: queueName, backend: b.name, jobId: job.jobId,
375
+ attempts: job.attempts, traceId: job.traceId,
376
+ },
377
+ reason: msg,
378
+ outcome: "failure",
379
+ });
380
+ }
381
+ });
382
+ })
383
+ .catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
384
+ .then(function () { state.inFlight.delete(job.jobId); });
385
+ }
386
+ );
387
+ })(jobs[i]);
388
+ }
389
+ await _pollSleep(fastPollMs);
390
+ }
391
+ })();
392
+
393
+ return state;
394
+ }
395
+
396
+ var _QUEUE_BACKOFF_OPTS = {
397
+ baseDelayMs: C.TIME.seconds(1),
398
+ maxDelayMs: C.TIME.minutes(5),
399
+ jitterFactor: 0, // queue uses deterministic backoff (no jitter) for predictability
400
+ };
401
+ function _backoffDelay(attempt) {
402
+ return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
403
+ }
404
+
405
+ function size(queueName, opts) {
406
+ _requireInit();
407
+ return _backendFor(opts).size(queueName);
408
+ }
409
+
410
+ function purge(queueName, opts) {
411
+ _requireInit();
412
+ var b = _backendFor(opts);
413
+ return b.purge(queueName).then(function (n) {
414
+ _emit("system.queue.purge", {
415
+ metadata: { queue: queueName, backend: b.name, deleted: n },
416
+ });
417
+ return n;
418
+ });
419
+ }
420
+
421
+ // ---- Dead-letter queue ----
422
+ //
423
+ // `dlqList` returns failed-after-retries jobs for operator review;
424
+ // `dlqRetry` resets a single job back to 'pending' so it gets picked
425
+ // up by consumers again (operator-driven — never automatic).
426
+
427
+ function dlqList(queueName, opts) {
428
+ _requireInit();
429
+ var b = _backendFor(opts);
430
+ if (typeof b.dlqList !== "function") {
431
+ return Promise.reject(_err("DLQ_UNSUPPORTED",
432
+ "queue backend '" + b.name + "' does not support dlqList", true));
433
+ }
434
+ return b.dlqList(queueName, opts);
435
+ }
436
+
437
+ function dlqRetry(jobId, opts) {
438
+ _requireInit();
439
+ var b = _backendFor(opts);
440
+ if (typeof b.dlqRetry !== "function") {
441
+ return Promise.reject(_err("DLQ_UNSUPPORTED",
442
+ "queue backend '" + b.name + "' does not support dlqRetry", true));
443
+ }
444
+ return b.dlqRetry(jobId).then(function (ok) {
445
+ if (ok) {
446
+ _emit("system.queue.dlq.retry", {
447
+ metadata: { jobId: jobId, backend: b.name },
448
+ });
449
+ }
450
+ return ok;
451
+ });
452
+ }
453
+
454
+ function dlqSize(queueName, opts) {
455
+ _requireInit();
456
+ var b = _backendFor(opts);
457
+ if (typeof b.dlqSize !== "function") {
458
+ return Promise.reject(_err("DLQ_UNSUPPORTED",
459
+ "queue backend '" + b.name + "' does not support dlqSize", true));
460
+ }
461
+ return b.dlqSize(queueName);
462
+ }
463
+
464
+ async function shutdown(opts) {
465
+ if (!initialized) return;
466
+ opts = opts || {};
467
+ var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
468
+ // Signal all consumers to stop
469
+ consumers.forEach(function (c) { c.cancel(); });
470
+ // Wait for in-flight handlers to complete
471
+ var deadline = Date.now() + timeoutMs;
472
+ while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
473
+ if (Date.now() > deadline) break;
474
+ await safeAsync.sleep(50);
475
+ }
476
+ consumers = [];
477
+ if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
478
+ }
479
+
480
+ function listBackends() {
481
+ _requireInit();
482
+ return Object.keys(backends).map(function (name) {
483
+ return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
484
+ });
485
+ }
486
+
487
+ function _emit(action, info) {
488
+ audit().safeEmit({ action: action, ...(info || {}) });
489
+ }
490
+
491
+ function _requireInit() {
492
+ if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
493
+ }
494
+
495
+ function _resetForTest() {
496
+ if (sweepTimer) { sweepTimer.stop(); sweepTimer = null; }
497
+ consumers.forEach(function (c) { c.cancel(); });
498
+ consumers = [];
499
+ backends = {};
500
+ defaultBackend = null;
501
+ initialized = false;
502
+ audit.reset();
503
+ }
504
+
505
+ // enqueueFlow — atomic registration of a parent-child job graph.
506
+ //
507
+ // await b.queue.enqueueFlow({
508
+ // queueName: "ingest",
509
+ // children: [
510
+ // { name: "fetch", payload: { url } },
511
+ // { name: "transform", payload: { ... }, dependsOn: ["fetch"] },
512
+ // { name: "publish", payload: { ... }, dependsOn: ["transform"] },
513
+ // ],
514
+ // });
515
+ //
516
+ // Cycle detection runs at registration (throws at call site). Each child enters
517
+ // the queue with availableAt = MAX_SAFE_INTEGER until parent completion
518
+ // bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
519
+ function enqueueFlow(spec) {
520
+ _requireInit();
521
+ if (!spec || typeof spec !== "object") {
522
+ return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
523
+ }
524
+ if (typeof spec.queueName !== "string" || !spec.queueName) {
525
+ return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
526
+ }
527
+ if (!Array.isArray(spec.children) || spec.children.length === 0) {
528
+ return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
529
+ }
530
+ // Validate each child's shape.
531
+ var byName = {};
532
+ for (var i = 0; i < spec.children.length; i++) {
533
+ var c = spec.children[i];
534
+ if (!c || typeof c !== "object") {
535
+ return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
536
+ }
537
+ if (typeof c.name !== "string" || !c.name) {
538
+ return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
539
+ }
540
+ if (byName[c.name]) {
541
+ return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
542
+ }
543
+ byName[c.name] = c;
544
+ if (c.dependsOn !== undefined) {
545
+ if (!Array.isArray(c.dependsOn)) {
546
+ return Promise.reject(_err("BAD_FLOW",
547
+ "children[" + i + "].dependsOn must be an array of names", true));
548
+ }
549
+ for (var di = 0; di < c.dependsOn.length; di++) {
550
+ if (typeof c.dependsOn[di] !== "string") {
551
+ return Promise.reject(_err("BAD_FLOW",
552
+ "children[" + i + "].dependsOn[" + di + "] must be a string name", true));
553
+ }
554
+ }
555
+ }
556
+ }
557
+ // Cycle detection — depth-first traversal with visited set.
558
+ function _visit(name, stack) {
559
+ if (stack.indexOf(name) !== -1) {
560
+ throw _err("FLOW_CYCLE", "flow cycle detected: " +
561
+ stack.concat([name]).join(" → "), true);
562
+ }
563
+ var child = byName[name];
564
+ if (!child || !child.dependsOn) return;
565
+ var nextStack = stack.concat([name]);
566
+ for (var k = 0; k < child.dependsOn.length; k++) {
567
+ var dep = child.dependsOn[k];
568
+ if (!byName[dep]) {
569
+ throw _err("FLOW_UNKNOWN_DEP",
570
+ "child '" + name + "' dependsOn unknown name '" + dep + "'", true);
571
+ }
572
+ _visit(dep, nextStack);
573
+ }
574
+ }
575
+ try {
576
+ var names = Object.keys(byName);
577
+ for (var n = 0; n < names.length; n++) _visit(names[n], []);
578
+ } catch (e) {
579
+ return Promise.reject(e);
580
+ }
581
+
582
+ var flowId = "flow-" + crypto.generateToken(C.BYTES.bytes(8));
583
+
584
+ return observability.tap("queue.enqueueFlow",
585
+ { queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
586
+ async function () {
587
+ var jobs = [];
588
+ // Two-pass insert: first pass enqueues all children with their
589
+ // names attached so the second pass can write dependsOn jobIds
590
+ // resolved by name. Children with deps land at MAX_SAFE_INTEGER
591
+ // availableAt automatically (see queue-local enqueue logic).
592
+ var nameToJobId = {};
593
+ for (var p = 0; p < spec.children.length; p++) {
594
+ var ch = spec.children[p];
595
+ // Hold off setting dependsOn until we know all sibling jobIds.
596
+ var enqOpts = {
597
+ flowId: flowId,
598
+ flowChildName: ch.name,
599
+ priority: ch.priority || 0,
600
+ classification: ch.classification || null,
601
+ traceId: ch.traceId || null,
602
+ maxAttempts: ch.maxAttempts,
603
+ // dependsOn intentionally omitted on first pass — will be patched
604
+ // in via direct UPDATE after all jobIds are known. This means
605
+ // root children (no deps) are immediately leaseable; deps-bearing
606
+ // children get patched to MAX_SAFE_INTEGER via second pass.
607
+ };
608
+ var result = await enqueue(spec.queueName, ch.payload, enqOpts);
609
+ nameToJobId[ch.name] = result.jobId;
610
+ jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
611
+ }
612
+ // Second pass: write dependsOn (translated to jobIds) for children
613
+ // that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
614
+ for (var q = 0; q < jobs.length; q++) {
615
+ var j = jobs[q];
616
+ if (j.dependsOn.length === 0) continue;
617
+ var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
618
+ await clusterStorage.execute(
619
+ "UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
620
+ [JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
621
+ );
622
+ }
623
+ _emit("system.queue.flow.enqueue", {
624
+ metadata: {
625
+ queue: spec.queueName,
626
+ flowId: flowId,
627
+ childCount: spec.children.length,
628
+ },
629
+ });
630
+ return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
631
+ }
632
+ );
633
+ }
634
+
635
+ // bootFromEnv — env-driven init mirroring b.network.bootFromEnv and
636
+ // b.logStream.bootFromEnv. Reads the BLAMEJS_QUEUE_* env vars and
637
+ // calls queue.init({ backends }) accordingly. Operators get a working
638
+ // queue backend without writing build-app code.
639
+ //
640
+ // BLAMEJS_QUEUE_PROTOCOL local | redis (default: local)
641
+ // BLAMEJS_QUEUE_REDIS_URL redis://host:port/db (required when protocol=redis)
642
+ // BLAMEJS_QUEUE_REDIS_PASSWORD auth password
643
+ // BLAMEJS_QUEUE_REDIS_USERNAME ACL username (optional)
644
+ // BLAMEJS_QUEUE_REDIS_TLS "1"/"true" forces TLS (else inferred from rediss://)
645
+ // BLAMEJS_QUEUE_REDIS_KEY_PREFIX key prefix (default "blamejs:queue")
646
+ function bootFromEnv(opts) {
647
+ opts = opts || {};
648
+ var env = opts.env || process.env;
649
+ if (initialized) return;
650
+ var protocol = env.BLAMEJS_QUEUE_PROTOCOL || "local";
651
+ var backendCfg;
652
+ if (protocol === "local") {
653
+ backendCfg = { protocol: "local" };
654
+ } else if (protocol === "redis") {
655
+ var url = env.BLAMEJS_QUEUE_REDIS_URL;
656
+ if (!url) {
657
+ throw _err("INVALID_CONFIG",
658
+ "queue.bootFromEnv: BLAMEJS_QUEUE_REDIS_URL is required when BLAMEJS_QUEUE_PROTOCOL=redis",
659
+ true);
660
+ }
661
+ var tlsRaw = env.BLAMEJS_QUEUE_REDIS_TLS;
662
+ var tls = tlsRaw === "1" || tlsRaw === "true";
663
+ backendCfg = {
664
+ protocol: "redis",
665
+ url: url,
666
+ password: env.BLAMEJS_QUEUE_REDIS_PASSWORD || null,
667
+ username: env.BLAMEJS_QUEUE_REDIS_USERNAME || null,
668
+ tls: tlsRaw !== undefined ? tls : undefined, // undefined → inferred from rediss://
669
+ keyPrefix: env.BLAMEJS_QUEUE_REDIS_KEY_PREFIX || undefined,
670
+ };
671
+ } else {
672
+ throw _err("INVALID_CONFIG",
673
+ "queue.bootFromEnv: BLAMEJS_QUEUE_PROTOCOL must be 'local' or 'redis', got '" + protocol + "'",
674
+ true);
675
+ }
676
+ init({ backends: { default: backendCfg }, defaultBackend: "default" });
677
+ }
678
+
679
+ module.exports = {
680
+ init: init,
681
+ bootFromEnv: bootFromEnv,
682
+ enqueue: enqueue,
683
+ enqueueFlow: enqueueFlow,
684
+ consume: consume,
685
+ size: size,
686
+ purge: purge,
687
+ shutdown: shutdown,
688
+ listBackends: listBackends,
689
+ dlqList: dlqList,
690
+ dlqRetry: dlqRetry,
691
+ dlqSize: dlqSize,
692
+ PROTOCOLS: dispatcher.protocols,
693
+ DEFERRED_PROTOCOLS: dispatcher.deferred,
694
+ _resetForTest: _resetForTest,
695
+ };