@blamejs/core 0.7.1 → 0.7.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +423 -389
- package/README.md +150 -149
- package/bin/blamejs.js +0 -0
- package/index.js +308 -282
- package/lib/api-key.js +660 -672
- package/lib/api-snapshot.js +338 -338
- package/lib/app-shutdown.js +385 -385
- package/lib/app.js +365 -365
- package/lib/archive.js +250 -250
- package/lib/atomic-file.js +544 -544
- package/lib/audit-chain.js +177 -177
- package/lib/audit-sign.js +344 -344
- package/lib/audit-tools.js +677 -677
- package/lib/audit.js +766 -766
- package/lib/auth/jwt.js +311 -311
- package/lib/auth/lockout.js +436 -436
- package/lib/auth/oauth.js +721 -721
- package/lib/auth/passkey.js +181 -181
- package/lib/auth/password.js +594 -594
- package/lib/backup/bundle.js +217 -217
- package/lib/backup/crypto.js +176 -176
- package/lib/backup/index.js +515 -515
- package/lib/backup/manifest.js +282 -282
- package/lib/break-glass.js +1338 -1338
- package/lib/bundler.js +441 -441
- package/lib/cache-redis.js +256 -266
- package/lib/cache.js +1206 -1211
- package/lib/canonical-json.js +115 -115
- package/lib/chain-writer.js +234 -234
- package/lib/cli-helpers.js +206 -206
- package/lib/cli.js +2334 -2334
- package/lib/cluster-provider-db.js +317 -317
- package/lib/cluster-storage.js +226 -226
- package/lib/cluster.js +703 -703
- package/lib/codepoint-class.js +196 -0
- package/lib/config-drift.js +301 -301
- package/lib/consent.js +222 -222
- package/lib/constants.js +191 -191
- package/lib/cookies.js +315 -315
- package/lib/credential-hash.js +322 -322
- package/lib/crypto.js +266 -266
- package/lib/csv.js +275 -286
- package/lib/db-declare-row-policy.js +267 -267
- package/lib/db-declare-view.js +420 -421
- package/lib/db-query.js +406 -406
- package/lib/db-schema.js +319 -319
- package/lib/db.js +1288 -1288
- package/lib/deprecate.js +222 -222
- package/lib/dev.js +335 -335
- package/lib/dual-control.js +473 -473
- package/lib/error-page.js +420 -420
- package/lib/external-db-migrate.js +441 -441
- package/lib/external-db.js +1061 -1061
- package/lib/file-type.js +273 -273
- package/lib/file-upload.js +1136 -0
- package/lib/forms.js +422 -422
- package/lib/framework-error.js +293 -202
- package/lib/framework-schema.js +717 -717
- package/lib/gate-contract.js +971 -0
- package/lib/guard-all.js +405 -0
- package/lib/guard-archive.js +739 -0
- package/lib/guard-csv.js +816 -0
- package/lib/guard-email.js +744 -0
- package/lib/guard-filename.js +724 -0
- package/lib/guard-html.js +976 -0
- package/lib/guard-json.js +729 -0
- package/lib/guard-markdown.js +586 -0
- package/lib/guard-svg.js +976 -0
- package/lib/guard-xml.js +405 -0
- package/lib/guard-yaml.js +529 -0
- package/lib/handlers.js +350 -350
- package/lib/http-client-cookie-jar.js +508 -508
- package/lib/http-client.js +1195 -1195
- package/lib/i18n.js +878 -878
- package/lib/jobs.js +185 -185
- package/lib/log-stream-cloudwatch.js +369 -369
- package/lib/log-stream-local.js +146 -146
- package/lib/log-stream-otlp-grpc.js +410 -410
- package/lib/log-stream-otlp.js +286 -286
- package/lib/log-stream-syslog.js +302 -302
- package/lib/log-stream-webhook.js +199 -199
- package/lib/log-stream.js +330 -330
- package/lib/log.js +500 -500
- package/lib/mail-bounce.js +528 -528
- package/lib/mail-dkim.js +369 -362
- package/lib/mail.js +981 -962
- package/lib/metrics.js +683 -683
- package/lib/middleware/api-encrypt.js +936 -573
- package/lib/middleware/attach-user.js +157 -157
- package/lib/middleware/body-parser.js +1170 -1091
- package/lib/middleware/bot-guard.js +178 -178
- package/lib/middleware/compression.js +452 -452
- package/lib/middleware/cors.js +314 -314
- package/lib/middleware/csp-nonce.js +348 -348
- package/lib/middleware/csrf-protect.js +316 -316
- package/lib/middleware/db-role-for.js +264 -269
- package/lib/middleware/health.js +392 -392
- package/lib/middleware/index.js +79 -79
- package/lib/middleware/rate-limit.js +358 -358
- package/lib/middleware/request-id.js +61 -61
- package/lib/middleware/request-log.js +168 -168
- package/lib/middleware/require-auth.js +104 -104
- package/lib/middleware/security-headers.js +116 -116
- package/lib/middleware/sse.js +166 -166
- package/lib/migrations.js +383 -383
- package/lib/mtls-ca.js +518 -518
- package/lib/mtls-engine-default.js +481 -481
- package/lib/network-dns.js +632 -632
- package/lib/network-heartbeat.js +290 -290
- package/lib/network-nts.js +574 -574
- package/lib/network-proxy.js +265 -265
- package/lib/network-tls.js +328 -328
- package/lib/network.js +233 -233
- package/lib/notify.js +612 -614
- package/lib/ntp-check.js +229 -229
- package/lib/numeric-bounds.js +111 -91
- package/lib/object-store/azure-blob-bucket-ops.js +349 -349
- package/lib/object-store/azure-blob.js +488 -451
- package/lib/object-store/gcs-bucket-ops.js +351 -351
- package/lib/object-store/gcs.js +519 -479
- package/lib/object-store/http-put.js +153 -153
- package/lib/object-store/index.js +197 -197
- package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
- package/lib/object-store/sigv4.js +903 -855
- package/lib/observability.js +151 -151
- package/lib/otel-export.js +269 -269
- package/lib/pagination.js +464 -464
- package/lib/parsers/index.js +80 -80
- package/lib/parsers/safe-env.js +642 -642
- package/lib/parsers/safe-ini.js +292 -292
- package/lib/parsers/safe-toml.js +784 -784
- package/lib/parsers/safe-xml.js +390 -390
- package/lib/parsers/safe-yaml.js +1015 -1015
- package/lib/permissions.js +708 -708
- package/lib/pqc-agent.js +87 -87
- package/lib/pqc-gate.js +279 -279
- package/lib/protobuf-encoder.js +190 -190
- package/lib/protocol-dispatcher.js +161 -161
- package/lib/pubsub-redis.js +167 -177
- package/lib/pubsub.js +429 -429
- package/lib/queue-local.js +476 -476
- package/lib/queue-redis.js +745 -752
- package/lib/queue-sqs.js +319 -319
- package/lib/queue.js +695 -695
- package/lib/redis-client.js +519 -489
- package/lib/request-helpers.js +340 -336
- package/lib/restore-bundle.js +237 -237
- package/lib/restore-rollback.js +259 -259
- package/lib/restore.js +409 -409
- package/lib/retry.js +376 -376
- package/lib/router.js +748 -748
- package/lib/safe-async.js +735 -735
- package/lib/safe-buffer.js +237 -237
- package/lib/safe-json.js +541 -541
- package/lib/safe-schema.js +1266 -1266
- package/lib/safe-url.js +159 -159
- package/lib/scheduler.js +706 -706
- package/lib/security-assert.js +373 -373
- package/lib/seeders.js +618 -630
- package/lib/session.js +478 -478
- package/lib/slug.js +269 -269
- package/lib/ssrf-guard.js +401 -401
- package/lib/static.js +879 -114
- package/lib/storage.js +471 -471
- package/lib/subject.js +281 -281
- package/lib/template.js +791 -791
- package/lib/testing.js +798 -798
- package/lib/time.js +310 -310
- package/lib/totp.js +302 -302
- package/lib/tracing.js +494 -494
- package/lib/uuid.js +132 -132
- package/lib/validate-opts.js +340 -270
- package/lib/vault/index.js +308 -308
- package/lib/vault/rotate.js +784 -784
- package/lib/vault/wrap.js +296 -296
- package/lib/vendor/noble-ciphers.cjs +9 -9
- package/lib/webhook.js +595 -598
- package/lib/websocket.js +1048 -1048
- package/package.json +77 -77
- package/sbom.cyclonedx.json +7 -7
package/lib/queue-local.js
CHANGED
|
@@ -1,476 +1,476 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Local-protocol queue adapter — DB-backed, dialect-portable.
|
|
4
|
-
*
|
|
5
|
-
* Single-node: backed by the framework's main DB (_blamejs_jobs in
|
|
6
|
-
* local SQLite, baked into FRAMEWORK_SCHEMA).
|
|
7
|
-
* Cluster mode: backed by external-db (_blamejs_jobs created via
|
|
8
|
-
* frameworkSchema.ensureSchema). cluster-storage.execute routes the
|
|
9
|
-
* same SQL to the right place based on cluster.isClusterMode().
|
|
10
|
-
*
|
|
11
|
-
* Lease semantics:
|
|
12
|
-
* enqueue → INSERT status='pending'
|
|
13
|
-
* lease → atomic UPDATE pending→inflight (single statement with
|
|
14
|
-
* RETURNING; no transaction needed — the WHERE clause's
|
|
15
|
-
* subquery + Postgres's row-lock-then-recheck behavior
|
|
16
|
-
* makes concurrent leasers safe automatically. The unlucky
|
|
17
|
-
* leaser sees zero rows and tries again on the next tick.)
|
|
18
|
-
* complete→ UPDATE inflight→done
|
|
19
|
-
* fail → single UPDATE with CASE WHEN attempts < maxAttempts
|
|
20
|
-
* (retry) ELSE (final failure) — collapsed into one
|
|
21
|
-
* statement so the same code path works in both single-
|
|
22
|
-
* node and cluster mode without a cross-dialect
|
|
23
|
-
* transaction primitive.
|
|
24
|
-
* sweep → orphaned 'inflight' rows whose lease expired → 'pending'
|
|
25
|
-
*
|
|
26
|
-
* Field-crypto integration: payload + lastError are sealed columns
|
|
27
|
-
* (declared in db.js's FRAMEWORK_SCHEMA registration). enqueue seals
|
|
28
|
-
* before INSERT; lease unseals the leased rows before returning to
|
|
29
|
-
* the caller. cluster-storage's RETURNING clause hands back sealed
|
|
30
|
-
* blobs which we run through cryptoField.unsealRow explicitly.
|
|
31
|
-
*/
|
|
32
|
-
var cluster = require("./cluster");
|
|
33
|
-
var clusterStorage = require("./cluster-storage");
|
|
34
|
-
var C = require("./constants");
|
|
35
|
-
var { generateToken } = require("./crypto");
|
|
36
|
-
var cryptoField = require("./crypto-field");
|
|
37
|
-
var lazyRequire = require("./lazy-require");
|
|
38
|
-
var nb = require("./numeric-bounds");
|
|
39
|
-
var safeJson = require("./safe-json");
|
|
40
|
-
var scheduler = require("./scheduler");
|
|
41
|
-
var { QueueError } = require("./framework-error");
|
|
42
|
-
|
|
43
|
-
var _err = QueueError.factory;
|
|
44
|
-
|
|
45
|
-
// vault is lazy-required because some flows (sealed lastError) only
|
|
46
|
-
// touch it on retry-with-error paths, and the import order
|
|
47
|
-
// (queue-local → vault → db → audit → cluster) tolerates the late bind.
|
|
48
|
-
var vault = lazyRequire(function () { return require("./vault"); });
|
|
49
|
-
|
|
50
|
-
// Column order kept as a constant so the placeholders + values lists
|
|
51
|
-
// stay in sync. Mirrors db.js's FRAMEWORK_SCHEMA for _blamejs_jobs.
|
|
52
|
-
var JOB_COLS = [
|
|
53
|
-
"_id", "queueName", "payload", "status",
|
|
54
|
-
"enqueuedAt", "availableAt", "leasedAt", "leaseExpiresAt",
|
|
55
|
-
"attempts", "maxAttempts", "lastError", "finishedAt",
|
|
56
|
-
"traceId", "classification", "priority",
|
|
57
|
-
"repeatCron", "repeatTimezone",
|
|
58
|
-
"flowId", "flowChildName", "dependsOn",
|
|
59
|
-
];
|
|
60
|
-
|
|
61
|
-
// Sentinel availableAt for flow children that haven't yet had their
|
|
62
|
-
// dependencies satisfied — far future so the lease query never picks
|
|
63
|
-
// them. Parent-completion sets a real availableAt when all deps complete.
|
|
64
|
-
var FLOW_BLOCKED_AVAILABLE_AT = Number.MAX_SAFE_INTEGER;
|
|
65
|
-
|
|
66
|
-
// Columns returned by lease() / used by RETURNING. Subset of JOB_COLS
|
|
67
|
-
// — only what callers need; fewer bytes over the wire in cluster mode.
|
|
68
|
-
var LEASE_RETURN_COLS = [
|
|
69
|
-
"_id", "queueName", "payload",
|
|
70
|
-
"attempts", "maxAttempts", "traceId", "classification",
|
|
71
|
-
"enqueuedAt", "leaseExpiresAt",
|
|
72
|
-
"repeatCron", "repeatTimezone", "flowId", "flowChildName",
|
|
73
|
-
];
|
|
74
|
-
|
|
75
|
-
function _quotedList(cols) {
|
|
76
|
-
return cols.map(function (c) { return '"' + c + '"'; }).join(", ");
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function _placeholders(cols) {
|
|
80
|
-
return cols.map(function () { return "?"; }).join(", ");
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
function _shapeLeasedRow(raw) {
|
|
84
|
-
// raw is a row coming back from RETURNING — payload is sealed if
|
|
85
|
-
// present. Run through cryptoField's unseal pipeline so the caller
|
|
86
|
-
// gets cleartext.
|
|
87
|
-
var unsealed = cryptoField.unsealRow("_blamejs_jobs", raw);
|
|
88
|
-
return {
|
|
89
|
-
jobId: unsealed._id,
|
|
90
|
-
queueName: unsealed.queueName,
|
|
91
|
-
payload: unsealed.payload ? safeJson.parse(unsealed.payload) : null,
|
|
92
|
-
attempts: Number(unsealed.attempts),
|
|
93
|
-
maxAttempts: Number(unsealed.maxAttempts),
|
|
94
|
-
traceId: unsealed.traceId,
|
|
95
|
-
classification: unsealed.classification,
|
|
96
|
-
enqueuedAt: Number(unsealed.enqueuedAt),
|
|
97
|
-
leaseExpiresAt: Number(unsealed.leaseExpiresAt),
|
|
98
|
-
repeatCron: unsealed.repeatCron || null,
|
|
99
|
-
repeatTimezone: unsealed.repeatTimezone || null,
|
|
100
|
-
flowId: unsealed.flowId || null,
|
|
101
|
-
flowChildName: unsealed.flowChildName || null,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function create(_config) {
|
|
106
|
-
// No protocol-level config — uses cluster-storage which dispatches
|
|
107
|
-
// to the framework's main DB (single-node) or external-db (cluster).
|
|
108
|
-
|
|
109
|
-
async function enqueue(queueName, payload, opts) {
|
|
110
|
-
cluster.requireLeader();
|
|
111
|
-
opts = opts || {};
|
|
112
|
-
var nowMs = Date.now();
|
|
113
|
-
// ----------------------------------------------------------------------
|
|
114
|
-
// SCHEDULING PRECEDENCE — the contract for every queue.enqueue caller
|
|
115
|
-
// ----------------------------------------------------------------------
|
|
116
|
-
// The queue accepts TWO ways to express "when should this job run":
|
|
117
|
-
// - opts.availableAt (absolute unix-ms) — precise, framework-internal
|
|
118
|
-
// - opts.delaySeconds (integer seconds) — operator shorthand
|
|
119
|
-
//
|
|
120
|
-
// PRECEDENCE: opts.availableAt wins when finite. Operators passing both
|
|
121
|
-
// get the absolute value; the relative form is computed from
|
|
122
|
-
// Date.now() and is therefore strictly less precise (loses sub-second
|
|
123
|
-
// resolution AND drifts on the internal-clock-vs-caller delta).
|
|
124
|
-
//
|
|
125
|
-
// If you're a caller computing a precise time (cron-repeat,
|
|
126
|
-
// scheduler.scheduleAt, notify.deferUntil), pass opts.availableAt
|
|
127
|
-
// ONLY. Don't pass both — the second one is noise (and was the
|
|
128
|
-
// shape that caused the v0.6.21 silent-drift bug). If you only have
|
|
129
|
-
// a relative offset, pass opts.delaySeconds ONLY.
|
|
130
|
-
// ----------------------------------------------------------------------
|
|
131
|
-
var availableAt;
|
|
132
|
-
if (typeof opts.availableAt === "number" && isFinite(opts.availableAt)) {
|
|
133
|
-
availableAt = opts.availableAt;
|
|
134
|
-
} else {
|
|
135
|
-
availableAt = nowMs + (opts.delaySeconds ? C.TIME.seconds(opts.delaySeconds) : 0);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
var priority = (typeof opts.priority === "number" && isFinite(opts.priority))
|
|
139
|
-
? Math.floor(opts.priority) : 0;
|
|
140
|
-
var repeatCron = opts.repeat && typeof opts.repeat.cron === "string"
|
|
141
|
-
? opts.repeat.cron : null;
|
|
142
|
-
var repeatTimezone = opts.repeat && typeof opts.repeat.timezone === "string"
|
|
143
|
-
? opts.repeat.timezone : null;
|
|
144
|
-
var flowId = typeof opts.flowId === "string" ? opts.flowId : null;
|
|
145
|
-
var flowChildName = typeof opts.flowChildName === "string" ? opts.flowChildName : null;
|
|
146
|
-
var dependsOn = Array.isArray(opts.dependsOn) && opts.dependsOn.length > 0
|
|
147
|
-
? JSON.stringify(opts.dependsOn) : null;
|
|
148
|
-
// Flow children with deps wait at MAX_SAFE_INTEGER until parent
|
|
149
|
-
// completion bumps availableAt — keeps them out of the lease index.
|
|
150
|
-
var effectiveAvailableAt = (dependsOn ? FLOW_BLOCKED_AVAILABLE_AT : availableAt);
|
|
151
|
-
|
|
152
|
-
var row = {
|
|
153
|
-
_id: generateToken(C.BYTES.bytes(16)),
|
|
154
|
-
queueName: queueName,
|
|
155
|
-
payload: payload === undefined ? null : JSON.stringify(payload),
|
|
156
|
-
status: "pending",
|
|
157
|
-
enqueuedAt: nowMs,
|
|
158
|
-
availableAt: effectiveAvailableAt,
|
|
159
|
-
leasedAt: null,
|
|
160
|
-
leaseExpiresAt: null,
|
|
161
|
-
attempts: 0,
|
|
162
|
-
maxAttempts: opts.maxAttempts != null ? opts.maxAttempts : 5,
|
|
163
|
-
lastError: null,
|
|
164
|
-
finishedAt: null,
|
|
165
|
-
traceId: opts.traceId || null,
|
|
166
|
-
classification: opts.classification || null,
|
|
167
|
-
priority: priority,
|
|
168
|
-
repeatCron: repeatCron,
|
|
169
|
-
repeatTimezone: repeatTimezone,
|
|
170
|
-
flowId: flowId,
|
|
171
|
-
flowChildName: flowChildName,
|
|
172
|
-
dependsOn: dependsOn,
|
|
173
|
-
};
|
|
174
|
-
var sealed = cryptoField.sealRow("_blamejs_jobs", row);
|
|
175
|
-
var values = JOB_COLS.map(function (c) { return c in sealed ? sealed[c] : null; });
|
|
176
|
-
|
|
177
|
-
await clusterStorage.execute(
|
|
178
|
-
"INSERT INTO _blamejs_jobs (" + _quotedList(JOB_COLS) + ") " +
|
|
179
|
-
"VALUES (" + _placeholders(JOB_COLS) + ")",
|
|
180
|
-
values
|
|
181
|
-
);
|
|
182
|
-
return {
|
|
183
|
-
jobId: row._id,
|
|
184
|
-
queueName: queueName,
|
|
185
|
-
enqueuedAt: nowMs,
|
|
186
|
-
availableAt: availableAt,
|
|
187
|
-
classification: row.classification,
|
|
188
|
-
};
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
async function lease(queueName, leaseMs, count) {
|
|
192
|
-
cluster.requireLeader();
|
|
193
|
-
var nowMs = Date.now();
|
|
194
|
-
var leaseExpiresAt = nowMs + leaseMs;
|
|
195
|
-
var maxRows = count != null ? count : 1;
|
|
196
|
-
|
|
197
|
-
// Single-statement atomic lease. The IN-subquery picks the head of
|
|
198
|
-
// the queue; the outer UPDATE locks those rows and only updates
|
|
199
|
-
// rows that still match status='pending' after the lock acquires
|
|
200
|
-
// (Postgres EvalPlanQual; SQLite is single-writer so the same row
|
|
201
|
-
// can't be picked twice). RETURNING hands back the leased columns
|
|
202
|
-
// so we don't need a separate SELECT after the UPDATE.
|
|
203
|
-
var sql =
|
|
204
|
-
"UPDATE _blamejs_jobs " +
|
|
205
|
-
"SET status = 'inflight', leasedAt = ?, leaseExpiresAt = ?, attempts = attempts + 1 " +
|
|
206
|
-
"WHERE _id IN (" +
|
|
207
|
-
" SELECT _id FROM _blamejs_jobs " +
|
|
208
|
-
" WHERE queueName = ? AND status = 'pending' AND availableAt <= ? " +
|
|
209
|
-
" ORDER BY priority DESC, availableAt ASC, enqueuedAt ASC " +
|
|
210
|
-
" LIMIT ?" +
|
|
211
|
-
") " +
|
|
212
|
-
"RETURNING " + _quotedList(LEASE_RETURN_COLS);
|
|
213
|
-
var result = await clusterStorage.execute(
|
|
214
|
-
sql,
|
|
215
|
-
[nowMs, leaseExpiresAt, queueName, nowMs, maxRows]
|
|
216
|
-
);
|
|
217
|
-
var leased = [];
|
|
218
|
-
for (var i = 0; i < result.rows.length; i++) {
|
|
219
|
-
leased.push(_shapeLeasedRow(result.rows[i]));
|
|
220
|
-
}
|
|
221
|
-
return leased;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
// extendLease — push the lease expiry forward for a long-running job.
|
|
225
|
-
// Handler context exposes this as `ctx.extendLease(ms)`. The job must
|
|
226
|
-
// still be in 'inflight' status (i.e. not yet swept by sweepExpired);
|
|
227
|
-
// otherwise the call no-ops and returns false.
|
|
228
|
-
async function extendLease(jobId, additionalMs) {
|
|
229
|
-
cluster.requireLeader();
|
|
230
|
-
if (typeof additionalMs !== "number" || additionalMs <= 0) {
|
|
231
|
-
throw _err("INVALID_LEASE_EXTENSION",
|
|
232
|
-
"extendLease: additionalMs must be a positive number", true);
|
|
233
|
-
}
|
|
234
|
-
var newExpiry = Date.now() + additionalMs;
|
|
235
|
-
var result = await clusterStorage.execute(
|
|
236
|
-
"UPDATE _blamejs_jobs SET leaseExpiresAt = ? " +
|
|
237
|
-
"WHERE _id = ? AND status = 'inflight'",
|
|
238
|
-
[newExpiry, jobId]
|
|
239
|
-
);
|
|
240
|
-
return (result.rowCount || 0) > 0;
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
async function complete(jobId) {
|
|
244
|
-
cluster.requireLeader();
|
|
245
|
-
var nowMs = Date.now();
|
|
246
|
-
// Read the row first so we can act on repeat / flow metadata after
|
|
247
|
-
// the status flip. Single SELECT + UPDATE pair under the same
|
|
248
|
-
// jobId — race-free under SQLite (single-writer); cluster-storage
|
|
249
|
-
// dispatches both calls to the same backend.
|
|
250
|
-
var rowRes = await clusterStorage.execute(
|
|
251
|
-
"SELECT _id, queueName, payload, repeatCron, repeatTimezone, " +
|
|
252
|
-
" flowId, flowChildName, priority, classification, traceId " +
|
|
253
|
-
"FROM _blamejs_jobs WHERE _id = ?",
|
|
254
|
-
[jobId]
|
|
255
|
-
);
|
|
256
|
-
var row = (rowRes && rowRes.rows && rowRes.rows[0]) || null;
|
|
257
|
-
|
|
258
|
-
await clusterStorage.execute(
|
|
259
|
-
"UPDATE _blamejs_jobs SET status = 'done', finishedAt = ?, leaseExpiresAt = NULL " +
|
|
260
|
-
"WHERE _id = ? AND status = 'inflight'",
|
|
261
|
-
[nowMs, jobId]
|
|
262
|
-
);
|
|
263
|
-
|
|
264
|
-
// Repeat-in-queue: cron-recurring job re-enqueues itself for the
|
|
265
|
-
// next firing time. Failures (which take the fail() path) don't
|
|
266
|
-
// re-enqueue — operators investigate before the cron resumes.
|
|
267
|
-
if (row && row.repeatCron) {
|
|
268
|
-
try {
|
|
269
|
-
var unsealedRow = cryptoField.unsealRow("_blamejs_jobs", row);
|
|
270
|
-
var cron = scheduler.parseCron(unsealedRow.repeatCron);
|
|
271
|
-
var nextMs = scheduler.nextCronFire(cron, new Date(nowMs), unsealedRow.repeatTimezone || null);
|
|
272
|
-
await enqueue(unsealedRow.queueName,
|
|
273
|
-
unsealedRow.payload ? safeJson.parse(unsealedRow.payload) : null,
|
|
274
|
-
{
|
|
275
|
-
// availableAt is the precise next-fire ms — pass it alone.
|
|
276
|
-
// Don't also pass delaySeconds (the v0.6.22 / v0.6.23 fix
|
|
277
|
-
// codified that opts.availableAt wins, but mixing both is
|
|
278
|
-
// the shape that masked the silent-drift bug; keep this
|
|
279
|
-
// call site clean as documentation by example).
|
|
280
|
-
availableAt: nextMs,
|
|
281
|
-
repeat: { cron: unsealedRow.repeatCron, timezone: unsealedRow.repeatTimezone },
|
|
282
|
-
priority: Number(unsealedRow.priority) || 0,
|
|
283
|
-
classification: unsealedRow.classification || null,
|
|
284
|
-
traceId: unsealedRow.traceId || null,
|
|
285
|
-
});
|
|
286
|
-
} catch (_e) { /* repeat re-enqueue best-effort — cron resumes next tick if op fixes the issue */ }
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
// Flow propagation: walk siblings whose dependsOn includes this
|
|
290
|
-
// jobId (or this job's flowChildName) and bump availableAt to now
|
|
291
|
-
// if ALL their deps are now complete.
|
|
292
|
-
if (row && row.flowId) {
|
|
293
|
-
await _maybeReleaseFlowChildren(row.flowId, jobId, row.flowChildName, nowMs);
|
|
294
|
-
}
|
|
295
|
-
return true;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
async function _maybeReleaseFlowChildren(flowId, completedJobId, completedChildName, nowMs) {
|
|
299
|
-
var siblingsRes = await clusterStorage.execute(
|
|
300
|
-
"SELECT _id, dependsOn, flowChildName, status, availableAt FROM _blamejs_jobs " +
|
|
301
|
-
"WHERE flowId = ? AND status = 'pending' AND availableAt > ?",
|
|
302
|
-
[flowId, nowMs]
|
|
303
|
-
);
|
|
304
|
-
var siblings = (siblingsRes && siblingsRes.rows) || [];
|
|
305
|
-
for (var i = 0; i < siblings.length; i++) {
|
|
306
|
-
var sib = siblings[i];
|
|
307
|
-
if (!sib.dependsOn) continue;
|
|
308
|
-
var deps;
|
|
309
|
-
try { deps = safeJson.parse(sib.dependsOn, { maxBytes: C.BYTES.mib(1) }); }
|
|
310
|
-
catch (_e) { continue; }
|
|
311
|
-
if (!Array.isArray(deps) || deps.length === 0) continue;
|
|
312
|
-
// Resolve which deps are satisfied. Each dep is either a jobId
|
|
313
|
-
// or a flowChildName; we accept both shapes against the flow.
|
|
314
|
-
var allDone = true;
|
|
315
|
-
for (var d = 0; d < deps.length; d++) {
|
|
316
|
-
var dep = deps[d];
|
|
317
|
-
// Quick path: just-completed job matches by id or child name.
|
|
318
|
-
if (dep === completedJobId || (completedChildName && dep === completedChildName)) continue;
|
|
319
|
-
// Otherwise SELECT to confirm done.
|
|
320
|
-
var depRes = await clusterStorage.execute(
|
|
321
|
-
"SELECT 1 FROM _blamejs_jobs WHERE flowId = ? AND status = 'done' AND " +
|
|
322
|
-
" (_id = ? OR flowChildName = ?) LIMIT 1",
|
|
323
|
-
[flowId, dep, dep]
|
|
324
|
-
);
|
|
325
|
-
if (!depRes || !depRes.rows || depRes.rows.length === 0) { allDone = false; break; }
|
|
326
|
-
}
|
|
327
|
-
if (allDone) {
|
|
328
|
-
await clusterStorage.execute(
|
|
329
|
-
"UPDATE _blamejs_jobs SET availableAt = ? WHERE _id = ?",
|
|
330
|
-
[nowMs, sib._id]
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
async function fail(jobId, errorMessage, opts) {
|
|
337
|
-
cluster.requireLeader();
|
|
338
|
-
opts = opts || {};
|
|
339
|
-
var retryDelayMs = opts.retryDelayMs != null ? opts.retryDelayMs : 0;
|
|
340
|
-
var nowMs = Date.now();
|
|
341
|
-
var sealedErr = errorMessage ? vault().seal(String(errorMessage)) : null;
|
|
342
|
-
|
|
343
|
-
// Single-statement decision: retry vs final failure based on the
|
|
344
|
-
// row's current attempts/maxAttempts. CASE expressions split the
|
|
345
|
-
// status / availableAt / finishedAt updates per branch — same
|
|
346
|
-
// semantics as the previous SELECT-then-UPDATE-in-transaction
|
|
347
|
-
// path, but no cross-dialect transaction primitive needed.
|
|
348
|
-
await clusterStorage.execute(
|
|
349
|
-
"UPDATE _blamejs_jobs SET " +
|
|
350
|
-
" status = CASE WHEN attempts < maxAttempts THEN 'pending' ELSE 'failed' END, " +
|
|
351
|
-
" lastError = ?, " +
|
|
352
|
-
" leaseExpiresAt = NULL, " +
|
|
353
|
-
" availableAt = CASE WHEN attempts < maxAttempts THEN ? ELSE availableAt END, " +
|
|
354
|
-
" finishedAt = CASE WHEN attempts < maxAttempts THEN NULL ELSE ? END " +
|
|
355
|
-
"WHERE _id = ?",
|
|
356
|
-
[sealedErr, nowMs + retryDelayMs, nowMs, jobId]
|
|
357
|
-
);
|
|
358
|
-
return true;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
async function sweepExpired() {
|
|
362
|
-
cluster.requireLeader();
|
|
363
|
-
var result = await clusterStorage.execute(
|
|
364
|
-
"UPDATE _blamejs_jobs SET status = 'pending', leaseExpiresAt = NULL " +
|
|
365
|
-
"WHERE status = 'inflight' AND leaseExpiresAt < ?",
|
|
366
|
-
[Date.now()]
|
|
367
|
-
);
|
|
368
|
-
return result.rowCount || 0;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
async function size(queueName) {
|
|
372
|
-
var row = await clusterStorage.executeOne(
|
|
373
|
-
"SELECT COUNT(*) AS n FROM _blamejs_jobs " +
|
|
374
|
-
"WHERE queueName = ? AND (status = 'pending' OR status = 'inflight')",
|
|
375
|
-
[queueName]
|
|
376
|
-
);
|
|
377
|
-
return row ? Number(row.n) : 0;
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// ---- DLQ (dead-letter queue) ----
|
|
381
|
-
//
|
|
382
|
-
// Jobs that exhausted their retry budget land in status='failed' with
|
|
383
|
-
// finishedAt set + lastError sealed. dlqList surfaces them for
|
|
384
|
-
// operator review; dlqRetry resets a job back to 'pending' so it can
|
|
385
|
-
// be reprocessed (operator-driven — never automatic).
|
|
386
|
-
|
|
387
|
-
async function dlqList(queueName, opts) {
|
|
388
|
-
opts = opts || {};
|
|
389
|
-
var limit = 100;
|
|
390
|
-
if (opts.limit !== undefined) {
|
|
391
|
-
if (!nb.isPositiveFiniteInt(opts.limit)) {
|
|
392
|
-
throw new QueueError("queue/bad-opt",
|
|
393
|
-
"queue.dlqList: limit must be a positive finite integer; got " +
|
|
394
|
-
nb.shape(opts.limit), true);
|
|
395
|
-
}
|
|
396
|
-
limit = opts.limit;
|
|
397
|
-
}
|
|
398
|
-
var rows = await clusterStorage.executeAll(
|
|
399
|
-
"SELECT _id, queueName, payload, status, enqueuedAt, finishedAt, " +
|
|
400
|
-
" attempts, maxAttempts, lastError, traceId, classification " +
|
|
401
|
-
"FROM _blamejs_jobs " +
|
|
402
|
-
"WHERE queueName = ? AND status = 'failed' " +
|
|
403
|
-
"ORDER BY finishedAt DESC LIMIT ?",
|
|
404
|
-
[queueName, limit]
|
|
405
|
-
);
|
|
406
|
-
return rows.map(function (row) {
|
|
407
|
-
var unsealed = cryptoField.unsealRow("_blamejs_jobs", row);
|
|
408
|
-
return {
|
|
409
|
-
jobId: row._id,
|
|
410
|
-
queueName: row.queueName,
|
|
411
|
-
payload: unsealed.payload ? safeJson.parse(unsealed.payload, { maxBytes: C.BYTES.mib(64) }) : null,
|
|
412
|
-
status: row.status,
|
|
413
|
-
enqueuedAt: Number(row.enqueuedAt),
|
|
414
|
-
finishedAt: row.finishedAt ? Number(row.finishedAt) : null,
|
|
415
|
-
attempts: Number(row.attempts),
|
|
416
|
-
maxAttempts: Number(row.maxAttempts),
|
|
417
|
-
lastError: unsealed.lastError || null,
|
|
418
|
-
traceId: row.traceId || null,
|
|
419
|
-
classification: row.classification || null,
|
|
420
|
-
};
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
async function dlqRetry(jobId) {
|
|
425
|
-
cluster.requireLeader();
|
|
426
|
-
var nowMs = Date.now();
|
|
427
|
-
var result = await clusterStorage.execute(
|
|
428
|
-
"UPDATE _blamejs_jobs SET " +
|
|
429
|
-
" status = 'pending', " +
|
|
430
|
-
" attempts = 0, " +
|
|
431
|
-
" availableAt = ?, " +
|
|
432
|
-
" finishedAt = NULL, " +
|
|
433
|
-
" leasedAt = NULL, " +
|
|
434
|
-
" leaseExpiresAt = NULL, " +
|
|
435
|
-
" lastError = NULL " +
|
|
436
|
-
"WHERE _id = ? AND status = 'failed'",
|
|
437
|
-
[nowMs, jobId]
|
|
438
|
-
);
|
|
439
|
-
return (result.rowCount || 0) > 0;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
async function dlqSize(queueName) {
|
|
443
|
-
var row = await clusterStorage.executeOne(
|
|
444
|
-
"SELECT COUNT(*) AS n FROM _blamejs_jobs " +
|
|
445
|
-
"WHERE queueName = ? AND status = 'failed'",
|
|
446
|
-
[queueName]
|
|
447
|
-
);
|
|
448
|
-
return row ? Number(row.n) : 0;
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
async function purge(queueName) {
|
|
452
|
-
cluster.requireLeader();
|
|
453
|
-
var result = await clusterStorage.execute(
|
|
454
|
-
"DELETE FROM _blamejs_jobs WHERE queueName = ?",
|
|
455
|
-
[queueName]
|
|
456
|
-
);
|
|
457
|
-
return result.rowCount || 0;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
return {
|
|
461
|
-
protocol: "local",
|
|
462
|
-
enqueue: enqueue,
|
|
463
|
-
lease: lease,
|
|
464
|
-
extendLease: extendLease,
|
|
465
|
-
complete: complete,
|
|
466
|
-
fail: fail,
|
|
467
|
-
sweepExpired: sweepExpired,
|
|
468
|
-
size: size,
|
|
469
|
-
purge: purge,
|
|
470
|
-
dlqList: dlqList,
|
|
471
|
-
dlqRetry: dlqRetry,
|
|
472
|
-
dlqSize: dlqSize,
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
module.exports = { create: create };
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Local-protocol queue adapter — DB-backed, dialect-portable.
|
|
4
|
+
*
|
|
5
|
+
* Single-node: backed by the framework's main DB (_blamejs_jobs in
|
|
6
|
+
* local SQLite, baked into FRAMEWORK_SCHEMA).
|
|
7
|
+
* Cluster mode: backed by external-db (_blamejs_jobs created via
|
|
8
|
+
* frameworkSchema.ensureSchema). cluster-storage.execute routes the
|
|
9
|
+
* same SQL to the right place based on cluster.isClusterMode().
|
|
10
|
+
*
|
|
11
|
+
* Lease semantics:
|
|
12
|
+
* enqueue → INSERT status='pending'
|
|
13
|
+
* lease → atomic UPDATE pending→inflight (single statement with
|
|
14
|
+
* RETURNING; no transaction needed — the WHERE clause's
|
|
15
|
+
* subquery + Postgres's row-lock-then-recheck behavior
|
|
16
|
+
* makes concurrent leasers safe automatically. The unlucky
|
|
17
|
+
* leaser sees zero rows and tries again on the next tick.)
|
|
18
|
+
* complete→ UPDATE inflight→done
|
|
19
|
+
* fail → single UPDATE with CASE WHEN attempts < maxAttempts
|
|
20
|
+
* (retry) ELSE (final failure) — collapsed into one
|
|
21
|
+
* statement so the same code path works in both single-
|
|
22
|
+
* node and cluster mode without a cross-dialect
|
|
23
|
+
* transaction primitive.
|
|
24
|
+
* sweep → orphaned 'inflight' rows whose lease expired → 'pending'
|
|
25
|
+
*
|
|
26
|
+
* Field-crypto integration: payload + lastError are sealed columns
|
|
27
|
+
* (declared in db.js's FRAMEWORK_SCHEMA registration). enqueue seals
|
|
28
|
+
* before INSERT; lease unseals the leased rows before returning to
|
|
29
|
+
* the caller. cluster-storage's RETURNING clause hands back sealed
|
|
30
|
+
* blobs which we run through cryptoField.unsealRow explicitly.
|
|
31
|
+
*/
|
|
32
|
+
var cluster = require("./cluster");
|
|
33
|
+
var clusterStorage = require("./cluster-storage");
|
|
34
|
+
var C = require("./constants");
|
|
35
|
+
var { generateToken } = require("./crypto");
|
|
36
|
+
var cryptoField = require("./crypto-field");
|
|
37
|
+
var lazyRequire = require("./lazy-require");
|
|
38
|
+
var nb = require("./numeric-bounds");
|
|
39
|
+
var safeJson = require("./safe-json");
|
|
40
|
+
var scheduler = require("./scheduler");
|
|
41
|
+
var { QueueError } = require("./framework-error");
|
|
42
|
+
|
|
43
|
+
var _err = QueueError.factory;
|
|
44
|
+
|
|
45
|
+
// vault is lazy-required because some flows (sealed lastError) only
|
|
46
|
+
// touch it on retry-with-error paths, and the import order
|
|
47
|
+
// (queue-local → vault → db → audit → cluster) tolerates the late bind.
|
|
48
|
+
var vault = lazyRequire(function () { return require("./vault"); });
|
|
49
|
+
|
|
50
|
+
// Column order kept as a constant so the placeholders + values lists
|
|
51
|
+
// stay in sync. Mirrors db.js's FRAMEWORK_SCHEMA for _blamejs_jobs.
|
|
52
|
+
var JOB_COLS = [
|
|
53
|
+
"_id", "queueName", "payload", "status",
|
|
54
|
+
"enqueuedAt", "availableAt", "leasedAt", "leaseExpiresAt",
|
|
55
|
+
"attempts", "maxAttempts", "lastError", "finishedAt",
|
|
56
|
+
"traceId", "classification", "priority",
|
|
57
|
+
"repeatCron", "repeatTimezone",
|
|
58
|
+
"flowId", "flowChildName", "dependsOn",
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
// Sentinel availableAt for flow children that haven't yet had their
|
|
62
|
+
// dependencies satisfied — far future so the lease query never picks
|
|
63
|
+
// them. Parent-completion sets a real availableAt when all deps complete.
|
|
64
|
+
var FLOW_BLOCKED_AVAILABLE_AT = Number.MAX_SAFE_INTEGER;
|
|
65
|
+
|
|
66
|
+
// Columns returned by lease() / used by RETURNING. Subset of JOB_COLS
|
|
67
|
+
// — only what callers need; fewer bytes over the wire in cluster mode.
|
|
68
|
+
var LEASE_RETURN_COLS = [
|
|
69
|
+
"_id", "queueName", "payload",
|
|
70
|
+
"attempts", "maxAttempts", "traceId", "classification",
|
|
71
|
+
"enqueuedAt", "leaseExpiresAt",
|
|
72
|
+
"repeatCron", "repeatTimezone", "flowId", "flowChildName",
|
|
73
|
+
];
|
|
74
|
+
|
|
75
|
+
function _quotedList(cols) {
|
|
76
|
+
return cols.map(function (c) { return '"' + c + '"'; }).join(", ");
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function _placeholders(cols) {
|
|
80
|
+
return cols.map(function () { return "?"; }).join(", ");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function _shapeLeasedRow(raw) {
|
|
84
|
+
// raw is a row coming back from RETURNING — payload is sealed if
|
|
85
|
+
// present. Run through cryptoField's unseal pipeline so the caller
|
|
86
|
+
// gets cleartext.
|
|
87
|
+
var unsealed = cryptoField.unsealRow("_blamejs_jobs", raw);
|
|
88
|
+
return {
|
|
89
|
+
jobId: unsealed._id,
|
|
90
|
+
queueName: unsealed.queueName,
|
|
91
|
+
payload: unsealed.payload ? safeJson.parse(unsealed.payload) : null,
|
|
92
|
+
attempts: Number(unsealed.attempts),
|
|
93
|
+
maxAttempts: Number(unsealed.maxAttempts),
|
|
94
|
+
traceId: unsealed.traceId,
|
|
95
|
+
classification: unsealed.classification,
|
|
96
|
+
enqueuedAt: Number(unsealed.enqueuedAt),
|
|
97
|
+
leaseExpiresAt: Number(unsealed.leaseExpiresAt),
|
|
98
|
+
repeatCron: unsealed.repeatCron || null,
|
|
99
|
+
repeatTimezone: unsealed.repeatTimezone || null,
|
|
100
|
+
flowId: unsealed.flowId || null,
|
|
101
|
+
flowChildName: unsealed.flowChildName || null,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function create(_config) {
|
|
106
|
+
// No protocol-level config — uses cluster-storage which dispatches
|
|
107
|
+
// to the framework's main DB (single-node) or external-db (cluster).
|
|
108
|
+
|
|
109
|
+
async function enqueue(queueName, payload, opts) {
|
|
110
|
+
cluster.requireLeader();
|
|
111
|
+
opts = opts || {};
|
|
112
|
+
var nowMs = Date.now();
|
|
113
|
+
// ----------------------------------------------------------------------
|
|
114
|
+
// SCHEDULING PRECEDENCE — the contract for every queue.enqueue caller
|
|
115
|
+
// ----------------------------------------------------------------------
|
|
116
|
+
// The queue accepts TWO ways to express "when should this job run":
|
|
117
|
+
// - opts.availableAt (absolute unix-ms) — precise, framework-internal
|
|
118
|
+
// - opts.delaySeconds (integer seconds) — operator shorthand
|
|
119
|
+
//
|
|
120
|
+
// PRECEDENCE: opts.availableAt wins when finite. Operators passing both
|
|
121
|
+
// get the absolute value; the relative form is computed from
|
|
122
|
+
// Date.now() and is therefore strictly less precise (loses sub-second
|
|
123
|
+
// resolution AND drifts on the internal-clock-vs-caller delta).
|
|
124
|
+
//
|
|
125
|
+
// If you're a caller computing a precise time (cron-repeat,
|
|
126
|
+
// scheduler.scheduleAt, notify.deferUntil), pass opts.availableAt
|
|
127
|
+
// ONLY. Don't pass both — the second one is noise (and was the
|
|
128
|
+
// shape that caused the v0.6.21 silent-drift bug). If you only have
|
|
129
|
+
// a relative offset, pass opts.delaySeconds ONLY.
|
|
130
|
+
// ----------------------------------------------------------------------
|
|
131
|
+
var availableAt;
|
|
132
|
+
if (typeof opts.availableAt === "number" && isFinite(opts.availableAt)) {
|
|
133
|
+
availableAt = opts.availableAt;
|
|
134
|
+
} else {
|
|
135
|
+
availableAt = nowMs + (opts.delaySeconds ? C.TIME.seconds(opts.delaySeconds) : 0);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
var priority = (typeof opts.priority === "number" && isFinite(opts.priority))
|
|
139
|
+
? Math.floor(opts.priority) : 0;
|
|
140
|
+
var repeatCron = opts.repeat && typeof opts.repeat.cron === "string"
|
|
141
|
+
? opts.repeat.cron : null;
|
|
142
|
+
var repeatTimezone = opts.repeat && typeof opts.repeat.timezone === "string"
|
|
143
|
+
? opts.repeat.timezone : null;
|
|
144
|
+
var flowId = typeof opts.flowId === "string" ? opts.flowId : null;
|
|
145
|
+
var flowChildName = typeof opts.flowChildName === "string" ? opts.flowChildName : null;
|
|
146
|
+
var dependsOn = Array.isArray(opts.dependsOn) && opts.dependsOn.length > 0
|
|
147
|
+
? JSON.stringify(opts.dependsOn) : null;
|
|
148
|
+
// Flow children with deps wait at MAX_SAFE_INTEGER until parent
|
|
149
|
+
// completion bumps availableAt — keeps them out of the lease index.
|
|
150
|
+
var effectiveAvailableAt = (dependsOn ? FLOW_BLOCKED_AVAILABLE_AT : availableAt);
|
|
151
|
+
|
|
152
|
+
var row = {
|
|
153
|
+
_id: generateToken(C.BYTES.bytes(16)),
|
|
154
|
+
queueName: queueName,
|
|
155
|
+
payload: payload === undefined ? null : JSON.stringify(payload),
|
|
156
|
+
status: "pending",
|
|
157
|
+
enqueuedAt: nowMs,
|
|
158
|
+
availableAt: effectiveAvailableAt,
|
|
159
|
+
leasedAt: null,
|
|
160
|
+
leaseExpiresAt: null,
|
|
161
|
+
attempts: 0,
|
|
162
|
+
maxAttempts: opts.maxAttempts != null ? opts.maxAttempts : 5,
|
|
163
|
+
lastError: null,
|
|
164
|
+
finishedAt: null,
|
|
165
|
+
traceId: opts.traceId || null,
|
|
166
|
+
classification: opts.classification || null,
|
|
167
|
+
priority: priority,
|
|
168
|
+
repeatCron: repeatCron,
|
|
169
|
+
repeatTimezone: repeatTimezone,
|
|
170
|
+
flowId: flowId,
|
|
171
|
+
flowChildName: flowChildName,
|
|
172
|
+
dependsOn: dependsOn,
|
|
173
|
+
};
|
|
174
|
+
var sealed = cryptoField.sealRow("_blamejs_jobs", row);
|
|
175
|
+
var values = JOB_COLS.map(function (c) { return c in sealed ? sealed[c] : null; });
|
|
176
|
+
|
|
177
|
+
await clusterStorage.execute(
|
|
178
|
+
"INSERT INTO _blamejs_jobs (" + _quotedList(JOB_COLS) + ") " +
|
|
179
|
+
"VALUES (" + _placeholders(JOB_COLS) + ")",
|
|
180
|
+
values
|
|
181
|
+
);
|
|
182
|
+
return {
|
|
183
|
+
jobId: row._id,
|
|
184
|
+
queueName: queueName,
|
|
185
|
+
enqueuedAt: nowMs,
|
|
186
|
+
availableAt: availableAt,
|
|
187
|
+
classification: row.classification,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async function lease(queueName, leaseMs, count) {
|
|
192
|
+
cluster.requireLeader();
|
|
193
|
+
var nowMs = Date.now();
|
|
194
|
+
var leaseExpiresAt = nowMs + leaseMs;
|
|
195
|
+
var maxRows = count != null ? count : 1;
|
|
196
|
+
|
|
197
|
+
// Single-statement atomic lease. The IN-subquery picks the head of
|
|
198
|
+
// the queue; the outer UPDATE locks those rows and only updates
|
|
199
|
+
// rows that still match status='pending' after the lock acquires
|
|
200
|
+
// (Postgres EvalPlanQual; SQLite is single-writer so the same row
|
|
201
|
+
// can't be picked twice). RETURNING hands back the leased columns
|
|
202
|
+
// so we don't need a separate SELECT after the UPDATE.
|
|
203
|
+
var sql =
|
|
204
|
+
"UPDATE _blamejs_jobs " +
|
|
205
|
+
"SET status = 'inflight', leasedAt = ?, leaseExpiresAt = ?, attempts = attempts + 1 " +
|
|
206
|
+
"WHERE _id IN (" +
|
|
207
|
+
" SELECT _id FROM _blamejs_jobs " +
|
|
208
|
+
" WHERE queueName = ? AND status = 'pending' AND availableAt <= ? " +
|
|
209
|
+
" ORDER BY priority DESC, availableAt ASC, enqueuedAt ASC " +
|
|
210
|
+
" LIMIT ?" +
|
|
211
|
+
") " +
|
|
212
|
+
"RETURNING " + _quotedList(LEASE_RETURN_COLS);
|
|
213
|
+
var result = await clusterStorage.execute(
|
|
214
|
+
sql,
|
|
215
|
+
[nowMs, leaseExpiresAt, queueName, nowMs, maxRows]
|
|
216
|
+
);
|
|
217
|
+
var leased = [];
|
|
218
|
+
for (var i = 0; i < result.rows.length; i++) {
|
|
219
|
+
leased.push(_shapeLeasedRow(result.rows[i]));
|
|
220
|
+
}
|
|
221
|
+
return leased;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// extendLease — push the lease expiry forward for a long-running job.
|
|
225
|
+
// Handler context exposes this as `ctx.extendLease(ms)`. The job must
|
|
226
|
+
// still be in 'inflight' status (i.e. not yet swept by sweepExpired);
|
|
227
|
+
// otherwise the call no-ops and returns false.
|
|
228
|
+
async function extendLease(jobId, additionalMs) {
|
|
229
|
+
cluster.requireLeader();
|
|
230
|
+
if (typeof additionalMs !== "number" || additionalMs <= 0) {
|
|
231
|
+
throw _err("INVALID_LEASE_EXTENSION",
|
|
232
|
+
"extendLease: additionalMs must be a positive number", true);
|
|
233
|
+
}
|
|
234
|
+
var newExpiry = Date.now() + additionalMs;
|
|
235
|
+
var result = await clusterStorage.execute(
|
|
236
|
+
"UPDATE _blamejs_jobs SET leaseExpiresAt = ? " +
|
|
237
|
+
"WHERE _id = ? AND status = 'inflight'",
|
|
238
|
+
[newExpiry, jobId]
|
|
239
|
+
);
|
|
240
|
+
return (result.rowCount || 0) > 0;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function complete(jobId) {
|
|
244
|
+
cluster.requireLeader();
|
|
245
|
+
var nowMs = Date.now();
|
|
246
|
+
// Read the row first so we can act on repeat / flow metadata after
|
|
247
|
+
// the status flip. Single SELECT + UPDATE pair under the same
|
|
248
|
+
// jobId — race-free under SQLite (single-writer); cluster-storage
|
|
249
|
+
// dispatches both calls to the same backend.
|
|
250
|
+
var rowRes = await clusterStorage.execute(
|
|
251
|
+
"SELECT _id, queueName, payload, repeatCron, repeatTimezone, " +
|
|
252
|
+
" flowId, flowChildName, priority, classification, traceId " +
|
|
253
|
+
"FROM _blamejs_jobs WHERE _id = ?",
|
|
254
|
+
[jobId]
|
|
255
|
+
);
|
|
256
|
+
var row = (rowRes && rowRes.rows && rowRes.rows[0]) || null;
|
|
257
|
+
|
|
258
|
+
await clusterStorage.execute(
|
|
259
|
+
"UPDATE _blamejs_jobs SET status = 'done', finishedAt = ?, leaseExpiresAt = NULL " +
|
|
260
|
+
"WHERE _id = ? AND status = 'inflight'",
|
|
261
|
+
[nowMs, jobId]
|
|
262
|
+
);
|
|
263
|
+
|
|
264
|
+
// Repeat-in-queue: cron-recurring job re-enqueues itself for the
|
|
265
|
+
// next firing time. Failures (which take the fail() path) don't
|
|
266
|
+
// re-enqueue — operators investigate before the cron resumes.
|
|
267
|
+
if (row && row.repeatCron) {
|
|
268
|
+
try {
|
|
269
|
+
var unsealedRow = cryptoField.unsealRow("_blamejs_jobs", row);
|
|
270
|
+
var cron = scheduler.parseCron(unsealedRow.repeatCron);
|
|
271
|
+
var nextMs = scheduler.nextCronFire(cron, new Date(nowMs), unsealedRow.repeatTimezone || null);
|
|
272
|
+
await enqueue(unsealedRow.queueName,
|
|
273
|
+
unsealedRow.payload ? safeJson.parse(unsealedRow.payload) : null,
|
|
274
|
+
{
|
|
275
|
+
// availableAt is the precise next-fire ms — pass it alone.
|
|
276
|
+
// Don't also pass delaySeconds (the v0.6.22 / v0.6.23 fix
|
|
277
|
+
// codified that opts.availableAt wins, but mixing both is
|
|
278
|
+
// the shape that masked the silent-drift bug; keep this
|
|
279
|
+
// call site clean as documentation by example).
|
|
280
|
+
availableAt: nextMs,
|
|
281
|
+
repeat: { cron: unsealedRow.repeatCron, timezone: unsealedRow.repeatTimezone },
|
|
282
|
+
priority: Number(unsealedRow.priority) || 0,
|
|
283
|
+
classification: unsealedRow.classification || null,
|
|
284
|
+
traceId: unsealedRow.traceId || null,
|
|
285
|
+
});
|
|
286
|
+
} catch (_e) { /* repeat re-enqueue best-effort — cron resumes next tick if op fixes the issue */ }
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// Flow propagation: walk siblings whose dependsOn includes this
|
|
290
|
+
// jobId (or this job's flowChildName) and bump availableAt to now
|
|
291
|
+
// if ALL their deps are now complete.
|
|
292
|
+
if (row && row.flowId) {
|
|
293
|
+
await _maybeReleaseFlowChildren(row.flowId, jobId, row.flowChildName, nowMs);
|
|
294
|
+
}
|
|
295
|
+
return true;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
async function _maybeReleaseFlowChildren(flowId, completedJobId, completedChildName, nowMs) {
|
|
299
|
+
var siblingsRes = await clusterStorage.execute(
|
|
300
|
+
"SELECT _id, dependsOn, flowChildName, status, availableAt FROM _blamejs_jobs " +
|
|
301
|
+
"WHERE flowId = ? AND status = 'pending' AND availableAt > ?",
|
|
302
|
+
[flowId, nowMs]
|
|
303
|
+
);
|
|
304
|
+
var siblings = (siblingsRes && siblingsRes.rows) || [];
|
|
305
|
+
for (var i = 0; i < siblings.length; i++) {
|
|
306
|
+
var sib = siblings[i];
|
|
307
|
+
if (!sib.dependsOn) continue;
|
|
308
|
+
var deps;
|
|
309
|
+
try { deps = safeJson.parse(sib.dependsOn, { maxBytes: C.BYTES.mib(1) }); }
|
|
310
|
+
catch (_e) { continue; }
|
|
311
|
+
if (!Array.isArray(deps) || deps.length === 0) continue;
|
|
312
|
+
// Resolve which deps are satisfied. Each dep is either a jobId
|
|
313
|
+
// or a flowChildName; we accept both shapes against the flow.
|
|
314
|
+
var allDone = true;
|
|
315
|
+
for (var d = 0; d < deps.length; d++) {
|
|
316
|
+
var dep = deps[d];
|
|
317
|
+
// Quick path: just-completed job matches by id or child name.
|
|
318
|
+
if (dep === completedJobId || (completedChildName && dep === completedChildName)) continue;
|
|
319
|
+
// Otherwise SELECT to confirm done.
|
|
320
|
+
var depRes = await clusterStorage.execute(
|
|
321
|
+
"SELECT 1 FROM _blamejs_jobs WHERE flowId = ? AND status = 'done' AND " +
|
|
322
|
+
" (_id = ? OR flowChildName = ?) LIMIT 1",
|
|
323
|
+
[flowId, dep, dep]
|
|
324
|
+
);
|
|
325
|
+
if (!depRes || !depRes.rows || depRes.rows.length === 0) { allDone = false; break; }
|
|
326
|
+
}
|
|
327
|
+
if (allDone) {
|
|
328
|
+
await clusterStorage.execute(
|
|
329
|
+
"UPDATE _blamejs_jobs SET availableAt = ? WHERE _id = ?",
|
|
330
|
+
[nowMs, sib._id]
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
async function fail(jobId, errorMessage, opts) {
|
|
337
|
+
cluster.requireLeader();
|
|
338
|
+
opts = opts || {};
|
|
339
|
+
var retryDelayMs = opts.retryDelayMs != null ? opts.retryDelayMs : 0;
|
|
340
|
+
var nowMs = Date.now();
|
|
341
|
+
var sealedErr = errorMessage ? vault().seal(String(errorMessage)) : null;
|
|
342
|
+
|
|
343
|
+
// Single-statement decision: retry vs final failure based on the
|
|
344
|
+
// row's current attempts/maxAttempts. CASE expressions split the
|
|
345
|
+
// status / availableAt / finishedAt updates per branch — same
|
|
346
|
+
// semantics as the previous SELECT-then-UPDATE-in-transaction
|
|
347
|
+
// path, but no cross-dialect transaction primitive needed.
|
|
348
|
+
await clusterStorage.execute(
|
|
349
|
+
"UPDATE _blamejs_jobs SET " +
|
|
350
|
+
" status = CASE WHEN attempts < maxAttempts THEN 'pending' ELSE 'failed' END, " +
|
|
351
|
+
" lastError = ?, " +
|
|
352
|
+
" leaseExpiresAt = NULL, " +
|
|
353
|
+
" availableAt = CASE WHEN attempts < maxAttempts THEN ? ELSE availableAt END, " +
|
|
354
|
+
" finishedAt = CASE WHEN attempts < maxAttempts THEN NULL ELSE ? END " +
|
|
355
|
+
"WHERE _id = ?",
|
|
356
|
+
[sealedErr, nowMs + retryDelayMs, nowMs, jobId]
|
|
357
|
+
);
|
|
358
|
+
return true;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function sweepExpired() {
|
|
362
|
+
cluster.requireLeader();
|
|
363
|
+
var result = await clusterStorage.execute(
|
|
364
|
+
"UPDATE _blamejs_jobs SET status = 'pending', leaseExpiresAt = NULL " +
|
|
365
|
+
"WHERE status = 'inflight' AND leaseExpiresAt < ?",
|
|
366
|
+
[Date.now()]
|
|
367
|
+
);
|
|
368
|
+
return result.rowCount || 0;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
async function size(queueName) {
|
|
372
|
+
var row = await clusterStorage.executeOne(
|
|
373
|
+
"SELECT COUNT(*) AS n FROM _blamejs_jobs " +
|
|
374
|
+
"WHERE queueName = ? AND (status = 'pending' OR status = 'inflight')",
|
|
375
|
+
[queueName]
|
|
376
|
+
);
|
|
377
|
+
return row ? Number(row.n) : 0;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// ---- DLQ (dead-letter queue) ----
|
|
381
|
+
//
|
|
382
|
+
// Jobs that exhausted their retry budget land in status='failed' with
|
|
383
|
+
// finishedAt set + lastError sealed. dlqList surfaces them for
|
|
384
|
+
// operator review; dlqRetry resets a job back to 'pending' so it can
|
|
385
|
+
// be reprocessed (operator-driven — never automatic).
|
|
386
|
+
|
|
387
|
+
async function dlqList(queueName, opts) {
|
|
388
|
+
opts = opts || {};
|
|
389
|
+
var limit = 100;
|
|
390
|
+
if (opts.limit !== undefined) {
|
|
391
|
+
if (!nb.isPositiveFiniteInt(opts.limit)) {
|
|
392
|
+
throw new QueueError("queue/bad-opt",
|
|
393
|
+
"queue.dlqList: limit must be a positive finite integer; got " +
|
|
394
|
+
nb.shape(opts.limit), true);
|
|
395
|
+
}
|
|
396
|
+
limit = opts.limit;
|
|
397
|
+
}
|
|
398
|
+
var rows = await clusterStorage.executeAll(
|
|
399
|
+
"SELECT _id, queueName, payload, status, enqueuedAt, finishedAt, " +
|
|
400
|
+
" attempts, maxAttempts, lastError, traceId, classification " +
|
|
401
|
+
"FROM _blamejs_jobs " +
|
|
402
|
+
"WHERE queueName = ? AND status = 'failed' " +
|
|
403
|
+
"ORDER BY finishedAt DESC LIMIT ?",
|
|
404
|
+
[queueName, limit]
|
|
405
|
+
);
|
|
406
|
+
return rows.map(function (row) {
|
|
407
|
+
var unsealed = cryptoField.unsealRow("_blamejs_jobs", row);
|
|
408
|
+
return {
|
|
409
|
+
jobId: row._id,
|
|
410
|
+
queueName: row.queueName,
|
|
411
|
+
payload: unsealed.payload ? safeJson.parse(unsealed.payload, { maxBytes: C.BYTES.mib(64) }) : null,
|
|
412
|
+
status: row.status,
|
|
413
|
+
enqueuedAt: Number(row.enqueuedAt),
|
|
414
|
+
finishedAt: row.finishedAt ? Number(row.finishedAt) : null,
|
|
415
|
+
attempts: Number(row.attempts),
|
|
416
|
+
maxAttempts: Number(row.maxAttempts),
|
|
417
|
+
lastError: unsealed.lastError || null,
|
|
418
|
+
traceId: row.traceId || null,
|
|
419
|
+
classification: row.classification || null,
|
|
420
|
+
};
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
async function dlqRetry(jobId) {
|
|
425
|
+
cluster.requireLeader();
|
|
426
|
+
var nowMs = Date.now();
|
|
427
|
+
var result = await clusterStorage.execute(
|
|
428
|
+
"UPDATE _blamejs_jobs SET " +
|
|
429
|
+
" status = 'pending', " +
|
|
430
|
+
" attempts = 0, " +
|
|
431
|
+
" availableAt = ?, " +
|
|
432
|
+
" finishedAt = NULL, " +
|
|
433
|
+
" leasedAt = NULL, " +
|
|
434
|
+
" leaseExpiresAt = NULL, " +
|
|
435
|
+
" lastError = NULL " +
|
|
436
|
+
"WHERE _id = ? AND status = 'failed'",
|
|
437
|
+
[nowMs, jobId]
|
|
438
|
+
);
|
|
439
|
+
return (result.rowCount || 0) > 0;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
async function dlqSize(queueName) {
|
|
443
|
+
var row = await clusterStorage.executeOne(
|
|
444
|
+
"SELECT COUNT(*) AS n FROM _blamejs_jobs " +
|
|
445
|
+
"WHERE queueName = ? AND status = 'failed'",
|
|
446
|
+
[queueName]
|
|
447
|
+
);
|
|
448
|
+
return row ? Number(row.n) : 0;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
async function purge(queueName) {
|
|
452
|
+
cluster.requireLeader();
|
|
453
|
+
var result = await clusterStorage.execute(
|
|
454
|
+
"DELETE FROM _blamejs_jobs WHERE queueName = ?",
|
|
455
|
+
[queueName]
|
|
456
|
+
);
|
|
457
|
+
return result.rowCount || 0;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
return {
|
|
461
|
+
protocol: "local",
|
|
462
|
+
enqueue: enqueue,
|
|
463
|
+
lease: lease,
|
|
464
|
+
extendLease: extendLease,
|
|
465
|
+
complete: complete,
|
|
466
|
+
fail: fail,
|
|
467
|
+
sweepExpired: sweepExpired,
|
|
468
|
+
size: size,
|
|
469
|
+
purge: purge,
|
|
470
|
+
dlqList: dlqList,
|
|
471
|
+
dlqRetry: dlqRetry,
|
|
472
|
+
dlqSize: dlqSize,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
module.exports = { create: create };
|