@blamejs/core 0.14.26 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +2 -2
- package/index.js +4 -0
- package/lib/agent-envelope-mac.js +104 -0
- package/lib/agent-event-bus.js +105 -4
- package/lib/agent-posture-chain.js +8 -42
- package/lib/ai-content-detect.js +9 -10
- package/lib/api-key.js +107 -74
- package/lib/atomic-file.js +62 -4
- package/lib/audit-chain.js +47 -11
- package/lib/audit-sign.js +77 -2
- package/lib/audit-tools.js +79 -51
- package/lib/audit.js +249 -123
- package/lib/auth/openid-federation.js +108 -47
- package/lib/backup/index.js +13 -10
- package/lib/break-glass.js +202 -144
- package/lib/cache.js +174 -105
- package/lib/chain-writer.js +38 -16
- package/lib/cli.js +19 -14
- package/lib/cluster-provider-db.js +130 -104
- package/lib/cluster-storage.js +119 -22
- package/lib/cluster.js +119 -71
- package/lib/compliance.js +169 -4
- package/lib/consent.js +73 -24
- package/lib/constants.js +16 -11
- package/lib/crypto-field.js +474 -92
- package/lib/db-declare-row-policy.js +35 -22
- package/lib/db-file-lifecycle.js +3 -2
- package/lib/db-query.js +497 -255
- package/lib/db-schema.js +209 -44
- package/lib/db.js +176 -95
- package/lib/error-page.js +14 -1
- package/lib/external-db-migrate.js +229 -139
- package/lib/external-db.js +25 -15
- package/lib/file-upload.js +52 -7
- package/lib/framework-error.js +14 -1
- package/lib/framework-files.js +73 -0
- package/lib/framework-schema.js +695 -394
- package/lib/gate-contract.js +649 -1
- package/lib/guard-agent-registry.js +26 -44
- package/lib/guard-all.js +1 -0
- package/lib/guard-auth.js +42 -112
- package/lib/guard-cidr.js +33 -154
- package/lib/guard-csv.js +46 -113
- package/lib/guard-domain.js +34 -157
- package/lib/guard-dsn.js +27 -43
- package/lib/guard-email.js +47 -69
- package/lib/guard-envelope.js +19 -32
- package/lib/guard-event-bus-payload.js +24 -42
- package/lib/guard-event-bus-topic.js +25 -43
- package/lib/guard-filename.js +42 -106
- package/lib/guard-graphql.js +42 -123
- package/lib/guard-html.js +53 -108
- package/lib/guard-idempotency-key.js +24 -42
- package/lib/guard-image.js +46 -103
- package/lib/guard-imap-command.js +18 -32
- package/lib/guard-jmap.js +16 -30
- package/lib/guard-json.js +38 -108
- package/lib/guard-jsonpath.js +38 -171
- package/lib/guard-jwt.js +49 -179
- package/lib/guard-list-id.js +25 -41
- package/lib/guard-list-unsubscribe.js +27 -43
- package/lib/guard-mail-compose.js +24 -42
- package/lib/guard-mail-move.js +26 -44
- package/lib/guard-mail-query.js +28 -46
- package/lib/guard-mail-reply.js +24 -42
- package/lib/guard-mail-sieve.js +24 -42
- package/lib/guard-managesieve-command.js +17 -31
- package/lib/guard-markdown.js +37 -104
- package/lib/guard-message-id.js +26 -45
- package/lib/guard-mime.js +39 -151
- package/lib/guard-oauth.js +54 -135
- package/lib/guard-pdf.js +45 -101
- package/lib/guard-pop3-command.js +21 -31
- package/lib/guard-posture-chain.js +24 -42
- package/lib/guard-regex.js +33 -107
- package/lib/guard-saga-config.js +24 -42
- package/lib/guard-shell.js +42 -172
- package/lib/guard-smtp-command.js +48 -54
- package/lib/guard-snapshot-envelope.js +24 -42
- package/lib/guard-sql.js +1491 -0
- package/lib/guard-stream-args.js +24 -43
- package/lib/guard-svg.js +47 -65
- package/lib/guard-template.js +35 -172
- package/lib/guard-tenant-id.js +26 -45
- package/lib/guard-time.js +32 -154
- package/lib/guard-trace-context.js +25 -44
- package/lib/guard-uuid.js +32 -153
- package/lib/guard-xml.js +38 -113
- package/lib/guard-yaml.js +51 -163
- package/lib/http-client.js +37 -9
- package/lib/inbox.js +120 -107
- package/lib/legal-hold.js +107 -50
- package/lib/log-stream-cloudwatch.js +47 -31
- package/lib/log-stream-otlp.js +32 -18
- package/lib/mail-crypto-smime.js +2 -6
- package/lib/mail-greylist.js +2 -6
- package/lib/mail-helo.js +2 -6
- package/lib/mail-journal.js +85 -64
- package/lib/mail-rbl.js +2 -6
- package/lib/mail-scan.js +2 -6
- package/lib/mail-server-jmap.js +117 -12
- package/lib/mail-spam-score.js +2 -6
- package/lib/mail-store.js +287 -154
- package/lib/middleware/body-parser.js +71 -25
- package/lib/middleware/csrf-protect.js +19 -8
- package/lib/middleware/fetch-metadata.js +17 -7
- package/lib/middleware/idempotency-key.js +54 -38
- package/lib/middleware/rate-limit.js +102 -32
- package/lib/middleware/security-headers.js +21 -5
- package/lib/migrations.js +108 -66
- package/lib/network-heartbeat.js +7 -0
- package/lib/nonce-store.js +31 -9
- package/lib/object-store/azure-blob-bucket-ops.js +9 -4
- package/lib/object-store/azure-blob.js +57 -3
- package/lib/object-store/sigv4.js +10 -0
- package/lib/observability.js +87 -0
- package/lib/otel-export.js +25 -1
- package/lib/outbox.js +136 -82
- package/lib/parsers/safe-xml.js +47 -7
- package/lib/pqc-agent.js +44 -0
- package/lib/pubsub-cluster.js +42 -20
- package/lib/queue-local.js +202 -139
- package/lib/queue-redis.js +9 -1
- package/lib/queue-sqs.js +6 -0
- package/lib/redact.js +68 -11
- package/lib/redis-client.js +160 -31
- package/lib/retention.js +82 -39
- package/lib/router.js +212 -5
- package/lib/safe-dns.js +29 -45
- package/lib/safe-ical.js +18 -33
- package/lib/safe-icap.js +27 -43
- package/lib/safe-sieve.js +21 -40
- package/lib/safe-sql.js +124 -3
- package/lib/safe-vcard.js +18 -33
- package/lib/scheduler.js +35 -12
- package/lib/seeders.js +122 -74
- package/lib/session-stores.js +42 -14
- package/lib/session.js +109 -72
- package/lib/sql.js +3885 -0
- package/lib/ssrf-guard.js +51 -4
- package/lib/static.js +177 -34
- package/lib/subject.js +55 -17
- package/lib/vault/index.js +3 -2
- package/lib/vault/passphrase-ops.js +3 -2
- package/lib/vault/rotate.js +104 -64
- package/lib/vendor-data.js +2 -0
- package/lib/websocket.js +35 -5
- package/package.json +1 -1
- package/sbom.cdx.json +6 -6
package/lib/otel-export.js
CHANGED
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
var C = require("./constants");
|
|
43
43
|
var canonicalJson = require("./canonical-json");
|
|
44
44
|
var httpClient = require("./http-client");
|
|
45
|
+
var observability = require("./observability");
|
|
45
46
|
var safeAsync = require("./safe-async");
|
|
46
47
|
var validateOpts = require("./validate-opts");
|
|
47
48
|
var { defineClass } = require("./framework-error");
|
|
@@ -64,6 +65,28 @@ var MAX_RESPONSE_BYTES = C.BYTES.mib(1);
|
|
|
64
65
|
// receiving backend handles the running sum.
|
|
65
66
|
var TEMPORALITY_DELTA = 1;
|
|
66
67
|
|
|
68
|
+
// Run a single attribute value through the active telemetry redactor.
|
|
69
|
+
// Telemetry is a first-class EGRESS sink — an attribute value holding a
|
|
70
|
+
// user email, bearer token, or vault-sealed ciphertext would otherwise
|
|
71
|
+
// be serialized verbatim onto the OTLP wire (CWE-532: insertion of
|
|
72
|
+
// sensitive information into an externally-shipped sink). The redactor is
|
|
73
|
+
// resolved per-call from b.observability so an operator-installed
|
|
74
|
+
// override (setRedactor) takes effect without re-creating the exporter.
|
|
75
|
+
//
|
|
76
|
+
// Drop-silent by design: this runs on the export hot path, where a throw
|
|
77
|
+
// from the redactor must never crash the request that produced the span.
|
|
78
|
+
// On a throw we DROP the attribute (signalled by the `_DROP` sentinel)
|
|
79
|
+
// rather than fall through to the raw value — failing toward dropping,
|
|
80
|
+
// not leaking.
|
|
81
|
+
var _DROP = {};
|
|
82
|
+
function _redactAttrValue(key, value) {
|
|
83
|
+
try {
|
|
84
|
+
return observability.getRedactor()(value, key);
|
|
85
|
+
} catch (_e) {
|
|
86
|
+
return _DROP; // redactor threw — drop the attribute, never export raw
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
67
90
|
// ---- attribute encoding ----
|
|
68
91
|
// OTLP attributes are KeyValue with typed `value` fields:
|
|
69
92
|
// { key, value: { stringValue | intValue | doubleValue | boolValue } }
|
|
@@ -72,7 +95,8 @@ function _attrsToOtlp(attrs) {
|
|
|
72
95
|
var out = [];
|
|
73
96
|
for (var k in attrs) {
|
|
74
97
|
if (!Object.prototype.hasOwnProperty.call(attrs, k)) continue;
|
|
75
|
-
var v = attrs[k];
|
|
98
|
+
var v = _redactAttrValue(k, attrs[k]);
|
|
99
|
+
if (v === _DROP) continue; // redactor threw — drop, don't leak
|
|
76
100
|
var kv;
|
|
77
101
|
if (typeof v === "string") kv = { stringValue: v };
|
|
78
102
|
else if (typeof v === "number") {
|
package/lib/outbox.js
CHANGED
|
@@ -70,6 +70,7 @@ var lazyRequire = require("./lazy-require");
|
|
|
70
70
|
var safeAsync = require("./safe-async");
|
|
71
71
|
var safeJson = require("./safe-json");
|
|
72
72
|
var safeSql = require("./safe-sql");
|
|
73
|
+
var sql = require("./sql");
|
|
73
74
|
var validateOpts = require("./validate-opts");
|
|
74
75
|
var { defineClass } = require("./framework-error");
|
|
75
76
|
|
|
@@ -93,6 +94,15 @@ function _validateTableName(name) {
|
|
|
93
94
|
return safeSql.quoteIdentifier(name);
|
|
94
95
|
}
|
|
95
96
|
|
|
97
|
+
// Map the operator backend's dialect tag to the b.sql dialect vocabulary.
|
|
98
|
+
// b.sql's terminal toExternalSql() then emits $1..$N for postgres and `?`
|
|
99
|
+
// for sqlite / mysql, matching what the operator-supplied driver expects.
|
|
100
|
+
function _sqlDialect(externalDb) {
|
|
101
|
+
var d = externalDb && externalDb.dialect;
|
|
102
|
+
if (d === "postgres" || d === "mysql") return d;
|
|
103
|
+
return "sqlite";
|
|
104
|
+
}
|
|
105
|
+
|
|
96
106
|
function _utcNowExpr(externalDb) {
|
|
97
107
|
// The framework's externalDb backends wrap Postgres + SQLite. Both
|
|
98
108
|
// accept a parameterized timestamp via JS Date → ISO string for
|
|
@@ -198,7 +208,10 @@ function create(opts) {
|
|
|
198
208
|
}
|
|
199
209
|
validateOpts.requireNonEmptyString(opts.table,
|
|
200
210
|
"outbox.create: table", OutboxError, "outbox/bad-table");
|
|
201
|
-
|
|
211
|
+
// Validate the table identifier at create-time so a bad name throws at
|
|
212
|
+
// boot, not at first query. b.sql re-quotes the name by construction on
|
|
213
|
+
// every emitted statement (the builder owns identifier quoting now).
|
|
214
|
+
_validateTableName(opts.table);
|
|
202
215
|
|
|
203
216
|
if (typeof opts.publisher !== "function") {
|
|
204
217
|
throw new OutboxError("outbox/bad-publisher",
|
|
@@ -302,38 +315,57 @@ function create(opts) {
|
|
|
302
315
|
"outbox.enqueue: payload/headers must be JSON-serializable: " + e.message);
|
|
303
316
|
}
|
|
304
317
|
|
|
305
|
-
var sql = "INSERT INTO " + quotedTable +
|
|
306
|
-
" (topic, payload, key, headers, enqueued_at, next_attempt_at, attempts, status)" +
|
|
307
|
-
" VALUES ($1, $2, $3, $4, $5, $5, 0, 'pending')";
|
|
308
318
|
var now = _utcNowExpr(externalDb);
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
319
|
+
// enqueued_at and next_attempt_at both take the same publisher-clock
|
|
320
|
+
// moment; b.sql binds it as two separate `?` so the placeholder/param
|
|
321
|
+
// parity gate holds (no $5-reused-twice shorthand).
|
|
322
|
+
var stmt = sql.insert(opts.table, { dialect: _sqlDialect(externalDb) })
|
|
323
|
+
.values({
|
|
324
|
+
topic: event.topic,
|
|
325
|
+
payload: payloadJson,
|
|
326
|
+
key: event.key || null,
|
|
327
|
+
headers: headersJson,
|
|
328
|
+
enqueued_at: now,
|
|
329
|
+
next_attempt_at: now,
|
|
330
|
+
attempts: 0,
|
|
331
|
+
status: "pending",
|
|
332
|
+
})
|
|
333
|
+
.toExternalSql(_sqlDialect(externalDb));
|
|
334
|
+
await txn.query(stmt.sql, stmt.params);
|
|
312
335
|
_emitMetric("enqueued", 1);
|
|
313
336
|
}
|
|
314
337
|
|
|
315
338
|
async function declareSchema(xdb) {
|
|
316
339
|
var target = xdb || externalDb;
|
|
317
|
-
var
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
")"
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
340
|
+
var dialect = _sqlDialect(target);
|
|
341
|
+
// The identity PK renders dialect-correct (BIGSERIAL on postgres,
|
|
342
|
+
// INTEGER PRIMARY KEY AUTOINCREMENT on sqlite, BIGINT AUTO_INCREMENT
|
|
343
|
+
// on mysql) - the prior hand-rolled DDL hardcoded Postgres BIGSERIAL /
|
|
344
|
+
// TIMESTAMPTZ even on a sqlite backend, which the dialect-aware type
|
|
345
|
+
// map now corrects. A varchar-with-length / timestamp-with-zone is
|
|
346
|
+
// passed verbatim by the type map (it sits in type position after a
|
|
347
|
+
// quoted column name, so no identifier injection is possible).
|
|
348
|
+
var tsType = dialect === "postgres" ? "TIMESTAMPTZ" : "TIMESTAMP";
|
|
349
|
+
var ddl = sql.toExternalSql(sql.createTable(opts.table, [
|
|
350
|
+
{ name: "id", serial: true },
|
|
351
|
+
{ name: "topic", type: "VARCHAR(255)", notNull: true },
|
|
352
|
+
{ name: "payload", type: "TEXT", notNull: true },
|
|
353
|
+
{ name: "key", type: "VARCHAR(255)" },
|
|
354
|
+
{ name: "headers", type: "TEXT" },
|
|
355
|
+
{ name: "enqueued_at", type: tsType, notNull: true },
|
|
356
|
+
{ name: "next_attempt_at", type: tsType, notNull: true },
|
|
357
|
+
{ name: "published_at", type: tsType },
|
|
358
|
+
{ name: "attempts", type: "INTEGER", notNull: true, default: 0 },
|
|
359
|
+
{ name: "last_error", type: "TEXT" },
|
|
360
|
+
{ name: "status", type: "VARCHAR(16)", notNull: true, default: "pending" },
|
|
361
|
+
], { dialect: dialect }), dialect);
|
|
362
|
+
// Partial index on the pending pool (the publisher's claim path scans
|
|
363
|
+
// status='pending' ORDER BY next_attempt_at). The 'pending' literal is
|
|
364
|
+
// a builder-emitted static predicate, opted in via allowLiterals.
|
|
365
|
+
var idx = sql.toExternalSql(sql.createIndex(opts.table + "_pending_idx", opts.table,
|
|
366
|
+
["next_attempt_at"], { dialect: dialect, where: "status = 'pending'" }), dialect);
|
|
367
|
+
await target.query(ddl.sql, ddl.params);
|
|
368
|
+
await target.query(idx.sql, idx.params);
|
|
337
369
|
}
|
|
338
370
|
|
|
339
371
|
// ---- Publisher worker ----
|
|
@@ -363,18 +395,24 @@ function create(opts) {
|
|
|
363
395
|
|
|
364
396
|
async function _claimBatch() {
|
|
365
397
|
var supportsSkipLocked = _supportsForUpdateSkipLocked();
|
|
398
|
+
var dialect = _sqlDialect(externalDb);
|
|
399
|
+
var CLAIM_COLS = ["id", "topic", "payload", "key", "headers", "attempts"];
|
|
366
400
|
return await externalDb.transaction(async function (xdb) {
|
|
367
401
|
var nowExpr = _utcNowExpr(externalDb);
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
"
|
|
373
|
-
"
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
402
|
+
// status='pending' is a builder-emitted static predicate (opted in
|
|
403
|
+
// via allowLiterals); next_attempt_at <= ? + the LIMIT both bind.
|
|
404
|
+
var selectBuilder = sql.select(opts.table, { dialect: dialect })
|
|
405
|
+
.columns(CLAIM_COLS)
|
|
406
|
+
.whereRaw("status = 'pending'", [], { allowLiterals: true })
|
|
407
|
+
.whereRaw("next_attempt_at <= ?", [nowExpr])
|
|
408
|
+
.orderBy("next_attempt_at")
|
|
409
|
+
.limit(batchSize);
|
|
410
|
+
// FOR UPDATE SKIP LOCKED on postgres / mysql; sqlite is a single
|
|
411
|
+
// writer with no row lock, so the claim there is the conservative
|
|
412
|
+
// mark-then-reselect path below (b.sql refuses forUpdate on sqlite).
|
|
413
|
+
if (supportsSkipLocked) selectBuilder.forUpdate({ skipLocked: true });
|
|
414
|
+
var selectSql = selectBuilder.toExternalSql(dialect);
|
|
415
|
+
var rows = await xdb.query(selectSql.sql, selectSql.params);
|
|
378
416
|
if (!rows || !rows.rows || rows.rows.length === 0) return [];
|
|
379
417
|
var ids = rows.rows.map(function (r) { return r.id; });
|
|
380
418
|
// Atomic claim: when the dialect lacks SKIP LOCKED, the UPDATE
|
|
@@ -386,30 +424,33 @@ function create(opts) {
|
|
|
386
424
|
// way Postgres does).
|
|
387
425
|
var actuallyClaimed;
|
|
388
426
|
if (supportsSkipLocked) {
|
|
389
|
-
// Postgres/MySQL: row lock held; ANY(
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
427
|
+
// Postgres/MySQL: row lock held; whereInArray emits `id = ANY(?)`
|
|
428
|
+
// on postgres (the whole id set as one bound array) / expanded
|
|
429
|
+
// `IN (?, ?, ...)` on mysql.
|
|
430
|
+
var claimUpdate = sql.update(opts.table, { dialect: dialect })
|
|
431
|
+
.set({ status: "in-flight" })
|
|
432
|
+
.whereInArray("id", ids)
|
|
433
|
+
.toExternalSql(dialect);
|
|
434
|
+
await xdb.query(claimUpdate.sql, claimUpdate.params);
|
|
394
435
|
actuallyClaimed = rows.rows;
|
|
395
436
|
} else {
|
|
396
437
|
// SQLite (or "other") path: emit a portable UPDATE that
|
|
397
438
|
// refuses overlap by gating on status='pending'. After the
|
|
398
439
|
// update we re-read the in-flight rows we own; rows that
|
|
399
|
-
// another publisher beat us to are skipped.
|
|
400
|
-
//
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
"
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
)
|
|
408
|
-
|
|
409
|
-
"
|
|
410
|
-
"
|
|
411
|
-
|
|
412
|
-
);
|
|
440
|
+
// another publisher beat us to are skipped. whereInArray expands
|
|
441
|
+
// to an `IN (?, ?, ...)` placeholder list on sqlite.
|
|
442
|
+
var markUpdate = sql.update(opts.table, { dialect: dialect })
|
|
443
|
+
.set({ status: "in-flight" })
|
|
444
|
+
.whereRaw("status = 'pending'", [], { allowLiterals: true })
|
|
445
|
+
.whereInArray("id", ids)
|
|
446
|
+
.toExternalSql(dialect);
|
|
447
|
+
await xdb.query(markUpdate.sql, markUpdate.params);
|
|
448
|
+
var afterSelect = sql.select(opts.table, { dialect: dialect })
|
|
449
|
+
.columns(CLAIM_COLS)
|
|
450
|
+
.whereRaw("status = 'in-flight'", [], { allowLiterals: true })
|
|
451
|
+
.whereInArray("id", ids)
|
|
452
|
+
.toExternalSql(dialect);
|
|
453
|
+
var afterRows = await xdb.query(afterSelect.sql, afterSelect.params);
|
|
413
454
|
actuallyClaimed = (afterRows && afterRows.rows) || [];
|
|
414
455
|
}
|
|
415
456
|
return actuallyClaimed.map(function (r) {
|
|
@@ -426,29 +467,40 @@ function create(opts) {
|
|
|
426
467
|
}
|
|
427
468
|
|
|
428
469
|
async function _markPublished(id) {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
470
|
+
var dialect = _sqlDialect(externalDb);
|
|
471
|
+
var stmt = sql.update(opts.table, { dialect: dialect })
|
|
472
|
+
.set({ status: "published", published_at: _utcNowExpr(externalDb) })
|
|
473
|
+
.where("id", id)
|
|
474
|
+
.toExternalSql(dialect);
|
|
475
|
+
await externalDb.query(stmt.sql, stmt.params);
|
|
434
476
|
}
|
|
435
477
|
|
|
436
478
|
async function _markRetry(id, attempts, errMsg) {
|
|
479
|
+
var dialect = _sqlDialect(externalDb);
|
|
437
480
|
var nextAt = new Date(Date.now() + _backoffMs(attempts + 1));
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
481
|
+
var stmt = sql.update(opts.table, { dialect: dialect })
|
|
482
|
+
.set({
|
|
483
|
+
status: "pending",
|
|
484
|
+
attempts: attempts + 1,
|
|
485
|
+
last_error: String(errMsg).slice(0, 1024), // error-message char cap
|
|
486
|
+
next_attempt_at: nextAt,
|
|
487
|
+
})
|
|
488
|
+
.where("id", id)
|
|
489
|
+
.toExternalSql(dialect);
|
|
490
|
+
await externalDb.query(stmt.sql, stmt.params);
|
|
444
491
|
}
|
|
445
492
|
|
|
446
493
|
async function _markDead(id, attempts, errMsg) {
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
494
|
+
var dialect = _sqlDialect(externalDb);
|
|
495
|
+
var stmt = sql.update(opts.table, { dialect: dialect })
|
|
496
|
+
.set({
|
|
497
|
+
status: "dead",
|
|
498
|
+
attempts: attempts + 1,
|
|
499
|
+
last_error: String(errMsg).slice(0, 1024), // error-message char cap
|
|
500
|
+
})
|
|
501
|
+
.where("id", id)
|
|
502
|
+
.toExternalSql(dialect);
|
|
503
|
+
await externalDb.query(stmt.sql, stmt.params);
|
|
452
504
|
_emitAudit("system.outbox.deadletter", "failure", { id: id, attempts: attempts + 1 });
|
|
453
505
|
_emitMetric("dead-letter", 1);
|
|
454
506
|
}
|
|
@@ -516,19 +568,21 @@ function create(opts) {
|
|
|
516
568
|
_emitAudit("system.outbox.stopped", "success", { name: name });
|
|
517
569
|
}
|
|
518
570
|
|
|
519
|
-
async function
|
|
520
|
-
var
|
|
521
|
-
|
|
522
|
-
)
|
|
571
|
+
async function _statusCount(status) {
|
|
572
|
+
var dialect = _sqlDialect(externalDb);
|
|
573
|
+
// status is a fixed builder-internal literal ('pending' / 'dead'),
|
|
574
|
+
// never operator input; opted in via allowLiterals. COUNT(*) AS n is
|
|
575
|
+
// the count aggregate with an alias.
|
|
576
|
+
var stmt = sql.select(opts.table, { dialect: dialect })
|
|
577
|
+
.count("*", "n")
|
|
578
|
+
.whereRaw("status = '" + status + "'", [], { allowLiterals: true })
|
|
579
|
+
.toExternalSql(dialect);
|
|
580
|
+
var res = await externalDb.query(stmt.sql, stmt.params);
|
|
523
581
|
return Number((res && res.rows && res.rows[0] && res.rows[0].n) || 0);
|
|
524
582
|
}
|
|
525
583
|
|
|
526
|
-
async function
|
|
527
|
-
|
|
528
|
-
"SELECT COUNT(*) AS n FROM " + quotedTable + " WHERE status = 'dead'", []
|
|
529
|
-
);
|
|
530
|
-
return Number((res && res.rows && res.rows[0] && res.rows[0].n) || 0);
|
|
531
|
-
}
|
|
584
|
+
async function pendingCount() { return await _statusCount("pending"); }
|
|
585
|
+
async function deadCount() { return await _statusCount("dead"); }
|
|
532
586
|
|
|
533
587
|
return {
|
|
534
588
|
enqueue: enqueue,
|
package/lib/parsers/safe-xml.js
CHANGED
|
@@ -13,10 +13,18 @@
|
|
|
13
13
|
* - Processing instructions referencing external resources
|
|
14
14
|
* - Unbounded recursion / element count / attribute count
|
|
15
15
|
* - CDATA sections of arbitrary length
|
|
16
|
+
* - Prototype pollution: an element or attribute named __proto__,
|
|
17
|
+
* constructor, or prototype landing as a key in the result tree
|
|
18
|
+
* (CWE-1321 / OWASP prototype-pollution)
|
|
16
19
|
*
|
|
17
20
|
* This parser closes all of them by default. DOCTYPE, external entities,
|
|
18
21
|
* and processing instructions other than '<?xml ?>' are REJECTED — apps
|
|
19
|
-
* that need them are using the wrong parser.
|
|
22
|
+
* that need them are using the wrong parser. Element and attribute names
|
|
23
|
+
* equal to __proto__ / constructor / prototype are REJECTED with
|
|
24
|
+
* xml/forbidden-name so they can never collide with an inherited member
|
|
25
|
+
* or reassign an accumulator's prototype; the result tree and every
|
|
26
|
+
* nested object it contains have a null prototype, so a consumer reading
|
|
27
|
+
* an absent key sees undefined rather than an inherited Object member.
|
|
20
28
|
*
|
|
21
29
|
* Output: a plain JS object. Element with attributes + children:
|
|
22
30
|
* <root id="x"><child>text</child></root>
|
|
@@ -75,6 +83,18 @@ var ABSOLUTE_MAX_ATTRIBUTES = 1_000;
|
|
|
75
83
|
// XML built-in entities (the ONLY entities allowed)
|
|
76
84
|
var BUILT_IN_ENTITIES = { lt: "<", gt: ">", amp: "&", quot: "\"", apos: "'" };
|
|
77
85
|
|
|
86
|
+
// Names that must never become a key in the result tree. A plain object
|
|
87
|
+
// inherits these from Object.prototype; an element/attribute named after
|
|
88
|
+
// one of them would otherwise collide with the inherited member (a
|
|
89
|
+
// consumer sees a function/object instead of undefined) or — for a
|
|
90
|
+
// computed-member write of an object value — reassign the accumulator's
|
|
91
|
+
// prototype (CWE-1321 / OWASP prototype-pollution). The accumulators are
|
|
92
|
+
// built with a null prototype, and these names are rejected outright so
|
|
93
|
+
// the result is always a clean key→value map. Mirrors the
|
|
94
|
+
// __proto__/constructor/prototype rejection the toml / yaml / ini
|
|
95
|
+
// parsers in this family already apply.
|
|
96
|
+
var FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
97
|
+
|
|
78
98
|
function _validateAndCap(name, value, defaultValue, ceiling) {
|
|
79
99
|
if (value === undefined) return defaultValue;
|
|
80
100
|
if (!numericBounds.isPositiveFiniteInt(value)) {
|
|
@@ -178,7 +198,12 @@ function parse(input, opts) {
|
|
|
178
198
|
} else break;
|
|
179
199
|
}
|
|
180
200
|
if (pos === start) throw _err("expected name", "xml/bad-name");
|
|
181
|
-
|
|
201
|
+
var parsed = input.substring(start, pos);
|
|
202
|
+
if (FORBIDDEN_KEYS.has(parsed)) {
|
|
203
|
+
throw _err("element/attribute name '" + parsed +
|
|
204
|
+
"' is reserved (prototype-pollution defense)", "xml/forbidden-name");
|
|
205
|
+
}
|
|
206
|
+
return parsed;
|
|
182
207
|
}
|
|
183
208
|
|
|
184
209
|
// Parse an attribute value (single- or double-quoted)
|
|
@@ -267,7 +292,12 @@ function parse(input, opts) {
|
|
|
267
292
|
|
|
268
293
|
expectChar("<");
|
|
269
294
|
var name = parseName();
|
|
270
|
-
|
|
295
|
+
// Null-prototype accumulator keyed by attacker-influenced attribute
|
|
296
|
+
// names — no inherited Object member can shadow a missing key, and the
|
|
297
|
+
// duplicate-attribute check below can't be fooled by an inherited
|
|
298
|
+
// function (CWE-1321). Forbidden names are already rejected in
|
|
299
|
+
// parseName.
|
|
300
|
+
var attrs = Object.create(null);
|
|
271
301
|
var attrCount = 0;
|
|
272
302
|
|
|
273
303
|
while (pos < len) {
|
|
@@ -351,10 +381,15 @@ function parse(input, opts) {
|
|
|
351
381
|
// Pure-text element → string
|
|
352
382
|
return _make(name, textParts.join("").trim() === "" ? textParts.join("") : textParts.join(""));
|
|
353
383
|
}
|
|
354
|
-
// Mixed / attributed element → object
|
|
355
|
-
|
|
384
|
+
// Mixed / attributed element → object. Both accumulators carry a null
|
|
385
|
+
// prototype: `grouped` is keyed by attacker-influenced child element
|
|
386
|
+
// names and `obj` receives them via Object.assign, so neither may
|
|
387
|
+
// expose an inherited Object member or be prototype-poisoned by a
|
|
388
|
+
// computed-member write (CWE-1321). Forbidden child names were already
|
|
389
|
+
// rejected in parseName.
|
|
390
|
+
var obj = Object.create(null);
|
|
356
391
|
if (hasAttrs) obj["@attrs"] = attrs;
|
|
357
|
-
var grouped =
|
|
392
|
+
var grouped = Object.create(null);
|
|
358
393
|
for (var i = 0; i < elementChildren.length; i++) {
|
|
359
394
|
var childWrap = elementChildren[i].value;
|
|
360
395
|
var childName = Object.keys(childWrap)[0];
|
|
@@ -374,7 +409,12 @@ function parse(input, opts) {
|
|
|
374
409
|
}
|
|
375
410
|
|
|
376
411
|
function _make(name, value) {
|
|
377
|
-
|
|
412
|
+
// Null-prototype wrapper keyed by the element name (parser-controlled,
|
|
413
|
+
// attacker-influenced). `out[name] = value` with a forbidden name
|
|
414
|
+
// would otherwise reassign the wrapper's prototype when value is an
|
|
415
|
+
// object; the name is already rejected in parseName and the null
|
|
416
|
+
// prototype removes the inherited-member surface entirely (CWE-1321).
|
|
417
|
+
var out = Object.create(null);
|
|
378
418
|
out[name] = value;
|
|
379
419
|
return out;
|
|
380
420
|
}
|
package/lib/pqc-agent.js
CHANGED
|
@@ -41,6 +41,33 @@ var PqcAgentError = defineClass("PqcAgentError", { alwaysPermanent: true });
|
|
|
41
41
|
// cycles when pqc-agent is required during framework bootstrap.
|
|
42
42
|
var audit = lazyRequire(function () { return require("./audit"); });
|
|
43
43
|
|
|
44
|
+
// Observe an outbound socket's negotiated TLS key-exchange group and audit a
|
|
45
|
+
// classical (non-PQC) downgrade. node:tls reports getEphemeralKeyInfo() as
|
|
46
|
+
// { type:"ECDH", name:"X25519", ... } for a classical group and as {} for an
|
|
47
|
+
// ML-KEM hybrid (it doesn't model the hybrid as ECDH). So a NON-empty name
|
|
48
|
+
// that doesn't carry "MLKEM" means the peer offered no hybrid and the
|
|
49
|
+
// handshake fell back to classical X25519 (the framework's last-resort
|
|
50
|
+
// group) — emit the downgrade so operators can see which dependencies are
|
|
51
|
+
// not yet PQC-ready. Best-effort + drop-silent: an audit failure must never
|
|
52
|
+
// break the request that triggered it.
|
|
53
|
+
function auditClassicalDowngrade(socket, meta) {
|
|
54
|
+
try {
|
|
55
|
+
if (!socket || typeof socket.getEphemeralKeyInfo !== "function") return;
|
|
56
|
+
var info = socket.getEphemeralKeyInfo() || {};
|
|
57
|
+
var group = info.name;
|
|
58
|
+
if (!group || /MLKEM/i.test(group)) return; // hybrid (or unreported) — not a downgrade
|
|
59
|
+
audit().safeEmit({
|
|
60
|
+
action: "tls.classical_downgrade",
|
|
61
|
+
outcome: "success",
|
|
62
|
+
metadata: {
|
|
63
|
+
group: group,
|
|
64
|
+
host: (meta && (meta.host || meta.servername)) || null,
|
|
65
|
+
port: (meta && meta.port) || null,
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
} catch (_e) { /* drop-silent — audit is best-effort; never break TLS */ }
|
|
69
|
+
}
|
|
70
|
+
|
|
44
71
|
// IANA TLS Supported Groups Registry — every named-group identifier
|
|
45
72
|
// the framework knows by name. Operators with `allowOperatorGroups:
|
|
46
73
|
// true` may pass any entry from this registry; entries outside it
|
|
@@ -186,6 +213,19 @@ function create(opts) {
|
|
|
186
213
|
var built = _buildAgentOpts(opts);
|
|
187
214
|
var agent = new https.Agent(built);
|
|
188
215
|
agent._builtOpts = built;
|
|
216
|
+
// Observe each NEW outbound socket's negotiated group (createConnection
|
|
217
|
+
// runs per fresh connection, not per keep-alive reuse). A classical
|
|
218
|
+
// negotiation means the peer offered no ML-KEM hybrid — audit the
|
|
219
|
+
// downgrade. Hybrid stays preferred on every handshake; this only fires on
|
|
220
|
+
// the classical fallback.
|
|
221
|
+
var _origCreateConnection = agent.createConnection.bind(agent);
|
|
222
|
+
agent.createConnection = function (options, cb) {
|
|
223
|
+
var socket = _origCreateConnection(options, cb);
|
|
224
|
+
if (socket && typeof socket.once === "function") {
|
|
225
|
+
socket.once("secureConnect", function () { auditClassicalDowngrade(socket, options); });
|
|
226
|
+
}
|
|
227
|
+
return socket;
|
|
228
|
+
};
|
|
189
229
|
// Per-instance cert rotation. The pre-v0.10.9 path required process
|
|
190
230
|
// restart for cert rotation on agents built via explicit `create()`
|
|
191
231
|
// (only the framework's lazy default had `b.pqcAgent.reload()`).
|
|
@@ -345,6 +385,10 @@ module.exports = {
|
|
|
345
385
|
create: create,
|
|
346
386
|
createHttp: createHttp,
|
|
347
387
|
reload: reload,
|
|
388
|
+
// Internal — shared with lib/http-client.js's h2 transport, which connects
|
|
389
|
+
// via node:http2 (not this agent) and so needs the same downgrade
|
|
390
|
+
// observation. Underscore-prefixed: not a public operator primitive.
|
|
391
|
+
_auditClassicalDowngrade: auditClassicalDowngrade,
|
|
348
392
|
DEFAULT_OPTS: DEFAULT_OPTS,
|
|
349
393
|
KNOWN_TLS_GROUPS: KNOWN_TLS_GROUPS,
|
|
350
394
|
enforced: true,
|
package/lib/pubsub-cluster.js
CHANGED
|
@@ -22,6 +22,8 @@
|
|
|
22
22
|
* publishedBy)`. Created by `lib/cluster-storage.js` migrations.
|
|
23
23
|
*/
|
|
24
24
|
var clusterStorage = require("./cluster-storage");
|
|
25
|
+
var frameworkSchema = require("./framework-schema");
|
|
26
|
+
var sql = require("./sql");
|
|
25
27
|
var C = require("./constants");
|
|
26
28
|
var lazyRequire = require("./lazy-require");
|
|
27
29
|
var validateOpts = require("./validate-opts");
|
|
@@ -31,6 +33,24 @@ var logger = lazyRequire(function () { return require("./log").boot("pubsub-clus
|
|
|
31
33
|
|
|
32
34
|
var PubsubError = defineClass("PubsubError");
|
|
33
35
|
|
|
36
|
+
// Resolved once: the fan-out table's concrete name, honoring the
|
|
37
|
+
// configurable framework-table prefix. clusterStorage.execute leaves
|
|
38
|
+
// this self-prefixed name unrewritten (its rewrite map is identity-
|
|
39
|
+
// filtered for already-prefixed tables) and translates `?` to `$N` for
|
|
40
|
+
// Postgres, so b.sql emits the bare resolved name + `?` placeholders.
|
|
41
|
+
var MESSAGES_TABLE = frameworkSchema.tableName("_blamejs_pubsub_messages"); // allow:hand-rolled-sql — single canonical logical-name reference
|
|
42
|
+
|
|
43
|
+
// b.sql opts for every fan-out statement: thread the ACTIVE backend
|
|
44
|
+
// dialect (clusterStorage.dialect() — "sqlite" single-node, "postgres" |
|
|
45
|
+
// "mysql" in cluster mode) so the emitted identifier quoting matches the
|
|
46
|
+
// backend the SQL dispatches to. Without it b.sql defaults to "sqlite"
|
|
47
|
+
// and double-quotes identifiers — correct on Postgres (both double-quote)
|
|
48
|
+
// but read as STRING LITERALS by MySQL (no ANSI_QUOTES), turning the
|
|
49
|
+
// INSERT/SELECT/DELETE into syntax errors. clusterStorage.execute still
|
|
50
|
+
// rewrites table names + translates `?` placeholders at dispatch; this
|
|
51
|
+
// controls only the builder-side identifier quoting.
|
|
52
|
+
function _sqlOpts() { return { dialect: clusterStorage.dialect() }; }
|
|
53
|
+
|
|
34
54
|
var DEFAULT_POLL_INTERVAL_MS = 100;
|
|
35
55
|
var DEFAULT_RETENTION_MS = C.TIME.minutes(1);
|
|
36
56
|
var DEFAULT_PRUNE_EVERY_MS = C.TIME.minutes(5);
|
|
@@ -65,11 +85,13 @@ function create(opts) {
|
|
|
65
85
|
|
|
66
86
|
async function publishRemote(scopedChannel, payload) {
|
|
67
87
|
var serialized = JSON.stringify(payload);
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
88
|
+
var built = sql.insert(MESSAGES_TABLE, _sqlOpts()).values({
|
|
89
|
+
topic: scopedChannel,
|
|
90
|
+
payload: serialized,
|
|
91
|
+
publishedAt: Date.now(),
|
|
92
|
+
publishedBy: _nodeId(),
|
|
93
|
+
}).toSql();
|
|
94
|
+
await clusterStorage.execute(built.sql, built.params);
|
|
73
95
|
return { remote: 1 };
|
|
74
96
|
}
|
|
75
97
|
|
|
@@ -78,24 +100,25 @@ function create(opts) {
|
|
|
78
100
|
var nodeId = _nodeId();
|
|
79
101
|
try {
|
|
80
102
|
// First poll: prime lastSeenId to the current MAX so we don't
|
|
81
|
-
// re-dispatch every historical row on startup.
|
|
103
|
+
// re-dispatch every historical row on startup. MAX(id) is NULL on
|
|
104
|
+
// an empty table; Number(null) || 0 below maps that to 0 (the same
|
|
105
|
+
// result the prior COALESCE(MAX(id), 0) produced).
|
|
82
106
|
if (!primed) {
|
|
83
|
-
var
|
|
84
|
-
|
|
85
|
-
[]
|
|
86
|
-
);
|
|
107
|
+
var primerBuilt = sql.select(MESSAGES_TABLE, _sqlOpts()).max("id", "maxId").toSql();
|
|
108
|
+
var primer = await clusterStorage.execute(primerBuilt.sql, primerBuilt.params);
|
|
87
109
|
if (primer.rows && primer.rows[0]) {
|
|
88
110
|
lastSeenId = Number(primer.rows[0].maxId) || 0;
|
|
89
111
|
}
|
|
90
112
|
primed = true;
|
|
91
113
|
return;
|
|
92
114
|
}
|
|
93
|
-
var
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
"
|
|
97
|
-
|
|
98
|
-
|
|
115
|
+
var pollBuilt = sql.select(MESSAGES_TABLE, _sqlOpts())
|
|
116
|
+
.columns(["id", "topic", "payload", "publishedAt", "publishedBy"])
|
|
117
|
+
.where("id", ">", lastSeenId)
|
|
118
|
+
.where("publishedBy", "<>", nodeId)
|
|
119
|
+
.orderBy("id", "asc")
|
|
120
|
+
.toSql();
|
|
121
|
+
var result = await clusterStorage.execute(pollBuilt.sql, pollBuilt.params);
|
|
99
122
|
var rows = result.rows || [];
|
|
100
123
|
for (var i = 0; i < rows.length; i++) {
|
|
101
124
|
var row = rows[i];
|
|
@@ -116,10 +139,9 @@ function create(opts) {
|
|
|
116
139
|
var now = Date.now();
|
|
117
140
|
if (now - lastPruneAt >= pruneEveryMs) {
|
|
118
141
|
lastPruneAt = now;
|
|
119
|
-
|
|
120
|
-
"
|
|
121
|
-
|
|
122
|
-
);
|
|
142
|
+
var pruneBuilt = sql.delete(MESSAGES_TABLE, _sqlOpts())
|
|
143
|
+
.where("publishedAt", "<", now - retentionMs).toSql();
|
|
144
|
+
await clusterStorage.execute(pruneBuilt.sql, pruneBuilt.params);
|
|
123
145
|
}
|
|
124
146
|
} catch (e) {
|
|
125
147
|
try { logger().warn("pubsub-cluster poll failed: " +
|