@blamejs/core 0.14.26 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +2 -2
- package/index.js +4 -0
- package/lib/agent-envelope-mac.js +104 -0
- package/lib/agent-event-bus.js +105 -4
- package/lib/agent-posture-chain.js +8 -42
- package/lib/ai-content-detect.js +9 -10
- package/lib/api-key.js +107 -74
- package/lib/atomic-file.js +62 -4
- package/lib/audit-chain.js +47 -11
- package/lib/audit-sign.js +77 -2
- package/lib/audit-tools.js +79 -51
- package/lib/audit.js +249 -123
- package/lib/auth/openid-federation.js +108 -47
- package/lib/backup/index.js +13 -10
- package/lib/break-glass.js +202 -144
- package/lib/cache.js +174 -105
- package/lib/chain-writer.js +38 -16
- package/lib/cli.js +19 -14
- package/lib/cluster-provider-db.js +130 -104
- package/lib/cluster-storage.js +119 -22
- package/lib/cluster.js +119 -71
- package/lib/compliance.js +169 -4
- package/lib/consent.js +73 -24
- package/lib/constants.js +16 -11
- package/lib/crypto-field.js +474 -92
- package/lib/db-declare-row-policy.js +35 -22
- package/lib/db-file-lifecycle.js +3 -2
- package/lib/db-query.js +497 -255
- package/lib/db-schema.js +209 -44
- package/lib/db.js +176 -95
- package/lib/error-page.js +14 -1
- package/lib/external-db-migrate.js +229 -139
- package/lib/external-db.js +25 -15
- package/lib/file-upload.js +52 -7
- package/lib/framework-error.js +14 -1
- package/lib/framework-files.js +73 -0
- package/lib/framework-schema.js +695 -394
- package/lib/gate-contract.js +649 -1
- package/lib/guard-agent-registry.js +26 -44
- package/lib/guard-all.js +1 -0
- package/lib/guard-auth.js +42 -112
- package/lib/guard-cidr.js +33 -154
- package/lib/guard-csv.js +46 -113
- package/lib/guard-domain.js +34 -157
- package/lib/guard-dsn.js +27 -43
- package/lib/guard-email.js +47 -69
- package/lib/guard-envelope.js +19 -32
- package/lib/guard-event-bus-payload.js +24 -42
- package/lib/guard-event-bus-topic.js +25 -43
- package/lib/guard-filename.js +42 -106
- package/lib/guard-graphql.js +42 -123
- package/lib/guard-html.js +53 -108
- package/lib/guard-idempotency-key.js +24 -42
- package/lib/guard-image.js +46 -103
- package/lib/guard-imap-command.js +18 -32
- package/lib/guard-jmap.js +16 -30
- package/lib/guard-json.js +38 -108
- package/lib/guard-jsonpath.js +38 -171
- package/lib/guard-jwt.js +49 -179
- package/lib/guard-list-id.js +25 -41
- package/lib/guard-list-unsubscribe.js +27 -43
- package/lib/guard-mail-compose.js +24 -42
- package/lib/guard-mail-move.js +26 -44
- package/lib/guard-mail-query.js +28 -46
- package/lib/guard-mail-reply.js +24 -42
- package/lib/guard-mail-sieve.js +24 -42
- package/lib/guard-managesieve-command.js +17 -31
- package/lib/guard-markdown.js +37 -104
- package/lib/guard-message-id.js +26 -45
- package/lib/guard-mime.js +39 -151
- package/lib/guard-oauth.js +54 -135
- package/lib/guard-pdf.js +45 -101
- package/lib/guard-pop3-command.js +21 -31
- package/lib/guard-posture-chain.js +24 -42
- package/lib/guard-regex.js +33 -107
- package/lib/guard-saga-config.js +24 -42
- package/lib/guard-shell.js +42 -172
- package/lib/guard-smtp-command.js +48 -54
- package/lib/guard-snapshot-envelope.js +24 -42
- package/lib/guard-sql.js +1491 -0
- package/lib/guard-stream-args.js +24 -43
- package/lib/guard-svg.js +47 -65
- package/lib/guard-template.js +35 -172
- package/lib/guard-tenant-id.js +26 -45
- package/lib/guard-time.js +32 -154
- package/lib/guard-trace-context.js +25 -44
- package/lib/guard-uuid.js +32 -153
- package/lib/guard-xml.js +38 -113
- package/lib/guard-yaml.js +51 -163
- package/lib/http-client.js +37 -9
- package/lib/inbox.js +120 -107
- package/lib/legal-hold.js +107 -50
- package/lib/log-stream-cloudwatch.js +47 -31
- package/lib/log-stream-otlp.js +32 -18
- package/lib/mail-crypto-smime.js +2 -6
- package/lib/mail-greylist.js +2 -6
- package/lib/mail-helo.js +2 -6
- package/lib/mail-journal.js +85 -64
- package/lib/mail-rbl.js +2 -6
- package/lib/mail-scan.js +2 -6
- package/lib/mail-server-jmap.js +117 -12
- package/lib/mail-spam-score.js +2 -6
- package/lib/mail-store.js +287 -154
- package/lib/middleware/body-parser.js +71 -25
- package/lib/middleware/csrf-protect.js +19 -8
- package/lib/middleware/fetch-metadata.js +17 -7
- package/lib/middleware/idempotency-key.js +54 -38
- package/lib/middleware/rate-limit.js +102 -32
- package/lib/middleware/security-headers.js +21 -5
- package/lib/migrations.js +108 -66
- package/lib/network-heartbeat.js +7 -0
- package/lib/nonce-store.js +31 -9
- package/lib/object-store/azure-blob-bucket-ops.js +9 -4
- package/lib/object-store/azure-blob.js +57 -3
- package/lib/object-store/sigv4.js +10 -0
- package/lib/observability.js +87 -0
- package/lib/otel-export.js +25 -1
- package/lib/outbox.js +136 -82
- package/lib/parsers/safe-xml.js +47 -7
- package/lib/pqc-agent.js +44 -0
- package/lib/pubsub-cluster.js +42 -20
- package/lib/queue-local.js +202 -139
- package/lib/queue-redis.js +9 -1
- package/lib/queue-sqs.js +6 -0
- package/lib/redact.js +68 -11
- package/lib/redis-client.js +160 -31
- package/lib/retention.js +82 -39
- package/lib/router.js +212 -5
- package/lib/safe-dns.js +29 -45
- package/lib/safe-ical.js +18 -33
- package/lib/safe-icap.js +27 -43
- package/lib/safe-sieve.js +21 -40
- package/lib/safe-sql.js +124 -3
- package/lib/safe-vcard.js +18 -33
- package/lib/scheduler.js +35 -12
- package/lib/seeders.js +122 -74
- package/lib/session-stores.js +42 -14
- package/lib/session.js +109 -72
- package/lib/sql.js +3885 -0
- package/lib/ssrf-guard.js +51 -4
- package/lib/static.js +177 -34
- package/lib/subject.js +55 -17
- package/lib/vault/index.js +3 -2
- package/lib/vault/passphrase-ops.js +3 -2
- package/lib/vault/rotate.js +104 -64
- package/lib/vendor-data.js +2 -0
- package/lib/websocket.js +35 -5
- package/package.json +1 -1
- package/sbom.cdx.json +6 -6
package/lib/guard-yaml.js
CHANGED
|
@@ -553,12 +553,8 @@ function parse(input, opts) {
|
|
|
553
553
|
throw _err("yaml.bad-input", "parse requires string input");
|
|
554
554
|
}
|
|
555
555
|
var issues = _detectIssues(input, opts);
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
throw _err(issues[i].ruleId || "yaml.refused",
|
|
559
|
-
"guardYaml.parse: " + issues[i].snippet);
|
|
560
|
-
}
|
|
561
|
-
}
|
|
556
|
+
gateContract.throwOnRefusalSeverity(issues,
|
|
557
|
+
{ errorClass: GuardYamlError, codePrefix: "yaml", severities: ["critical"], op: "parse" });
|
|
562
558
|
return safeYamlLazy().parse(input, {
|
|
563
559
|
maxBytes: opts.maxBytes,
|
|
564
560
|
maxDepth: opts.maxDepth,
|
|
@@ -566,161 +562,53 @@ function parse(input, opts) {
|
|
|
566
562
|
});
|
|
567
563
|
}
|
|
568
564
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
* var yamlGate = b.guardYaml.gate({ profile: "strict" });
|
|
594
|
-
* var hostile = Buffer.from("!!python/object/new:cls\nargs: [x]\n", "utf8");
|
|
595
|
-
* var verdict = await yamlGate.check({ bytes: hostile });
|
|
596
|
-
* verdict.action; // → "refuse"
|
|
597
|
-
*/
|
|
598
|
-
function gate(opts) {
|
|
599
|
-
opts = _resolveOpts(opts);
|
|
600
|
-
return gateContract.buildGuardGate(
|
|
601
|
-
opts.name || "guardYaml:" + (opts.profile || "default"),
|
|
602
|
-
opts,
|
|
603
|
-
async function (ctx) {
|
|
604
|
-
var text = gateContract.extractBytesAsText(ctx);
|
|
605
|
-
if (!text) return { ok: true, action: "serve" };
|
|
606
|
-
var rv = validate(text, opts);
|
|
607
|
-
if (rv.issues.length === 0) return { ok: true, action: "serve" };
|
|
608
|
-
var hasCritical = rv.issues.some(function (i) {
|
|
609
|
-
return i.severity === "critical" || i.severity === "high";
|
|
610
|
-
});
|
|
611
|
-
if (!hasCritical) return { ok: true, action: "audit-only", issues: rv.issues };
|
|
612
|
-
return { ok: false, action: "refuse", issues: rv.issues };
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
/**
|
|
617
|
-
* @primitive b.guardYaml.buildProfile
|
|
618
|
-
* @signature b.guardYaml.buildProfile(opts)
|
|
619
|
-
* @since 0.7.14
|
|
620
|
-
* @status stable
|
|
621
|
-
* @related b.guardYaml.gate, b.guardYaml.compliancePosture
|
|
622
|
-
*
|
|
623
|
-
* Compose a derived profile from one or more named bases plus
|
|
624
|
-
* inline overrides. `opts.extends` is a profile name (`"strict"` /
|
|
625
|
-
* `"balanced"` / `"permissive"`) or an array of names; later entries
|
|
626
|
-
* shadow earlier ones. Inline `opts` keys win last. Used to keep
|
|
627
|
-
* operator-defined profiles traceable to a baseline rather than re-
|
|
628
|
-
* typing every key.
|
|
629
|
-
*
|
|
630
|
-
* @opts
|
|
631
|
-
* extends: string|string[], // base profile name(s) to compose
|
|
632
|
-
* ...: any guard-yaml key, // inline override of resolved keys
|
|
633
|
-
*
|
|
634
|
-
* @example
|
|
635
|
-
* var custom = b.guardYaml.buildProfile({
|
|
636
|
-
* extends: "balanced",
|
|
637
|
-
* tagPolicy: "reject",
|
|
638
|
-
* maxAnchors: 8,
|
|
639
|
-
* });
|
|
640
|
-
* custom.tagPolicy; // → "reject"
|
|
641
|
-
* custom.maxAnchors; // → 8
|
|
642
|
-
*/
|
|
643
|
-
var buildProfile = gateContract.makeProfileBuilder(PROFILES);
|
|
644
|
-
|
|
645
|
-
/**
|
|
646
|
-
* @primitive b.guardYaml.compliancePosture
|
|
647
|
-
* @signature b.guardYaml.compliancePosture(name)
|
|
648
|
-
* @since 0.7.14
|
|
649
|
-
* @status stable
|
|
650
|
-
* @compliance hipaa, pci-dss, gdpr, soc2
|
|
651
|
-
* @related b.guardYaml.gate, b.guardYaml.buildProfile
|
|
652
|
-
*
|
|
653
|
-
* Look up a compliance-posture overlay by name (`"hipaa"` /
|
|
654
|
-
* `"pci-dss"` / `"gdpr"` / `"soc2"`). Returns a shallow clone of the
|
|
655
|
-
* posture object — the caller may mutate freely. Throws
|
|
656
|
-
* `GuardYamlError("yaml.bad-posture")` on unknown name.
|
|
657
|
-
*
|
|
658
|
-
* @example
|
|
659
|
-
* var posture = b.guardYaml.compliancePosture("hipaa");
|
|
660
|
-
* posture.tagPolicy; // → "reject"
|
|
661
|
-
* posture.forensicSnippetBytes; // → 256
|
|
662
|
-
*/
|
|
663
|
-
function compliancePosture(name) {
|
|
664
|
-
return gateContract.lookupCompliancePosture(name, COMPLIANCE_POSTURES, _err, "yaml");
|
|
665
|
-
}
|
|
565
|
+
// The gate is the standard serve -> audit-only -> refuse chain (content
|
|
566
|
+
// kind, reading ctx.bytes); it is assembled by gateContract.defineGuard's
|
|
567
|
+
// default gate below. YAML sanitize is intentionally not offered — there's
|
|
568
|
+
// no safe re-emit for tag-injection / alias-explosion shapes; the only
|
|
569
|
+
// correct response is refusal, which the default chain (no sanitize action)
|
|
570
|
+
// matches exactly. Its "guardYaml:<profile>" gate name and
|
|
571
|
+
// serve/audit-only/refuse decisions are identical to the hand-written gate
|
|
572
|
+
// this replaced.
|
|
573
|
+
|
|
574
|
+
// buildProfile / compliancePosture / loadRulePack are assembled by
|
|
575
|
+
// gateContract.defineGuard below; their wiki sections render from the
|
|
576
|
+
// single-sourced @abiTemplate (defineGuard) blocks in gate-contract.js,
|
|
577
|
+
// instantiated per guard by the page generator.
|
|
578
|
+
|
|
579
|
+
var INTEGRATION_FIXTURES = Object.freeze({
|
|
580
|
+
kind: "content",
|
|
581
|
+
contentType: "application/yaml",
|
|
582
|
+
extension: ".yaml",
|
|
583
|
+
benignBytes: Buffer.from('name: alice\nage: 30\n', "utf8"),
|
|
584
|
+
// Hostile: deserialization-tag injection (CVE-2026-24009 PyYAML
|
|
585
|
+
// class). Parser-runtime would attempt to instantiate the named
|
|
586
|
+
// language-specific class.
|
|
587
|
+
hostileBytes: Buffer.from("!!python/object/new:cls\nargs: [\"x\"]\n", "utf8"),
|
|
588
|
+
});
|
|
666
589
|
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
*/
|
|
693
|
-
var loadRulePack = _yamlRulePacks.load;
|
|
694
|
-
|
|
695
|
-
module.exports = {
|
|
696
|
-
// ---- guard-* family registry exports ----
|
|
697
|
-
NAME: "yaml",
|
|
698
|
-
KIND: "content",
|
|
699
|
-
MIME_TYPES: Object.freeze([
|
|
700
|
-
"application/yaml", "application/x-yaml", "text/yaml", "text/x-yaml",
|
|
701
|
-
]),
|
|
702
|
-
EXTENSIONS: Object.freeze([".yml", ".yaml"]),
|
|
703
|
-
INTEGRATION_FIXTURES: Object.freeze({
|
|
704
|
-
kind: "content",
|
|
705
|
-
contentType: "application/yaml",
|
|
706
|
-
extension: ".yaml",
|
|
707
|
-
benignBytes: Buffer.from('name: alice\nage: 30\n', "utf8"),
|
|
708
|
-
// Hostile: deserialization-tag injection (CVE-2026-24009 PyYAML
|
|
709
|
-
// class). Parser-runtime would attempt to instantiate the named
|
|
710
|
-
// language-specific class.
|
|
711
|
-
hostileBytes: Buffer.from("!!python/object/new:cls\nargs: [\"x\"]\n", "utf8"),
|
|
712
|
-
}),
|
|
713
|
-
// ---- primitive surface ----
|
|
714
|
-
validate: validate,
|
|
715
|
-
parse: parse,
|
|
716
|
-
gate: gate,
|
|
717
|
-
buildProfile: buildProfile,
|
|
718
|
-
compliancePosture: compliancePosture,
|
|
719
|
-
loadRulePack: loadRulePack,
|
|
720
|
-
PROFILES: PROFILES,
|
|
721
|
-
DEFAULTS: DEFAULTS,
|
|
722
|
-
COMPLIANCE_POSTURES: COMPLIANCE_POSTURES,
|
|
723
|
-
DANGEROUS_TAG_PREFIXES: DANGEROUS_TAG_PREFIXES,
|
|
724
|
-
SAFE_CORE_TAGS: SAFE_CORE_TAGS,
|
|
725
|
-
GuardYamlError: GuardYamlError,
|
|
726
|
-
};
|
|
590
|
+
// Assembled from the gate-contract guard factory: error class, registry
|
|
591
|
+
// exports (NAME / KIND / MIME_TYPES / EXTENSIONS / INTEGRATION_FIXTURES),
|
|
592
|
+
// buildProfile / compliancePosture / loadRulePack wiring, plus the
|
|
593
|
+
// per-guard inspection surface (validate) and YAML extras
|
|
594
|
+
// (parse / DANGEROUS_TAG_PREFIXES / SAFE_CORE_TAGS) passed through
|
|
595
|
+
// verbatim. The gate is the factory default serve/audit-only/refuse chain
|
|
596
|
+
// (content kind, no sanitize action — there's no safe re-emit for
|
|
597
|
+
// tag-injection / alias-explosion shapes).
|
|
598
|
+
module.exports = gateContract.defineGuard({
|
|
599
|
+
name: "yaml",
|
|
600
|
+
kind: "content",
|
|
601
|
+
errorClass: GuardYamlError,
|
|
602
|
+
profiles: PROFILES,
|
|
603
|
+
defaults: DEFAULTS,
|
|
604
|
+
postures: COMPLIANCE_POSTURES,
|
|
605
|
+
mimeTypes: ["application/yaml", "application/x-yaml", "text/yaml", "text/x-yaml"],
|
|
606
|
+
extensions: [".yml", ".yaml"],
|
|
607
|
+
integrationFixtures: INTEGRATION_FIXTURES,
|
|
608
|
+
validate: validate,
|
|
609
|
+
extra: {
|
|
610
|
+
parse: parse,
|
|
611
|
+
DANGEROUS_TAG_PREFIXES: DANGEROUS_TAG_PREFIXES,
|
|
612
|
+
SAFE_CORE_TAGS: SAFE_CORE_TAGS,
|
|
613
|
+
},
|
|
614
|
+
});
|
package/lib/http-client.js
CHANGED
|
@@ -275,6 +275,9 @@ function _connectHttpsWithAlpn(u, ips) {
|
|
|
275
275
|
session.once("connect", function () {
|
|
276
276
|
var alpn = session.alpnProtocol;
|
|
277
277
|
if (alpn === "h2") {
|
|
278
|
+
// node:http2 connects directly (not via pqcAgent), so observe the
|
|
279
|
+
// negotiated group here too and audit a classical (non-PQC) downgrade.
|
|
280
|
+
pqcAgent._auditClassicalDowngrade(session.socket, { host: u.hostname, port: u.port });
|
|
278
281
|
_wireH2Session(session, _originKey(u));
|
|
279
282
|
_done({ kind: "h2", session: session });
|
|
280
283
|
return;
|
|
@@ -285,6 +288,17 @@ function _connectHttpsWithAlpn(u, ips) {
|
|
|
285
288
|
});
|
|
286
289
|
session.once("error", function (err) {
|
|
287
290
|
_tearDownH2Session(session);
|
|
291
|
+
// An HTTP/1.1-only TLS server has no "h2" to select and replies with
|
|
292
|
+
// the no_application_protocol alert (RFC 7301). node:http2 forces an
|
|
293
|
+
// h2-only ALPN offer, so the connect event never fires for such a
|
|
294
|
+
// server and the http/1.1-selection fallback above can't run — catch
|
|
295
|
+
// the alert here and fall back to an h1 transport for this origin.
|
|
296
|
+
// Real Azure / S3 / Keycloak support h2; Azurite, Azure Stack, and
|
|
297
|
+
// many private / older endpoints are h1-only.
|
|
298
|
+
if (err && err.code === "ERR_SSL_TLSV1_ALERT_NO_APPLICATION_PROTOCOL") {
|
|
299
|
+
_done(_makeH1Transport(u, ips));
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
288
302
|
_fail(err);
|
|
289
303
|
});
|
|
290
304
|
});
|
|
@@ -1925,7 +1939,21 @@ async function downloadStream(opts) {
|
|
|
1925
1939
|
});
|
|
1926
1940
|
counter.bytesWritten = 0;
|
|
1927
1941
|
|
|
1928
|
-
|
|
1942
|
+
// CWE-377 (insecure temporary file) / CWE-59 (symlink follow): stage
|
|
1943
|
+
// the download into the sibling tmp file with an EXCLUSIVE, no-follow
|
|
1944
|
+
// create. The legacy "w" flag is O_WRONLY|O_CREAT|O_TRUNC — it would
|
|
1945
|
+
// open (and truncate, or write through) a file an attacker pre-planted
|
|
1946
|
+
// at tmpPath, including a symlink aimed at a victim path this process
|
|
1947
|
+
// can write. O_EXCL fails with EEXIST if anything already exists at
|
|
1948
|
+
// tmpPath; O_NOFOLLOW rejects a symlink in the final path component
|
|
1949
|
+
// where the platform defines it (Windows leaves it undefined → `|| 0`).
|
|
1950
|
+
// tmpPath already carries a 64-bit CSPRNG suffix (line above), so an
|
|
1951
|
+
// EEXIST here is a hostile-collision signal, not a benign retry.
|
|
1952
|
+
var fileStream = nodeFs.createWriteStream(tmpPath, {
|
|
1953
|
+
mode: DEFAULT_DOWNLOAD_FILE_MODE,
|
|
1954
|
+
flags: nodeFs.constants.O_WRONLY | nodeFs.constants.O_CREAT |
|
|
1955
|
+
nodeFs.constants.O_EXCL | (nodeFs.constants.O_NOFOLLOW || 0),
|
|
1956
|
+
});
|
|
1929
1957
|
|
|
1930
1958
|
try {
|
|
1931
1959
|
await streamPromises.pipeline(res.body, counter, fileStream);
|
|
@@ -1944,14 +1972,14 @@ async function downloadStream(opts) {
|
|
|
1944
1972
|
// across platforms but matches the discipline of the rest of the
|
|
1945
1973
|
// framework's atomic-write paths.
|
|
1946
1974
|
//
|
|
1947
|
-
// CodeQL js/insecure-temporary-file
|
|
1948
|
-
//
|
|
1949
|
-
//
|
|
1950
|
-
//
|
|
1951
|
-
// (downloadStream contract — never under os.tmpdir())
|
|
1952
|
-
//
|
|
1953
|
-
//
|
|
1954
|
-
//
|
|
1975
|
+
// CodeQL js/insecure-temporary-file (CWE-377 / CWE-59): the tmp file
|
|
1976
|
+
// was already created above with O_EXCL | O_NOFOLLOW, so this reopen
|
|
1977
|
+
// binds to an inode this process exclusively created at a path
|
|
1978
|
+
// carrying a 64-bit CSPRNG suffix (line above) next to operator-
|
|
1979
|
+
// supplied `dest` (downloadStream contract — never under os.tmpdir()).
|
|
1980
|
+
// The earlier exclusive create — not the reopen — is the symlink-
|
|
1981
|
+
// pre-creation defense; this fd is used solely for fsync, after the
|
|
1982
|
+
// upstream pipeline already wrote the bytes.
|
|
1955
1983
|
try {
|
|
1956
1984
|
var fd = nodeFs.openSync(tmpPath, "r+");
|
|
1957
1985
|
try { atomicFile.fsync(fd); } finally { try { nodeFs.closeSync(fd); } catch (_c) { /* best-effort fd close */ } }
|
package/lib/inbox.js
CHANGED
|
@@ -46,6 +46,7 @@ var C = require("./constants");
|
|
|
46
46
|
var lazyRequire = require("./lazy-require");
|
|
47
47
|
var safeJson = require("./safe-json");
|
|
48
48
|
var safeSql = require("./safe-sql");
|
|
49
|
+
var sql = require("./sql");
|
|
49
50
|
var validateOpts = require("./validate-opts");
|
|
50
51
|
var { defineClass } = require("./framework-error");
|
|
51
52
|
|
|
@@ -63,14 +64,19 @@ function _validateTableName(name) {
|
|
|
63
64
|
}
|
|
64
65
|
}
|
|
65
66
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
67
|
+
// Map the operator backend's dialect tag to the b.sql dialect vocabulary
|
|
68
|
+
// (postgres -> $1..$N at toExternalSql; sqlite/other -> `?`).
|
|
69
|
+
function _sqlDialect(externalDb) {
|
|
70
|
+
return (externalDb && externalDb.dialect === "postgres") ? "postgres" : "sqlite";
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// The server-clock timestamp expression, as an allowlisted b.sql function
|
|
74
|
+
// cell (emits the keyword the engine evaluates, binds no param). Postgres
|
|
75
|
+
// returns timestamptz from NOW(); the portable CURRENT_TIMESTAMP serves
|
|
76
|
+
// sqlite. Used directly in a b.sql values()/set() cell.
|
|
77
|
+
function _utcNowCell(externalDb) {
|
|
78
|
+
return (externalDb && externalDb.dialect === "postgres")
|
|
79
|
+
? sql.fn("NOW") : sql.fn("CURRENT_TIMESTAMP");
|
|
74
80
|
}
|
|
75
81
|
|
|
76
82
|
/**
|
|
@@ -144,12 +150,9 @@ function create(opts) {
|
|
|
144
150
|
|
|
145
151
|
var externalDb = opts.externalDb;
|
|
146
152
|
var tableRaw = opts.table;
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
//
|
|
150
|
-
// lib/safe-sql.js), so one quoted form serves both inbox paths.
|
|
151
|
-
var qTable = safeSql.quoteIdentifier(tableRaw, "sqlite");
|
|
152
|
-
var qIndex = safeSql.quoteIdentifier(tableRaw + "_received_at_idx", "sqlite");
|
|
153
|
+
// The table identifier reaches SQL through b.sql, which validates +
|
|
154
|
+
// quotes it by construction on every emitted statement; _validateTableName
|
|
155
|
+
// above fails fast at create() time on a bad name.
|
|
153
156
|
var retentionDays = (typeof opts.retentionDays === "number" && opts.retentionDays > 0) // allow:numeric-opt-Infinity
|
|
154
157
|
? opts.retentionDays : 30; // default retention days
|
|
155
158
|
var auditOn = opts.audit !== false;
|
|
@@ -227,43 +230,38 @@ function create(opts) {
|
|
|
227
230
|
}
|
|
228
231
|
metaJson = serialized;
|
|
229
232
|
}
|
|
230
|
-
var
|
|
231
|
-
var
|
|
233
|
+
var dialect = _sqlDialect(externalDb);
|
|
234
|
+
var nowCell = _utcNowCell(externalDb);
|
|
232
235
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
var
|
|
257
|
-
"INSERT OR IGNORE INTO " + qTable +
|
|
258
|
-
" (message_id, source, received_at, metadata_json) " +
|
|
259
|
-
" VALUES (?, ?, " + nowExpr + ", ?) RETURNING 1",
|
|
260
|
-
[receiveOpts.messageId, receiveOpts.source, metaJson]);
|
|
261
|
-
var sqlFresh = !!(sqlInsert && sqlInsert.rows && sqlInsert.rows.length === 1);
|
|
236
|
+
// ON CONFLICT (source, message_id) DO NOTHING RETURNING message_id:
|
|
237
|
+
// a fresh insert returns one row, a duplicate redelivery returns none
|
|
238
|
+
// (the collision short-circuits). received_at takes the server-clock
|
|
239
|
+
// function cell (NOW() / CURRENT_TIMESTAMP, no param); metadata_json
|
|
240
|
+
// binds with a ::jsonb cast on Postgres and as a plain `?` on sqlite.
|
|
241
|
+
// RETURNING message_id (rather than RETURNING 1) is the portable
|
|
242
|
+
// presence sentinel - one row iff the insert landed - across both
|
|
243
|
+
// dialects, collapsing the prior INSERT-then-SELECT-changes() race
|
|
244
|
+
// into one round-trip on sqlite too.
|
|
245
|
+
var metaCell = (dialect === "postgres") ? sql.cast(metaJson, "jsonb") : metaJson;
|
|
246
|
+
var stmt = sql.upsert(tableRaw, { dialect: dialect })
|
|
247
|
+
.columns(["message_id", "source", "received_at", "metadata_json"])
|
|
248
|
+
.values({
|
|
249
|
+
message_id: receiveOpts.messageId,
|
|
250
|
+
source: receiveOpts.source,
|
|
251
|
+
received_at: nowCell,
|
|
252
|
+
metadata_json: metaCell,
|
|
253
|
+
})
|
|
254
|
+
.onConflict(["source", "message_id"])
|
|
255
|
+
.doNothing()
|
|
256
|
+
.returning(["message_id"])
|
|
257
|
+
.toExternalSql(dialect);
|
|
258
|
+
var rs = await txn.query(stmt.sql, stmt.params);
|
|
259
|
+
var fresh = !!(rs && rs.rows && rs.rows.length === 1);
|
|
262
260
|
_emitAudit("inbox.received", "success", {
|
|
263
261
|
source: receiveOpts.source, messageId: receiveOpts.messageId,
|
|
264
|
-
fresh:
|
|
262
|
+
fresh: fresh,
|
|
265
263
|
});
|
|
266
|
-
return
|
|
264
|
+
return fresh;
|
|
267
265
|
}
|
|
268
266
|
|
|
269
267
|
async function markProcessed(receiveOpts, txn) {
|
|
@@ -272,13 +270,13 @@ function create(opts) {
|
|
|
272
270
|
"markProcessed: txn must be a transaction handle");
|
|
273
271
|
}
|
|
274
272
|
_validateReceiveOpts(receiveOpts, "markProcessed");
|
|
275
|
-
var
|
|
276
|
-
var
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
await txn.query(sql,
|
|
273
|
+
var dialect = _sqlDialect(externalDb);
|
|
274
|
+
var stmt = sql.update(tableRaw, { dialect: dialect })
|
|
275
|
+
.set({ processed_at: _utcNowCell(externalDb) })
|
|
276
|
+
.where("source", receiveOpts.source)
|
|
277
|
+
.where("message_id", receiveOpts.messageId)
|
|
278
|
+
.toExternalSql(dialect);
|
|
279
|
+
await txn.query(stmt.sql, stmt.params);
|
|
282
280
|
}
|
|
283
281
|
|
|
284
282
|
async function handle(receiveOpts, handler) {
|
|
@@ -321,56 +319,65 @@ function create(opts) {
|
|
|
321
319
|
}
|
|
322
320
|
|
|
323
321
|
async function declareSchema(xdb) {
|
|
324
|
-
var dialect = (xdb
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
await xdb.query(
|
|
349
|
-
"CREATE INDEX IF NOT EXISTS " + qIndex + " " +
|
|
350
|
-
"ON " + qTable + " (received_at)");
|
|
351
|
-
}
|
|
322
|
+
var dialect = _sqlDialect(xdb);
|
|
323
|
+
// received_at / processed_at are a timestamp-with-zone column on
|
|
324
|
+
// Postgres and a TEXT (ISO-8601) column on sqlite; metadata_json is
|
|
325
|
+
// JSONB on Postgres and TEXT on sqlite. The composite (source,
|
|
326
|
+
// message_id) primary key is the dedupe collision boundary. Verbatim
|
|
327
|
+
// type strings sit in type position after the quoted column name.
|
|
328
|
+
var tsType = dialect === "postgres" ? "TIMESTAMPTZ" : "TEXT";
|
|
329
|
+
var tsDefault = dialect === "postgres" ? "NOW()" : "CURRENT_TIMESTAMP";
|
|
330
|
+
var jsonType = dialect === "postgres" ? "JSONB" : "TEXT";
|
|
331
|
+
var ddl = sql.toExternalSql(sql.createTable(tableRaw, [
|
|
332
|
+
{ name: "message_id", type: "TEXT", notNull: true },
|
|
333
|
+
{ name: "source", type: "TEXT", notNull: true },
|
|
334
|
+
// The DEFAULT here is a SQL function keyword, not a bound literal -
|
|
335
|
+
// expressed via the `constraints` verbatim clause so the type map
|
|
336
|
+
// does not quote NOW() / CURRENT_TIMESTAMP as a string default.
|
|
337
|
+
{ name: "received_at", type: tsType, notNull: true, constraints: "DEFAULT " + tsDefault },
|
|
338
|
+
{ name: "processed_at", type: tsType },
|
|
339
|
+
{ name: "metadata_json", type: jsonType },
|
|
340
|
+
], { dialect: dialect, primaryKey: ["source", "message_id"] }), dialect);
|
|
341
|
+
await xdb.query(ddl.sql, ddl.params);
|
|
342
|
+
|
|
343
|
+
var idx = sql.toExternalSql(sql.createIndex(tableRaw + "_received_at_idx", tableRaw,
|
|
344
|
+
["received_at"], { dialect: dialect }), dialect);
|
|
345
|
+
await xdb.query(idx.sql, idx.params);
|
|
352
346
|
}
|
|
353
347
|
|
|
354
348
|
async function sweep() {
|
|
355
|
-
var dialect = (externalDb
|
|
349
|
+
var dialect = _sqlDialect(externalDb);
|
|
356
350
|
var deleted = 0;
|
|
357
351
|
await externalDb.transaction(async function (xdb) {
|
|
358
352
|
if (dialect === "postgres") {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
353
|
+
// received_at < NOW() - <retention>::interval, with the
|
|
354
|
+
// unprocessed grace of 2x retention. The interval STRINGS bind as
|
|
355
|
+
// a ::interval-cast `?` (the cast value-cell form); NOW() is the
|
|
356
|
+
// server-clock function token in the predicate (a guarded raw
|
|
357
|
+
// fragment - no string literal, no bound value).
|
|
358
|
+
var delStmt = sql.delete(tableRaw, { dialect: "postgres" })
|
|
359
|
+
.whereRaw("received_at < NOW() - ?::interval", [retentionDays + " days"])
|
|
360
|
+
.whereRaw("(processed_at IS NOT NULL OR received_at < NOW() - ?::interval)",
|
|
361
|
+
[(retentionDays * 2) + " days"])
|
|
362
|
+
.toExternalSql("postgres");
|
|
363
|
+
var rs = await xdb.query(delStmt.sql, delStmt.params);
|
|
364
364
|
deleted = (rs && typeof rs.rowCount === "number") ? rs.rowCount : 0;
|
|
365
365
|
} else {
|
|
366
366
|
var staleDate = new Date(Date.now() - retentionDays * C.TIME.days(1)).toISOString();
|
|
367
367
|
var unprocStaleDate = new Date(Date.now() - retentionDays * 2 * C.TIME.days(1)).toISOString();
|
|
368
|
-
|
|
369
|
-
"
|
|
370
|
-
"
|
|
371
|
-
"
|
|
372
|
-
|
|
373
|
-
|
|
368
|
+
var delSqlite = sql.delete(tableRaw, { dialect: "sqlite" })
|
|
369
|
+
.where("received_at", "<", staleDate)
|
|
370
|
+
.whereRaw("(processed_at IS NOT NULL OR received_at < ?)", [unprocStaleDate])
|
|
371
|
+
.toExternalSql("sqlite");
|
|
372
|
+
await xdb.query(delSqlite.sql, delSqlite.params);
|
|
373
|
+
// SELECT changes() reports the row count of the last sqlite write
|
|
374
|
+
// on this connection. b.sql.catalog has no changes() builder (it
|
|
375
|
+
// is a sqlite-internal scalar with no table), so emit it through
|
|
376
|
+
// the same dialect-final terminal as a guarded raw projection on a
|
|
377
|
+
// zero-table SELECT is not expressible; the single-statement
|
|
378
|
+
// changes() probe is a fixed, parameter-free sqlite scalar query.
|
|
379
|
+
var changedResult = await xdb.query(
|
|
380
|
+
sql.toExternalSql(sql.catalog.changes(), "sqlite").sql);
|
|
374
381
|
var changedRow = changedResult.rows && changedResult.rows[0];
|
|
375
382
|
deleted = changedRow ? Number(changedRow.c) : 0;
|
|
376
383
|
}
|
|
@@ -383,12 +390,16 @@ function create(opts) {
|
|
|
383
390
|
|
|
384
391
|
async function isFresh(receiveOpts) {
|
|
385
392
|
_validateReceiveOpts(receiveOpts, "isFresh");
|
|
386
|
-
var dialect = (externalDb
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
393
|
+
var dialect = _sqlDialect(externalDb);
|
|
394
|
+
// SELECT 1 ... is a presence probe; the constant 1 projection is a
|
|
395
|
+
// builder-emitted raw scalar (no column, no bound value).
|
|
396
|
+
var stmt = sql.select(tableRaw, { dialect: dialect })
|
|
397
|
+
.selectRaw("1")
|
|
398
|
+
.where("source", receiveOpts.source)
|
|
399
|
+
.where("message_id", receiveOpts.messageId)
|
|
400
|
+
.toExternalSql(dialect);
|
|
390
401
|
var rs = await externalDb.transaction(async function (xdb) {
|
|
391
|
-
return await xdb.query(sql,
|
|
402
|
+
return await xdb.query(stmt.sql, stmt.params);
|
|
392
403
|
});
|
|
393
404
|
return !rs || !rs.rows || rs.rows.length === 0;
|
|
394
405
|
}
|
|
@@ -397,15 +408,17 @@ function create(opts) {
|
|
|
397
408
|
opts2 = opts2 || {};
|
|
398
409
|
var sourceFilter = (typeof opts2.source === "string" && opts2.source.length > 0)
|
|
399
410
|
? opts2.source : null;
|
|
400
|
-
var dialect = (externalDb
|
|
411
|
+
var dialect = _sqlDialect(externalDb);
|
|
401
412
|
var stats = await externalDb.transaction(async function (xdb) {
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
413
|
+
// COUNT(*) total + COUNT(processed_at) processed (the latter counts
|
|
414
|
+
// only non-NULL processed_at, i.e. handled rows), optionally scoped
|
|
415
|
+
// to one source.
|
|
416
|
+
var builder = sql.select(tableRaw, { dialect: dialect })
|
|
417
|
+
.count("*", "total")
|
|
418
|
+
.count("processed_at", "processed");
|
|
419
|
+
if (sourceFilter) builder.where("source", sourceFilter);
|
|
420
|
+
var stmt = builder.toExternalSql(dialect);
|
|
421
|
+
var rs = await xdb.query(stmt.sql, stmt.params);
|
|
409
422
|
var row = rs.rows && rs.rows[0];
|
|
410
423
|
return {
|
|
411
424
|
total: row ? Number(row.total) : 0,
|