@blamejs/core 0.14.27 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +2 -2
- package/index.js +4 -0
- package/lib/ai-content-detect.js +9 -10
- package/lib/api-key.js +107 -74
- package/lib/atomic-file.js +29 -1
- package/lib/audit-chain.js +47 -11
- package/lib/audit-sign.js +77 -2
- package/lib/audit-tools.js +79 -51
- package/lib/audit.js +218 -100
- package/lib/backup/index.js +13 -10
- package/lib/break-glass.js +202 -144
- package/lib/cache.js +174 -105
- package/lib/chain-writer.js +38 -16
- package/lib/cli.js +19 -14
- package/lib/cluster-provider-db.js +130 -104
- package/lib/cluster-storage.js +119 -22
- package/lib/cluster.js +119 -71
- package/lib/compliance.js +22 -0
- package/lib/consent.js +73 -24
- package/lib/constants.js +16 -11
- package/lib/crypto-field.js +387 -91
- package/lib/db-declare-row-policy.js +35 -22
- package/lib/db-file-lifecycle.js +3 -2
- package/lib/db-query.js +497 -255
- package/lib/db-schema.js +209 -44
- package/lib/db.js +176 -95
- package/lib/external-db-migrate.js +229 -139
- package/lib/external-db.js +25 -15
- package/lib/framework-error.js +11 -0
- package/lib/framework-files.js +73 -0
- package/lib/framework-schema.js +695 -394
- package/lib/gate-contract.js +596 -1
- package/lib/guard-agent-registry.js +26 -44
- package/lib/guard-all.js +1 -0
- package/lib/guard-auth.js +42 -112
- package/lib/guard-cidr.js +33 -154
- package/lib/guard-csv.js +46 -113
- package/lib/guard-domain.js +34 -157
- package/lib/guard-dsn.js +27 -43
- package/lib/guard-email.js +47 -69
- package/lib/guard-envelope.js +19 -32
- package/lib/guard-event-bus-payload.js +24 -42
- package/lib/guard-event-bus-topic.js +25 -43
- package/lib/guard-filename.js +42 -106
- package/lib/guard-graphql.js +42 -123
- package/lib/guard-html.js +53 -108
- package/lib/guard-idempotency-key.js +24 -42
- package/lib/guard-image.js +46 -103
- package/lib/guard-imap-command.js +18 -32
- package/lib/guard-jmap.js +16 -30
- package/lib/guard-json.js +38 -108
- package/lib/guard-jsonpath.js +38 -171
- package/lib/guard-jwt.js +49 -179
- package/lib/guard-list-id.js +25 -41
- package/lib/guard-list-unsubscribe.js +27 -43
- package/lib/guard-mail-compose.js +24 -42
- package/lib/guard-mail-move.js +26 -44
- package/lib/guard-mail-query.js +28 -46
- package/lib/guard-mail-reply.js +24 -42
- package/lib/guard-mail-sieve.js +24 -42
- package/lib/guard-managesieve-command.js +17 -31
- package/lib/guard-markdown.js +37 -104
- package/lib/guard-message-id.js +26 -45
- package/lib/guard-mime.js +39 -151
- package/lib/guard-oauth.js +54 -135
- package/lib/guard-pdf.js +45 -101
- package/lib/guard-pop3-command.js +21 -31
- package/lib/guard-posture-chain.js +24 -42
- package/lib/guard-regex.js +33 -107
- package/lib/guard-saga-config.js +24 -42
- package/lib/guard-shell.js +42 -172
- package/lib/guard-smtp-command.js +48 -54
- package/lib/guard-snapshot-envelope.js +24 -42
- package/lib/guard-sql.js +1491 -0
- package/lib/guard-stream-args.js +24 -43
- package/lib/guard-svg.js +47 -65
- package/lib/guard-template.js +35 -172
- package/lib/guard-tenant-id.js +26 -45
- package/lib/guard-time.js +32 -154
- package/lib/guard-trace-context.js +25 -44
- package/lib/guard-uuid.js +32 -153
- package/lib/guard-xml.js +38 -113
- package/lib/guard-yaml.js +51 -163
- package/lib/http-client.js +14 -0
- package/lib/inbox.js +120 -107
- package/lib/legal-hold.js +107 -50
- package/lib/log-stream-cloudwatch.js +47 -31
- package/lib/log-stream-otlp.js +32 -18
- package/lib/mail-crypto-smime.js +2 -6
- package/lib/mail-greylist.js +2 -6
- package/lib/mail-helo.js +2 -6
- package/lib/mail-journal.js +85 -64
- package/lib/mail-rbl.js +2 -6
- package/lib/mail-scan.js +2 -6
- package/lib/mail-spam-score.js +2 -6
- package/lib/mail-store.js +287 -154
- package/lib/middleware/fetch-metadata.js +17 -7
- package/lib/middleware/idempotency-key.js +54 -38
- package/lib/middleware/rate-limit.js +102 -32
- package/lib/middleware/security-headers.js +21 -5
- package/lib/migrations.js +108 -66
- package/lib/network-heartbeat.js +7 -0
- package/lib/nonce-store.js +31 -9
- package/lib/object-store/azure-blob-bucket-ops.js +9 -4
- package/lib/object-store/azure-blob.js +31 -3
- package/lib/object-store/sigv4.js +10 -0
- package/lib/outbox.js +136 -82
- package/lib/pqc-agent.js +44 -0
- package/lib/pubsub-cluster.js +42 -20
- package/lib/queue-local.js +202 -139
- package/lib/queue-redis.js +9 -1
- package/lib/queue-sqs.js +6 -0
- package/lib/retention.js +82 -39
- package/lib/safe-dns.js +29 -45
- package/lib/safe-ical.js +18 -33
- package/lib/safe-icap.js +27 -43
- package/lib/safe-sieve.js +21 -40
- package/lib/safe-sql.js +124 -3
- package/lib/safe-vcard.js +18 -33
- package/lib/scheduler.js +35 -12
- package/lib/seeders.js +122 -74
- package/lib/session-stores.js +42 -14
- package/lib/session.js +109 -72
- package/lib/sql.js +3885 -0
- package/lib/static.js +45 -7
- package/lib/subject.js +55 -17
- package/lib/vault/index.js +3 -2
- package/lib/vault/passphrase-ops.js +3 -2
- package/lib/vault/rotate.js +104 -64
- package/lib/vendor-data.js +2 -0
- package/lib/websocket.js +16 -0
- package/package.json +1 -1
- package/sbom.cdx.json +6 -6
package/lib/guard-yaml.js
CHANGED
|
@@ -553,12 +553,8 @@ function parse(input, opts) {
|
|
|
553
553
|
throw _err("yaml.bad-input", "parse requires string input");
|
|
554
554
|
}
|
|
555
555
|
var issues = _detectIssues(input, opts);
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
throw _err(issues[i].ruleId || "yaml.refused",
|
|
559
|
-
"guardYaml.parse: " + issues[i].snippet);
|
|
560
|
-
}
|
|
561
|
-
}
|
|
556
|
+
gateContract.throwOnRefusalSeverity(issues,
|
|
557
|
+
{ errorClass: GuardYamlError, codePrefix: "yaml", severities: ["critical"], op: "parse" });
|
|
562
558
|
return safeYamlLazy().parse(input, {
|
|
563
559
|
maxBytes: opts.maxBytes,
|
|
564
560
|
maxDepth: opts.maxDepth,
|
|
@@ -566,161 +562,53 @@ function parse(input, opts) {
|
|
|
566
562
|
});
|
|
567
563
|
}
|
|
568
564
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
* var yamlGate = b.guardYaml.gate({ profile: "strict" });
|
|
594
|
-
* var hostile = Buffer.from("!!python/object/new:cls\nargs: [x]\n", "utf8");
|
|
595
|
-
* var verdict = await yamlGate.check({ bytes: hostile });
|
|
596
|
-
* verdict.action; // → "refuse"
|
|
597
|
-
*/
|
|
598
|
-
function gate(opts) {
|
|
599
|
-
opts = _resolveOpts(opts);
|
|
600
|
-
return gateContract.buildGuardGate(
|
|
601
|
-
opts.name || "guardYaml:" + (opts.profile || "default"),
|
|
602
|
-
opts,
|
|
603
|
-
async function (ctx) {
|
|
604
|
-
var text = gateContract.extractBytesAsText(ctx);
|
|
605
|
-
if (!text) return { ok: true, action: "serve" };
|
|
606
|
-
var rv = validate(text, opts);
|
|
607
|
-
if (rv.issues.length === 0) return { ok: true, action: "serve" };
|
|
608
|
-
var hasCritical = rv.issues.some(function (i) {
|
|
609
|
-
return i.severity === "critical" || i.severity === "high";
|
|
610
|
-
});
|
|
611
|
-
if (!hasCritical) return { ok: true, action: "audit-only", issues: rv.issues };
|
|
612
|
-
return { ok: false, action: "refuse", issues: rv.issues };
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
/**
|
|
617
|
-
* @primitive b.guardYaml.buildProfile
|
|
618
|
-
* @signature b.guardYaml.buildProfile(opts)
|
|
619
|
-
* @since 0.7.14
|
|
620
|
-
* @status stable
|
|
621
|
-
* @related b.guardYaml.gate, b.guardYaml.compliancePosture
|
|
622
|
-
*
|
|
623
|
-
* Compose a derived profile from one or more named bases plus
|
|
624
|
-
* inline overrides. `opts.extends` is a profile name (`"strict"` /
|
|
625
|
-
* `"balanced"` / `"permissive"`) or an array of names; later entries
|
|
626
|
-
* shadow earlier ones. Inline `opts` keys win last. Used to keep
|
|
627
|
-
* operator-defined profiles traceable to a baseline rather than re-
|
|
628
|
-
* typing every key.
|
|
629
|
-
*
|
|
630
|
-
* @opts
|
|
631
|
-
* extends: string|string[], // base profile name(s) to compose
|
|
632
|
-
* ...: any guard-yaml key, // inline override of resolved keys
|
|
633
|
-
*
|
|
634
|
-
* @example
|
|
635
|
-
* var custom = b.guardYaml.buildProfile({
|
|
636
|
-
* extends: "balanced",
|
|
637
|
-
* tagPolicy: "reject",
|
|
638
|
-
* maxAnchors: 8,
|
|
639
|
-
* });
|
|
640
|
-
* custom.tagPolicy; // → "reject"
|
|
641
|
-
* custom.maxAnchors; // → 8
|
|
642
|
-
*/
|
|
643
|
-
var buildProfile = gateContract.makeProfileBuilder(PROFILES);
|
|
644
|
-
|
|
645
|
-
/**
|
|
646
|
-
* @primitive b.guardYaml.compliancePosture
|
|
647
|
-
* @signature b.guardYaml.compliancePosture(name)
|
|
648
|
-
* @since 0.7.14
|
|
649
|
-
* @status stable
|
|
650
|
-
* @compliance hipaa, pci-dss, gdpr, soc2
|
|
651
|
-
* @related b.guardYaml.gate, b.guardYaml.buildProfile
|
|
652
|
-
*
|
|
653
|
-
* Look up a compliance-posture overlay by name (`"hipaa"` /
|
|
654
|
-
* `"pci-dss"` / `"gdpr"` / `"soc2"`). Returns a shallow clone of the
|
|
655
|
-
* posture object — the caller may mutate freely. Throws
|
|
656
|
-
* `GuardYamlError("yaml.bad-posture")` on unknown name.
|
|
657
|
-
*
|
|
658
|
-
* @example
|
|
659
|
-
* var posture = b.guardYaml.compliancePosture("hipaa");
|
|
660
|
-
* posture.tagPolicy; // → "reject"
|
|
661
|
-
* posture.forensicSnippetBytes; // → 256
|
|
662
|
-
*/
|
|
663
|
-
function compliancePosture(name) {
|
|
664
|
-
return gateContract.lookupCompliancePosture(name, COMPLIANCE_POSTURES, _err, "yaml");
|
|
665
|
-
}
|
|
565
|
+
// The gate is the standard serve -> audit-only -> refuse chain (content
|
|
566
|
+
// kind, reading ctx.bytes); it is assembled by gateContract.defineGuard's
|
|
567
|
+
// default gate below. YAML sanitize is intentionally not offered — there's
|
|
568
|
+
// no safe re-emit for tag-injection / alias-explosion shapes; the only
|
|
569
|
+
// correct response is refusal, which the default chain (no sanitize action)
|
|
570
|
+
// matches exactly. Its "guardYaml:<profile>" gate name and
|
|
571
|
+
// serve/audit-only/refuse decisions are identical to the hand-written gate
|
|
572
|
+
// this replaced.
|
|
573
|
+
|
|
574
|
+
// buildProfile / compliancePosture / loadRulePack are assembled by
|
|
575
|
+
// gateContract.defineGuard below; their wiki sections render from the
|
|
576
|
+
// single-sourced @abiTemplate (defineGuard) blocks in gate-contract.js,
|
|
577
|
+
// instantiated per guard by the page generator.
|
|
578
|
+
|
|
579
|
+
var INTEGRATION_FIXTURES = Object.freeze({
|
|
580
|
+
kind: "content",
|
|
581
|
+
contentType: "application/yaml",
|
|
582
|
+
extension: ".yaml",
|
|
583
|
+
benignBytes: Buffer.from('name: alice\nage: 30\n', "utf8"),
|
|
584
|
+
// Hostile: deserialization-tag injection (CVE-2026-24009 PyYAML
|
|
585
|
+
// class). Parser-runtime would attempt to instantiate the named
|
|
586
|
+
// language-specific class.
|
|
587
|
+
hostileBytes: Buffer.from("!!python/object/new:cls\nargs: [\"x\"]\n", "utf8"),
|
|
588
|
+
});
|
|
666
589
|
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
*/
|
|
693
|
-
var loadRulePack = _yamlRulePacks.load;
|
|
694
|
-
|
|
695
|
-
module.exports = {
|
|
696
|
-
// ---- guard-* family registry exports ----
|
|
697
|
-
NAME: "yaml",
|
|
698
|
-
KIND: "content",
|
|
699
|
-
MIME_TYPES: Object.freeze([
|
|
700
|
-
"application/yaml", "application/x-yaml", "text/yaml", "text/x-yaml",
|
|
701
|
-
]),
|
|
702
|
-
EXTENSIONS: Object.freeze([".yml", ".yaml"]),
|
|
703
|
-
INTEGRATION_FIXTURES: Object.freeze({
|
|
704
|
-
kind: "content",
|
|
705
|
-
contentType: "application/yaml",
|
|
706
|
-
extension: ".yaml",
|
|
707
|
-
benignBytes: Buffer.from('name: alice\nage: 30\n', "utf8"),
|
|
708
|
-
// Hostile: deserialization-tag injection (CVE-2026-24009 PyYAML
|
|
709
|
-
// class). Parser-runtime would attempt to instantiate the named
|
|
710
|
-
// language-specific class.
|
|
711
|
-
hostileBytes: Buffer.from("!!python/object/new:cls\nargs: [\"x\"]\n", "utf8"),
|
|
712
|
-
}),
|
|
713
|
-
// ---- primitive surface ----
|
|
714
|
-
validate: validate,
|
|
715
|
-
parse: parse,
|
|
716
|
-
gate: gate,
|
|
717
|
-
buildProfile: buildProfile,
|
|
718
|
-
compliancePosture: compliancePosture,
|
|
719
|
-
loadRulePack: loadRulePack,
|
|
720
|
-
PROFILES: PROFILES,
|
|
721
|
-
DEFAULTS: DEFAULTS,
|
|
722
|
-
COMPLIANCE_POSTURES: COMPLIANCE_POSTURES,
|
|
723
|
-
DANGEROUS_TAG_PREFIXES: DANGEROUS_TAG_PREFIXES,
|
|
724
|
-
SAFE_CORE_TAGS: SAFE_CORE_TAGS,
|
|
725
|
-
GuardYamlError: GuardYamlError,
|
|
726
|
-
};
|
|
590
|
+
// Assembled from the gate-contract guard factory: error class, registry
|
|
591
|
+
// exports (NAME / KIND / MIME_TYPES / EXTENSIONS / INTEGRATION_FIXTURES),
|
|
592
|
+
// buildProfile / compliancePosture / loadRulePack wiring, plus the
|
|
593
|
+
// per-guard inspection surface (validate) and YAML extras
|
|
594
|
+
// (parse / DANGEROUS_TAG_PREFIXES / SAFE_CORE_TAGS) passed through
|
|
595
|
+
// verbatim. The gate is the factory default serve/audit-only/refuse chain
|
|
596
|
+
// (content kind, no sanitize action — there's no safe re-emit for
|
|
597
|
+
// tag-injection / alias-explosion shapes).
|
|
598
|
+
module.exports = gateContract.defineGuard({
|
|
599
|
+
name: "yaml",
|
|
600
|
+
kind: "content",
|
|
601
|
+
errorClass: GuardYamlError,
|
|
602
|
+
profiles: PROFILES,
|
|
603
|
+
defaults: DEFAULTS,
|
|
604
|
+
postures: COMPLIANCE_POSTURES,
|
|
605
|
+
mimeTypes: ["application/yaml", "application/x-yaml", "text/yaml", "text/x-yaml"],
|
|
606
|
+
extensions: [".yml", ".yaml"],
|
|
607
|
+
integrationFixtures: INTEGRATION_FIXTURES,
|
|
608
|
+
validate: validate,
|
|
609
|
+
extra: {
|
|
610
|
+
parse: parse,
|
|
611
|
+
DANGEROUS_TAG_PREFIXES: DANGEROUS_TAG_PREFIXES,
|
|
612
|
+
SAFE_CORE_TAGS: SAFE_CORE_TAGS,
|
|
613
|
+
},
|
|
614
|
+
});
|
package/lib/http-client.js
CHANGED
|
@@ -275,6 +275,9 @@ function _connectHttpsWithAlpn(u, ips) {
|
|
|
275
275
|
session.once("connect", function () {
|
|
276
276
|
var alpn = session.alpnProtocol;
|
|
277
277
|
if (alpn === "h2") {
|
|
278
|
+
// node:http2 connects directly (not via pqcAgent), so observe the
|
|
279
|
+
// negotiated group here too and audit a classical (non-PQC) downgrade.
|
|
280
|
+
pqcAgent._auditClassicalDowngrade(session.socket, { host: u.hostname, port: u.port });
|
|
278
281
|
_wireH2Session(session, _originKey(u));
|
|
279
282
|
_done({ kind: "h2", session: session });
|
|
280
283
|
return;
|
|
@@ -285,6 +288,17 @@ function _connectHttpsWithAlpn(u, ips) {
|
|
|
285
288
|
});
|
|
286
289
|
session.once("error", function (err) {
|
|
287
290
|
_tearDownH2Session(session);
|
|
291
|
+
// An HTTP/1.1-only TLS server has no "h2" to select and replies with
|
|
292
|
+
// the no_application_protocol alert (RFC 7301). node:http2 forces an
|
|
293
|
+
// h2-only ALPN offer, so the connect event never fires for such a
|
|
294
|
+
// server and the http/1.1-selection fallback above can't run — catch
|
|
295
|
+
// the alert here and fall back to an h1 transport for this origin.
|
|
296
|
+
// Real Azure / S3 / Keycloak support h2; Azurite, Azure Stack, and
|
|
297
|
+
// many private / older endpoints are h1-only.
|
|
298
|
+
if (err && err.code === "ERR_SSL_TLSV1_ALERT_NO_APPLICATION_PROTOCOL") {
|
|
299
|
+
_done(_makeH1Transport(u, ips));
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
288
302
|
_fail(err);
|
|
289
303
|
});
|
|
290
304
|
});
|
package/lib/inbox.js
CHANGED
|
@@ -46,6 +46,7 @@ var C = require("./constants");
|
|
|
46
46
|
var lazyRequire = require("./lazy-require");
|
|
47
47
|
var safeJson = require("./safe-json");
|
|
48
48
|
var safeSql = require("./safe-sql");
|
|
49
|
+
var sql = require("./sql");
|
|
49
50
|
var validateOpts = require("./validate-opts");
|
|
50
51
|
var { defineClass } = require("./framework-error");
|
|
51
52
|
|
|
@@ -63,14 +64,19 @@ function _validateTableName(name) {
|
|
|
63
64
|
}
|
|
64
65
|
}
|
|
65
66
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
67
|
+
// Map the operator backend's dialect tag to the b.sql dialect vocabulary
|
|
68
|
+
// (postgres -> $1..$N at toExternalSql; sqlite/other -> `?`).
|
|
69
|
+
function _sqlDialect(externalDb) {
|
|
70
|
+
return (externalDb && externalDb.dialect === "postgres") ? "postgres" : "sqlite";
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// The server-clock timestamp expression, as an allowlisted b.sql function
|
|
74
|
+
// cell (emits the keyword the engine evaluates, binds no param). Postgres
|
|
75
|
+
// returns timestamptz from NOW(); the portable CURRENT_TIMESTAMP serves
|
|
76
|
+
// sqlite. Used directly in a b.sql values()/set() cell.
|
|
77
|
+
function _utcNowCell(externalDb) {
|
|
78
|
+
return (externalDb && externalDb.dialect === "postgres")
|
|
79
|
+
? sql.fn("NOW") : sql.fn("CURRENT_TIMESTAMP");
|
|
74
80
|
}
|
|
75
81
|
|
|
76
82
|
/**
|
|
@@ -144,12 +150,9 @@ function create(opts) {
|
|
|
144
150
|
|
|
145
151
|
var externalDb = opts.externalDb;
|
|
146
152
|
var tableRaw = opts.table;
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
//
|
|
150
|
-
// lib/safe-sql.js), so one quoted form serves both inbox paths.
|
|
151
|
-
var qTable = safeSql.quoteIdentifier(tableRaw, "sqlite");
|
|
152
|
-
var qIndex = safeSql.quoteIdentifier(tableRaw + "_received_at_idx", "sqlite");
|
|
153
|
+
// The table identifier reaches SQL through b.sql, which validates +
|
|
154
|
+
// quotes it by construction on every emitted statement; _validateTableName
|
|
155
|
+
// above fails fast at create() time on a bad name.
|
|
153
156
|
var retentionDays = (typeof opts.retentionDays === "number" && opts.retentionDays > 0) // allow:numeric-opt-Infinity
|
|
154
157
|
? opts.retentionDays : 30; // default retention days
|
|
155
158
|
var auditOn = opts.audit !== false;
|
|
@@ -227,43 +230,38 @@ function create(opts) {
|
|
|
227
230
|
}
|
|
228
231
|
metaJson = serialized;
|
|
229
232
|
}
|
|
230
|
-
var
|
|
231
|
-
var
|
|
233
|
+
var dialect = _sqlDialect(externalDb);
|
|
234
|
+
var nowCell = _utcNowCell(externalDb);
|
|
232
235
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
var
|
|
257
|
-
"INSERT OR IGNORE INTO " + qTable +
|
|
258
|
-
" (message_id, source, received_at, metadata_json) " +
|
|
259
|
-
" VALUES (?, ?, " + nowExpr + ", ?) RETURNING 1",
|
|
260
|
-
[receiveOpts.messageId, receiveOpts.source, metaJson]);
|
|
261
|
-
var sqlFresh = !!(sqlInsert && sqlInsert.rows && sqlInsert.rows.length === 1);
|
|
236
|
+
// ON CONFLICT (source, message_id) DO NOTHING RETURNING message_id:
|
|
237
|
+
// a fresh insert returns one row, a duplicate redelivery returns none
|
|
238
|
+
// (the collision short-circuits). received_at takes the server-clock
|
|
239
|
+
// function cell (NOW() / CURRENT_TIMESTAMP, no param); metadata_json
|
|
240
|
+
// binds with a ::jsonb cast on Postgres and as a plain `?` on sqlite.
|
|
241
|
+
// RETURNING message_id (rather than RETURNING 1) is the portable
|
|
242
|
+
// presence sentinel - one row iff the insert landed - across both
|
|
243
|
+
// dialects, collapsing the prior INSERT-then-SELECT-changes() race
|
|
244
|
+
// into one round-trip on sqlite too.
|
|
245
|
+
var metaCell = (dialect === "postgres") ? sql.cast(metaJson, "jsonb") : metaJson;
|
|
246
|
+
var stmt = sql.upsert(tableRaw, { dialect: dialect })
|
|
247
|
+
.columns(["message_id", "source", "received_at", "metadata_json"])
|
|
248
|
+
.values({
|
|
249
|
+
message_id: receiveOpts.messageId,
|
|
250
|
+
source: receiveOpts.source,
|
|
251
|
+
received_at: nowCell,
|
|
252
|
+
metadata_json: metaCell,
|
|
253
|
+
})
|
|
254
|
+
.onConflict(["source", "message_id"])
|
|
255
|
+
.doNothing()
|
|
256
|
+
.returning(["message_id"])
|
|
257
|
+
.toExternalSql(dialect);
|
|
258
|
+
var rs = await txn.query(stmt.sql, stmt.params);
|
|
259
|
+
var fresh = !!(rs && rs.rows && rs.rows.length === 1);
|
|
262
260
|
_emitAudit("inbox.received", "success", {
|
|
263
261
|
source: receiveOpts.source, messageId: receiveOpts.messageId,
|
|
264
|
-
fresh:
|
|
262
|
+
fresh: fresh,
|
|
265
263
|
});
|
|
266
|
-
return
|
|
264
|
+
return fresh;
|
|
267
265
|
}
|
|
268
266
|
|
|
269
267
|
async function markProcessed(receiveOpts, txn) {
|
|
@@ -272,13 +270,13 @@ function create(opts) {
|
|
|
272
270
|
"markProcessed: txn must be a transaction handle");
|
|
273
271
|
}
|
|
274
272
|
_validateReceiveOpts(receiveOpts, "markProcessed");
|
|
275
|
-
var
|
|
276
|
-
var
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
await txn.query(sql,
|
|
273
|
+
var dialect = _sqlDialect(externalDb);
|
|
274
|
+
var stmt = sql.update(tableRaw, { dialect: dialect })
|
|
275
|
+
.set({ processed_at: _utcNowCell(externalDb) })
|
|
276
|
+
.where("source", receiveOpts.source)
|
|
277
|
+
.where("message_id", receiveOpts.messageId)
|
|
278
|
+
.toExternalSql(dialect);
|
|
279
|
+
await txn.query(stmt.sql, stmt.params);
|
|
282
280
|
}
|
|
283
281
|
|
|
284
282
|
async function handle(receiveOpts, handler) {
|
|
@@ -321,56 +319,65 @@ function create(opts) {
|
|
|
321
319
|
}
|
|
322
320
|
|
|
323
321
|
async function declareSchema(xdb) {
|
|
324
|
-
var dialect = (xdb
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
await xdb.query(
|
|
349
|
-
"CREATE INDEX IF NOT EXISTS " + qIndex + " " +
|
|
350
|
-
"ON " + qTable + " (received_at)");
|
|
351
|
-
}
|
|
322
|
+
var dialect = _sqlDialect(xdb);
|
|
323
|
+
// received_at / processed_at are a timestamp-with-zone column on
|
|
324
|
+
// Postgres and a TEXT (ISO-8601) column on sqlite; metadata_json is
|
|
325
|
+
// JSONB on Postgres and TEXT on sqlite. The composite (source,
|
|
326
|
+
// message_id) primary key is the dedupe collision boundary. Verbatim
|
|
327
|
+
// type strings sit in type position after the quoted column name.
|
|
328
|
+
var tsType = dialect === "postgres" ? "TIMESTAMPTZ" : "TEXT";
|
|
329
|
+
var tsDefault = dialect === "postgres" ? "NOW()" : "CURRENT_TIMESTAMP";
|
|
330
|
+
var jsonType = dialect === "postgres" ? "JSONB" : "TEXT";
|
|
331
|
+
var ddl = sql.toExternalSql(sql.createTable(tableRaw, [
|
|
332
|
+
{ name: "message_id", type: "TEXT", notNull: true },
|
|
333
|
+
{ name: "source", type: "TEXT", notNull: true },
|
|
334
|
+
// The DEFAULT here is a SQL function keyword, not a bound literal -
|
|
335
|
+
// expressed via the `constraints` verbatim clause so the type map
|
|
336
|
+
// does not quote NOW() / CURRENT_TIMESTAMP as a string default.
|
|
337
|
+
{ name: "received_at", type: tsType, notNull: true, constraints: "DEFAULT " + tsDefault },
|
|
338
|
+
{ name: "processed_at", type: tsType },
|
|
339
|
+
{ name: "metadata_json", type: jsonType },
|
|
340
|
+
], { dialect: dialect, primaryKey: ["source", "message_id"] }), dialect);
|
|
341
|
+
await xdb.query(ddl.sql, ddl.params);
|
|
342
|
+
|
|
343
|
+
var idx = sql.toExternalSql(sql.createIndex(tableRaw + "_received_at_idx", tableRaw,
|
|
344
|
+
["received_at"], { dialect: dialect }), dialect);
|
|
345
|
+
await xdb.query(idx.sql, idx.params);
|
|
352
346
|
}
|
|
353
347
|
|
|
354
348
|
async function sweep() {
|
|
355
|
-
var dialect = (externalDb
|
|
349
|
+
var dialect = _sqlDialect(externalDb);
|
|
356
350
|
var deleted = 0;
|
|
357
351
|
await externalDb.transaction(async function (xdb) {
|
|
358
352
|
if (dialect === "postgres") {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
353
|
+
// received_at < NOW() - <retention>::interval, with the
|
|
354
|
+
// unprocessed grace of 2x retention. The interval STRINGS bind as
|
|
355
|
+
// a ::interval-cast `?` (the cast value-cell form); NOW() is the
|
|
356
|
+
// server-clock function token in the predicate (a guarded raw
|
|
357
|
+
// fragment - no string literal, no bound value).
|
|
358
|
+
var delStmt = sql.delete(tableRaw, { dialect: "postgres" })
|
|
359
|
+
.whereRaw("received_at < NOW() - ?::interval", [retentionDays + " days"])
|
|
360
|
+
.whereRaw("(processed_at IS NOT NULL OR received_at < NOW() - ?::interval)",
|
|
361
|
+
[(retentionDays * 2) + " days"])
|
|
362
|
+
.toExternalSql("postgres");
|
|
363
|
+
var rs = await xdb.query(delStmt.sql, delStmt.params);
|
|
364
364
|
deleted = (rs && typeof rs.rowCount === "number") ? rs.rowCount : 0;
|
|
365
365
|
} else {
|
|
366
366
|
var staleDate = new Date(Date.now() - retentionDays * C.TIME.days(1)).toISOString();
|
|
367
367
|
var unprocStaleDate = new Date(Date.now() - retentionDays * 2 * C.TIME.days(1)).toISOString();
|
|
368
|
-
|
|
369
|
-
"
|
|
370
|
-
"
|
|
371
|
-
"
|
|
372
|
-
|
|
373
|
-
|
|
368
|
+
var delSqlite = sql.delete(tableRaw, { dialect: "sqlite" })
|
|
369
|
+
.where("received_at", "<", staleDate)
|
|
370
|
+
.whereRaw("(processed_at IS NOT NULL OR received_at < ?)", [unprocStaleDate])
|
|
371
|
+
.toExternalSql("sqlite");
|
|
372
|
+
await xdb.query(delSqlite.sql, delSqlite.params);
|
|
373
|
+
// SELECT changes() reports the row count of the last sqlite write
|
|
374
|
+
// on this connection. b.sql.catalog has no changes() builder (it
|
|
375
|
+
// is a sqlite-internal scalar with no table), so emit it through
|
|
376
|
+
// the same dialect-final terminal as a guarded raw projection on a
|
|
377
|
+
// zero-table SELECT is not expressible; the single-statement
|
|
378
|
+
// changes() probe is a fixed, parameter-free sqlite scalar query.
|
|
379
|
+
var changedResult = await xdb.query(
|
|
380
|
+
sql.toExternalSql(sql.catalog.changes(), "sqlite").sql);
|
|
374
381
|
var changedRow = changedResult.rows && changedResult.rows[0];
|
|
375
382
|
deleted = changedRow ? Number(changedRow.c) : 0;
|
|
376
383
|
}
|
|
@@ -383,12 +390,16 @@ function create(opts) {
|
|
|
383
390
|
|
|
384
391
|
async function isFresh(receiveOpts) {
|
|
385
392
|
_validateReceiveOpts(receiveOpts, "isFresh");
|
|
386
|
-
var dialect = (externalDb
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
393
|
+
var dialect = _sqlDialect(externalDb);
|
|
394
|
+
// SELECT 1 ... is a presence probe; the constant 1 projection is a
|
|
395
|
+
// builder-emitted raw scalar (no column, no bound value).
|
|
396
|
+
var stmt = sql.select(tableRaw, { dialect: dialect })
|
|
397
|
+
.selectRaw("1")
|
|
398
|
+
.where("source", receiveOpts.source)
|
|
399
|
+
.where("message_id", receiveOpts.messageId)
|
|
400
|
+
.toExternalSql(dialect);
|
|
390
401
|
var rs = await externalDb.transaction(async function (xdb) {
|
|
391
|
-
return await xdb.query(sql,
|
|
402
|
+
return await xdb.query(stmt.sql, stmt.params);
|
|
392
403
|
});
|
|
393
404
|
return !rs || !rs.rows || rs.rows.length === 0;
|
|
394
405
|
}
|
|
@@ -397,15 +408,17 @@ function create(opts) {
|
|
|
397
408
|
opts2 = opts2 || {};
|
|
398
409
|
var sourceFilter = (typeof opts2.source === "string" && opts2.source.length > 0)
|
|
399
410
|
? opts2.source : null;
|
|
400
|
-
var dialect = (externalDb
|
|
411
|
+
var dialect = _sqlDialect(externalDb);
|
|
401
412
|
var stats = await externalDb.transaction(async function (xdb) {
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
413
|
+
// COUNT(*) total + COUNT(processed_at) processed (the latter counts
|
|
414
|
+
// only non-NULL processed_at, i.e. handled rows), optionally scoped
|
|
415
|
+
// to one source.
|
|
416
|
+
var builder = sql.select(tableRaw, { dialect: dialect })
|
|
417
|
+
.count("*", "total")
|
|
418
|
+
.count("processed_at", "processed");
|
|
419
|
+
if (sourceFilter) builder.where("source", sourceFilter);
|
|
420
|
+
var stmt = builder.toExternalSql(dialect);
|
|
421
|
+
var rs = await xdb.query(stmt.sql, stmt.params);
|
|
409
422
|
var row = rs.rows && rs.rows[0];
|
|
410
423
|
return {
|
|
411
424
|
total: row ? Number(row.total) : 0,
|