@blamejs/core 0.7.18 → 0.7.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/CHANGELOG.md +427 -423
  2. package/README.md +150 -150
  3. package/bin/blamejs.js +0 -0
  4. package/index.js +310 -308
  5. package/lib/api-key.js +660 -660
  6. package/lib/api-snapshot.js +338 -338
  7. package/lib/app-shutdown.js +385 -385
  8. package/lib/app.js +365 -365
  9. package/lib/archive.js +250 -250
  10. package/lib/atomic-file.js +544 -544
  11. package/lib/audit-chain.js +177 -177
  12. package/lib/audit-sign.js +344 -344
  13. package/lib/audit-tools.js +677 -677
  14. package/lib/audit.js +766 -766
  15. package/lib/auth/jwt-external.js +365 -0
  16. package/lib/auth/jwt.js +337 -311
  17. package/lib/auth/lockout.js +436 -436
  18. package/lib/auth/oauth.js +721 -721
  19. package/lib/auth/passkey.js +181 -181
  20. package/lib/auth/password.js +628 -594
  21. package/lib/backup/bundle.js +217 -217
  22. package/lib/backup/crypto.js +176 -176
  23. package/lib/backup/index.js +515 -515
  24. package/lib/backup/manifest.js +282 -282
  25. package/lib/break-glass.js +1338 -1338
  26. package/lib/bundler.js +441 -441
  27. package/lib/cache-redis.js +256 -256
  28. package/lib/cache.js +1206 -1206
  29. package/lib/canonical-json.js +115 -115
  30. package/lib/chain-writer.js +234 -234
  31. package/lib/cli-helpers.js +206 -206
  32. package/lib/cli.js +2334 -2334
  33. package/lib/cluster-provider-db.js +317 -317
  34. package/lib/cluster-storage.js +226 -226
  35. package/lib/cluster.js +703 -703
  36. package/lib/config-drift.js +301 -301
  37. package/lib/consent.js +222 -222
  38. package/lib/constants.js +191 -191
  39. package/lib/cookies.js +350 -315
  40. package/lib/credential-hash.js +322 -322
  41. package/lib/crypto.js +266 -266
  42. package/lib/csv.js +275 -275
  43. package/lib/db-declare-row-policy.js +267 -267
  44. package/lib/db-declare-view.js +420 -420
  45. package/lib/db-query.js +406 -406
  46. package/lib/db-schema.js +319 -319
  47. package/lib/db.js +1288 -1288
  48. package/lib/deprecate.js +222 -222
  49. package/lib/dev.js +335 -335
  50. package/lib/dual-control.js +473 -473
  51. package/lib/error-page.js +420 -420
  52. package/lib/external-db-migrate.js +441 -441
  53. package/lib/external-db.js +1061 -1061
  54. package/lib/file-type.js +273 -273
  55. package/lib/forms.js +422 -422
  56. package/lib/framework-error.js +293 -293
  57. package/lib/framework-schema.js +717 -717
  58. package/lib/handlers.js +350 -350
  59. package/lib/http-client-cookie-jar.js +508 -508
  60. package/lib/http-client.js +1195 -1195
  61. package/lib/i18n.js +878 -878
  62. package/lib/jobs.js +185 -185
  63. package/lib/log-stream-cloudwatch.js +369 -369
  64. package/lib/log-stream-local.js +146 -146
  65. package/lib/log-stream-otlp-grpc.js +410 -410
  66. package/lib/log-stream-otlp.js +286 -286
  67. package/lib/log-stream-syslog.js +302 -302
  68. package/lib/log-stream-webhook.js +199 -199
  69. package/lib/log-stream.js +330 -330
  70. package/lib/log.js +500 -500
  71. package/lib/mail-bounce.js +528 -528
  72. package/lib/mail-dkim.js +369 -369
  73. package/lib/mail.js +981 -981
  74. package/lib/metrics.js +683 -683
  75. package/lib/middleware/api-encrypt.js +936 -936
  76. package/lib/middleware/attach-user.js +157 -157
  77. package/lib/middleware/bearer-auth.js +152 -0
  78. package/lib/middleware/body-parser.js +1170 -1170
  79. package/lib/middleware/bot-guard.js +178 -178
  80. package/lib/middleware/compression.js +452 -452
  81. package/lib/middleware/cors.js +314 -314
  82. package/lib/middleware/csp-nonce.js +348 -348
  83. package/lib/middleware/csrf-protect.js +399 -316
  84. package/lib/middleware/db-role-for.js +264 -264
  85. package/lib/middleware/fetch-metadata.js +129 -0
  86. package/lib/middleware/health.js +392 -392
  87. package/lib/middleware/index.js +85 -79
  88. package/lib/middleware/rate-limit.js +358 -358
  89. package/lib/middleware/request-id.js +61 -61
  90. package/lib/middleware/request-log.js +168 -168
  91. package/lib/middleware/require-auth.js +104 -104
  92. package/lib/middleware/security-headers.js +121 -116
  93. package/lib/middleware/sse.js +166 -166
  94. package/lib/migrations.js +383 -383
  95. package/lib/mtls-ca.js +518 -518
  96. package/lib/mtls-engine-default.js +481 -481
  97. package/lib/network-dns.js +632 -632
  98. package/lib/network-heartbeat.js +290 -290
  99. package/lib/network-nts.js +574 -574
  100. package/lib/network-proxy.js +265 -265
  101. package/lib/network-tls.js +328 -328
  102. package/lib/network.js +233 -233
  103. package/lib/notify.js +612 -612
  104. package/lib/ntp-check.js +229 -229
  105. package/lib/numeric-bounds.js +111 -111
  106. package/lib/object-store/azure-blob-bucket-ops.js +349 -349
  107. package/lib/object-store/azure-blob.js +488 -488
  108. package/lib/object-store/gcs-bucket-ops.js +351 -351
  109. package/lib/object-store/gcs.js +519 -519
  110. package/lib/object-store/http-put.js +153 -153
  111. package/lib/object-store/index.js +197 -197
  112. package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
  113. package/lib/object-store/sigv4.js +903 -903
  114. package/lib/observability.js +151 -151
  115. package/lib/otel-export.js +269 -269
  116. package/lib/pagination.js +464 -464
  117. package/lib/parsers/index.js +80 -80
  118. package/lib/parsers/safe-env.js +642 -642
  119. package/lib/parsers/safe-ini.js +292 -292
  120. package/lib/parsers/safe-toml.js +784 -784
  121. package/lib/parsers/safe-xml.js +390 -390
  122. package/lib/parsers/safe-yaml.js +1015 -1015
  123. package/lib/permissions.js +708 -708
  124. package/lib/pqc-agent.js +87 -87
  125. package/lib/pqc-gate.js +279 -279
  126. package/lib/protobuf-encoder.js +190 -190
  127. package/lib/protocol-dispatcher.js +161 -161
  128. package/lib/pubsub-redis.js +167 -167
  129. package/lib/pubsub.js +429 -429
  130. package/lib/queue-local.js +476 -476
  131. package/lib/queue-redis.js +745 -745
  132. package/lib/queue-sqs.js +319 -319
  133. package/lib/queue.js +695 -695
  134. package/lib/redis-client.js +519 -519
  135. package/lib/request-helpers.js +340 -340
  136. package/lib/restore-bundle.js +237 -237
  137. package/lib/restore-rollback.js +259 -259
  138. package/lib/restore.js +409 -409
  139. package/lib/retry.js +376 -376
  140. package/lib/router.js +748 -748
  141. package/lib/safe-async.js +735 -735
  142. package/lib/safe-buffer.js +237 -237
  143. package/lib/safe-json.js +541 -541
  144. package/lib/safe-schema.js +1266 -1266
  145. package/lib/safe-url.js +159 -159
  146. package/lib/scheduler.js +706 -706
  147. package/lib/security-assert.js +373 -373
  148. package/lib/seeders.js +618 -618
  149. package/lib/session.js +535 -478
  150. package/lib/slug.js +269 -269
  151. package/lib/ssrf-guard.js +401 -401
  152. package/lib/static.js +7 -5
  153. package/lib/storage.js +471 -471
  154. package/lib/subject.js +281 -281
  155. package/lib/template.js +791 -791
  156. package/lib/testing.js +798 -798
  157. package/lib/time.js +310 -310
  158. package/lib/totp.js +302 -302
  159. package/lib/tracing.js +494 -494
  160. package/lib/uuid.js +132 -132
  161. package/lib/validate-opts.js +340 -340
  162. package/lib/vault/index.js +308 -308
  163. package/lib/vault/rotate.js +784 -784
  164. package/lib/vault/wrap.js +296 -296
  165. package/lib/vendor/noble-ciphers.cjs +9 -9
  166. package/lib/webhook.js +595 -595
  167. package/lib/websocket.js +1048 -1048
  168. package/package.json +77 -77
  169. package/sbom.cyclonedx.json +7 -7
package/lib/cluster.js CHANGED
@@ -1,703 +1,703 @@
1
- "use strict";
2
- /**
3
- * Cluster coordination — leader election + fencing tokens.
4
- *
5
- * Opt-in via `b.cluster.init(...)`. When init is never called, the
6
- * local process behaves as a permanent single leader: `isLeader()`
7
- * always returns true, `fencingToken()` returns 0, no heartbeat thread
8
- * runs, no DB is touched. Single-node deployments pay zero overhead.
9
- *
10
- * When init IS called, the framework starts a heartbeat that renews
11
- * the leader lease via the configured provider. On lease loss (network
12
- * partition, takeover, lease expiry) the node transitions to follower
13
- * and write-side framework primitives throw `NotLeaderError`.
14
- *
15
- * Threat model:
16
- * - Two leaders writing simultaneously: prevented by fencing tokens.
17
- * Every leader-only DB write includes the current token; a
18
- * CHECK constraint on the audit-tip row rejects a stale token.
19
- * The application-layer `requireLeader()` gate is just an early
20
- * rejection optimisation; the DB constraint is the canonical guard.
21
- * - Follower receiving a write: rejected at the framework boundary.
22
- * Operators front the cluster with a load balancer that routes
23
- * write paths to the current leader.
24
- * - External-db unreachable: heartbeat fails; after `leaseTtl` no
25
- * leader exists and writes fail closed. When the DB recovers,
26
- * election resumes.
27
- *
28
- * Public API:
29
- * await cluster.init(opts) one-time bootstrap
30
- * cluster.isLeader() sync; true on leader (or single-node)
31
- * cluster.currentNodeId() sync; configured nodeId
32
- * cluster.endpoint() sync; this node's routable URL
33
- * (operator-supplied at init), or
34
- * null if unconfigured. Stored in
35
- * the leader-election row so
36
- * external observers can resolve
37
- * "where is the current leader?"
38
- * cluster.fencingToken() sync; current monotonic token
39
- * cluster.requireLeader() sync; throws NotLeaderError
40
- * cluster.currentLeader() async; { nodeId, leaseExpiresAt,
41
- * fencingToken,
42
- * endpoint } | null
43
- * cluster.discoveryHandler() returns an HTTP request handler
44
- * (req, res) → JSON. Mount on any
45
- * route to expose the current
46
- * leader for service-mesh / LB
47
- * consumption. 200 with leader,
48
- * 503 with `{ leader: null }`
49
- * when no leader.
50
- * cluster.onTransition(fn) register transition handler
51
- * await cluster.shutdown() releases lease, stops heartbeat
52
- */
53
- var C = require("./constants");
54
- var clusterProviderDb = require("./cluster-provider-db");
55
- var crypto = require("./crypto");
56
- var lazyRequire = require("./lazy-require");
57
- var { boot } = require("./log");
58
- var safeAsync = require("./safe-async");
59
- var safeJson = require("./safe-json");
60
- var safeSql = require("./safe-sql");
61
- var safeUrl = require("./safe-url");
62
- var { FrameworkError, ClusterError } = require("./framework-error");
63
-
64
- // Lazy: vault → db → cluster forms a load-time chain, and external-db is
65
- // loaded before its init has run; both are safe to call once cluster
66
- // reaches runtime, but eager require here would deadlock the load order.
67
- var externalDb = lazyRequire(function () { return require("./external-db"); });
68
- var vault = lazyRequire(function () { return require("./vault"); });
69
-
70
- var DEFAULT_LEASE_TTL = C.TIME.seconds(30);
71
- var DEFAULT_HEARTBEAT = C.TIME.seconds(10);
72
- var MIN_LEASE_TTL = C.TIME.seconds(5);
73
- var MIN_HEARTBEAT = C.TIME.seconds(1);
74
-
75
- var initialized = false;
76
- var terminated = false; // set true by shutdown() so the
77
- // permanent-leader fallback isn't
78
- // re-engaged after a graceful exit
79
- var nodeId = null;
80
- var role = null; // 'leader' | 'follower'
81
- var provider = null;
82
- var lease = null; // current lease (if leader)
83
- var heartbeatTimer = null;
84
- var heartbeatMs = null;
85
- var leaseTtlMs = null;
86
- var transitionHandlers = [];
87
- // Backend coordinates for write-dispatch code in audit/consent/etc.
88
- // These are set when cluster.init is called with `externalDbBackend`
89
- // (the default DB-row provider path); operators using a custom
90
- // provider can set them via init opts directly.
91
- var configuredExternalDbBackend = null;
92
- var configuredDialect = null;
93
- // Operator-supplied routable endpoint for THIS node, used by external
94
- // load balancers / service meshes to learn where to send write traffic.
95
- // Stored in the leader-election row on every acquire/renew so any node
96
- // (or external observer) can resolve "where is the current leader?"
97
- // via cluster.currentLeader() / cluster.discoveryHandler().
98
- var configuredEndpoint = null;
99
-
100
- var log = boot("cluster");
101
-
102
- class NotLeaderError extends FrameworkError {
103
- constructor(message) {
104
- super(message || "not leader: write rejected by cluster gate", "NOT_LEADER");
105
- this.name = "NotLeaderError";
106
- this.statusCode = 503; // operator's load balancer should retry on the leader
107
- this.isClusterError = true;
108
- this.isNotLeaderError = true;
109
- }
110
- }
111
-
112
- var _err = ClusterError.factory;
113
-
114
- function _emitTransition(kind, detail) {
115
- var event = Object.assign({ kind: kind, nodeId: nodeId, at: Date.now() }, detail || {});
116
- for (var i = 0; i < transitionHandlers.length; i++) {
117
- try { transitionHandlers[i](event); }
118
- catch (e) { log.error("transition handler threw: " + e.message); }
119
- }
120
- }
121
-
122
- // ---- init ----
123
-
124
- async function init(opts) {
125
- if (initialized) {
126
- throw _err("ALREADY_INITIALIZED", "cluster.init() called twice", true);
127
- }
128
- opts = opts || {};
129
- if (!opts.nodeId) {
130
- throw _err("INVALID_CONFIG", "cluster.init({ nodeId }) is required", true);
131
- }
132
- nodeId = String(opts.nodeId);
133
-
134
- leaseTtlMs = opts.leaseTtl != null ? Number(opts.leaseTtl) : DEFAULT_LEASE_TTL;
135
- if (leaseTtlMs < MIN_LEASE_TTL) {
136
- throw _err("INVALID_TTL",
137
- "leaseTtl must be >= " + MIN_LEASE_TTL + "ms (got " + leaseTtlMs + ")",
138
- true);
139
- }
140
- heartbeatMs = opts.heartbeatInterval != null
141
- ? Number(opts.heartbeatInterval)
142
- : DEFAULT_HEARTBEAT;
143
- if (heartbeatMs < MIN_HEARTBEAT) {
144
- throw _err("INVALID_HEARTBEAT",
145
- "heartbeatInterval must be >= " + MIN_HEARTBEAT + "ms (got " + heartbeatMs + ")",
146
- true);
147
- }
148
- if (heartbeatMs >= leaseTtlMs) {
149
- throw _err("INVALID_HEARTBEAT",
150
- "heartbeatInterval must be < leaseTtl (got heartbeat=" + heartbeatMs +
151
- ", leaseTtl=" + leaseTtlMs + "); recommend ~1/3 of leaseTtl",
152
- true);
153
- }
154
-
155
- role = (opts.role || "leader").toLowerCase();
156
- if (role !== "leader" && role !== "follower") {
157
- throw _err("INVALID_ROLE", "role must be 'leader' or 'follower'", true);
158
- }
159
-
160
- // Optional endpoint. If provided, validate scheme + shape via url-safe
161
- // — HTTPS-only by default since this is the URL external services use
162
- // to reach the leader. Operators with internal cleartext clusters opt
163
- // in via opts.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
164
- if (opts.endpoint != null) {
165
- try {
166
- safeUrl.parse(opts.endpoint, {
167
- allowedProtocols: opts.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
168
- errorClass: ClusterError,
169
- });
170
- } catch (e) {
171
- // Re-throw with a config-shaped error so operators see the cluster.init
172
- // boundary, not a bare url-safe trace.
173
- throw _err("INVALID_ENDPOINT",
174
- "cluster.init({ endpoint }) rejected: " + e.message, true);
175
- }
176
- configuredEndpoint = String(opts.endpoint);
177
- } else {
178
- configuredEndpoint = null;
179
- }
180
-
181
- if (typeof opts.onTransition === "function") {
182
- transitionHandlers.push(opts.onTransition);
183
- }
184
-
185
- // Provider: either operator-supplied, or build the default DB-row
186
- // provider against an externalDb backend.
187
- if (opts.provider) {
188
- provider = opts.provider;
189
- // Operator-custom provider: they may still be writing framework
190
- // state to an externalDb backend, in which case they pass these
191
- // separately so write-dispatch code knows where to go.
192
- configuredExternalDbBackend = opts.externalDbBackend || null;
193
- configuredDialect = (opts.dialect || "postgres").toLowerCase();
194
- } else {
195
- if (!opts.externalDbBackend) {
196
- throw _err("INVALID_CONFIG",
197
- "cluster.init requires either { provider } or { externalDbBackend }", true);
198
- }
199
- provider = clusterProviderDb.create({
200
- externalDbBackend: opts.externalDbBackend,
201
- dialect: opts.dialect,
202
- });
203
- configuredExternalDbBackend = opts.externalDbBackend;
204
- configuredDialect = (opts.dialect || "postgres").toLowerCase();
205
- }
206
-
207
- if (typeof provider.ensureSchema === "function") {
208
- await provider.ensureSchema();
209
- }
210
-
211
- initialized = true;
212
- log("initialized as nodeId='" + nodeId + "', role='" + role + "'");
213
-
214
- // Initial acquisition attempt (only if role === 'leader')
215
- if (role === "leader") {
216
- await _tryAcquire();
217
- }
218
-
219
- // Boot-time rollback detection on the audit + consent chains. Runs
220
- // regardless of role — every node should refuse to participate in a
221
- // cluster whose shared chains have been rolled back (a follower
222
- // would face the same chain integrity failure if it later took
223
- // over). Skipped when configuredExternalDbBackend is unset, which
224
- // means a custom provider is in use without externalDb-resident
225
- // framework state — the operator owns rollback detection in that
226
- // case.
227
- if (configuredExternalDbBackend) {
228
- await _checkChainTipRollback("audit", "_blamejs_audit_log", "_blamejs_audit_tip");
229
- await _checkChainTipRollback("consent", "_blamejs_consent_log", "_blamejs_consent_tip");
230
- // Vault-key consistency: every node in a cluster must hold the
231
- // SAME vault key. A node booting with a different key would seal
232
- // new writes under a key the rest of the cluster can't unseal,
233
- // and (on takeover) be unable to unseal the rest of the cluster's
234
- // sealed columns — silent corruption. Compare a fingerprint of
235
- // this node's vault keys against the canonical one stored at
236
- // first cluster boot; refuse to participate on mismatch.
237
- await _checkVaultKeyConsistency();
238
- }
239
-
240
- // Start heartbeat
241
- heartbeatTimer = safeAsync.repeating(_heartbeat, heartbeatMs, { name: "cluster-heartbeat" });
242
- }
243
-
244
- // Cluster-mode equivalent of db.js's single-node audit.tip-sidecar
245
- // rollback check. Reads the persistent _blamejs_audit_tip row and
246
- // compares to the current chain head in _blamejs_audit_log:
247
- //
248
- // - No tip row: first cluster boot or operator-cleared. Skip
249
- // silently (matches the single-node sidecar-missing path).
250
- // - Tip recorded a counter > current MAX: chain was truncated /
251
- // restored from older snapshot. FATAL — refuse boot.
252
- // - Tip recorded a hash that doesn't match the row at that
253
- // counter: the row at that counter was substituted (different
254
- // hash for same counter). FATAL — refuse boot.
255
- //
256
- // process.exit(1) is the framework's convention for boot-time
257
- // integrity failures (audit chain, checkpoints, single-node
258
- // rollback). Cluster mode keeps the same posture so operators see
259
- // a single boot-time failure mode regardless of deployment shape.
260
- // Generalized boot-time rollback check used by both audit and consent
261
- // chains. chainName is the human-readable label included in log
262
- // output ("audit" / "consent"). logTable is the chain table
263
- // (_blamejs_audit_log / _blamejs_consent_log). tipTable is the
264
- // single-row coordination table that records the latest counter +
265
- // rowHash + fencingToken (_blamejs_audit_tip / _blamejs_consent_tip).
266
- //
267
- // Surfaces three outcomes:
268
- // - tip table missing → operator running cluster gates-only mode
269
- // (cluster wired for leader election but framework state still
270
- // lives in per-node SQLite without `frameworkSchema.ensureSchema`);
271
- // skip silently.
272
- // - no tip row → first cluster boot or operator-cleared; skip.
273
- // - currentMax < tipCounter, or tip rowHash != row-at-counter
274
- // hash → FATAL via process.exit(1). Same posture as the
275
- // single-node audit.tip sidecar rollback check.
276
- async function _checkChainTipRollback(chainName, logTable, tipTable) {
277
- // Both tables are framework-internal constants from the call sites
278
- // (`_blamejs_audit_log`, `_blamejs_consent_log`, etc.). Validate +
279
- // quote per the framework's identifier-quoting convention so a
280
- // future rename can't silently break the query.
281
- safeSql.validateIdentifier(logTable, { allowReserved: true });
282
- safeSql.validateIdentifier(tipTable, { allowReserved: true });
283
- var qLogTable = safeSql.quoteIdentifier(logTable);
284
- var qTipTable = safeSql.quoteIdentifier(tipTable);
285
-
286
- var tipRows;
287
- try {
288
- tipRows = await externalDb().query(
289
- "SELECT atMonotonicCounter, rowHash FROM " + qTipTable +
290
- " WHERE scope = " + (configuredDialect === "postgres" ? "$1" : "?"),
291
- [chainName],
292
- { backend: configuredExternalDbBackend }
293
- );
294
- } catch (e) {
295
- var msg = (e && e.message) || "";
296
- if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
297
- log(chainName + "-tip table not present — skipping rollback check (cluster gates-only mode)");
298
- return;
299
- }
300
- throw e;
301
- }
302
- if (!tipRows.rows || tipRows.rows.length === 0) {
303
- log("no " + chainName + "-tip row — skipping rollback check (first cluster boot or operator-cleared)");
304
- return;
305
- }
306
- var tip = tipRows.rows[0];
307
- var tipCounter = Number(tip.atMonotonicCounter);
308
- var tipHash = tip.rowHash;
309
-
310
- var currentRows = await externalDb().query(
311
- "SELECT MAX(monotonicCounter) AS m FROM " + qLogTable,
312
- [],
313
- { backend: configuredExternalDbBackend }
314
- );
315
- var currentMax = (currentRows.rows && currentRows.rows[0] && currentRows.rows[0].m)
316
- ? Number(currentRows.rows[0].m)
317
- : 0;
318
-
319
- if (currentMax < tipCounter) {
320
- throw _err("ROLLBACK_DETECTED",
321
- "FATAL: cluster-mode " + chainName + "-log rollback detected. " +
322
- chainName + "-tip counter: " + tipCounter +
323
- "; current external-db max: " + currentMax +
324
- ". Either external-db was restored from an older snapshot, or " +
325
- logTable + " rows have been deleted. Investigate before continuing.",
326
- true);
327
- }
328
-
329
- if (tipHash) {
330
- var hashRows = await externalDb().query(
331
- "SELECT rowHash FROM " + qLogTable + " WHERE monotonicCounter = " +
332
- (configuredDialect === "postgres" ? "$1" : "?"),
333
- [tipCounter],
334
- { backend: configuredExternalDbBackend }
335
- );
336
- if (hashRows.rows && hashRows.rows.length > 0) {
337
- var rowAtTip = hashRows.rows[0].rowHash;
338
- if (rowAtTip !== tipHash) {
339
- throw _err("ROLLBACK_DETECTED",
340
- "FATAL: cluster-mode " + chainName + "-log rollback detected (row-hash mismatch). " +
341
- chainName + "-tip counter: " + tipCounter +
342
- "; " + chainName + "-tip rowHash: " + tipHash +
343
- "; current row rowHash: " + rowAtTip +
344
- ". The row at the recorded tip counter has a different hash — " +
345
- "indicates row substitution at the chain head. Investigate before continuing.",
346
- true);
347
- }
348
- }
349
- }
350
- log("cluster " + chainName + "-tip rollback check ok (tip counter " + tipCounter +
351
- ", current " + currentMax + ")");
352
- }
353
-
354
- // Compute a deterministic fingerprint of THIS node's vault keys.
355
- // SHA3-512 of the concatenated public keys (PQC + classical halves of
356
- // the hybrid encryption keypair). One-way: nothing about the private
357
- // key material is recoverable from the fingerprint, so it's safe to
358
- // store in the coordination table that all cluster nodes can read.
359
- //
360
- // Returns null if vault.init() hasn't been called — cluster gates-only
361
- // mode (no sealed-column work) doesn't need this check, same defensive
362
- // posture as the audit-tip rollback check skipping when there's no
363
- // audit-tip table.
364
- function _vaultKeyFingerprint() {
365
- var keysJson;
366
- try {
367
- keysJson = vault().getKeysJson();
368
- } catch (e) {
369
- // vault.init() not called — gates-only mode. Skip silently.
370
- if (/vault.init\(\) must be awaited/.test((e && e.message) || "")) {
371
- return null;
372
- }
373
- throw e;
374
- }
375
- // vault.getKeysJson() returns the keys serialized as JSON (the same
376
- // format vault writes to disk). Parse to extract the public halves;
377
- // we never touch privateKey/ecPrivateKey here.
378
- var keys = safeJson.parse(keysJson);
379
- if (!keys || !keys.publicKey || !keys.ecPublicKey) return null;
380
- // Domain-separation prefix so this fingerprint can't be confused
381
- // with a hash of the same bytes computed elsewhere in the framework.
382
- return crypto.sha3Hash("blamejs/cluster-state/v1\n" +
383
- keys.publicKey + "\n" +
384
- keys.ecPublicKey);
385
- }
386
-
387
- async function _checkVaultKeyConsistency() {
388
- var localFp = _vaultKeyFingerprint();
389
- if (localFp === null) {
390
- log("vault not initialized — skipping vault-key consistency check (cluster gates-only mode)");
391
- return;
392
- }
393
- var nowMs = Date.now();
394
- var ph = configuredDialect === "postgres";
395
-
396
- // First boot: try to record THIS node's fingerprint. ON CONFLICT DO
397
- // NOTHING means the FIRST node to boot wins; subsequent nodes
398
- // observe whatever's already there. Every node then SELECTs and
399
- // compares — any mismatch (including ours after a losing race)
400
- // surfaces the drift.
401
- try {
402
- await externalDb().query(
403
- "INSERT INTO _blamejs_cluster_state " +
404
- " (scope, vaultKeyFp, recordedAt, recordedByNode) " +
405
- "VALUES ('state', " +
406
- (ph ? "$1, $2, $3" : "?, ?, ?") + ") " +
407
- "ON CONFLICT (scope) DO NOTHING",
408
- [localFp, nowMs, nodeId],
409
- { backend: configuredExternalDbBackend }
410
- );
411
- } catch (e) {
412
- // Table missing → the cluster-provider-db ensureSchema didn't run
413
- // (custom provider that doesn't create _blamejs_cluster_state).
414
- // Skip silently — same defensive posture as the audit-tip check.
415
- var msg = (e && e.message) || "";
416
- if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
417
- log("cluster-state table not present — skipping vault-key consistency check (custom provider)");
418
- return;
419
- }
420
- throw e;
421
- }
422
-
423
- // Read whatever fingerprint is canonical (ours if first boot,
424
- // someone else's if we lost the race or are joining an existing cluster).
425
- var rows = await externalDb().query(
426
- "SELECT vaultKeyFp, recordedByNode, recordedAt FROM _blamejs_cluster_state " +
427
- "WHERE scope = 'state'",
428
- [],
429
- { backend: configuredExternalDbBackend }
430
- );
431
- if (!rows.rows || rows.rows.length === 0) {
432
- // Should never happen — we just INSERTed. Surface as fatal so the
433
- // condition isn't silently ignored.
434
- throw _err("CLUSTER_STATE_MISSING",
435
- "FATAL: cluster-state row missing immediately after INSERT — " +
436
- "external-db may not be honoring writes. Refusing boot.",
437
- true);
438
- }
439
- var canonical = rows.rows[0];
440
- if (canonical.vaultKeyFp !== localFp) {
441
- var fpPrefix = C.BYTES.bytes(16);
442
- throw _err("VAULT_KEY_DRIFT",
443
- "FATAL: vault-key drift detected. " +
444
- "local node: " + nodeId +
445
- "; local fingerprint: " + localFp.slice(0, fpPrefix) + "…" +
446
- "; canonical recorded by: " + canonical.recordedByNode +
447
- "; canonical fingerprint: " + canonical.vaultKeyFp.slice(0, fpPrefix) + "…" +
448
- ". This node holds a DIFFERENT vault key than the rest of the " +
449
- "cluster. Sealed-column writes from this node would be unreadable " +
450
- "by the others (and vice versa). Restore the same vault key file " +
451
- "before booting this node into the cluster.",
452
- true);
453
- }
454
- log("cluster vault-key consistency ok (fingerprint " +
455
- localFp.slice(0, C.BYTES.bytes(16)) + "… recorded by " + canonical.recordedByNode + ")");
456
- }
457
-
458
- async function _tryAcquire() {
459
- if (role !== "leader") return; // pinned-follower role: never claim
460
- try {
461
- var got = await provider.acquireLease(nodeId, leaseTtlMs, {
462
- endpoint: configuredEndpoint,
463
- });
464
- if (got) {
465
- var wasLeader = !!lease;
466
- lease = got;
467
- if (!wasLeader) {
468
- log("acquired lease — fencingToken=" + lease.fencingToken);
469
- _emitTransition("lease-acquired", { fencingToken: lease.fencingToken });
470
- }
471
- }
472
- } catch (e) {
473
- log.error("acquire failed: " + e.message);
474
- }
475
- }
476
-
477
- async function _heartbeat() {
478
- if (!initialized) return;
479
- if (!lease) {
480
- // Not currently leader — try to acquire (lease may have expired
481
- // on the previous holder).
482
- await _tryAcquire();
483
- return;
484
- }
485
- // We hold a lease — renew it. Re-supply the configured endpoint so a
486
- // hot-reload of the operator's config (e.g. node moves to a new
487
- // routable URL after a restart) eventually reaches the discovery row.
488
- try {
489
- lease = await provider.renewLease(lease, { endpoint: configuredEndpoint });
490
- } catch (e) {
491
- if (e.code === "LEASE_LOST") {
492
- log.error("lease lost: " + e.message);
493
- var lostToken = lease ? lease.fencingToken : null;
494
- lease = null;
495
- _emitTransition("lease-lost", { fencingToken: lostToken });
496
- // Attempt to re-acquire on the next heartbeat naturally.
497
- } else {
498
- // Transient error — retry on next heartbeat. If it persists past
499
- // leaseTtl another node will steal, and we'll detect via LEASE_LOST.
500
- log.error("renew failed transiently: " + e.message);
501
- }
502
- }
503
- }
504
-
505
- // ---- public sync surface ----
506
-
507
- function isLeader() {
508
- if (terminated) return false; // post-shutdown: never leader
509
- if (!initialized) return true; // never-initialized: permanent leader
510
- return !!lease && Date.now() < lease.expiresAt;
511
- }
512
-
513
- // Has cluster.init been called with a real configuration? Used by
514
- // write-dispatch code (audit, consent, …) to decide whether framework
515
- // state should go to local SQLite or external-db.
516
- function isClusterMode() {
517
- return initialized && !!configuredExternalDbBackend;
518
- }
519
-
520
- function externalDbBackend() {
521
- return configuredExternalDbBackend;
522
- }
523
-
524
- function dialect() {
525
- return configuredDialect;
526
- }
527
-
528
- function currentNodeId() {
529
- return initialized ? nodeId : "single-node-local";
530
- }
531
-
532
- // This node's routable endpoint (operator-configured at cluster.init).
533
- // Returns null when not configured or in single-node fallback. External
534
- // observers should call discoveryHandler() / currentLeader() instead —
535
- // this getter is for the local node's own self-identity.
536
- function endpoint() {
537
- return configuredEndpoint;
538
- }
539
-
540
- function fencingToken() {
541
- if (!initialized) return 0;
542
- return lease ? lease.fencingToken : 0;
543
- }
544
-
545
- function requireLeader() {
546
- if (!isLeader()) {
547
- throw new NotLeaderError(
548
- "node '" + currentNodeId() + "' is not currently leader" +
549
- (initialized ? "" : " (cluster not initialized)")
550
- );
551
- }
552
- }
553
-
554
- async function currentLeader() {
555
- if (!initialized) {
556
- return {
557
- nodeId: "single-node-local",
558
- leaseExpiresAt: Infinity,
559
- fencingToken: 0,
560
- endpoint: null,
561
- };
562
- }
563
- return await provider.currentLeader();
564
- }
565
-
566
- // HTTP request handler — replies with the current cluster leader for
567
- // service-mesh / load-balancer discovery. Operators mount this at
568
- // whatever route they want (`/cluster/leader`, `/health/leader`, etc.).
569
- //
570
- // 200 application/json — leader present
571
- // { leader: { nodeId, endpoint, fencingToken, leaseExpiresAt },
572
- // self: { nodeId, endpoint, isLeader } }
573
- //
574
- // 503 application/json — no leader (no row, expired lease, DB
575
- // unreachable, single-node not initialized with cluster mode)
576
- // { leader: null, self: { nodeId, endpoint, isLeader } }
577
- //
578
- // No auth — this endpoint is intended to be called by infrastructure
579
- // inside the trust boundary (LB, healthcheck, dashboard). Operators
580
- // who expose it externally should layer auth via their own middleware.
581
- //
582
- // Handler is method-agnostic so it works behind any HTTP probe shape
583
- // (GET, HEAD, etc.). Cache-Control: no-store to avoid stale-leader
584
- // responses pinned by a caching proxy during a takeover.
585
- function discoveryHandler() {
586
- return async function (req, res) {
587
- var selfInfo = {
588
- nodeId: currentNodeId(),
589
- endpoint: configuredEndpoint,
590
- isLeader: isLeader(),
591
- };
592
- var body;
593
- var status;
594
- try {
595
- var leader = await currentLeader();
596
- if (leader && leader.nodeId && leader.nodeId !== "single-node-local") {
597
- body = { leader: leader, self: selfInfo };
598
- status = 200;
599
- } else if (leader && leader.nodeId === "single-node-local") {
600
- // Permanent-leader fallback (cluster.init never called). Reply
601
- // 200 — the operator's app is healthy and the "leader" is this
602
- // process. Useful so the discovery endpoint is never a false
603
- // negative in single-node deployments.
604
- body = { leader: leader, self: selfInfo };
605
- status = 200;
606
- } else {
607
- body = { leader: null, self: selfInfo };
608
- status = 503;
609
- }
610
- } catch (e) {
611
- body = { leader: null, self: selfInfo, error: e.message };
612
- status = 503;
613
- }
614
- var json = JSON.stringify(body);
615
- res.writeHead(status, {
616
- "Content-Type": "application/json; charset=utf-8",
617
- "Content-Length": Buffer.byteLength(json),
618
- "Cache-Control": "no-store",
619
- });
620
- res.end(json);
621
- };
622
- }
623
-
624
- function onTransition(handler) {
625
- if (typeof handler !== "function") {
626
- throw _err("INVALID_HANDLER", "onTransition expects a function", true);
627
- }
628
- transitionHandlers.push(handler);
629
- }
630
-
631
- async function shutdown() {
632
- if (!initialized) return;
633
- if (heartbeatTimer) {
634
- heartbeatTimer.stop();
635
- heartbeatTimer = null;
636
- }
637
- if (lease) {
638
- try {
639
- await provider.releaseLease(lease);
640
- _emitTransition("lease-released", { fencingToken: lease.fencingToken });
641
- log("lease released on shutdown");
642
- } catch (e) {
643
- log.error("release on shutdown failed: " + e.message);
644
- }
645
- lease = null;
646
- }
647
- initialized = false;
648
- terminated = true;
649
- provider = null;
650
- role = null;
651
- leaseTtlMs = null;
652
- heartbeatMs = null;
653
- configuredExternalDbBackend = null;
654
- configuredDialect = null;
655
- configuredEndpoint = null;
656
- transitionHandlers = [];
657
- // nodeId is preserved post-shutdown so audit metadata still reflects
658
- // who this process was; cleared only by _resetForTest.
659
- }
660
-
661
- // ---- test helpers — not part of public contract ----
662
-
663
- function _resetForTest() {
664
- if (heartbeatTimer) heartbeatTimer.stop();
665
- heartbeatTimer = null;
666
- initialized = false;
667
- terminated = false;
668
- nodeId = null;
669
- role = null;
670
- provider = null;
671
- lease = null;
672
- leaseTtlMs = null;
673
- heartbeatMs = null;
674
- configuredExternalDbBackend = null;
675
- configuredDialect = null;
676
- configuredEndpoint = null;
677
- transitionHandlers = [];
678
- }
679
-
680
- async function _heartbeatNowForTest() {
681
- // Drive one heartbeat synchronously without waiting for the timer —
682
- // lets tests deterministically observe lease state transitions.
683
- await _heartbeat();
684
- }
685
-
686
- module.exports = {
687
- init: init,
688
- isLeader: isLeader,
689
- isClusterMode: isClusterMode,
690
- externalDbBackend: externalDbBackend,
691
- dialect: dialect,
692
- currentNodeId: currentNodeId,
693
- endpoint: endpoint,
694
- fencingToken: fencingToken,
695
- requireLeader: requireLeader,
696
- currentLeader: currentLeader,
697
- discoveryHandler: discoveryHandler,
698
- onTransition: onTransition,
699
- shutdown: shutdown,
700
- NotLeaderError: NotLeaderError,
701
- _resetForTest: _resetForTest,
702
- _heartbeatNowForTest: _heartbeatNowForTest,
703
- };
1
+ "use strict";
2
+ /**
3
+ * Cluster coordination — leader election + fencing tokens.
4
+ *
5
+ * Opt-in via `b.cluster.init(...)`. When init is never called, the
6
+ * local process behaves as a permanent single leader: `isLeader()`
7
+ * always returns true, `fencingToken()` returns 0, no heartbeat thread
8
+ * runs, no DB is touched. Single-node deployments pay zero overhead.
9
+ *
10
+ * When init IS called, the framework starts a heartbeat that renews
11
+ * the leader lease via the configured provider. On lease loss (network
12
+ * partition, takeover, lease expiry) the node transitions to follower
13
+ * and write-side framework primitives throw `NotLeaderError`.
14
+ *
15
+ * Threat model:
16
+ * - Two leaders writing simultaneously: prevented by fencing tokens.
17
+ * Every leader-only DB write includes the current token; a
18
+ * CHECK constraint on the audit-tip row rejects a stale token.
19
+ * The application-layer `requireLeader()` gate is just an early
20
+ * rejection optimisation; the DB constraint is the canonical guard.
21
+ * - Follower receiving a write: rejected at the framework boundary.
22
+ * Operators front the cluster with a load balancer that routes
23
+ * write paths to the current leader.
24
+ * - External-db unreachable: heartbeat fails; after `leaseTtl` no
25
+ * leader exists and writes fail closed. When the DB recovers,
26
+ * election resumes.
27
+ *
28
+ * Public API:
29
+ * await cluster.init(opts) one-time bootstrap
30
+ * cluster.isLeader() sync; true on leader (or single-node)
31
+ * cluster.currentNodeId() sync; configured nodeId
32
+ * cluster.endpoint() sync; this node's routable URL
33
+ * (operator-supplied at init), or
34
+ * null if unconfigured. Stored in
35
+ * the leader-election row so
36
+ * external observers can resolve
37
+ * "where is the current leader?"
38
+ * cluster.fencingToken() sync; current monotonic token
39
+ * cluster.requireLeader() sync; throws NotLeaderError
40
+ * cluster.currentLeader() async; { nodeId, leaseExpiresAt,
41
+ * fencingToken,
42
+ * endpoint } | null
43
+ * cluster.discoveryHandler() returns an HTTP request handler
44
+ * (req, res) → JSON. Mount on any
45
+ * route to expose the current
46
+ * leader for service-mesh / LB
47
+ * consumption. 200 with leader,
48
+ * 503 with `{ leader: null }`
49
+ * when no leader.
50
+ * cluster.onTransition(fn) register transition handler
51
+ * await cluster.shutdown() releases lease, stops heartbeat
52
+ */
53
+ var C = require("./constants");
54
+ var clusterProviderDb = require("./cluster-provider-db");
55
+ var crypto = require("./crypto");
56
+ var lazyRequire = require("./lazy-require");
57
+ var { boot } = require("./log");
58
+ var safeAsync = require("./safe-async");
59
+ var safeJson = require("./safe-json");
60
+ var safeSql = require("./safe-sql");
61
+ var safeUrl = require("./safe-url");
62
+ var { FrameworkError, ClusterError } = require("./framework-error");
63
+
64
+ // Lazy: vault → db → cluster forms a load-time chain, and external-db is
65
+ // loaded before its init has run; both are safe to call once cluster
66
+ // reaches runtime, but eager require here would deadlock the load order.
67
+ var externalDb = lazyRequire(function () { return require("./external-db"); });
68
+ var vault = lazyRequire(function () { return require("./vault"); });
69
+
70
+ var DEFAULT_LEASE_TTL = C.TIME.seconds(30);
71
+ var DEFAULT_HEARTBEAT = C.TIME.seconds(10);
72
+ var MIN_LEASE_TTL = C.TIME.seconds(5);
73
+ var MIN_HEARTBEAT = C.TIME.seconds(1);
74
+
75
+ var initialized = false;
76
+ var terminated = false; // set true by shutdown() so the
77
+ // permanent-leader fallback isn't
78
+ // re-engaged after a graceful exit
79
+ var nodeId = null;
80
+ var role = null; // 'leader' | 'follower'
81
+ var provider = null;
82
+ var lease = null; // current lease (if leader)
83
+ var heartbeatTimer = null;
84
+ var heartbeatMs = null;
85
+ var leaseTtlMs = null;
86
+ var transitionHandlers = [];
87
+ // Backend coordinates for write-dispatch code in audit/consent/etc.
88
+ // These are set when cluster.init is called with `externalDbBackend`
89
+ // (the default DB-row provider path); operators using a custom
90
+ // provider can set them via init opts directly.
91
+ var configuredExternalDbBackend = null;
92
+ var configuredDialect = null;
93
+ // Operator-supplied routable endpoint for THIS node, used by external
94
+ // load balancers / service meshes to learn where to send write traffic.
95
+ // Stored in the leader-election row on every acquire/renew so any node
96
+ // (or external observer) can resolve "where is the current leader?"
97
+ // via cluster.currentLeader() / cluster.discoveryHandler().
98
+ var configuredEndpoint = null;
99
+
100
+ var log = boot("cluster");
101
+
102
+ class NotLeaderError extends FrameworkError {
103
+ constructor(message) {
104
+ super(message || "not leader: write rejected by cluster gate", "NOT_LEADER");
105
+ this.name = "NotLeaderError";
106
+ this.statusCode = 503; // operator's load balancer should retry on the leader
107
+ this.isClusterError = true;
108
+ this.isNotLeaderError = true;
109
+ }
110
+ }
111
+
112
+ var _err = ClusterError.factory;
113
+
114
+ function _emitTransition(kind, detail) {
115
+ var event = Object.assign({ kind: kind, nodeId: nodeId, at: Date.now() }, detail || {});
116
+ for (var i = 0; i < transitionHandlers.length; i++) {
117
+ try { transitionHandlers[i](event); }
118
+ catch (e) { log.error("transition handler threw: " + e.message); }
119
+ }
120
+ }
121
+
122
+ // ---- init ----
123
+
124
+ async function init(opts) {
125
+ if (initialized) {
126
+ throw _err("ALREADY_INITIALIZED", "cluster.init() called twice", true);
127
+ }
128
+ opts = opts || {};
129
+ if (!opts.nodeId) {
130
+ throw _err("INVALID_CONFIG", "cluster.init({ nodeId }) is required", true);
131
+ }
132
+ nodeId = String(opts.nodeId);
133
+
134
+ leaseTtlMs = opts.leaseTtl != null ? Number(opts.leaseTtl) : DEFAULT_LEASE_TTL;
135
+ if (leaseTtlMs < MIN_LEASE_TTL) {
136
+ throw _err("INVALID_TTL",
137
+ "leaseTtl must be >= " + MIN_LEASE_TTL + "ms (got " + leaseTtlMs + ")",
138
+ true);
139
+ }
140
+ heartbeatMs = opts.heartbeatInterval != null
141
+ ? Number(opts.heartbeatInterval)
142
+ : DEFAULT_HEARTBEAT;
143
+ if (heartbeatMs < MIN_HEARTBEAT) {
144
+ throw _err("INVALID_HEARTBEAT",
145
+ "heartbeatInterval must be >= " + MIN_HEARTBEAT + "ms (got " + heartbeatMs + ")",
146
+ true);
147
+ }
148
+ if (heartbeatMs >= leaseTtlMs) {
149
+ throw _err("INVALID_HEARTBEAT",
150
+ "heartbeatInterval must be < leaseTtl (got heartbeat=" + heartbeatMs +
151
+ ", leaseTtl=" + leaseTtlMs + "); recommend ~1/3 of leaseTtl",
152
+ true);
153
+ }
154
+
155
+ role = (opts.role || "leader").toLowerCase();
156
+ if (role !== "leader" && role !== "follower") {
157
+ throw _err("INVALID_ROLE", "role must be 'leader' or 'follower'", true);
158
+ }
159
+
160
+ // Optional endpoint. If provided, validate scheme + shape via url-safe
161
+ // — HTTPS-only by default since this is the URL external services use
162
+ // to reach the leader. Operators with internal cleartext clusters opt
163
+ // in via opts.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
164
+ if (opts.endpoint != null) {
165
+ try {
166
+ safeUrl.parse(opts.endpoint, {
167
+ allowedProtocols: opts.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
168
+ errorClass: ClusterError,
169
+ });
170
+ } catch (e) {
171
+ // Re-throw with a config-shaped error so operators see the cluster.init
172
+ // boundary, not a bare url-safe trace.
173
+ throw _err("INVALID_ENDPOINT",
174
+ "cluster.init({ endpoint }) rejected: " + e.message, true);
175
+ }
176
+ configuredEndpoint = String(opts.endpoint);
177
+ } else {
178
+ configuredEndpoint = null;
179
+ }
180
+
181
+ if (typeof opts.onTransition === "function") {
182
+ transitionHandlers.push(opts.onTransition);
183
+ }
184
+
185
+ // Provider: either operator-supplied, or build the default DB-row
186
+ // provider against an externalDb backend.
187
+ if (opts.provider) {
188
+ provider = opts.provider;
189
+ // Operator-custom provider: they may still be writing framework
190
+ // state to an externalDb backend, in which case they pass these
191
+ // separately so write-dispatch code knows where to go.
192
+ configuredExternalDbBackend = opts.externalDbBackend || null;
193
+ configuredDialect = (opts.dialect || "postgres").toLowerCase();
194
+ } else {
195
+ if (!opts.externalDbBackend) {
196
+ throw _err("INVALID_CONFIG",
197
+ "cluster.init requires either { provider } or { externalDbBackend }", true);
198
+ }
199
+ provider = clusterProviderDb.create({
200
+ externalDbBackend: opts.externalDbBackend,
201
+ dialect: opts.dialect,
202
+ });
203
+ configuredExternalDbBackend = opts.externalDbBackend;
204
+ configuredDialect = (opts.dialect || "postgres").toLowerCase();
205
+ }
206
+
207
+ if (typeof provider.ensureSchema === "function") {
208
+ await provider.ensureSchema();
209
+ }
210
+
211
+ initialized = true;
212
+ log("initialized as nodeId='" + nodeId + "', role='" + role + "'");
213
+
214
+ // Initial acquisition attempt (only if role === 'leader')
215
+ if (role === "leader") {
216
+ await _tryAcquire();
217
+ }
218
+
219
+ // Boot-time rollback detection on the audit + consent chains. Runs
220
+ // regardless of role — every node should refuse to participate in a
221
+ // cluster whose shared chains have been rolled back (a follower
222
+ // would face the same chain integrity failure if it later took
223
+ // over). Skipped when configuredExternalDbBackend is unset, which
224
+ // means a custom provider is in use without externalDb-resident
225
+ // framework state — the operator owns rollback detection in that
226
+ // case.
227
+ if (configuredExternalDbBackend) {
228
+ await _checkChainTipRollback("audit", "_blamejs_audit_log", "_blamejs_audit_tip");
229
+ await _checkChainTipRollback("consent", "_blamejs_consent_log", "_blamejs_consent_tip");
230
+ // Vault-key consistency: every node in a cluster must hold the
231
+ // SAME vault key. A node booting with a different key would seal
232
+ // new writes under a key the rest of the cluster can't unseal,
233
+ // and (on takeover) be unable to unseal the rest of the cluster's
234
+ // sealed columns — silent corruption. Compare a fingerprint of
235
+ // this node's vault keys against the canonical one stored at
236
+ // first cluster boot; refuse to participate on mismatch.
237
+ await _checkVaultKeyConsistency();
238
+ }
239
+
240
+ // Start heartbeat
241
+ heartbeatTimer = safeAsync.repeating(_heartbeat, heartbeatMs, { name: "cluster-heartbeat" });
242
+ }
243
+
244
+ // Cluster-mode equivalent of db.js's single-node audit.tip-sidecar
245
+ // rollback check. Reads the persistent _blamejs_audit_tip row and
246
+ // compares to the current chain head in _blamejs_audit_log:
247
+ //
248
+ // - No tip row: first cluster boot or operator-cleared. Skip
249
+ // silently (matches the single-node sidecar-missing path).
250
+ // - Tip recorded a counter > current MAX: chain was truncated /
251
+ // restored from older snapshot. FATAL — refuse boot.
252
+ // - Tip recorded a hash that doesn't match the row at that
253
+ // counter: the row at that counter was substituted (different
254
+ // hash for same counter). FATAL — refuse boot.
255
+ //
256
+ // process.exit(1) is the framework's convention for boot-time
257
+ // integrity failures (audit chain, checkpoints, single-node
258
+ // rollback). Cluster mode keeps the same posture so operators see
259
+ // a single boot-time failure mode regardless of deployment shape.
260
+ // Generalized boot-time rollback check used by both audit and consent
261
+ // chains. chainName is the human-readable label included in log
262
+ // output ("audit" / "consent"). logTable is the chain table
263
+ // (_blamejs_audit_log / _blamejs_consent_log). tipTable is the
264
+ // single-row coordination table that records the latest counter +
265
+ // rowHash + fencingToken (_blamejs_audit_tip / _blamejs_consent_tip).
266
+ //
267
+ // Surfaces three outcomes:
268
+ // - tip table missing → operator running cluster gates-only mode
269
+ // (cluster wired for leader election but framework state still
270
+ // lives in per-node SQLite without `frameworkSchema.ensureSchema`);
271
+ // skip silently.
272
+ // - no tip row → first cluster boot or operator-cleared; skip.
273
+ // - currentMax < tipCounter, or tip rowHash != row-at-counter
274
+ // hash → FATAL via process.exit(1). Same posture as the
275
+ // single-node audit.tip sidecar rollback check.
276
+ async function _checkChainTipRollback(chainName, logTable, tipTable) {
277
+ // Both tables are framework-internal constants from the call sites
278
+ // (`_blamejs_audit_log`, `_blamejs_consent_log`, etc.). Validate +
279
+ // quote per the framework's identifier-quoting convention so a
280
+ // future rename can't silently break the query.
281
+ safeSql.validateIdentifier(logTable, { allowReserved: true });
282
+ safeSql.validateIdentifier(tipTable, { allowReserved: true });
283
+ var qLogTable = safeSql.quoteIdentifier(logTable);
284
+ var qTipTable = safeSql.quoteIdentifier(tipTable);
285
+
286
+ var tipRows;
287
+ try {
288
+ tipRows = await externalDb().query(
289
+ "SELECT atMonotonicCounter, rowHash FROM " + qTipTable +
290
+ " WHERE scope = " + (configuredDialect === "postgres" ? "$1" : "?"),
291
+ [chainName],
292
+ { backend: configuredExternalDbBackend }
293
+ );
294
+ } catch (e) {
295
+ var msg = (e && e.message) || "";
296
+ if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
297
+ log(chainName + "-tip table not present — skipping rollback check (cluster gates-only mode)");
298
+ return;
299
+ }
300
+ throw e;
301
+ }
302
+ if (!tipRows.rows || tipRows.rows.length === 0) {
303
+ log("no " + chainName + "-tip row — skipping rollback check (first cluster boot or operator-cleared)");
304
+ return;
305
+ }
306
+ var tip = tipRows.rows[0];
307
+ var tipCounter = Number(tip.atMonotonicCounter);
308
+ var tipHash = tip.rowHash;
309
+
310
+ var currentRows = await externalDb().query(
311
+ "SELECT MAX(monotonicCounter) AS m FROM " + qLogTable,
312
+ [],
313
+ { backend: configuredExternalDbBackend }
314
+ );
315
+ var currentMax = (currentRows.rows && currentRows.rows[0] && currentRows.rows[0].m)
316
+ ? Number(currentRows.rows[0].m)
317
+ : 0;
318
+
319
+ if (currentMax < tipCounter) {
320
+ throw _err("ROLLBACK_DETECTED",
321
+ "FATAL: cluster-mode " + chainName + "-log rollback detected. " +
322
+ chainName + "-tip counter: " + tipCounter +
323
+ "; current external-db max: " + currentMax +
324
+ ". Either external-db was restored from an older snapshot, or " +
325
+ logTable + " rows have been deleted. Investigate before continuing.",
326
+ true);
327
+ }
328
+
329
+ if (tipHash) {
330
+ var hashRows = await externalDb().query(
331
+ "SELECT rowHash FROM " + qLogTable + " WHERE monotonicCounter = " +
332
+ (configuredDialect === "postgres" ? "$1" : "?"),
333
+ [tipCounter],
334
+ { backend: configuredExternalDbBackend }
335
+ );
336
+ if (hashRows.rows && hashRows.rows.length > 0) {
337
+ var rowAtTip = hashRows.rows[0].rowHash;
338
+ if (rowAtTip !== tipHash) {
339
+ throw _err("ROLLBACK_DETECTED",
340
+ "FATAL: cluster-mode " + chainName + "-log rollback detected (row-hash mismatch). " +
341
+ chainName + "-tip counter: " + tipCounter +
342
+ "; " + chainName + "-tip rowHash: " + tipHash +
343
+ "; current row rowHash: " + rowAtTip +
344
+ ". The row at the recorded tip counter has a different hash — " +
345
+ "indicates row substitution at the chain head. Investigate before continuing.",
346
+ true);
347
+ }
348
+ }
349
+ }
350
+ log("cluster " + chainName + "-tip rollback check ok (tip counter " + tipCounter +
351
+ ", current " + currentMax + ")");
352
+ }
353
+
354
+ // Compute a deterministic fingerprint of THIS node's vault keys.
355
+ // SHA3-512 of the concatenated public keys (PQC + classical halves of
356
+ // the hybrid encryption keypair). One-way: nothing about the private
357
+ // key material is recoverable from the fingerprint, so it's safe to
358
+ // store in the coordination table that all cluster nodes can read.
359
+ //
360
+ // Returns null if vault.init() hasn't been called — cluster gates-only
361
+ // mode (no sealed-column work) doesn't need this check, same defensive
362
+ // posture as the audit-tip rollback check skipping when there's no
363
+ // audit-tip table.
364
+ function _vaultKeyFingerprint() {
365
+ var keysJson;
366
+ try {
367
+ keysJson = vault().getKeysJson();
368
+ } catch (e) {
369
+ // vault.init() not called — gates-only mode. Skip silently.
370
+ if (/vault.init\(\) must be awaited/.test((e && e.message) || "")) {
371
+ return null;
372
+ }
373
+ throw e;
374
+ }
375
+ // vault.getKeysJson() returns the keys serialized as JSON (the same
376
+ // format vault writes to disk). Parse to extract the public halves;
377
+ // we never touch privateKey/ecPrivateKey here.
378
+ var keys = safeJson.parse(keysJson);
379
+ if (!keys || !keys.publicKey || !keys.ecPublicKey) return null;
380
+ // Domain-separation prefix so this fingerprint can't be confused
381
+ // with a hash of the same bytes computed elsewhere in the framework.
382
+ return crypto.sha3Hash("blamejs/cluster-state/v1\n" +
383
+ keys.publicKey + "\n" +
384
+ keys.ecPublicKey);
385
+ }
386
+
387
+ async function _checkVaultKeyConsistency() {
388
+ var localFp = _vaultKeyFingerprint();
389
+ if (localFp === null) {
390
+ log("vault not initialized — skipping vault-key consistency check (cluster gates-only mode)");
391
+ return;
392
+ }
393
+ var nowMs = Date.now();
394
+ var ph = configuredDialect === "postgres";
395
+
396
+ // First boot: try to record THIS node's fingerprint. ON CONFLICT DO
397
+ // NOTHING means the FIRST node to boot wins; subsequent nodes
398
+ // observe whatever's already there. Every node then SELECTs and
399
+ // compares — any mismatch (including ours after a losing race)
400
+ // surfaces the drift.
401
+ try {
402
+ await externalDb().query(
403
+ "INSERT INTO _blamejs_cluster_state " +
404
+ " (scope, vaultKeyFp, recordedAt, recordedByNode) " +
405
+ "VALUES ('state', " +
406
+ (ph ? "$1, $2, $3" : "?, ?, ?") + ") " +
407
+ "ON CONFLICT (scope) DO NOTHING",
408
+ [localFp, nowMs, nodeId],
409
+ { backend: configuredExternalDbBackend }
410
+ );
411
+ } catch (e) {
412
+ // Table missing → the cluster-provider-db ensureSchema didn't run
413
+ // (custom provider that doesn't create _blamejs_cluster_state).
414
+ // Skip silently — same defensive posture as the audit-tip check.
415
+ var msg = (e && e.message) || "";
416
+ if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
417
+ log("cluster-state table not present — skipping vault-key consistency check (custom provider)");
418
+ return;
419
+ }
420
+ throw e;
421
+ }
422
+
423
+ // Read whatever fingerprint is canonical (ours if first boot,
424
+ // someone else's if we lost the race or are joining an existing cluster).
425
+ var rows = await externalDb().query(
426
+ "SELECT vaultKeyFp, recordedByNode, recordedAt FROM _blamejs_cluster_state " +
427
+ "WHERE scope = 'state'",
428
+ [],
429
+ { backend: configuredExternalDbBackend }
430
+ );
431
+ if (!rows.rows || rows.rows.length === 0) {
432
+ // Should never happen — we just INSERTed. Surface as fatal so the
433
+ // condition isn't silently ignored.
434
+ throw _err("CLUSTER_STATE_MISSING",
435
+ "FATAL: cluster-state row missing immediately after INSERT — " +
436
+ "external-db may not be honoring writes. Refusing boot.",
437
+ true);
438
+ }
439
+ var canonical = rows.rows[0];
440
+ if (canonical.vaultKeyFp !== localFp) {
441
+ var fpPrefix = C.BYTES.bytes(16);
442
+ throw _err("VAULT_KEY_DRIFT",
443
+ "FATAL: vault-key drift detected. " +
444
+ "local node: " + nodeId +
445
+ "; local fingerprint: " + localFp.slice(0, fpPrefix) + "…" +
446
+ "; canonical recorded by: " + canonical.recordedByNode +
447
+ "; canonical fingerprint: " + canonical.vaultKeyFp.slice(0, fpPrefix) + "…" +
448
+ ". This node holds a DIFFERENT vault key than the rest of the " +
449
+ "cluster. Sealed-column writes from this node would be unreadable " +
450
+ "by the others (and vice versa). Restore the same vault key file " +
451
+ "before booting this node into the cluster.",
452
+ true);
453
+ }
454
+ log("cluster vault-key consistency ok (fingerprint " +
455
+ localFp.slice(0, C.BYTES.bytes(16)) + "… recorded by " + canonical.recordedByNode + ")");
456
+ }
457
+
458
+ async function _tryAcquire() {
459
+ if (role !== "leader") return; // pinned-follower role: never claim
460
+ try {
461
+ var got = await provider.acquireLease(nodeId, leaseTtlMs, {
462
+ endpoint: configuredEndpoint,
463
+ });
464
+ if (got) {
465
+ var wasLeader = !!lease;
466
+ lease = got;
467
+ if (!wasLeader) {
468
+ log("acquired lease — fencingToken=" + lease.fencingToken);
469
+ _emitTransition("lease-acquired", { fencingToken: lease.fencingToken });
470
+ }
471
+ }
472
+ } catch (e) {
473
+ log.error("acquire failed: " + e.message);
474
+ }
475
+ }
476
+
477
+ async function _heartbeat() {
478
+ if (!initialized) return;
479
+ if (!lease) {
480
+ // Not currently leader — try to acquire (lease may have expired
481
+ // on the previous holder).
482
+ await _tryAcquire();
483
+ return;
484
+ }
485
+ // We hold a lease — renew it. Re-supply the configured endpoint so a
486
+ // hot-reload of the operator's config (e.g. node moves to a new
487
+ // routable URL after a restart) eventually reaches the discovery row.
488
+ try {
489
+ lease = await provider.renewLease(lease, { endpoint: configuredEndpoint });
490
+ } catch (e) {
491
+ if (e.code === "LEASE_LOST") {
492
+ log.error("lease lost: " + e.message);
493
+ var lostToken = lease ? lease.fencingToken : null;
494
+ lease = null;
495
+ _emitTransition("lease-lost", { fencingToken: lostToken });
496
+ // Attempt to re-acquire on the next heartbeat naturally.
497
+ } else {
498
+ // Transient error — retry on next heartbeat. If it persists past
499
+ // leaseTtl another node will steal, and we'll detect via LEASE_LOST.
500
+ log.error("renew failed transiently: " + e.message);
501
+ }
502
+ }
503
+ }
504
+
505
+ // ---- public sync surface ----
506
+
507
+ function isLeader() {
508
+ if (terminated) return false; // post-shutdown: never leader
509
+ if (!initialized) return true; // never-initialized: permanent leader
510
+ return !!lease && Date.now() < lease.expiresAt;
511
+ }
512
+
513
+ // Has cluster.init been called with a real configuration? Used by
514
+ // write-dispatch code (audit, consent, …) to decide whether framework
515
+ // state should go to local SQLite or external-db.
516
+ function isClusterMode() {
517
+ return initialized && !!configuredExternalDbBackend;
518
+ }
519
+
520
+ function externalDbBackend() {
521
+ return configuredExternalDbBackend;
522
+ }
523
+
524
+ function dialect() {
525
+ return configuredDialect;
526
+ }
527
+
528
+ function currentNodeId() {
529
+ return initialized ? nodeId : "single-node-local";
530
+ }
531
+
532
+ // This node's routable endpoint (operator-configured at cluster.init).
533
+ // Returns null when not configured or in single-node fallback. External
534
+ // observers should call discoveryHandler() / currentLeader() instead —
535
+ // this getter is for the local node's own self-identity.
536
+ function endpoint() {
537
+ return configuredEndpoint;
538
+ }
539
+
540
+ function fencingToken() {
541
+ if (!initialized) return 0;
542
+ return lease ? lease.fencingToken : 0;
543
+ }
544
+
545
+ function requireLeader() {
546
+ if (!isLeader()) {
547
+ throw new NotLeaderError(
548
+ "node '" + currentNodeId() + "' is not currently leader" +
549
+ (initialized ? "" : " (cluster not initialized)")
550
+ );
551
+ }
552
+ }
553
+
554
+ async function currentLeader() {
555
+ if (!initialized) {
556
+ return {
557
+ nodeId: "single-node-local",
558
+ leaseExpiresAt: Infinity,
559
+ fencingToken: 0,
560
+ endpoint: null,
561
+ };
562
+ }
563
+ return await provider.currentLeader();
564
+ }
565
+
566
+ // HTTP request handler — replies with the current cluster leader for
567
+ // service-mesh / load-balancer discovery. Operators mount this at
568
+ // whatever route they want (`/cluster/leader`, `/health/leader`, etc.).
569
+ //
570
+ // 200 application/json — leader present
571
+ // { leader: { nodeId, endpoint, fencingToken, leaseExpiresAt },
572
+ // self: { nodeId, endpoint, isLeader } }
573
+ //
574
+ // 503 application/json — no leader (no row, expired lease, DB
575
+ // unreachable, single-node not initialized with cluster mode)
576
+ // { leader: null, self: { nodeId, endpoint, isLeader } }
577
+ //
578
+ // No auth — this endpoint is intended to be called by infrastructure
579
+ // inside the trust boundary (LB, healthcheck, dashboard). Operators
580
+ // who expose it externally should layer auth via their own middleware.
581
+ //
582
+ // Handler is method-agnostic so it works behind any HTTP probe shape
583
+ // (GET, HEAD, etc.). Cache-Control: no-store to avoid stale-leader
584
+ // responses pinned by a caching proxy during a takeover.
585
+ function discoveryHandler() {
586
+ return async function (req, res) {
587
+ var selfInfo = {
588
+ nodeId: currentNodeId(),
589
+ endpoint: configuredEndpoint,
590
+ isLeader: isLeader(),
591
+ };
592
+ var body;
593
+ var status;
594
+ try {
595
+ var leader = await currentLeader();
596
+ if (leader && leader.nodeId && leader.nodeId !== "single-node-local") {
597
+ body = { leader: leader, self: selfInfo };
598
+ status = 200;
599
+ } else if (leader && leader.nodeId === "single-node-local") {
600
+ // Permanent-leader fallback (cluster.init never called). Reply
601
+ // 200 — the operator's app is healthy and the "leader" is this
602
+ // process. Useful so the discovery endpoint is never a false
603
+ // negative in single-node deployments.
604
+ body = { leader: leader, self: selfInfo };
605
+ status = 200;
606
+ } else {
607
+ body = { leader: null, self: selfInfo };
608
+ status = 503;
609
+ }
610
+ } catch (e) {
611
+ body = { leader: null, self: selfInfo, error: e.message };
612
+ status = 503;
613
+ }
614
+ var json = JSON.stringify(body);
615
+ res.writeHead(status, {
616
+ "Content-Type": "application/json; charset=utf-8",
617
+ "Content-Length": Buffer.byteLength(json),
618
+ "Cache-Control": "no-store",
619
+ });
620
+ res.end(json);
621
+ };
622
+ }
623
+
624
+ function onTransition(handler) {
625
+ if (typeof handler !== "function") {
626
+ throw _err("INVALID_HANDLER", "onTransition expects a function", true);
627
+ }
628
+ transitionHandlers.push(handler);
629
+ }
630
+
631
+ async function shutdown() {
632
+ if (!initialized) return;
633
+ if (heartbeatTimer) {
634
+ heartbeatTimer.stop();
635
+ heartbeatTimer = null;
636
+ }
637
+ if (lease) {
638
+ try {
639
+ await provider.releaseLease(lease);
640
+ _emitTransition("lease-released", { fencingToken: lease.fencingToken });
641
+ log("lease released on shutdown");
642
+ } catch (e) {
643
+ log.error("release on shutdown failed: " + e.message);
644
+ }
645
+ lease = null;
646
+ }
647
+ initialized = false;
648
+ terminated = true;
649
+ provider = null;
650
+ role = null;
651
+ leaseTtlMs = null;
652
+ heartbeatMs = null;
653
+ configuredExternalDbBackend = null;
654
+ configuredDialect = null;
655
+ configuredEndpoint = null;
656
+ transitionHandlers = [];
657
+ // nodeId is preserved post-shutdown so audit metadata still reflects
658
+ // who this process was; cleared only by _resetForTest.
659
+ }
660
+
661
+ // ---- test helpers — not part of public contract ----
662
+
663
+ function _resetForTest() {
664
+ if (heartbeatTimer) heartbeatTimer.stop();
665
+ heartbeatTimer = null;
666
+ initialized = false;
667
+ terminated = false;
668
+ nodeId = null;
669
+ role = null;
670
+ provider = null;
671
+ lease = null;
672
+ leaseTtlMs = null;
673
+ heartbeatMs = null;
674
+ configuredExternalDbBackend = null;
675
+ configuredDialect = null;
676
+ configuredEndpoint = null;
677
+ transitionHandlers = [];
678
+ }
679
+
680
+ async function _heartbeatNowForTest() {
681
+ // Drive one heartbeat synchronously without waiting for the timer —
682
+ // lets tests deterministically observe lease state transitions.
683
+ await _heartbeat();
684
+ }
685
+
686
+ module.exports = {
687
+ init: init,
688
+ isLeader: isLeader,
689
+ isClusterMode: isClusterMode,
690
+ externalDbBackend: externalDbBackend,
691
+ dialect: dialect,
692
+ currentNodeId: currentNodeId,
693
+ endpoint: endpoint,
694
+ fencingToken: fencingToken,
695
+ requireLeader: requireLeader,
696
+ currentLeader: currentLeader,
697
+ discoveryHandler: discoveryHandler,
698
+ onTransition: onTransition,
699
+ shutdown: shutdown,
700
+ NotLeaderError: NotLeaderError,
701
+ _resetForTest: _resetForTest,
702
+ _heartbeatNowForTest: _heartbeatNowForTest,
703
+ };