@blamejs/core 0.7.18 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/CHANGELOG.md +425 -423
  2. package/README.md +150 -150
  3. package/bin/blamejs.js +0 -0
  4. package/index.js +310 -308
  5. package/lib/api-key.js +660 -660
  6. package/lib/api-snapshot.js +338 -338
  7. package/lib/app-shutdown.js +385 -385
  8. package/lib/app.js +365 -365
  9. package/lib/archive.js +250 -250
  10. package/lib/atomic-file.js +544 -544
  11. package/lib/audit-chain.js +177 -177
  12. package/lib/audit-sign.js +344 -344
  13. package/lib/audit-tools.js +677 -677
  14. package/lib/audit.js +766 -766
  15. package/lib/auth/jwt-external.js +365 -0
  16. package/lib/auth/jwt.js +337 -311
  17. package/lib/auth/lockout.js +436 -436
  18. package/lib/auth/oauth.js +721 -721
  19. package/lib/auth/passkey.js +181 -181
  20. package/lib/auth/password.js +628 -594
  21. package/lib/backup/bundle.js +217 -217
  22. package/lib/backup/crypto.js +176 -176
  23. package/lib/backup/index.js +515 -515
  24. package/lib/backup/manifest.js +282 -282
  25. package/lib/break-glass.js +1338 -1338
  26. package/lib/bundler.js +441 -441
  27. package/lib/cache-redis.js +256 -256
  28. package/lib/cache.js +1206 -1206
  29. package/lib/canonical-json.js +115 -115
  30. package/lib/chain-writer.js +234 -234
  31. package/lib/cli-helpers.js +206 -206
  32. package/lib/cli.js +2334 -2334
  33. package/lib/cluster-provider-db.js +317 -317
  34. package/lib/cluster-storage.js +226 -226
  35. package/lib/cluster.js +703 -703
  36. package/lib/config-drift.js +301 -301
  37. package/lib/consent.js +222 -222
  38. package/lib/constants.js +191 -191
  39. package/lib/cookies.js +315 -315
  40. package/lib/credential-hash.js +322 -322
  41. package/lib/crypto.js +266 -266
  42. package/lib/csv.js +275 -275
  43. package/lib/db-declare-row-policy.js +267 -267
  44. package/lib/db-declare-view.js +420 -420
  45. package/lib/db-query.js +406 -406
  46. package/lib/db-schema.js +319 -319
  47. package/lib/db.js +1288 -1288
  48. package/lib/deprecate.js +222 -222
  49. package/lib/dev.js +335 -335
  50. package/lib/dual-control.js +473 -473
  51. package/lib/error-page.js +420 -420
  52. package/lib/external-db-migrate.js +441 -441
  53. package/lib/external-db.js +1061 -1061
  54. package/lib/file-type.js +273 -273
  55. package/lib/forms.js +422 -422
  56. package/lib/framework-error.js +293 -293
  57. package/lib/framework-schema.js +717 -717
  58. package/lib/handlers.js +350 -350
  59. package/lib/http-client-cookie-jar.js +508 -508
  60. package/lib/http-client.js +1195 -1195
  61. package/lib/i18n.js +878 -878
  62. package/lib/jobs.js +185 -185
  63. package/lib/log-stream-cloudwatch.js +369 -369
  64. package/lib/log-stream-local.js +146 -146
  65. package/lib/log-stream-otlp-grpc.js +410 -410
  66. package/lib/log-stream-otlp.js +286 -286
  67. package/lib/log-stream-syslog.js +302 -302
  68. package/lib/log-stream-webhook.js +199 -199
  69. package/lib/log-stream.js +330 -330
  70. package/lib/log.js +500 -500
  71. package/lib/mail-bounce.js +528 -528
  72. package/lib/mail-dkim.js +369 -369
  73. package/lib/mail.js +981 -981
  74. package/lib/metrics.js +683 -683
  75. package/lib/middleware/api-encrypt.js +936 -936
  76. package/lib/middleware/attach-user.js +157 -157
  77. package/lib/middleware/bearer-auth.js +152 -0
  78. package/lib/middleware/body-parser.js +1170 -1170
  79. package/lib/middleware/bot-guard.js +178 -178
  80. package/lib/middleware/compression.js +452 -452
  81. package/lib/middleware/cors.js +314 -314
  82. package/lib/middleware/csp-nonce.js +348 -348
  83. package/lib/middleware/csrf-protect.js +316 -316
  84. package/lib/middleware/db-role-for.js +264 -264
  85. package/lib/middleware/health.js +392 -392
  86. package/lib/middleware/index.js +82 -79
  87. package/lib/middleware/rate-limit.js +358 -358
  88. package/lib/middleware/request-id.js +61 -61
  89. package/lib/middleware/request-log.js +168 -168
  90. package/lib/middleware/require-auth.js +104 -104
  91. package/lib/middleware/security-headers.js +116 -116
  92. package/lib/middleware/sse.js +166 -166
  93. package/lib/migrations.js +383 -383
  94. package/lib/mtls-ca.js +518 -518
  95. package/lib/mtls-engine-default.js +481 -481
  96. package/lib/network-dns.js +632 -632
  97. package/lib/network-heartbeat.js +290 -290
  98. package/lib/network-nts.js +574 -574
  99. package/lib/network-proxy.js +265 -265
  100. package/lib/network-tls.js +328 -328
  101. package/lib/network.js +233 -233
  102. package/lib/notify.js +612 -612
  103. package/lib/ntp-check.js +229 -229
  104. package/lib/numeric-bounds.js +111 -111
  105. package/lib/object-store/azure-blob-bucket-ops.js +349 -349
  106. package/lib/object-store/azure-blob.js +488 -488
  107. package/lib/object-store/gcs-bucket-ops.js +351 -351
  108. package/lib/object-store/gcs.js +519 -519
  109. package/lib/object-store/http-put.js +153 -153
  110. package/lib/object-store/index.js +197 -197
  111. package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
  112. package/lib/object-store/sigv4.js +903 -903
  113. package/lib/observability.js +151 -151
  114. package/lib/otel-export.js +269 -269
  115. package/lib/pagination.js +464 -464
  116. package/lib/parsers/index.js +80 -80
  117. package/lib/parsers/safe-env.js +642 -642
  118. package/lib/parsers/safe-ini.js +292 -292
  119. package/lib/parsers/safe-toml.js +784 -784
  120. package/lib/parsers/safe-xml.js +390 -390
  121. package/lib/parsers/safe-yaml.js +1015 -1015
  122. package/lib/permissions.js +708 -708
  123. package/lib/pqc-agent.js +87 -87
  124. package/lib/pqc-gate.js +279 -279
  125. package/lib/protobuf-encoder.js +190 -190
  126. package/lib/protocol-dispatcher.js +161 -161
  127. package/lib/pubsub-redis.js +167 -167
  128. package/lib/pubsub.js +429 -429
  129. package/lib/queue-local.js +476 -476
  130. package/lib/queue-redis.js +745 -745
  131. package/lib/queue-sqs.js +319 -319
  132. package/lib/queue.js +695 -695
  133. package/lib/redis-client.js +519 -519
  134. package/lib/request-helpers.js +340 -340
  135. package/lib/restore-bundle.js +237 -237
  136. package/lib/restore-rollback.js +259 -259
  137. package/lib/restore.js +409 -409
  138. package/lib/retry.js +376 -376
  139. package/lib/router.js +748 -748
  140. package/lib/safe-async.js +735 -735
  141. package/lib/safe-buffer.js +237 -237
  142. package/lib/safe-json.js +541 -541
  143. package/lib/safe-schema.js +1266 -1266
  144. package/lib/safe-url.js +159 -159
  145. package/lib/scheduler.js +706 -706
  146. package/lib/security-assert.js +373 -373
  147. package/lib/seeders.js +618 -618
  148. package/lib/session.js +535 -478
  149. package/lib/slug.js +269 -269
  150. package/lib/ssrf-guard.js +401 -401
  151. package/lib/storage.js +471 -471
  152. package/lib/subject.js +281 -281
  153. package/lib/template.js +791 -791
  154. package/lib/testing.js +798 -798
  155. package/lib/time.js +310 -310
  156. package/lib/totp.js +302 -302
  157. package/lib/tracing.js +494 -494
  158. package/lib/uuid.js +132 -132
  159. package/lib/validate-opts.js +340 -340
  160. package/lib/vault/index.js +308 -308
  161. package/lib/vault/rotate.js +784 -784
  162. package/lib/vault/wrap.js +296 -296
  163. package/lib/vendor/noble-ciphers.cjs +9 -9
  164. package/lib/webhook.js +595 -595
  165. package/lib/websocket.js +1048 -1048
  166. package/package.json +77 -77
  167. package/sbom.cyclonedx.json +7 -7
package/lib/cluster.js CHANGED
@@ -1,703 +1,703 @@
1
- "use strict";
2
- /**
3
- * Cluster coordination — leader election + fencing tokens.
4
- *
5
- * Opt-in via `b.cluster.init(...)`. When init is never called, the
6
- * local process behaves as a permanent single leader: `isLeader()`
7
- * always returns true, `fencingToken()` returns 0, no heartbeat thread
8
- * runs, no DB is touched. Single-node deployments pay zero overhead.
9
- *
10
- * When init IS called, the framework starts a heartbeat that renews
11
- * the leader lease via the configured provider. On lease loss (network
12
- * partition, takeover, lease expiry) the node transitions to follower
13
- * and write-side framework primitives throw `NotLeaderError`.
14
- *
15
- * Threat model:
16
- * - Two leaders writing simultaneously: prevented by fencing tokens.
17
- * Every leader-only DB write includes the current token; a
18
- * CHECK constraint on the audit-tip row rejects a stale token.
19
- * The application-layer `requireLeader()` gate is just an early
20
- * rejection optimisation; the DB constraint is the canonical guard.
21
- * - Follower receiving a write: rejected at the framework boundary.
22
- * Operators front the cluster with a load balancer that routes
23
- * write paths to the current leader.
24
- * - External-db unreachable: heartbeat fails; after `leaseTtl` no
25
- * leader exists and writes fail closed. When the DB recovers,
26
- * election resumes.
27
- *
28
- * Public API:
29
- * await cluster.init(opts) one-time bootstrap
30
- * cluster.isLeader() sync; true on leader (or single-node)
31
- * cluster.currentNodeId() sync; configured nodeId
32
- * cluster.endpoint() sync; this node's routable URL
33
- * (operator-supplied at init), or
34
- * null if unconfigured. Stored in
35
- * the leader-election row so
36
- * external observers can resolve
37
- * "where is the current leader?"
38
- * cluster.fencingToken() sync; current monotonic token
39
- * cluster.requireLeader() sync; throws NotLeaderError
40
- * cluster.currentLeader() async; { nodeId, leaseExpiresAt,
41
- * fencingToken,
42
- * endpoint } | null
43
- * cluster.discoveryHandler() returns an HTTP request handler
44
- * (req, res) → JSON. Mount on any
45
- * route to expose the current
46
- * leader for service-mesh / LB
47
- * consumption. 200 with leader,
48
- * 503 with `{ leader: null }`
49
- * when no leader.
50
- * cluster.onTransition(fn) register transition handler
51
- * await cluster.shutdown() releases lease, stops heartbeat
52
- */
53
- var C = require("./constants");
54
- var clusterProviderDb = require("./cluster-provider-db");
55
- var crypto = require("./crypto");
56
- var lazyRequire = require("./lazy-require");
57
- var { boot } = require("./log");
58
- var safeAsync = require("./safe-async");
59
- var safeJson = require("./safe-json");
60
- var safeSql = require("./safe-sql");
61
- var safeUrl = require("./safe-url");
62
- var { FrameworkError, ClusterError } = require("./framework-error");
63
-
64
- // Lazy: vault → db → cluster forms a load-time chain, and external-db is
65
- // loaded before its init has run; both are safe to call once cluster
66
- // reaches runtime, but eager require here would deadlock the load order.
67
- var externalDb = lazyRequire(function () { return require("./external-db"); });
68
- var vault = lazyRequire(function () { return require("./vault"); });
69
-
70
- var DEFAULT_LEASE_TTL = C.TIME.seconds(30);
71
- var DEFAULT_HEARTBEAT = C.TIME.seconds(10);
72
- var MIN_LEASE_TTL = C.TIME.seconds(5);
73
- var MIN_HEARTBEAT = C.TIME.seconds(1);
74
-
75
- var initialized = false;
76
- var terminated = false; // set true by shutdown() so the
77
- // permanent-leader fallback isn't
78
- // re-engaged after a graceful exit
79
- var nodeId = null;
80
- var role = null; // 'leader' | 'follower'
81
- var provider = null;
82
- var lease = null; // current lease (if leader)
83
- var heartbeatTimer = null;
84
- var heartbeatMs = null;
85
- var leaseTtlMs = null;
86
- var transitionHandlers = [];
87
- // Backend coordinates for write-dispatch code in audit/consent/etc.
88
- // These are set when cluster.init is called with `externalDbBackend`
89
- // (the default DB-row provider path); operators using a custom
90
- // provider can set them via init opts directly.
91
- var configuredExternalDbBackend = null;
92
- var configuredDialect = null;
93
- // Operator-supplied routable endpoint for THIS node, used by external
94
- // load balancers / service meshes to learn where to send write traffic.
95
- // Stored in the leader-election row on every acquire/renew so any node
96
- // (or external observer) can resolve "where is the current leader?"
97
- // via cluster.currentLeader() / cluster.discoveryHandler().
98
- var configuredEndpoint = null;
99
-
100
- var log = boot("cluster");
101
-
102
- class NotLeaderError extends FrameworkError {
103
- constructor(message) {
104
- super(message || "not leader: write rejected by cluster gate", "NOT_LEADER");
105
- this.name = "NotLeaderError";
106
- this.statusCode = 503; // operator's load balancer should retry on the leader
107
- this.isClusterError = true;
108
- this.isNotLeaderError = true;
109
- }
110
- }
111
-
112
- var _err = ClusterError.factory;
113
-
114
- function _emitTransition(kind, detail) {
115
- var event = Object.assign({ kind: kind, nodeId: nodeId, at: Date.now() }, detail || {});
116
- for (var i = 0; i < transitionHandlers.length; i++) {
117
- try { transitionHandlers[i](event); }
118
- catch (e) { log.error("transition handler threw: " + e.message); }
119
- }
120
- }
121
-
122
- // ---- init ----
123
-
124
- async function init(opts) {
125
- if (initialized) {
126
- throw _err("ALREADY_INITIALIZED", "cluster.init() called twice", true);
127
- }
128
- opts = opts || {};
129
- if (!opts.nodeId) {
130
- throw _err("INVALID_CONFIG", "cluster.init({ nodeId }) is required", true);
131
- }
132
- nodeId = String(opts.nodeId);
133
-
134
- leaseTtlMs = opts.leaseTtl != null ? Number(opts.leaseTtl) : DEFAULT_LEASE_TTL;
135
- if (leaseTtlMs < MIN_LEASE_TTL) {
136
- throw _err("INVALID_TTL",
137
- "leaseTtl must be >= " + MIN_LEASE_TTL + "ms (got " + leaseTtlMs + ")",
138
- true);
139
- }
140
- heartbeatMs = opts.heartbeatInterval != null
141
- ? Number(opts.heartbeatInterval)
142
- : DEFAULT_HEARTBEAT;
143
- if (heartbeatMs < MIN_HEARTBEAT) {
144
- throw _err("INVALID_HEARTBEAT",
145
- "heartbeatInterval must be >= " + MIN_HEARTBEAT + "ms (got " + heartbeatMs + ")",
146
- true);
147
- }
148
- if (heartbeatMs >= leaseTtlMs) {
149
- throw _err("INVALID_HEARTBEAT",
150
- "heartbeatInterval must be < leaseTtl (got heartbeat=" + heartbeatMs +
151
- ", leaseTtl=" + leaseTtlMs + "); recommend ~1/3 of leaseTtl",
152
- true);
153
- }
154
-
155
- role = (opts.role || "leader").toLowerCase();
156
- if (role !== "leader" && role !== "follower") {
157
- throw _err("INVALID_ROLE", "role must be 'leader' or 'follower'", true);
158
- }
159
-
160
- // Optional endpoint. If provided, validate scheme + shape via url-safe
161
- // — HTTPS-only by default since this is the URL external services use
162
- // to reach the leader. Operators with internal cleartext clusters opt
163
- // in via opts.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
164
- if (opts.endpoint != null) {
165
- try {
166
- safeUrl.parse(opts.endpoint, {
167
- allowedProtocols: opts.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
168
- errorClass: ClusterError,
169
- });
170
- } catch (e) {
171
- // Re-throw with a config-shaped error so operators see the cluster.init
172
- // boundary, not a bare url-safe trace.
173
- throw _err("INVALID_ENDPOINT",
174
- "cluster.init({ endpoint }) rejected: " + e.message, true);
175
- }
176
- configuredEndpoint = String(opts.endpoint);
177
- } else {
178
- configuredEndpoint = null;
179
- }
180
-
181
- if (typeof opts.onTransition === "function") {
182
- transitionHandlers.push(opts.onTransition);
183
- }
184
-
185
- // Provider: either operator-supplied, or build the default DB-row
186
- // provider against an externalDb backend.
187
- if (opts.provider) {
188
- provider = opts.provider;
189
- // Operator-custom provider: they may still be writing framework
190
- // state to an externalDb backend, in which case they pass these
191
- // separately so write-dispatch code knows where to go.
192
- configuredExternalDbBackend = opts.externalDbBackend || null;
193
- configuredDialect = (opts.dialect || "postgres").toLowerCase();
194
- } else {
195
- if (!opts.externalDbBackend) {
196
- throw _err("INVALID_CONFIG",
197
- "cluster.init requires either { provider } or { externalDbBackend }", true);
198
- }
199
- provider = clusterProviderDb.create({
200
- externalDbBackend: opts.externalDbBackend,
201
- dialect: opts.dialect,
202
- });
203
- configuredExternalDbBackend = opts.externalDbBackend;
204
- configuredDialect = (opts.dialect || "postgres").toLowerCase();
205
- }
206
-
207
- if (typeof provider.ensureSchema === "function") {
208
- await provider.ensureSchema();
209
- }
210
-
211
- initialized = true;
212
- log("initialized as nodeId='" + nodeId + "', role='" + role + "'");
213
-
214
- // Initial acquisition attempt (only if role === 'leader')
215
- if (role === "leader") {
216
- await _tryAcquire();
217
- }
218
-
219
- // Boot-time rollback detection on the audit + consent chains. Runs
220
- // regardless of role — every node should refuse to participate in a
221
- // cluster whose shared chains have been rolled back (a follower
222
- // would face the same chain integrity failure if it later took
223
- // over). Skipped when configuredExternalDbBackend is unset, which
224
- // means a custom provider is in use without externalDb-resident
225
- // framework state — the operator owns rollback detection in that
226
- // case.
227
- if (configuredExternalDbBackend) {
228
- await _checkChainTipRollback("audit", "_blamejs_audit_log", "_blamejs_audit_tip");
229
- await _checkChainTipRollback("consent", "_blamejs_consent_log", "_blamejs_consent_tip");
230
- // Vault-key consistency: every node in a cluster must hold the
231
- // SAME vault key. A node booting with a different key would seal
232
- // new writes under a key the rest of the cluster can't unseal,
233
- // and (on takeover) be unable to unseal the rest of the cluster's
234
- // sealed columns — silent corruption. Compare a fingerprint of
235
- // this node's vault keys against the canonical one stored at
236
- // first cluster boot; refuse to participate on mismatch.
237
- await _checkVaultKeyConsistency();
238
- }
239
-
240
- // Start heartbeat
241
- heartbeatTimer = safeAsync.repeating(_heartbeat, heartbeatMs, { name: "cluster-heartbeat" });
242
- }
243
-
244
- // Cluster-mode equivalent of db.js's single-node audit.tip-sidecar
245
- // rollback check. Reads the persistent _blamejs_audit_tip row and
246
- // compares to the current chain head in _blamejs_audit_log:
247
- //
248
- // - No tip row: first cluster boot or operator-cleared. Skip
249
- // silently (matches the single-node sidecar-missing path).
250
- // - Tip recorded a counter > current MAX: chain was truncated /
251
- // restored from older snapshot. FATAL — refuse boot.
252
- // - Tip recorded a hash that doesn't match the row at that
253
- // counter: the row at that counter was substituted (different
254
- // hash for same counter). FATAL — refuse boot.
255
- //
256
- // process.exit(1) is the framework's convention for boot-time
257
- // integrity failures (audit chain, checkpoints, single-node
258
- // rollback). Cluster mode keeps the same posture so operators see
259
- // a single boot-time failure mode regardless of deployment shape.
260
- // Generalized boot-time rollback check used by both audit and consent
261
- // chains. chainName is the human-readable label included in log
262
- // output ("audit" / "consent"). logTable is the chain table
263
- // (_blamejs_audit_log / _blamejs_consent_log). tipTable is the
264
- // single-row coordination table that records the latest counter +
265
- // rowHash + fencingToken (_blamejs_audit_tip / _blamejs_consent_tip).
266
- //
267
- // Surfaces three outcomes:
268
- // - tip table missing → operator running cluster gates-only mode
269
- // (cluster wired for leader election but framework state still
270
- // lives in per-node SQLite without `frameworkSchema.ensureSchema`);
271
- // skip silently.
272
- // - no tip row → first cluster boot or operator-cleared; skip.
273
- // - currentMax < tipCounter, or tip rowHash != row-at-counter
274
- // hash → FATAL via process.exit(1). Same posture as the
275
- // single-node audit.tip sidecar rollback check.
276
- async function _checkChainTipRollback(chainName, logTable, tipTable) {
277
- // Both tables are framework-internal constants from the call sites
278
- // (`_blamejs_audit_log`, `_blamejs_consent_log`, etc.). Validate +
279
- // quote per the framework's identifier-quoting convention so a
280
- // future rename can't silently break the query.
281
- safeSql.validateIdentifier(logTable, { allowReserved: true });
282
- safeSql.validateIdentifier(tipTable, { allowReserved: true });
283
- var qLogTable = safeSql.quoteIdentifier(logTable);
284
- var qTipTable = safeSql.quoteIdentifier(tipTable);
285
-
286
- var tipRows;
287
- try {
288
- tipRows = await externalDb().query(
289
- "SELECT atMonotonicCounter, rowHash FROM " + qTipTable +
290
- " WHERE scope = " + (configuredDialect === "postgres" ? "$1" : "?"),
291
- [chainName],
292
- { backend: configuredExternalDbBackend }
293
- );
294
- } catch (e) {
295
- var msg = (e && e.message) || "";
296
- if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
297
- log(chainName + "-tip table not present — skipping rollback check (cluster gates-only mode)");
298
- return;
299
- }
300
- throw e;
301
- }
302
- if (!tipRows.rows || tipRows.rows.length === 0) {
303
- log("no " + chainName + "-tip row — skipping rollback check (first cluster boot or operator-cleared)");
304
- return;
305
- }
306
- var tip = tipRows.rows[0];
307
- var tipCounter = Number(tip.atMonotonicCounter);
308
- var tipHash = tip.rowHash;
309
-
310
- var currentRows = await externalDb().query(
311
- "SELECT MAX(monotonicCounter) AS m FROM " + qLogTable,
312
- [],
313
- { backend: configuredExternalDbBackend }
314
- );
315
- var currentMax = (currentRows.rows && currentRows.rows[0] && currentRows.rows[0].m)
316
- ? Number(currentRows.rows[0].m)
317
- : 0;
318
-
319
- if (currentMax < tipCounter) {
320
- throw _err("ROLLBACK_DETECTED",
321
- "FATAL: cluster-mode " + chainName + "-log rollback detected. " +
322
- chainName + "-tip counter: " + tipCounter +
323
- "; current external-db max: " + currentMax +
324
- ". Either external-db was restored from an older snapshot, or " +
325
- logTable + " rows have been deleted. Investigate before continuing.",
326
- true);
327
- }
328
-
329
- if (tipHash) {
330
- var hashRows = await externalDb().query(
331
- "SELECT rowHash FROM " + qLogTable + " WHERE monotonicCounter = " +
332
- (configuredDialect === "postgres" ? "$1" : "?"),
333
- [tipCounter],
334
- { backend: configuredExternalDbBackend }
335
- );
336
- if (hashRows.rows && hashRows.rows.length > 0) {
337
- var rowAtTip = hashRows.rows[0].rowHash;
338
- if (rowAtTip !== tipHash) {
339
- throw _err("ROLLBACK_DETECTED",
340
- "FATAL: cluster-mode " + chainName + "-log rollback detected (row-hash mismatch). " +
341
- chainName + "-tip counter: " + tipCounter +
342
- "; " + chainName + "-tip rowHash: " + tipHash +
343
- "; current row rowHash: " + rowAtTip +
344
- ". The row at the recorded tip counter has a different hash — " +
345
- "indicates row substitution at the chain head. Investigate before continuing.",
346
- true);
347
- }
348
- }
349
- }
350
- log("cluster " + chainName + "-tip rollback check ok (tip counter " + tipCounter +
351
- ", current " + currentMax + ")");
352
- }
353
-
354
- // Compute a deterministic fingerprint of THIS node's vault keys.
355
- // SHA3-512 of the concatenated public keys (PQC + classical halves of
356
- // the hybrid encryption keypair). One-way: nothing about the private
357
- // key material is recoverable from the fingerprint, so it's safe to
358
- // store in the coordination table that all cluster nodes can read.
359
- //
360
- // Returns null if vault.init() hasn't been called — cluster gates-only
361
- // mode (no sealed-column work) doesn't need this check, same defensive
362
- // posture as the audit-tip rollback check skipping when there's no
363
- // audit-tip table.
364
- function _vaultKeyFingerprint() {
365
- var keysJson;
366
- try {
367
- keysJson = vault().getKeysJson();
368
- } catch (e) {
369
- // vault.init() not called — gates-only mode. Skip silently.
370
- if (/vault.init\(\) must be awaited/.test((e && e.message) || "")) {
371
- return null;
372
- }
373
- throw e;
374
- }
375
- // vault.getKeysJson() returns the keys serialized as JSON (the same
376
- // format vault writes to disk). Parse to extract the public halves;
377
- // we never touch privateKey/ecPrivateKey here.
378
- var keys = safeJson.parse(keysJson);
379
- if (!keys || !keys.publicKey || !keys.ecPublicKey) return null;
380
- // Domain-separation prefix so this fingerprint can't be confused
381
- // with a hash of the same bytes computed elsewhere in the framework.
382
- return crypto.sha3Hash("blamejs/cluster-state/v1\n" +
383
- keys.publicKey + "\n" +
384
- keys.ecPublicKey);
385
- }
386
-
387
- async function _checkVaultKeyConsistency() {
388
- var localFp = _vaultKeyFingerprint();
389
- if (localFp === null) {
390
- log("vault not initialized — skipping vault-key consistency check (cluster gates-only mode)");
391
- return;
392
- }
393
- var nowMs = Date.now();
394
- var ph = configuredDialect === "postgres";
395
-
396
- // First boot: try to record THIS node's fingerprint. ON CONFLICT DO
397
- // NOTHING means the FIRST node to boot wins; subsequent nodes
398
- // observe whatever's already there. Every node then SELECTs and
399
- // compares — any mismatch (including ours after a losing race)
400
- // surfaces the drift.
401
- try {
402
- await externalDb().query(
403
- "INSERT INTO _blamejs_cluster_state " +
404
- " (scope, vaultKeyFp, recordedAt, recordedByNode) " +
405
- "VALUES ('state', " +
406
- (ph ? "$1, $2, $3" : "?, ?, ?") + ") " +
407
- "ON CONFLICT (scope) DO NOTHING",
408
- [localFp, nowMs, nodeId],
409
- { backend: configuredExternalDbBackend }
410
- );
411
- } catch (e) {
412
- // Table missing → the cluster-provider-db ensureSchema didn't run
413
- // (custom provider that doesn't create _blamejs_cluster_state).
414
- // Skip silently — same defensive posture as the audit-tip check.
415
- var msg = (e && e.message) || "";
416
- if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
417
- log("cluster-state table not present — skipping vault-key consistency check (custom provider)");
418
- return;
419
- }
420
- throw e;
421
- }
422
-
423
- // Read whatever fingerprint is canonical (ours if first boot,
424
- // someone else's if we lost the race or are joining an existing cluster).
425
- var rows = await externalDb().query(
426
- "SELECT vaultKeyFp, recordedByNode, recordedAt FROM _blamejs_cluster_state " +
427
- "WHERE scope = 'state'",
428
- [],
429
- { backend: configuredExternalDbBackend }
430
- );
431
- if (!rows.rows || rows.rows.length === 0) {
432
- // Should never happen — we just INSERTed. Surface as fatal so the
433
- // condition isn't silently ignored.
434
- throw _err("CLUSTER_STATE_MISSING",
435
- "FATAL: cluster-state row missing immediately after INSERT — " +
436
- "external-db may not be honoring writes. Refusing boot.",
437
- true);
438
- }
439
- var canonical = rows.rows[0];
440
- if (canonical.vaultKeyFp !== localFp) {
441
- var fpPrefix = C.BYTES.bytes(16);
442
- throw _err("VAULT_KEY_DRIFT",
443
- "FATAL: vault-key drift detected. " +
444
- "local node: " + nodeId +
445
- "; local fingerprint: " + localFp.slice(0, fpPrefix) + "…" +
446
- "; canonical recorded by: " + canonical.recordedByNode +
447
- "; canonical fingerprint: " + canonical.vaultKeyFp.slice(0, fpPrefix) + "…" +
448
- ". This node holds a DIFFERENT vault key than the rest of the " +
449
- "cluster. Sealed-column writes from this node would be unreadable " +
450
- "by the others (and vice versa). Restore the same vault key file " +
451
- "before booting this node into the cluster.",
452
- true);
453
- }
454
- log("cluster vault-key consistency ok (fingerprint " +
455
- localFp.slice(0, C.BYTES.bytes(16)) + "… recorded by " + canonical.recordedByNode + ")");
456
- }
457
-
458
- async function _tryAcquire() {
459
- if (role !== "leader") return; // pinned-follower role: never claim
460
- try {
461
- var got = await provider.acquireLease(nodeId, leaseTtlMs, {
462
- endpoint: configuredEndpoint,
463
- });
464
- if (got) {
465
- var wasLeader = !!lease;
466
- lease = got;
467
- if (!wasLeader) {
468
- log("acquired lease — fencingToken=" + lease.fencingToken);
469
- _emitTransition("lease-acquired", { fencingToken: lease.fencingToken });
470
- }
471
- }
472
- } catch (e) {
473
- log.error("acquire failed: " + e.message);
474
- }
475
- }
476
-
477
- async function _heartbeat() {
478
- if (!initialized) return;
479
- if (!lease) {
480
- // Not currently leader — try to acquire (lease may have expired
481
- // on the previous holder).
482
- await _tryAcquire();
483
- return;
484
- }
485
- // We hold a lease — renew it. Re-supply the configured endpoint so a
486
- // hot-reload of the operator's config (e.g. node moves to a new
487
- // routable URL after a restart) eventually reaches the discovery row.
488
- try {
489
- lease = await provider.renewLease(lease, { endpoint: configuredEndpoint });
490
- } catch (e) {
491
- if (e.code === "LEASE_LOST") {
492
- log.error("lease lost: " + e.message);
493
- var lostToken = lease ? lease.fencingToken : null;
494
- lease = null;
495
- _emitTransition("lease-lost", { fencingToken: lostToken });
496
- // Attempt to re-acquire on the next heartbeat naturally.
497
- } else {
498
- // Transient error — retry on next heartbeat. If it persists past
499
- // leaseTtl another node will steal, and we'll detect via LEASE_LOST.
500
- log.error("renew failed transiently: " + e.message);
501
- }
502
- }
503
- }
504
-
505
- // ---- public sync surface ----
506
-
507
- function isLeader() {
508
- if (terminated) return false; // post-shutdown: never leader
509
- if (!initialized) return true; // never-initialized: permanent leader
510
- return !!lease && Date.now() < lease.expiresAt;
511
- }
512
-
513
- // Has cluster.init been called with a real configuration? Used by
514
- // write-dispatch code (audit, consent, …) to decide whether framework
515
- // state should go to local SQLite or external-db.
516
- function isClusterMode() {
517
- return initialized && !!configuredExternalDbBackend;
518
- }
519
-
520
- function externalDbBackend() {
521
- return configuredExternalDbBackend;
522
- }
523
-
524
- function dialect() {
525
- return configuredDialect;
526
- }
527
-
528
- function currentNodeId() {
529
- return initialized ? nodeId : "single-node-local";
530
- }
531
-
532
- // This node's routable endpoint (operator-configured at cluster.init).
533
- // Returns null when not configured or in single-node fallback. External
534
- // observers should call discoveryHandler() / currentLeader() instead —
535
- // this getter is for the local node's own self-identity.
536
- function endpoint() {
537
- return configuredEndpoint;
538
- }
539
-
540
- function fencingToken() {
541
- if (!initialized) return 0;
542
- return lease ? lease.fencingToken : 0;
543
- }
544
-
545
- function requireLeader() {
546
- if (!isLeader()) {
547
- throw new NotLeaderError(
548
- "node '" + currentNodeId() + "' is not currently leader" +
549
- (initialized ? "" : " (cluster not initialized)")
550
- );
551
- }
552
- }
553
-
554
- async function currentLeader() {
555
- if (!initialized) {
556
- return {
557
- nodeId: "single-node-local",
558
- leaseExpiresAt: Infinity,
559
- fencingToken: 0,
560
- endpoint: null,
561
- };
562
- }
563
- return await provider.currentLeader();
564
- }
565
-
566
- // HTTP request handler — replies with the current cluster leader for
567
- // service-mesh / load-balancer discovery. Operators mount this at
568
- // whatever route they want (`/cluster/leader`, `/health/leader`, etc.).
569
- //
570
- // 200 application/json — leader present
571
- // { leader: { nodeId, endpoint, fencingToken, leaseExpiresAt },
572
- // self: { nodeId, endpoint, isLeader } }
573
- //
574
- // 503 application/json — no leader (no row, expired lease, DB
575
- // unreachable, single-node not initialized with cluster mode)
576
- // { leader: null, self: { nodeId, endpoint, isLeader } }
577
- //
578
- // No auth — this endpoint is intended to be called by infrastructure
579
- // inside the trust boundary (LB, healthcheck, dashboard). Operators
580
- // who expose it externally should layer auth via their own middleware.
581
- //
582
- // Handler is method-agnostic so it works behind any HTTP probe shape
583
- // (GET, HEAD, etc.). Cache-Control: no-store to avoid stale-leader
584
- // responses pinned by a caching proxy during a takeover.
585
- function discoveryHandler() {
586
- return async function (req, res) {
587
- var selfInfo = {
588
- nodeId: currentNodeId(),
589
- endpoint: configuredEndpoint,
590
- isLeader: isLeader(),
591
- };
592
- var body;
593
- var status;
594
- try {
595
- var leader = await currentLeader();
596
- if (leader && leader.nodeId && leader.nodeId !== "single-node-local") {
597
- body = { leader: leader, self: selfInfo };
598
- status = 200;
599
- } else if (leader && leader.nodeId === "single-node-local") {
600
- // Permanent-leader fallback (cluster.init never called). Reply
601
- // 200 — the operator's app is healthy and the "leader" is this
602
- // process. Useful so the discovery endpoint is never a false
603
- // negative in single-node deployments.
604
- body = { leader: leader, self: selfInfo };
605
- status = 200;
606
- } else {
607
- body = { leader: null, self: selfInfo };
608
- status = 503;
609
- }
610
- } catch (e) {
611
- body = { leader: null, self: selfInfo, error: e.message };
612
- status = 503;
613
- }
614
- var json = JSON.stringify(body);
615
- res.writeHead(status, {
616
- "Content-Type": "application/json; charset=utf-8",
617
- "Content-Length": Buffer.byteLength(json),
618
- "Cache-Control": "no-store",
619
- });
620
- res.end(json);
621
- };
622
- }
623
-
624
- function onTransition(handler) {
625
- if (typeof handler !== "function") {
626
- throw _err("INVALID_HANDLER", "onTransition expects a function", true);
627
- }
628
- transitionHandlers.push(handler);
629
- }
630
-
631
- async function shutdown() {
632
- if (!initialized) return;
633
- if (heartbeatTimer) {
634
- heartbeatTimer.stop();
635
- heartbeatTimer = null;
636
- }
637
- if (lease) {
638
- try {
639
- await provider.releaseLease(lease);
640
- _emitTransition("lease-released", { fencingToken: lease.fencingToken });
641
- log("lease released on shutdown");
642
- } catch (e) {
643
- log.error("release on shutdown failed: " + e.message);
644
- }
645
- lease = null;
646
- }
647
- initialized = false;
648
- terminated = true;
649
- provider = null;
650
- role = null;
651
- leaseTtlMs = null;
652
- heartbeatMs = null;
653
- configuredExternalDbBackend = null;
654
- configuredDialect = null;
655
- configuredEndpoint = null;
656
- transitionHandlers = [];
657
- // nodeId is preserved post-shutdown so audit metadata still reflects
658
- // who this process was; cleared only by _resetForTest.
659
- }
660
-
661
- // ---- test helpers — not part of public contract ----
662
-
663
- function _resetForTest() {
664
- if (heartbeatTimer) heartbeatTimer.stop();
665
- heartbeatTimer = null;
666
- initialized = false;
667
- terminated = false;
668
- nodeId = null;
669
- role = null;
670
- provider = null;
671
- lease = null;
672
- leaseTtlMs = null;
673
- heartbeatMs = null;
674
- configuredExternalDbBackend = null;
675
- configuredDialect = null;
676
- configuredEndpoint = null;
677
- transitionHandlers = [];
678
- }
679
-
680
- async function _heartbeatNowForTest() {
681
- // Drive one heartbeat synchronously without waiting for the timer —
682
- // lets tests deterministically observe lease state transitions.
683
- await _heartbeat();
684
- }
685
-
686
- module.exports = {
687
- init: init,
688
- isLeader: isLeader,
689
- isClusterMode: isClusterMode,
690
- externalDbBackend: externalDbBackend,
691
- dialect: dialect,
692
- currentNodeId: currentNodeId,
693
- endpoint: endpoint,
694
- fencingToken: fencingToken,
695
- requireLeader: requireLeader,
696
- currentLeader: currentLeader,
697
- discoveryHandler: discoveryHandler,
698
- onTransition: onTransition,
699
- shutdown: shutdown,
700
- NotLeaderError: NotLeaderError,
701
- _resetForTest: _resetForTest,
702
- _heartbeatNowForTest: _heartbeatNowForTest,
703
- };
1
+ "use strict";
2
+ /**
3
+ * Cluster coordination — leader election + fencing tokens.
4
+ *
5
+ * Opt-in via `b.cluster.init(...)`. When init is never called, the
6
+ * local process behaves as a permanent single leader: `isLeader()`
7
+ * always returns true, `fencingToken()` returns 0, no heartbeat thread
8
+ * runs, no DB is touched. Single-node deployments pay zero overhead.
9
+ *
10
+ * When init IS called, the framework starts a heartbeat that renews
11
+ * the leader lease via the configured provider. On lease loss (network
12
+ * partition, takeover, lease expiry) the node transitions to follower
13
+ * and write-side framework primitives throw `NotLeaderError`.
14
+ *
15
+ * Threat model:
16
+ * - Two leaders writing simultaneously: prevented by fencing tokens.
17
+ * Every leader-only DB write includes the current token; a
18
+ * CHECK constraint on the audit-tip row rejects a stale token.
19
+ * The application-layer `requireLeader()` gate is just an early
20
+ * rejection optimisation; the DB constraint is the canonical guard.
21
+ * - Follower receiving a write: rejected at the framework boundary.
22
+ * Operators front the cluster with a load balancer that routes
23
+ * write paths to the current leader.
24
+ * - External-db unreachable: heartbeat fails; after `leaseTtl` no
25
+ * leader exists and writes fail closed. When the DB recovers,
26
+ * election resumes.
27
+ *
28
+ * Public API:
29
+ * await cluster.init(opts) one-time bootstrap
30
+ * cluster.isLeader() sync; true on leader (or single-node)
31
+ * cluster.currentNodeId() sync; configured nodeId
32
+ * cluster.endpoint() sync; this node's routable URL
33
+ * (operator-supplied at init), or
34
+ * null if unconfigured. Stored in
35
+ * the leader-election row so
36
+ * external observers can resolve
37
+ * "where is the current leader?"
38
+ * cluster.fencingToken() sync; current monotonic token
39
+ * cluster.requireLeader() sync; throws NotLeaderError
40
+ * cluster.currentLeader() async; { nodeId, leaseExpiresAt,
41
+ * fencingToken,
42
+ * endpoint } | null
43
+ * cluster.discoveryHandler() returns an HTTP request handler
44
+ * (req, res) → JSON. Mount on any
45
+ * route to expose the current
46
+ * leader for service-mesh / LB
47
+ * consumption. 200 with leader,
48
+ * 503 with `{ leader: null }`
49
+ * when no leader.
50
+ * cluster.onTransition(fn) register transition handler
51
+ * await cluster.shutdown() releases lease, stops heartbeat
52
+ */
53
+ var C = require("./constants");
54
+ var clusterProviderDb = require("./cluster-provider-db");
55
+ var crypto = require("./crypto");
56
+ var lazyRequire = require("./lazy-require");
57
+ var { boot } = require("./log");
58
+ var safeAsync = require("./safe-async");
59
+ var safeJson = require("./safe-json");
60
+ var safeSql = require("./safe-sql");
61
+ var safeUrl = require("./safe-url");
62
+ var { FrameworkError, ClusterError } = require("./framework-error");
63
+
64
+ // Lazy: vault → db → cluster forms a load-time chain, and external-db is
65
+ // loaded before its init has run; both are safe to call once cluster
66
+ // reaches runtime, but eager require here would deadlock the load order.
67
+ var externalDb = lazyRequire(function () { return require("./external-db"); });
68
+ var vault = lazyRequire(function () { return require("./vault"); });
69
+
70
+ var DEFAULT_LEASE_TTL = C.TIME.seconds(30);
71
+ var DEFAULT_HEARTBEAT = C.TIME.seconds(10);
72
+ var MIN_LEASE_TTL = C.TIME.seconds(5);
73
+ var MIN_HEARTBEAT = C.TIME.seconds(1);
74
+
75
+ var initialized = false;
76
+ var terminated = false; // set true by shutdown() so the
77
+ // permanent-leader fallback isn't
78
+ // re-engaged after a graceful exit
79
+ var nodeId = null;
80
+ var role = null; // 'leader' | 'follower'
81
+ var provider = null;
82
+ var lease = null; // current lease (if leader)
83
+ var heartbeatTimer = null;
84
+ var heartbeatMs = null;
85
+ var leaseTtlMs = null;
86
+ var transitionHandlers = [];
87
+ // Backend coordinates for write-dispatch code in audit/consent/etc.
88
+ // These are set when cluster.init is called with `externalDbBackend`
89
+ // (the default DB-row provider path); operators using a custom
90
+ // provider can set them via init opts directly.
91
+ var configuredExternalDbBackend = null;
92
+ var configuredDialect = null;
93
+ // Operator-supplied routable endpoint for THIS node, used by external
94
+ // load balancers / service meshes to learn where to send write traffic.
95
+ // Stored in the leader-election row on every acquire/renew so any node
96
+ // (or external observer) can resolve "where is the current leader?"
97
+ // via cluster.currentLeader() / cluster.discoveryHandler().
98
+ var configuredEndpoint = null;
99
+
100
+ var log = boot("cluster");
101
+
102
+ class NotLeaderError extends FrameworkError {
103
+ constructor(message) {
104
+ super(message || "not leader: write rejected by cluster gate", "NOT_LEADER");
105
+ this.name = "NotLeaderError";
106
+ this.statusCode = 503; // operator's load balancer should retry on the leader
107
+ this.isClusterError = true;
108
+ this.isNotLeaderError = true;
109
+ }
110
+ }
111
+
112
+ var _err = ClusterError.factory;
113
+
114
+ function _emitTransition(kind, detail) {
115
+ var event = Object.assign({ kind: kind, nodeId: nodeId, at: Date.now() }, detail || {});
116
+ for (var i = 0; i < transitionHandlers.length; i++) {
117
+ try { transitionHandlers[i](event); }
118
+ catch (e) { log.error("transition handler threw: " + e.message); }
119
+ }
120
+ }
121
+
122
+ // ---- init ----
123
+
124
+ async function init(opts) {
125
+ if (initialized) {
126
+ throw _err("ALREADY_INITIALIZED", "cluster.init() called twice", true);
127
+ }
128
+ opts = opts || {};
129
+ if (!opts.nodeId) {
130
+ throw _err("INVALID_CONFIG", "cluster.init({ nodeId }) is required", true);
131
+ }
132
+ nodeId = String(opts.nodeId);
133
+
134
+ leaseTtlMs = opts.leaseTtl != null ? Number(opts.leaseTtl) : DEFAULT_LEASE_TTL;
135
+ if (leaseTtlMs < MIN_LEASE_TTL) {
136
+ throw _err("INVALID_TTL",
137
+ "leaseTtl must be >= " + MIN_LEASE_TTL + "ms (got " + leaseTtlMs + ")",
138
+ true);
139
+ }
140
+ heartbeatMs = opts.heartbeatInterval != null
141
+ ? Number(opts.heartbeatInterval)
142
+ : DEFAULT_HEARTBEAT;
143
+ if (heartbeatMs < MIN_HEARTBEAT) {
144
+ throw _err("INVALID_HEARTBEAT",
145
+ "heartbeatInterval must be >= " + MIN_HEARTBEAT + "ms (got " + heartbeatMs + ")",
146
+ true);
147
+ }
148
+ if (heartbeatMs >= leaseTtlMs) {
149
+ throw _err("INVALID_HEARTBEAT",
150
+ "heartbeatInterval must be < leaseTtl (got heartbeat=" + heartbeatMs +
151
+ ", leaseTtl=" + leaseTtlMs + "); recommend ~1/3 of leaseTtl",
152
+ true);
153
+ }
154
+
155
+ role = (opts.role || "leader").toLowerCase();
156
+ if (role !== "leader" && role !== "follower") {
157
+ throw _err("INVALID_ROLE", "role must be 'leader' or 'follower'", true);
158
+ }
159
+
160
+ // Optional endpoint. If provided, validate scheme + shape via url-safe
161
+ // — HTTPS-only by default since this is the URL external services use
162
+ // to reach the leader. Operators with internal cleartext clusters opt
163
+ // in via opts.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
164
+ if (opts.endpoint != null) {
165
+ try {
166
+ safeUrl.parse(opts.endpoint, {
167
+ allowedProtocols: opts.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
168
+ errorClass: ClusterError,
169
+ });
170
+ } catch (e) {
171
+ // Re-throw with a config-shaped error so operators see the cluster.init
172
+ // boundary, not a bare url-safe trace.
173
+ throw _err("INVALID_ENDPOINT",
174
+ "cluster.init({ endpoint }) rejected: " + e.message, true);
175
+ }
176
+ configuredEndpoint = String(opts.endpoint);
177
+ } else {
178
+ configuredEndpoint = null;
179
+ }
180
+
181
+ if (typeof opts.onTransition === "function") {
182
+ transitionHandlers.push(opts.onTransition);
183
+ }
184
+
185
+ // Provider: either operator-supplied, or build the default DB-row
186
+ // provider against an externalDb backend.
187
+ if (opts.provider) {
188
+ provider = opts.provider;
189
+ // Operator-custom provider: they may still be writing framework
190
+ // state to an externalDb backend, in which case they pass these
191
+ // separately so write-dispatch code knows where to go.
192
+ configuredExternalDbBackend = opts.externalDbBackend || null;
193
+ configuredDialect = (opts.dialect || "postgres").toLowerCase();
194
+ } else {
195
+ if (!opts.externalDbBackend) {
196
+ throw _err("INVALID_CONFIG",
197
+ "cluster.init requires either { provider } or { externalDbBackend }", true);
198
+ }
199
+ provider = clusterProviderDb.create({
200
+ externalDbBackend: opts.externalDbBackend,
201
+ dialect: opts.dialect,
202
+ });
203
+ configuredExternalDbBackend = opts.externalDbBackend;
204
+ configuredDialect = (opts.dialect || "postgres").toLowerCase();
205
+ }
206
+
207
+ if (typeof provider.ensureSchema === "function") {
208
+ await provider.ensureSchema();
209
+ }
210
+
211
+ initialized = true;
212
+ log("initialized as nodeId='" + nodeId + "', role='" + role + "'");
213
+
214
+ // Initial acquisition attempt (only if role === 'leader')
215
+ if (role === "leader") {
216
+ await _tryAcquire();
217
+ }
218
+
219
+ // Boot-time rollback detection on the audit + consent chains. Runs
220
+ // regardless of role — every node should refuse to participate in a
221
+ // cluster whose shared chains have been rolled back (a follower
222
+ // would face the same chain integrity failure if it later took
223
+ // over). Skipped when configuredExternalDbBackend is unset, which
224
+ // means a custom provider is in use without externalDb-resident
225
+ // framework state — the operator owns rollback detection in that
226
+ // case.
227
+ if (configuredExternalDbBackend) {
228
+ await _checkChainTipRollback("audit", "_blamejs_audit_log", "_blamejs_audit_tip");
229
+ await _checkChainTipRollback("consent", "_blamejs_consent_log", "_blamejs_consent_tip");
230
+ // Vault-key consistency: every node in a cluster must hold the
231
+ // SAME vault key. A node booting with a different key would seal
232
+ // new writes under a key the rest of the cluster can't unseal,
233
+ // and (on takeover) be unable to unseal the rest of the cluster's
234
+ // sealed columns — silent corruption. Compare a fingerprint of
235
+ // this node's vault keys against the canonical one stored at
236
+ // first cluster boot; refuse to participate on mismatch.
237
+ await _checkVaultKeyConsistency();
238
+ }
239
+
240
+ // Start heartbeat
241
+ heartbeatTimer = safeAsync.repeating(_heartbeat, heartbeatMs, { name: "cluster-heartbeat" });
242
+ }
243
+
244
+ // Cluster-mode equivalent of db.js's single-node audit.tip-sidecar
245
+ // rollback check. Reads the persistent _blamejs_audit_tip row and
246
+ // compares to the current chain head in _blamejs_audit_log:
247
+ //
248
+ // - No tip row: first cluster boot or operator-cleared. Skip
249
+ // silently (matches the single-node sidecar-missing path).
250
+ // - Tip recorded a counter > current MAX: chain was truncated /
251
+ // restored from older snapshot. FATAL — refuse boot.
252
+ // - Tip recorded a hash that doesn't match the row at that
253
+ // counter: the row at that counter was substituted (different
254
+ // hash for same counter). FATAL — refuse boot.
255
+ //
256
+ // process.exit(1) is the framework's convention for boot-time
257
+ // integrity failures (audit chain, checkpoints, single-node
258
+ // rollback). Cluster mode keeps the same posture so operators see
259
+ // a single boot-time failure mode regardless of deployment shape.
260
+ // Generalized boot-time rollback check used by both audit and consent
261
+ // chains. chainName is the human-readable label included in log
262
+ // output ("audit" / "consent"). logTable is the chain table
263
+ // (_blamejs_audit_log / _blamejs_consent_log). tipTable is the
264
+ // single-row coordination table that records the latest counter +
265
+ // rowHash + fencingToken (_blamejs_audit_tip / _blamejs_consent_tip).
266
+ //
267
+ // Surfaces three outcomes:
268
+ // - tip table missing → operator running cluster gates-only mode
269
+ // (cluster wired for leader election but framework state still
270
+ // lives in per-node SQLite without `frameworkSchema.ensureSchema`);
271
+ // skip silently.
272
+ // - no tip row → first cluster boot or operator-cleared; skip.
273
+ // - currentMax < tipCounter, or tip rowHash != row-at-counter
274
+ // hash → FATAL via process.exit(1). Same posture as the
275
+ // single-node audit.tip sidecar rollback check.
276
+ async function _checkChainTipRollback(chainName, logTable, tipTable) {
277
+ // Both tables are framework-internal constants from the call sites
278
+ // (`_blamejs_audit_log`, `_blamejs_consent_log`, etc.). Validate +
279
+ // quote per the framework's identifier-quoting convention so a
280
+ // future rename can't silently break the query.
281
+ safeSql.validateIdentifier(logTable, { allowReserved: true });
282
+ safeSql.validateIdentifier(tipTable, { allowReserved: true });
283
+ var qLogTable = safeSql.quoteIdentifier(logTable);
284
+ var qTipTable = safeSql.quoteIdentifier(tipTable);
285
+
286
+ var tipRows;
287
+ try {
288
+ tipRows = await externalDb().query(
289
+ "SELECT atMonotonicCounter, rowHash FROM " + qTipTable +
290
+ " WHERE scope = " + (configuredDialect === "postgres" ? "$1" : "?"),
291
+ [chainName],
292
+ { backend: configuredExternalDbBackend }
293
+ );
294
+ } catch (e) {
295
+ var msg = (e && e.message) || "";
296
+ if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
297
+ log(chainName + "-tip table not present — skipping rollback check (cluster gates-only mode)");
298
+ return;
299
+ }
300
+ throw e;
301
+ }
302
+ if (!tipRows.rows || tipRows.rows.length === 0) {
303
+ log("no " + chainName + "-tip row — skipping rollback check (first cluster boot or operator-cleared)");
304
+ return;
305
+ }
306
+ var tip = tipRows.rows[0];
307
+ var tipCounter = Number(tip.atMonotonicCounter);
308
+ var tipHash = tip.rowHash;
309
+
310
+ var currentRows = await externalDb().query(
311
+ "SELECT MAX(monotonicCounter) AS m FROM " + qLogTable,
312
+ [],
313
+ { backend: configuredExternalDbBackend }
314
+ );
315
+ var currentMax = (currentRows.rows && currentRows.rows[0] && currentRows.rows[0].m)
316
+ ? Number(currentRows.rows[0].m)
317
+ : 0;
318
+
319
+ if (currentMax < tipCounter) {
320
+ throw _err("ROLLBACK_DETECTED",
321
+ "FATAL: cluster-mode " + chainName + "-log rollback detected. " +
322
+ chainName + "-tip counter: " + tipCounter +
323
+ "; current external-db max: " + currentMax +
324
+ ". Either external-db was restored from an older snapshot, or " +
325
+ logTable + " rows have been deleted. Investigate before continuing.",
326
+ true);
327
+ }
328
+
329
+ if (tipHash) {
330
+ var hashRows = await externalDb().query(
331
+ "SELECT rowHash FROM " + qLogTable + " WHERE monotonicCounter = " +
332
+ (configuredDialect === "postgres" ? "$1" : "?"),
333
+ [tipCounter],
334
+ { backend: configuredExternalDbBackend }
335
+ );
336
+ if (hashRows.rows && hashRows.rows.length > 0) {
337
+ var rowAtTip = hashRows.rows[0].rowHash;
338
+ if (rowAtTip !== tipHash) {
339
+ throw _err("ROLLBACK_DETECTED",
340
+ "FATAL: cluster-mode " + chainName + "-log rollback detected (row-hash mismatch). " +
341
+ chainName + "-tip counter: " + tipCounter +
342
+ "; " + chainName + "-tip rowHash: " + tipHash +
343
+ "; current row rowHash: " + rowAtTip +
344
+ ". The row at the recorded tip counter has a different hash — " +
345
+ "indicates row substitution at the chain head. Investigate before continuing.",
346
+ true);
347
+ }
348
+ }
349
+ }
350
+ log("cluster " + chainName + "-tip rollback check ok (tip counter " + tipCounter +
351
+ ", current " + currentMax + ")");
352
+ }
353
+
354
+ // Compute a deterministic fingerprint of THIS node's vault keys.
355
+ // SHA3-512 of the concatenated public keys (PQC + classical halves of
356
+ // the hybrid encryption keypair). One-way: nothing about the private
357
+ // key material is recoverable from the fingerprint, so it's safe to
358
+ // store in the coordination table that all cluster nodes can read.
359
+ //
360
+ // Returns null if vault.init() hasn't been called — cluster gates-only
361
+ // mode (no sealed-column work) doesn't need this check, same defensive
362
+ // posture as the audit-tip rollback check skipping when there's no
363
+ // audit-tip table.
364
+ function _vaultKeyFingerprint() {
365
+ var keysJson;
366
+ try {
367
+ keysJson = vault().getKeysJson();
368
+ } catch (e) {
369
+ // vault.init() not called — gates-only mode. Skip silently.
370
+ if (/vault.init\(\) must be awaited/.test((e && e.message) || "")) {
371
+ return null;
372
+ }
373
+ throw e;
374
+ }
375
+ // vault.getKeysJson() returns the keys serialized as JSON (the same
376
+ // format vault writes to disk). Parse to extract the public halves;
377
+ // we never touch privateKey/ecPrivateKey here.
378
+ var keys = safeJson.parse(keysJson);
379
+ if (!keys || !keys.publicKey || !keys.ecPublicKey) return null;
380
+ // Domain-separation prefix so this fingerprint can't be confused
381
+ // with a hash of the same bytes computed elsewhere in the framework.
382
+ return crypto.sha3Hash("blamejs/cluster-state/v1\n" +
383
+ keys.publicKey + "\n" +
384
+ keys.ecPublicKey);
385
+ }
386
+
387
+ async function _checkVaultKeyConsistency() {
388
+ var localFp = _vaultKeyFingerprint();
389
+ if (localFp === null) {
390
+ log("vault not initialized — skipping vault-key consistency check (cluster gates-only mode)");
391
+ return;
392
+ }
393
+ var nowMs = Date.now();
394
+ var ph = configuredDialect === "postgres";
395
+
396
+ // First boot: try to record THIS node's fingerprint. ON CONFLICT DO
397
+ // NOTHING means the FIRST node to boot wins; subsequent nodes
398
+ // observe whatever's already there. Every node then SELECTs and
399
+ // compares — any mismatch (including ours after a losing race)
400
+ // surfaces the drift.
401
+ try {
402
+ await externalDb().query(
403
+ "INSERT INTO _blamejs_cluster_state " +
404
+ " (scope, vaultKeyFp, recordedAt, recordedByNode) " +
405
+ "VALUES ('state', " +
406
+ (ph ? "$1, $2, $3" : "?, ?, ?") + ") " +
407
+ "ON CONFLICT (scope) DO NOTHING",
408
+ [localFp, nowMs, nodeId],
409
+ { backend: configuredExternalDbBackend }
410
+ );
411
+ } catch (e) {
412
+ // Table missing → the cluster-provider-db ensureSchema didn't run
413
+ // (custom provider that doesn't create _blamejs_cluster_state).
414
+ // Skip silently — same defensive posture as the audit-tip check.
415
+ var msg = (e && e.message) || "";
416
+ if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
417
+ log("cluster-state table not present — skipping vault-key consistency check (custom provider)");
418
+ return;
419
+ }
420
+ throw e;
421
+ }
422
+
423
+ // Read whatever fingerprint is canonical (ours if first boot,
424
+ // someone else's if we lost the race or are joining an existing cluster).
425
+ var rows = await externalDb().query(
426
+ "SELECT vaultKeyFp, recordedByNode, recordedAt FROM _blamejs_cluster_state " +
427
+ "WHERE scope = 'state'",
428
+ [],
429
+ { backend: configuredExternalDbBackend }
430
+ );
431
+ if (!rows.rows || rows.rows.length === 0) {
432
+ // Should never happen — we just INSERTed. Surface as fatal so the
433
+ // condition isn't silently ignored.
434
+ throw _err("CLUSTER_STATE_MISSING",
435
+ "FATAL: cluster-state row missing immediately after INSERT — " +
436
+ "external-db may not be honoring writes. Refusing boot.",
437
+ true);
438
+ }
439
+ var canonical = rows.rows[0];
440
+ if (canonical.vaultKeyFp !== localFp) {
441
+ var fpPrefix = C.BYTES.bytes(16);
442
+ throw _err("VAULT_KEY_DRIFT",
443
+ "FATAL: vault-key drift detected. " +
444
+ "local node: " + nodeId +
445
+ "; local fingerprint: " + localFp.slice(0, fpPrefix) + "…" +
446
+ "; canonical recorded by: " + canonical.recordedByNode +
447
+ "; canonical fingerprint: " + canonical.vaultKeyFp.slice(0, fpPrefix) + "…" +
448
+ ". This node holds a DIFFERENT vault key than the rest of the " +
449
+ "cluster. Sealed-column writes from this node would be unreadable " +
450
+ "by the others (and vice versa). Restore the same vault key file " +
451
+ "before booting this node into the cluster.",
452
+ true);
453
+ }
454
+ log("cluster vault-key consistency ok (fingerprint " +
455
+ localFp.slice(0, C.BYTES.bytes(16)) + "… recorded by " + canonical.recordedByNode + ")");
456
+ }
457
+
458
+ async function _tryAcquire() {
459
+ if (role !== "leader") return; // pinned-follower role: never claim
460
+ try {
461
+ var got = await provider.acquireLease(nodeId, leaseTtlMs, {
462
+ endpoint: configuredEndpoint,
463
+ });
464
+ if (got) {
465
+ var wasLeader = !!lease;
466
+ lease = got;
467
+ if (!wasLeader) {
468
+ log("acquired lease — fencingToken=" + lease.fencingToken);
469
+ _emitTransition("lease-acquired", { fencingToken: lease.fencingToken });
470
+ }
471
+ }
472
+ } catch (e) {
473
+ log.error("acquire failed: " + e.message);
474
+ }
475
+ }
476
+
477
+ async function _heartbeat() {
478
+ if (!initialized) return;
479
+ if (!lease) {
480
+ // Not currently leader — try to acquire (lease may have expired
481
+ // on the previous holder).
482
+ await _tryAcquire();
483
+ return;
484
+ }
485
+ // We hold a lease — renew it. Re-supply the configured endpoint so a
486
+ // hot-reload of the operator's config (e.g. node moves to a new
487
+ // routable URL after a restart) eventually reaches the discovery row.
488
+ try {
489
+ lease = await provider.renewLease(lease, { endpoint: configuredEndpoint });
490
+ } catch (e) {
491
+ if (e.code === "LEASE_LOST") {
492
+ log.error("lease lost: " + e.message);
493
+ var lostToken = lease ? lease.fencingToken : null;
494
+ lease = null;
495
+ _emitTransition("lease-lost", { fencingToken: lostToken });
496
+ // Attempt to re-acquire on the next heartbeat naturally.
497
+ } else {
498
+ // Transient error — retry on next heartbeat. If it persists past
499
+ // leaseTtl another node will steal, and we'll detect via LEASE_LOST.
500
+ log.error("renew failed transiently: " + e.message);
501
+ }
502
+ }
503
+ }
504
+
505
+ // ---- public sync surface ----
506
+
507
+ function isLeader() {
508
+ if (terminated) return false; // post-shutdown: never leader
509
+ if (!initialized) return true; // never-initialized: permanent leader
510
+ return !!lease && Date.now() < lease.expiresAt;
511
+ }
512
+
513
+ // Has cluster.init been called with a real configuration? Used by
514
+ // write-dispatch code (audit, consent, …) to decide whether framework
515
+ // state should go to local SQLite or external-db.
516
+ function isClusterMode() {
517
+ return initialized && !!configuredExternalDbBackend;
518
+ }
519
+
520
+ function externalDbBackend() {
521
+ return configuredExternalDbBackend;
522
+ }
523
+
524
+ function dialect() {
525
+ return configuredDialect;
526
+ }
527
+
528
+ function currentNodeId() {
529
+ return initialized ? nodeId : "single-node-local";
530
+ }
531
+
532
+ // This node's routable endpoint (operator-configured at cluster.init).
533
+ // Returns null when not configured or in single-node fallback. External
534
+ // observers should call discoveryHandler() / currentLeader() instead —
535
+ // this getter is for the local node's own self-identity.
536
+ function endpoint() {
537
+ return configuredEndpoint;
538
+ }
539
+
540
+ function fencingToken() {
541
+ if (!initialized) return 0;
542
+ return lease ? lease.fencingToken : 0;
543
+ }
544
+
545
+ function requireLeader() {
546
+ if (!isLeader()) {
547
+ throw new NotLeaderError(
548
+ "node '" + currentNodeId() + "' is not currently leader" +
549
+ (initialized ? "" : " (cluster not initialized)")
550
+ );
551
+ }
552
+ }
553
+
554
+ async function currentLeader() {
555
+ if (!initialized) {
556
+ return {
557
+ nodeId: "single-node-local",
558
+ leaseExpiresAt: Infinity,
559
+ fencingToken: 0,
560
+ endpoint: null,
561
+ };
562
+ }
563
+ return await provider.currentLeader();
564
+ }
565
+
566
+ // HTTP request handler — replies with the current cluster leader for
567
+ // service-mesh / load-balancer discovery. Operators mount this at
568
+ // whatever route they want (`/cluster/leader`, `/health/leader`, etc.).
569
+ //
570
+ // 200 application/json — leader present
571
+ // { leader: { nodeId, endpoint, fencingToken, leaseExpiresAt },
572
+ // self: { nodeId, endpoint, isLeader } }
573
+ //
574
+ // 503 application/json — no leader (no row, expired lease, DB
575
+ // unreachable, single-node not initialized with cluster mode)
576
+ // { leader: null, self: { nodeId, endpoint, isLeader } }
577
+ //
578
+ // No auth — this endpoint is intended to be called by infrastructure
579
+ // inside the trust boundary (LB, healthcheck, dashboard). Operators
580
+ // who expose it externally should layer auth via their own middleware.
581
+ //
582
+ // Handler is method-agnostic so it works behind any HTTP probe shape
583
+ // (GET, HEAD, etc.). Cache-Control: no-store to avoid stale-leader
584
+ // responses pinned by a caching proxy during a takeover.
585
+ function discoveryHandler() {
586
+ return async function (req, res) {
587
+ var selfInfo = {
588
+ nodeId: currentNodeId(),
589
+ endpoint: configuredEndpoint,
590
+ isLeader: isLeader(),
591
+ };
592
+ var body;
593
+ var status;
594
+ try {
595
+ var leader = await currentLeader();
596
+ if (leader && leader.nodeId && leader.nodeId !== "single-node-local") {
597
+ body = { leader: leader, self: selfInfo };
598
+ status = 200;
599
+ } else if (leader && leader.nodeId === "single-node-local") {
600
+ // Permanent-leader fallback (cluster.init never called). Reply
601
+ // 200 — the operator's app is healthy and the "leader" is this
602
+ // process. Useful so the discovery endpoint is never a false
603
+ // negative in single-node deployments.
604
+ body = { leader: leader, self: selfInfo };
605
+ status = 200;
606
+ } else {
607
+ body = { leader: null, self: selfInfo };
608
+ status = 503;
609
+ }
610
+ } catch (e) {
611
+ body = { leader: null, self: selfInfo, error: e.message };
612
+ status = 503;
613
+ }
614
+ var json = JSON.stringify(body);
615
+ res.writeHead(status, {
616
+ "Content-Type": "application/json; charset=utf-8",
617
+ "Content-Length": Buffer.byteLength(json),
618
+ "Cache-Control": "no-store",
619
+ });
620
+ res.end(json);
621
+ };
622
+ }
623
+
624
+ function onTransition(handler) {
625
+ if (typeof handler !== "function") {
626
+ throw _err("INVALID_HANDLER", "onTransition expects a function", true);
627
+ }
628
+ transitionHandlers.push(handler);
629
+ }
630
+
631
+ async function shutdown() {
632
+ if (!initialized) return;
633
+ if (heartbeatTimer) {
634
+ heartbeatTimer.stop();
635
+ heartbeatTimer = null;
636
+ }
637
+ if (lease) {
638
+ try {
639
+ await provider.releaseLease(lease);
640
+ _emitTransition("lease-released", { fencingToken: lease.fencingToken });
641
+ log("lease released on shutdown");
642
+ } catch (e) {
643
+ log.error("release on shutdown failed: " + e.message);
644
+ }
645
+ lease = null;
646
+ }
647
+ initialized = false;
648
+ terminated = true;
649
+ provider = null;
650
+ role = null;
651
+ leaseTtlMs = null;
652
+ heartbeatMs = null;
653
+ configuredExternalDbBackend = null;
654
+ configuredDialect = null;
655
+ configuredEndpoint = null;
656
+ transitionHandlers = [];
657
+ // nodeId is preserved post-shutdown so audit metadata still reflects
658
+ // who this process was; cleared only by _resetForTest.
659
+ }
660
+
661
+ // ---- test helpers — not part of public contract ----
662
+
663
+ function _resetForTest() {
664
+ if (heartbeatTimer) heartbeatTimer.stop();
665
+ heartbeatTimer = null;
666
+ initialized = false;
667
+ terminated = false;
668
+ nodeId = null;
669
+ role = null;
670
+ provider = null;
671
+ lease = null;
672
+ leaseTtlMs = null;
673
+ heartbeatMs = null;
674
+ configuredExternalDbBackend = null;
675
+ configuredDialect = null;
676
+ configuredEndpoint = null;
677
+ transitionHandlers = [];
678
+ }
679
+
680
+ async function _heartbeatNowForTest() {
681
+ // Drive one heartbeat synchronously without waiting for the timer —
682
+ // lets tests deterministically observe lease state transitions.
683
+ await _heartbeat();
684
+ }
685
+
686
+ module.exports = {
687
+ init: init,
688
+ isLeader: isLeader,
689
+ isClusterMode: isClusterMode,
690
+ externalDbBackend: externalDbBackend,
691
+ dialect: dialect,
692
+ currentNodeId: currentNodeId,
693
+ endpoint: endpoint,
694
+ fencingToken: fencingToken,
695
+ requireLeader: requireLeader,
696
+ currentLeader: currentLeader,
697
+ discoveryHandler: discoveryHandler,
698
+ onTransition: onTransition,
699
+ shutdown: shutdown,
700
+ NotLeaderError: NotLeaderError,
701
+ _resetForTest: _resetForTest,
702
+ _heartbeatNowForTest: _heartbeatNowForTest,
703
+ };