@blamejs/core 0.7.4 → 0.7.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +423 -395
- package/README.md +150 -149
- package/bin/blamejs.js +0 -0
- package/index.js +308 -284
- package/lib/api-key.js +660 -663
- package/lib/api-snapshot.js +338 -338
- package/lib/app-shutdown.js +385 -385
- package/lib/app.js +365 -365
- package/lib/archive.js +250 -250
- package/lib/atomic-file.js +544 -544
- package/lib/audit-chain.js +177 -177
- package/lib/audit-sign.js +344 -344
- package/lib/audit-tools.js +677 -677
- package/lib/audit.js +766 -766
- package/lib/auth/jwt.js +311 -311
- package/lib/auth/lockout.js +436 -436
- package/lib/auth/oauth.js +721 -721
- package/lib/auth/passkey.js +181 -181
- package/lib/auth/password.js +594 -594
- package/lib/backup/bundle.js +217 -217
- package/lib/backup/crypto.js +176 -176
- package/lib/backup/index.js +515 -515
- package/lib/backup/manifest.js +282 -282
- package/lib/break-glass.js +1338 -1338
- package/lib/bundler.js +441 -441
- package/lib/cache-redis.js +256 -256
- package/lib/cache.js +1206 -1206
- package/lib/canonical-json.js +115 -115
- package/lib/chain-writer.js +234 -234
- package/lib/cli-helpers.js +206 -206
- package/lib/cli.js +2334 -2334
- package/lib/cluster-provider-db.js +317 -317
- package/lib/cluster-storage.js +226 -226
- package/lib/cluster.js +703 -703
- package/lib/codepoint-class.js +196 -0
- package/lib/config-drift.js +301 -301
- package/lib/consent.js +222 -222
- package/lib/constants.js +191 -191
- package/lib/cookies.js +315 -315
- package/lib/credential-hash.js +322 -322
- package/lib/crypto.js +266 -266
- package/lib/csv.js +275 -286
- package/lib/db-declare-row-policy.js +267 -267
- package/lib/db-declare-view.js +420 -421
- package/lib/db-query.js +406 -406
- package/lib/db-schema.js +319 -319
- package/lib/db.js +1288 -1288
- package/lib/deprecate.js +222 -222
- package/lib/dev.js +335 -335
- package/lib/dual-control.js +473 -473
- package/lib/error-page.js +420 -420
- package/lib/external-db-migrate.js +441 -441
- package/lib/external-db.js +1061 -1061
- package/lib/file-type.js +273 -273
- package/lib/file-upload.js +213 -10
- package/lib/forms.js +422 -422
- package/lib/framework-error.js +293 -215
- package/lib/framework-schema.js +717 -717
- package/lib/gate-contract.js +971 -0
- package/lib/guard-all.js +405 -0
- package/lib/guard-archive.js +739 -0
- package/lib/guard-csv.js +816 -0
- package/lib/guard-email.js +744 -0
- package/lib/guard-filename.js +724 -0
- package/lib/guard-html.js +976 -0
- package/lib/guard-json.js +729 -0
- package/lib/guard-markdown.js +586 -0
- package/lib/guard-svg.js +976 -0
- package/lib/guard-xml.js +405 -0
- package/lib/guard-yaml.js +529 -0
- package/lib/handlers.js +350 -350
- package/lib/http-client-cookie-jar.js +508 -508
- package/lib/http-client.js +1195 -1195
- package/lib/i18n.js +878 -878
- package/lib/jobs.js +185 -185
- package/lib/log-stream-cloudwatch.js +369 -369
- package/lib/log-stream-local.js +146 -146
- package/lib/log-stream-otlp-grpc.js +410 -410
- package/lib/log-stream-otlp.js +286 -286
- package/lib/log-stream-syslog.js +302 -302
- package/lib/log-stream-webhook.js +199 -199
- package/lib/log-stream.js +330 -330
- package/lib/log.js +500 -500
- package/lib/mail-bounce.js +528 -528
- package/lib/mail-dkim.js +369 -362
- package/lib/mail.js +981 -962
- package/lib/metrics.js +683 -683
- package/lib/middleware/api-encrypt.js +936 -936
- package/lib/middleware/attach-user.js +157 -157
- package/lib/middleware/body-parser.js +1170 -1091
- package/lib/middleware/bot-guard.js +178 -178
- package/lib/middleware/compression.js +452 -452
- package/lib/middleware/cors.js +314 -314
- package/lib/middleware/csp-nonce.js +348 -348
- package/lib/middleware/csrf-protect.js +316 -316
- package/lib/middleware/db-role-for.js +264 -264
- package/lib/middleware/health.js +392 -392
- package/lib/middleware/index.js +79 -79
- package/lib/middleware/rate-limit.js +358 -358
- package/lib/middleware/request-id.js +61 -61
- package/lib/middleware/request-log.js +168 -168
- package/lib/middleware/require-auth.js +104 -104
- package/lib/middleware/security-headers.js +116 -116
- package/lib/middleware/sse.js +166 -166
- package/lib/migrations.js +383 -383
- package/lib/mtls-ca.js +518 -518
- package/lib/mtls-engine-default.js +481 -481
- package/lib/network-dns.js +632 -632
- package/lib/network-heartbeat.js +290 -290
- package/lib/network-nts.js +574 -574
- package/lib/network-proxy.js +265 -265
- package/lib/network-tls.js +328 -328
- package/lib/network.js +233 -233
- package/lib/notify.js +612 -612
- package/lib/ntp-check.js +229 -229
- package/lib/numeric-bounds.js +111 -91
- package/lib/object-store/azure-blob-bucket-ops.js +349 -349
- package/lib/object-store/azure-blob.js +488 -488
- package/lib/object-store/gcs-bucket-ops.js +351 -351
- package/lib/object-store/gcs.js +519 -519
- package/lib/object-store/http-put.js +153 -153
- package/lib/object-store/index.js +197 -197
- package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
- package/lib/object-store/sigv4.js +903 -903
- package/lib/observability.js +151 -151
- package/lib/otel-export.js +269 -269
- package/lib/pagination.js +464 -464
- package/lib/parsers/index.js +80 -80
- package/lib/parsers/safe-env.js +642 -642
- package/lib/parsers/safe-ini.js +292 -292
- package/lib/parsers/safe-toml.js +784 -784
- package/lib/parsers/safe-xml.js +390 -390
- package/lib/parsers/safe-yaml.js +1015 -1015
- package/lib/permissions.js +708 -708
- package/lib/pqc-agent.js +87 -87
- package/lib/pqc-gate.js +279 -279
- package/lib/protobuf-encoder.js +190 -190
- package/lib/protocol-dispatcher.js +161 -161
- package/lib/pubsub-redis.js +167 -167
- package/lib/pubsub.js +429 -429
- package/lib/queue-local.js +476 -476
- package/lib/queue-redis.js +745 -745
- package/lib/queue-sqs.js +319 -319
- package/lib/queue.js +695 -695
- package/lib/redis-client.js +519 -519
- package/lib/request-helpers.js +340 -340
- package/lib/restore-bundle.js +237 -237
- package/lib/restore-rollback.js +259 -259
- package/lib/restore.js +409 -409
- package/lib/retry.js +376 -376
- package/lib/router.js +748 -748
- package/lib/safe-async.js +735 -735
- package/lib/safe-buffer.js +237 -237
- package/lib/safe-json.js +541 -541
- package/lib/safe-schema.js +1266 -1266
- package/lib/safe-url.js +159 -159
- package/lib/scheduler.js +706 -706
- package/lib/security-assert.js +373 -373
- package/lib/seeders.js +618 -618
- package/lib/session.js +478 -478
- package/lib/slug.js +269 -269
- package/lib/ssrf-guard.js +401 -401
- package/lib/static.js +184 -4
- package/lib/storage.js +471 -471
- package/lib/subject.js +281 -281
- package/lib/template.js +791 -791
- package/lib/testing.js +798 -798
- package/lib/time.js +310 -310
- package/lib/totp.js +302 -302
- package/lib/tracing.js +494 -494
- package/lib/uuid.js +132 -132
- package/lib/validate-opts.js +340 -319
- package/lib/vault/index.js +308 -308
- package/lib/vault/rotate.js +784 -784
- package/lib/vault/wrap.js +296 -296
- package/lib/vendor/noble-ciphers.cjs +9 -9
- package/lib/webhook.js +595 -595
- package/lib/websocket.js +1048 -1048
- package/package.json +77 -77
- package/sbom.cyclonedx.json +7 -7
package/lib/cluster.js
CHANGED
|
@@ -1,703 +1,703 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Cluster coordination — leader election + fencing tokens.
|
|
4
|
-
*
|
|
5
|
-
* Opt-in via `b.cluster.init(...)`. When init is never called, the
|
|
6
|
-
* local process behaves as a permanent single leader: `isLeader()`
|
|
7
|
-
* always returns true, `fencingToken()` returns 0, no heartbeat thread
|
|
8
|
-
* runs, no DB is touched. Single-node deployments pay zero overhead.
|
|
9
|
-
*
|
|
10
|
-
* When init IS called, the framework starts a heartbeat that renews
|
|
11
|
-
* the leader lease via the configured provider. On lease loss (network
|
|
12
|
-
* partition, takeover, lease expiry) the node transitions to follower
|
|
13
|
-
* and write-side framework primitives throw `NotLeaderError`.
|
|
14
|
-
*
|
|
15
|
-
* Threat model:
|
|
16
|
-
* - Two leaders writing simultaneously: prevented by fencing tokens.
|
|
17
|
-
* Every leader-only DB write includes the current token; a
|
|
18
|
-
* CHECK constraint on the audit-tip row rejects a stale token.
|
|
19
|
-
* The application-layer `requireLeader()` gate is just an early
|
|
20
|
-
* rejection optimisation; the DB constraint is the canonical guard.
|
|
21
|
-
* - Follower receiving a write: rejected at the framework boundary.
|
|
22
|
-
* Operators front the cluster with a load balancer that routes
|
|
23
|
-
* write paths to the current leader.
|
|
24
|
-
* - External-db unreachable: heartbeat fails; after `leaseTtl` no
|
|
25
|
-
* leader exists and writes fail closed. When the DB recovers,
|
|
26
|
-
* election resumes.
|
|
27
|
-
*
|
|
28
|
-
* Public API:
|
|
29
|
-
* await cluster.init(opts) one-time bootstrap
|
|
30
|
-
* cluster.isLeader() sync; true on leader (or single-node)
|
|
31
|
-
* cluster.currentNodeId() sync; configured nodeId
|
|
32
|
-
* cluster.endpoint() sync; this node's routable URL
|
|
33
|
-
* (operator-supplied at init), or
|
|
34
|
-
* null if unconfigured. Stored in
|
|
35
|
-
* the leader-election row so
|
|
36
|
-
* external observers can resolve
|
|
37
|
-
* "where is the current leader?"
|
|
38
|
-
* cluster.fencingToken() sync; current monotonic token
|
|
39
|
-
* cluster.requireLeader() sync; throws NotLeaderError
|
|
40
|
-
* cluster.currentLeader() async; { nodeId, leaseExpiresAt,
|
|
41
|
-
* fencingToken,
|
|
42
|
-
* endpoint } | null
|
|
43
|
-
* cluster.discoveryHandler() returns an HTTP request handler
|
|
44
|
-
* (req, res) → JSON. Mount on any
|
|
45
|
-
* route to expose the current
|
|
46
|
-
* leader for service-mesh / LB
|
|
47
|
-
* consumption. 200 with leader,
|
|
48
|
-
* 503 with `{ leader: null }`
|
|
49
|
-
* when no leader.
|
|
50
|
-
* cluster.onTransition(fn) register transition handler
|
|
51
|
-
* await cluster.shutdown() releases lease, stops heartbeat
|
|
52
|
-
*/
|
|
53
|
-
var C = require("./constants");
|
|
54
|
-
var clusterProviderDb = require("./cluster-provider-db");
|
|
55
|
-
var crypto = require("./crypto");
|
|
56
|
-
var lazyRequire = require("./lazy-require");
|
|
57
|
-
var { boot } = require("./log");
|
|
58
|
-
var safeAsync = require("./safe-async");
|
|
59
|
-
var safeJson = require("./safe-json");
|
|
60
|
-
var safeSql = require("./safe-sql");
|
|
61
|
-
var safeUrl = require("./safe-url");
|
|
62
|
-
var { FrameworkError, ClusterError } = require("./framework-error");
|
|
63
|
-
|
|
64
|
-
// Lazy: vault → db → cluster forms a load-time chain, and external-db is
|
|
65
|
-
// loaded before its init has run; both are safe to call once cluster
|
|
66
|
-
// reaches runtime, but eager require here would deadlock the load order.
|
|
67
|
-
var externalDb = lazyRequire(function () { return require("./external-db"); });
|
|
68
|
-
var vault = lazyRequire(function () { return require("./vault"); });
|
|
69
|
-
|
|
70
|
-
var DEFAULT_LEASE_TTL = C.TIME.seconds(30);
|
|
71
|
-
var DEFAULT_HEARTBEAT = C.TIME.seconds(10);
|
|
72
|
-
var MIN_LEASE_TTL = C.TIME.seconds(5);
|
|
73
|
-
var MIN_HEARTBEAT = C.TIME.seconds(1);
|
|
74
|
-
|
|
75
|
-
var initialized = false;
|
|
76
|
-
var terminated = false; // set true by shutdown() so the
|
|
77
|
-
// permanent-leader fallback isn't
|
|
78
|
-
// re-engaged after a graceful exit
|
|
79
|
-
var nodeId = null;
|
|
80
|
-
var role = null; // 'leader' | 'follower'
|
|
81
|
-
var provider = null;
|
|
82
|
-
var lease = null; // current lease (if leader)
|
|
83
|
-
var heartbeatTimer = null;
|
|
84
|
-
var heartbeatMs = null;
|
|
85
|
-
var leaseTtlMs = null;
|
|
86
|
-
var transitionHandlers = [];
|
|
87
|
-
// Backend coordinates for write-dispatch code in audit/consent/etc.
|
|
88
|
-
// These are set when cluster.init is called with `externalDbBackend`
|
|
89
|
-
// (the default DB-row provider path); operators using a custom
|
|
90
|
-
// provider can set them via init opts directly.
|
|
91
|
-
var configuredExternalDbBackend = null;
|
|
92
|
-
var configuredDialect = null;
|
|
93
|
-
// Operator-supplied routable endpoint for THIS node, used by external
|
|
94
|
-
// load balancers / service meshes to learn where to send write traffic.
|
|
95
|
-
// Stored in the leader-election row on every acquire/renew so any node
|
|
96
|
-
// (or external observer) can resolve "where is the current leader?"
|
|
97
|
-
// via cluster.currentLeader() / cluster.discoveryHandler().
|
|
98
|
-
var configuredEndpoint = null;
|
|
99
|
-
|
|
100
|
-
var log = boot("cluster");
|
|
101
|
-
|
|
102
|
-
class NotLeaderError extends FrameworkError {
|
|
103
|
-
constructor(message) {
|
|
104
|
-
super(message || "not leader: write rejected by cluster gate", "NOT_LEADER");
|
|
105
|
-
this.name = "NotLeaderError";
|
|
106
|
-
this.statusCode = 503; // operator's load balancer should retry on the leader
|
|
107
|
-
this.isClusterError = true;
|
|
108
|
-
this.isNotLeaderError = true;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
var _err = ClusterError.factory;
|
|
113
|
-
|
|
114
|
-
function _emitTransition(kind, detail) {
|
|
115
|
-
var event = Object.assign({ kind: kind, nodeId: nodeId, at: Date.now() }, detail || {});
|
|
116
|
-
for (var i = 0; i < transitionHandlers.length; i++) {
|
|
117
|
-
try { transitionHandlers[i](event); }
|
|
118
|
-
catch (e) { log.error("transition handler threw: " + e.message); }
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// ---- init ----
|
|
123
|
-
|
|
124
|
-
async function init(opts) {
|
|
125
|
-
if (initialized) {
|
|
126
|
-
throw _err("ALREADY_INITIALIZED", "cluster.init() called twice", true);
|
|
127
|
-
}
|
|
128
|
-
opts = opts || {};
|
|
129
|
-
if (!opts.nodeId) {
|
|
130
|
-
throw _err("INVALID_CONFIG", "cluster.init({ nodeId }) is required", true);
|
|
131
|
-
}
|
|
132
|
-
nodeId = String(opts.nodeId);
|
|
133
|
-
|
|
134
|
-
leaseTtlMs = opts.leaseTtl != null ? Number(opts.leaseTtl) : DEFAULT_LEASE_TTL;
|
|
135
|
-
if (leaseTtlMs < MIN_LEASE_TTL) {
|
|
136
|
-
throw _err("INVALID_TTL",
|
|
137
|
-
"leaseTtl must be >= " + MIN_LEASE_TTL + "ms (got " + leaseTtlMs + ")",
|
|
138
|
-
true);
|
|
139
|
-
}
|
|
140
|
-
heartbeatMs = opts.heartbeatInterval != null
|
|
141
|
-
? Number(opts.heartbeatInterval)
|
|
142
|
-
: DEFAULT_HEARTBEAT;
|
|
143
|
-
if (heartbeatMs < MIN_HEARTBEAT) {
|
|
144
|
-
throw _err("INVALID_HEARTBEAT",
|
|
145
|
-
"heartbeatInterval must be >= " + MIN_HEARTBEAT + "ms (got " + heartbeatMs + ")",
|
|
146
|
-
true);
|
|
147
|
-
}
|
|
148
|
-
if (heartbeatMs >= leaseTtlMs) {
|
|
149
|
-
throw _err("INVALID_HEARTBEAT",
|
|
150
|
-
"heartbeatInterval must be < leaseTtl (got heartbeat=" + heartbeatMs +
|
|
151
|
-
", leaseTtl=" + leaseTtlMs + "); recommend ~1/3 of leaseTtl",
|
|
152
|
-
true);
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
role = (opts.role || "leader").toLowerCase();
|
|
156
|
-
if (role !== "leader" && role !== "follower") {
|
|
157
|
-
throw _err("INVALID_ROLE", "role must be 'leader' or 'follower'", true);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// Optional endpoint. If provided, validate scheme + shape via url-safe
|
|
161
|
-
// — HTTPS-only by default since this is the URL external services use
|
|
162
|
-
// to reach the leader. Operators with internal cleartext clusters opt
|
|
163
|
-
// in via opts.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
|
|
164
|
-
if (opts.endpoint != null) {
|
|
165
|
-
try {
|
|
166
|
-
safeUrl.parse(opts.endpoint, {
|
|
167
|
-
allowedProtocols: opts.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
|
|
168
|
-
errorClass: ClusterError,
|
|
169
|
-
});
|
|
170
|
-
} catch (e) {
|
|
171
|
-
// Re-throw with a config-shaped error so operators see the cluster.init
|
|
172
|
-
// boundary, not a bare url-safe trace.
|
|
173
|
-
throw _err("INVALID_ENDPOINT",
|
|
174
|
-
"cluster.init({ endpoint }) rejected: " + e.message, true);
|
|
175
|
-
}
|
|
176
|
-
configuredEndpoint = String(opts.endpoint);
|
|
177
|
-
} else {
|
|
178
|
-
configuredEndpoint = null;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
if (typeof opts.onTransition === "function") {
|
|
182
|
-
transitionHandlers.push(opts.onTransition);
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
// Provider: either operator-supplied, or build the default DB-row
|
|
186
|
-
// provider against an externalDb backend.
|
|
187
|
-
if (opts.provider) {
|
|
188
|
-
provider = opts.provider;
|
|
189
|
-
// Operator-custom provider: they may still be writing framework
|
|
190
|
-
// state to an externalDb backend, in which case they pass these
|
|
191
|
-
// separately so write-dispatch code knows where to go.
|
|
192
|
-
configuredExternalDbBackend = opts.externalDbBackend || null;
|
|
193
|
-
configuredDialect = (opts.dialect || "postgres").toLowerCase();
|
|
194
|
-
} else {
|
|
195
|
-
if (!opts.externalDbBackend) {
|
|
196
|
-
throw _err("INVALID_CONFIG",
|
|
197
|
-
"cluster.init requires either { provider } or { externalDbBackend }", true);
|
|
198
|
-
}
|
|
199
|
-
provider = clusterProviderDb.create({
|
|
200
|
-
externalDbBackend: opts.externalDbBackend,
|
|
201
|
-
dialect: opts.dialect,
|
|
202
|
-
});
|
|
203
|
-
configuredExternalDbBackend = opts.externalDbBackend;
|
|
204
|
-
configuredDialect = (opts.dialect || "postgres").toLowerCase();
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
if (typeof provider.ensureSchema === "function") {
|
|
208
|
-
await provider.ensureSchema();
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
initialized = true;
|
|
212
|
-
log("initialized as nodeId='" + nodeId + "', role='" + role + "'");
|
|
213
|
-
|
|
214
|
-
// Initial acquisition attempt (only if role === 'leader')
|
|
215
|
-
if (role === "leader") {
|
|
216
|
-
await _tryAcquire();
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
// Boot-time rollback detection on the audit + consent chains. Runs
|
|
220
|
-
// regardless of role — every node should refuse to participate in a
|
|
221
|
-
// cluster whose shared chains have been rolled back (a follower
|
|
222
|
-
// would face the same chain integrity failure if it later took
|
|
223
|
-
// over). Skipped when configuredExternalDbBackend is unset, which
|
|
224
|
-
// means a custom provider is in use without externalDb-resident
|
|
225
|
-
// framework state — the operator owns rollback detection in that
|
|
226
|
-
// case.
|
|
227
|
-
if (configuredExternalDbBackend) {
|
|
228
|
-
await _checkChainTipRollback("audit", "_blamejs_audit_log", "_blamejs_audit_tip");
|
|
229
|
-
await _checkChainTipRollback("consent", "_blamejs_consent_log", "_blamejs_consent_tip");
|
|
230
|
-
// Vault-key consistency: every node in a cluster must hold the
|
|
231
|
-
// SAME vault key. A node booting with a different key would seal
|
|
232
|
-
// new writes under a key the rest of the cluster can't unseal,
|
|
233
|
-
// and (on takeover) be unable to unseal the rest of the cluster's
|
|
234
|
-
// sealed columns — silent corruption. Compare a fingerprint of
|
|
235
|
-
// this node's vault keys against the canonical one stored at
|
|
236
|
-
// first cluster boot; refuse to participate on mismatch.
|
|
237
|
-
await _checkVaultKeyConsistency();
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
// Start heartbeat
|
|
241
|
-
heartbeatTimer = safeAsync.repeating(_heartbeat, heartbeatMs, { name: "cluster-heartbeat" });
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// Cluster-mode equivalent of db.js's single-node audit.tip-sidecar
|
|
245
|
-
// rollback check. Reads the persistent _blamejs_audit_tip row and
|
|
246
|
-
// compares to the current chain head in _blamejs_audit_log:
|
|
247
|
-
//
|
|
248
|
-
// - No tip row: first cluster boot or operator-cleared. Skip
|
|
249
|
-
// silently (matches the single-node sidecar-missing path).
|
|
250
|
-
// - Tip recorded a counter > current MAX: chain was truncated /
|
|
251
|
-
// restored from older snapshot. FATAL — refuse boot.
|
|
252
|
-
// - Tip recorded a hash that doesn't match the row at that
|
|
253
|
-
// counter: the row at that counter was substituted (different
|
|
254
|
-
// hash for same counter). FATAL — refuse boot.
|
|
255
|
-
//
|
|
256
|
-
// process.exit(1) is the framework's convention for boot-time
|
|
257
|
-
// integrity failures (audit chain, checkpoints, single-node
|
|
258
|
-
// rollback). Cluster mode keeps the same posture so operators see
|
|
259
|
-
// a single boot-time failure mode regardless of deployment shape.
|
|
260
|
-
// Generalized boot-time rollback check used by both audit and consent
|
|
261
|
-
// chains. chainName is the human-readable label included in log
|
|
262
|
-
// output ("audit" / "consent"). logTable is the chain table
|
|
263
|
-
// (_blamejs_audit_log / _blamejs_consent_log). tipTable is the
|
|
264
|
-
// single-row coordination table that records the latest counter +
|
|
265
|
-
// rowHash + fencingToken (_blamejs_audit_tip / _blamejs_consent_tip).
|
|
266
|
-
//
|
|
267
|
-
// Surfaces three outcomes:
|
|
268
|
-
// - tip table missing → operator running cluster gates-only mode
|
|
269
|
-
// (cluster wired for leader election but framework state still
|
|
270
|
-
// lives in per-node SQLite without `frameworkSchema.ensureSchema`);
|
|
271
|
-
// skip silently.
|
|
272
|
-
// - no tip row → first cluster boot or operator-cleared; skip.
|
|
273
|
-
// - currentMax < tipCounter, or tip rowHash != row-at-counter
|
|
274
|
-
// hash → FATAL via process.exit(1). Same posture as the
|
|
275
|
-
// single-node audit.tip sidecar rollback check.
|
|
276
|
-
async function _checkChainTipRollback(chainName, logTable, tipTable) {
|
|
277
|
-
// Both tables are framework-internal constants from the call sites
|
|
278
|
-
// (`_blamejs_audit_log`, `_blamejs_consent_log`, etc.). Validate +
|
|
279
|
-
// quote per the framework's identifier-quoting convention so a
|
|
280
|
-
// future rename can't silently break the query.
|
|
281
|
-
safeSql.validateIdentifier(logTable, { allowReserved: true });
|
|
282
|
-
safeSql.validateIdentifier(tipTable, { allowReserved: true });
|
|
283
|
-
var qLogTable = safeSql.quoteIdentifier(logTable);
|
|
284
|
-
var qTipTable = safeSql.quoteIdentifier(tipTable);
|
|
285
|
-
|
|
286
|
-
var tipRows;
|
|
287
|
-
try {
|
|
288
|
-
tipRows = await externalDb().query(
|
|
289
|
-
"SELECT atMonotonicCounter, rowHash FROM " + qTipTable +
|
|
290
|
-
" WHERE scope = " + (configuredDialect === "postgres" ? "$1" : "?"),
|
|
291
|
-
[chainName],
|
|
292
|
-
{ backend: configuredExternalDbBackend }
|
|
293
|
-
);
|
|
294
|
-
} catch (e) {
|
|
295
|
-
var msg = (e && e.message) || "";
|
|
296
|
-
if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
|
|
297
|
-
log(chainName + "-tip table not present — skipping rollback check (cluster gates-only mode)");
|
|
298
|
-
return;
|
|
299
|
-
}
|
|
300
|
-
throw e;
|
|
301
|
-
}
|
|
302
|
-
if (!tipRows.rows || tipRows.rows.length === 0) {
|
|
303
|
-
log("no " + chainName + "-tip row — skipping rollback check (first cluster boot or operator-cleared)");
|
|
304
|
-
return;
|
|
305
|
-
}
|
|
306
|
-
var tip = tipRows.rows[0];
|
|
307
|
-
var tipCounter = Number(tip.atMonotonicCounter);
|
|
308
|
-
var tipHash = tip.rowHash;
|
|
309
|
-
|
|
310
|
-
var currentRows = await externalDb().query(
|
|
311
|
-
"SELECT MAX(monotonicCounter) AS m FROM " + qLogTable,
|
|
312
|
-
[],
|
|
313
|
-
{ backend: configuredExternalDbBackend }
|
|
314
|
-
);
|
|
315
|
-
var currentMax = (currentRows.rows && currentRows.rows[0] && currentRows.rows[0].m)
|
|
316
|
-
? Number(currentRows.rows[0].m)
|
|
317
|
-
: 0;
|
|
318
|
-
|
|
319
|
-
if (currentMax < tipCounter) {
|
|
320
|
-
throw _err("ROLLBACK_DETECTED",
|
|
321
|
-
"FATAL: cluster-mode " + chainName + "-log rollback detected. " +
|
|
322
|
-
chainName + "-tip counter: " + tipCounter +
|
|
323
|
-
"; current external-db max: " + currentMax +
|
|
324
|
-
". Either external-db was restored from an older snapshot, or " +
|
|
325
|
-
logTable + " rows have been deleted. Investigate before continuing.",
|
|
326
|
-
true);
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
if (tipHash) {
|
|
330
|
-
var hashRows = await externalDb().query(
|
|
331
|
-
"SELECT rowHash FROM " + qLogTable + " WHERE monotonicCounter = " +
|
|
332
|
-
(configuredDialect === "postgres" ? "$1" : "?"),
|
|
333
|
-
[tipCounter],
|
|
334
|
-
{ backend: configuredExternalDbBackend }
|
|
335
|
-
);
|
|
336
|
-
if (hashRows.rows && hashRows.rows.length > 0) {
|
|
337
|
-
var rowAtTip = hashRows.rows[0].rowHash;
|
|
338
|
-
if (rowAtTip !== tipHash) {
|
|
339
|
-
throw _err("ROLLBACK_DETECTED",
|
|
340
|
-
"FATAL: cluster-mode " + chainName + "-log rollback detected (row-hash mismatch). " +
|
|
341
|
-
chainName + "-tip counter: " + tipCounter +
|
|
342
|
-
"; " + chainName + "-tip rowHash: " + tipHash +
|
|
343
|
-
"; current row rowHash: " + rowAtTip +
|
|
344
|
-
". The row at the recorded tip counter has a different hash — " +
|
|
345
|
-
"indicates row substitution at the chain head. Investigate before continuing.",
|
|
346
|
-
true);
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
log("cluster " + chainName + "-tip rollback check ok (tip counter " + tipCounter +
|
|
351
|
-
", current " + currentMax + ")");
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
// Compute a deterministic fingerprint of THIS node's vault keys.
|
|
355
|
-
// SHA3-512 of the concatenated public keys (PQC + classical halves of
|
|
356
|
-
// the hybrid encryption keypair). One-way: nothing about the private
|
|
357
|
-
// key material is recoverable from the fingerprint, so it's safe to
|
|
358
|
-
// store in the coordination table that all cluster nodes can read.
|
|
359
|
-
//
|
|
360
|
-
// Returns null if vault.init() hasn't been called — cluster gates-only
|
|
361
|
-
// mode (no sealed-column work) doesn't need this check, same defensive
|
|
362
|
-
// posture as the audit-tip rollback check skipping when there's no
|
|
363
|
-
// audit-tip table.
|
|
364
|
-
function _vaultKeyFingerprint() {
|
|
365
|
-
var keysJson;
|
|
366
|
-
try {
|
|
367
|
-
keysJson = vault().getKeysJson();
|
|
368
|
-
} catch (e) {
|
|
369
|
-
// vault.init() not called — gates-only mode. Skip silently.
|
|
370
|
-
if (/vault.init\(\) must be awaited/.test((e && e.message) || "")) {
|
|
371
|
-
return null;
|
|
372
|
-
}
|
|
373
|
-
throw e;
|
|
374
|
-
}
|
|
375
|
-
// vault.getKeysJson() returns the keys serialized as JSON (the same
|
|
376
|
-
// format vault writes to disk). Parse to extract the public halves;
|
|
377
|
-
// we never touch privateKey/ecPrivateKey here.
|
|
378
|
-
var keys = safeJson.parse(keysJson);
|
|
379
|
-
if (!keys || !keys.publicKey || !keys.ecPublicKey) return null;
|
|
380
|
-
// Domain-separation prefix so this fingerprint can't be confused
|
|
381
|
-
// with a hash of the same bytes computed elsewhere in the framework.
|
|
382
|
-
return crypto.sha3Hash("blamejs/cluster-state/v1\n" +
|
|
383
|
-
keys.publicKey + "\n" +
|
|
384
|
-
keys.ecPublicKey);
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
async function _checkVaultKeyConsistency() {
|
|
388
|
-
var localFp = _vaultKeyFingerprint();
|
|
389
|
-
if (localFp === null) {
|
|
390
|
-
log("vault not initialized — skipping vault-key consistency check (cluster gates-only mode)");
|
|
391
|
-
return;
|
|
392
|
-
}
|
|
393
|
-
var nowMs = Date.now();
|
|
394
|
-
var ph = configuredDialect === "postgres";
|
|
395
|
-
|
|
396
|
-
// First boot: try to record THIS node's fingerprint. ON CONFLICT DO
|
|
397
|
-
// NOTHING means the FIRST node to boot wins; subsequent nodes
|
|
398
|
-
// observe whatever's already there. Every node then SELECTs and
|
|
399
|
-
// compares — any mismatch (including ours after a losing race)
|
|
400
|
-
// surfaces the drift.
|
|
401
|
-
try {
|
|
402
|
-
await externalDb().query(
|
|
403
|
-
"INSERT INTO _blamejs_cluster_state " +
|
|
404
|
-
" (scope, vaultKeyFp, recordedAt, recordedByNode) " +
|
|
405
|
-
"VALUES ('state', " +
|
|
406
|
-
(ph ? "$1, $2, $3" : "?, ?, ?") + ") " +
|
|
407
|
-
"ON CONFLICT (scope) DO NOTHING",
|
|
408
|
-
[localFp, nowMs, nodeId],
|
|
409
|
-
{ backend: configuredExternalDbBackend }
|
|
410
|
-
);
|
|
411
|
-
} catch (e) {
|
|
412
|
-
// Table missing → the cluster-provider-db ensureSchema didn't run
|
|
413
|
-
// (custom provider that doesn't create _blamejs_cluster_state).
|
|
414
|
-
// Skip silently — same defensive posture as the audit-tip check.
|
|
415
|
-
var msg = (e && e.message) || "";
|
|
416
|
-
if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
|
|
417
|
-
log("cluster-state table not present — skipping vault-key consistency check (custom provider)");
|
|
418
|
-
return;
|
|
419
|
-
}
|
|
420
|
-
throw e;
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
// Read whatever fingerprint is canonical (ours if first boot,
|
|
424
|
-
// someone else's if we lost the race or are joining an existing cluster).
|
|
425
|
-
var rows = await externalDb().query(
|
|
426
|
-
"SELECT vaultKeyFp, recordedByNode, recordedAt FROM _blamejs_cluster_state " +
|
|
427
|
-
"WHERE scope = 'state'",
|
|
428
|
-
[],
|
|
429
|
-
{ backend: configuredExternalDbBackend }
|
|
430
|
-
);
|
|
431
|
-
if (!rows.rows || rows.rows.length === 0) {
|
|
432
|
-
// Should never happen — we just INSERTed. Surface as fatal so the
|
|
433
|
-
// condition isn't silently ignored.
|
|
434
|
-
throw _err("CLUSTER_STATE_MISSING",
|
|
435
|
-
"FATAL: cluster-state row missing immediately after INSERT — " +
|
|
436
|
-
"external-db may not be honoring writes. Refusing boot.",
|
|
437
|
-
true);
|
|
438
|
-
}
|
|
439
|
-
var canonical = rows.rows[0];
|
|
440
|
-
if (canonical.vaultKeyFp !== localFp) {
|
|
441
|
-
var fpPrefix = C.BYTES.bytes(16);
|
|
442
|
-
throw _err("VAULT_KEY_DRIFT",
|
|
443
|
-
"FATAL: vault-key drift detected. " +
|
|
444
|
-
"local node: " + nodeId +
|
|
445
|
-
"; local fingerprint: " + localFp.slice(0, fpPrefix) + "…" +
|
|
446
|
-
"; canonical recorded by: " + canonical.recordedByNode +
|
|
447
|
-
"; canonical fingerprint: " + canonical.vaultKeyFp.slice(0, fpPrefix) + "…" +
|
|
448
|
-
". This node holds a DIFFERENT vault key than the rest of the " +
|
|
449
|
-
"cluster. Sealed-column writes from this node would be unreadable " +
|
|
450
|
-
"by the others (and vice versa). Restore the same vault key file " +
|
|
451
|
-
"before booting this node into the cluster.",
|
|
452
|
-
true);
|
|
453
|
-
}
|
|
454
|
-
log("cluster vault-key consistency ok (fingerprint " +
|
|
455
|
-
localFp.slice(0, C.BYTES.bytes(16)) + "… recorded by " + canonical.recordedByNode + ")");
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
async function _tryAcquire() {
|
|
459
|
-
if (role !== "leader") return; // pinned-follower role: never claim
|
|
460
|
-
try {
|
|
461
|
-
var got = await provider.acquireLease(nodeId, leaseTtlMs, {
|
|
462
|
-
endpoint: configuredEndpoint,
|
|
463
|
-
});
|
|
464
|
-
if (got) {
|
|
465
|
-
var wasLeader = !!lease;
|
|
466
|
-
lease = got;
|
|
467
|
-
if (!wasLeader) {
|
|
468
|
-
log("acquired lease — fencingToken=" + lease.fencingToken);
|
|
469
|
-
_emitTransition("lease-acquired", { fencingToken: lease.fencingToken });
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
} catch (e) {
|
|
473
|
-
log.error("acquire failed: " + e.message);
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
async function _heartbeat() {
|
|
478
|
-
if (!initialized) return;
|
|
479
|
-
if (!lease) {
|
|
480
|
-
// Not currently leader — try to acquire (lease may have expired
|
|
481
|
-
// on the previous holder).
|
|
482
|
-
await _tryAcquire();
|
|
483
|
-
return;
|
|
484
|
-
}
|
|
485
|
-
// We hold a lease — renew it. Re-supply the configured endpoint so a
|
|
486
|
-
// hot-reload of the operator's config (e.g. node moves to a new
|
|
487
|
-
// routable URL after a restart) eventually reaches the discovery row.
|
|
488
|
-
try {
|
|
489
|
-
lease = await provider.renewLease(lease, { endpoint: configuredEndpoint });
|
|
490
|
-
} catch (e) {
|
|
491
|
-
if (e.code === "LEASE_LOST") {
|
|
492
|
-
log.error("lease lost: " + e.message);
|
|
493
|
-
var lostToken = lease ? lease.fencingToken : null;
|
|
494
|
-
lease = null;
|
|
495
|
-
_emitTransition("lease-lost", { fencingToken: lostToken });
|
|
496
|
-
// Attempt to re-acquire on the next heartbeat naturally.
|
|
497
|
-
} else {
|
|
498
|
-
// Transient error — retry on next heartbeat. If it persists past
|
|
499
|
-
// leaseTtl another node will steal, and we'll detect via LEASE_LOST.
|
|
500
|
-
log.error("renew failed transiently: " + e.message);
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
// ---- public sync surface ----
|
|
506
|
-
|
|
507
|
-
function isLeader() {
|
|
508
|
-
if (terminated) return false; // post-shutdown: never leader
|
|
509
|
-
if (!initialized) return true; // never-initialized: permanent leader
|
|
510
|
-
return !!lease && Date.now() < lease.expiresAt;
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
// Has cluster.init been called with a real configuration? Used by
|
|
514
|
-
// write-dispatch code (audit, consent, …) to decide whether framework
|
|
515
|
-
// state should go to local SQLite or external-db.
|
|
516
|
-
function isClusterMode() {
|
|
517
|
-
return initialized && !!configuredExternalDbBackend;
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
function externalDbBackend() {
|
|
521
|
-
return configuredExternalDbBackend;
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
function dialect() {
|
|
525
|
-
return configuredDialect;
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
function currentNodeId() {
|
|
529
|
-
return initialized ? nodeId : "single-node-local";
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
// This node's routable endpoint (operator-configured at cluster.init).
|
|
533
|
-
// Returns null when not configured or in single-node fallback. External
|
|
534
|
-
// observers should call discoveryHandler() / currentLeader() instead —
|
|
535
|
-
// this getter is for the local node's own self-identity.
|
|
536
|
-
function endpoint() {
|
|
537
|
-
return configuredEndpoint;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
function fencingToken() {
|
|
541
|
-
if (!initialized) return 0;
|
|
542
|
-
return lease ? lease.fencingToken : 0;
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
function requireLeader() {
|
|
546
|
-
if (!isLeader()) {
|
|
547
|
-
throw new NotLeaderError(
|
|
548
|
-
"node '" + currentNodeId() + "' is not currently leader" +
|
|
549
|
-
(initialized ? "" : " (cluster not initialized)")
|
|
550
|
-
);
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
async function currentLeader() {
|
|
555
|
-
if (!initialized) {
|
|
556
|
-
return {
|
|
557
|
-
nodeId: "single-node-local",
|
|
558
|
-
leaseExpiresAt: Infinity,
|
|
559
|
-
fencingToken: 0,
|
|
560
|
-
endpoint: null,
|
|
561
|
-
};
|
|
562
|
-
}
|
|
563
|
-
return await provider.currentLeader();
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
// HTTP request handler — replies with the current cluster leader for
|
|
567
|
-
// service-mesh / load-balancer discovery. Operators mount this at
|
|
568
|
-
// whatever route they want (`/cluster/leader`, `/health/leader`, etc.).
|
|
569
|
-
//
|
|
570
|
-
// 200 application/json — leader present
|
|
571
|
-
// { leader: { nodeId, endpoint, fencingToken, leaseExpiresAt },
|
|
572
|
-
// self: { nodeId, endpoint, isLeader } }
|
|
573
|
-
//
|
|
574
|
-
// 503 application/json — no leader (no row, expired lease, DB
|
|
575
|
-
// unreachable, single-node not initialized with cluster mode)
|
|
576
|
-
// { leader: null, self: { nodeId, endpoint, isLeader } }
|
|
577
|
-
//
|
|
578
|
-
// No auth — this endpoint is intended to be called by infrastructure
|
|
579
|
-
// inside the trust boundary (LB, healthcheck, dashboard). Operators
|
|
580
|
-
// who expose it externally should layer auth via their own middleware.
|
|
581
|
-
//
|
|
582
|
-
// Handler is method-agnostic so it works behind any HTTP probe shape
|
|
583
|
-
// (GET, HEAD, etc.). Cache-Control: no-store to avoid stale-leader
|
|
584
|
-
// responses pinned by a caching proxy during a takeover.
|
|
585
|
-
function discoveryHandler() {
|
|
586
|
-
return async function (req, res) {
|
|
587
|
-
var selfInfo = {
|
|
588
|
-
nodeId: currentNodeId(),
|
|
589
|
-
endpoint: configuredEndpoint,
|
|
590
|
-
isLeader: isLeader(),
|
|
591
|
-
};
|
|
592
|
-
var body;
|
|
593
|
-
var status;
|
|
594
|
-
try {
|
|
595
|
-
var leader = await currentLeader();
|
|
596
|
-
if (leader && leader.nodeId && leader.nodeId !== "single-node-local") {
|
|
597
|
-
body = { leader: leader, self: selfInfo };
|
|
598
|
-
status = 200;
|
|
599
|
-
} else if (leader && leader.nodeId === "single-node-local") {
|
|
600
|
-
// Permanent-leader fallback (cluster.init never called). Reply
|
|
601
|
-
// 200 — the operator's app is healthy and the "leader" is this
|
|
602
|
-
// process. Useful so the discovery endpoint is never a false
|
|
603
|
-
// negative in single-node deployments.
|
|
604
|
-
body = { leader: leader, self: selfInfo };
|
|
605
|
-
status = 200;
|
|
606
|
-
} else {
|
|
607
|
-
body = { leader: null, self: selfInfo };
|
|
608
|
-
status = 503;
|
|
609
|
-
}
|
|
610
|
-
} catch (e) {
|
|
611
|
-
body = { leader: null, self: selfInfo, error: e.message };
|
|
612
|
-
status = 503;
|
|
613
|
-
}
|
|
614
|
-
var json = JSON.stringify(body);
|
|
615
|
-
res.writeHead(status, {
|
|
616
|
-
"Content-Type": "application/json; charset=utf-8",
|
|
617
|
-
"Content-Length": Buffer.byteLength(json),
|
|
618
|
-
"Cache-Control": "no-store",
|
|
619
|
-
});
|
|
620
|
-
res.end(json);
|
|
621
|
-
};
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
function onTransition(handler) {
|
|
625
|
-
if (typeof handler !== "function") {
|
|
626
|
-
throw _err("INVALID_HANDLER", "onTransition expects a function", true);
|
|
627
|
-
}
|
|
628
|
-
transitionHandlers.push(handler);
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
async function shutdown() {
|
|
632
|
-
if (!initialized) return;
|
|
633
|
-
if (heartbeatTimer) {
|
|
634
|
-
heartbeatTimer.stop();
|
|
635
|
-
heartbeatTimer = null;
|
|
636
|
-
}
|
|
637
|
-
if (lease) {
|
|
638
|
-
try {
|
|
639
|
-
await provider.releaseLease(lease);
|
|
640
|
-
_emitTransition("lease-released", { fencingToken: lease.fencingToken });
|
|
641
|
-
log("lease released on shutdown");
|
|
642
|
-
} catch (e) {
|
|
643
|
-
log.error("release on shutdown failed: " + e.message);
|
|
644
|
-
}
|
|
645
|
-
lease = null;
|
|
646
|
-
}
|
|
647
|
-
initialized = false;
|
|
648
|
-
terminated = true;
|
|
649
|
-
provider = null;
|
|
650
|
-
role = null;
|
|
651
|
-
leaseTtlMs = null;
|
|
652
|
-
heartbeatMs = null;
|
|
653
|
-
configuredExternalDbBackend = null;
|
|
654
|
-
configuredDialect = null;
|
|
655
|
-
configuredEndpoint = null;
|
|
656
|
-
transitionHandlers = [];
|
|
657
|
-
// nodeId is preserved post-shutdown so audit metadata still reflects
|
|
658
|
-
// who this process was; cleared only by _resetForTest.
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
// ---- test helpers — not part of public contract ----
|
|
662
|
-
|
|
663
|
-
function _resetForTest() {
|
|
664
|
-
if (heartbeatTimer) heartbeatTimer.stop();
|
|
665
|
-
heartbeatTimer = null;
|
|
666
|
-
initialized = false;
|
|
667
|
-
terminated = false;
|
|
668
|
-
nodeId = null;
|
|
669
|
-
role = null;
|
|
670
|
-
provider = null;
|
|
671
|
-
lease = null;
|
|
672
|
-
leaseTtlMs = null;
|
|
673
|
-
heartbeatMs = null;
|
|
674
|
-
configuredExternalDbBackend = null;
|
|
675
|
-
configuredDialect = null;
|
|
676
|
-
configuredEndpoint = null;
|
|
677
|
-
transitionHandlers = [];
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
async function _heartbeatNowForTest() {
|
|
681
|
-
// Drive one heartbeat synchronously without waiting for the timer —
|
|
682
|
-
// lets tests deterministically observe lease state transitions.
|
|
683
|
-
await _heartbeat();
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
module.exports = {
|
|
687
|
-
init: init,
|
|
688
|
-
isLeader: isLeader,
|
|
689
|
-
isClusterMode: isClusterMode,
|
|
690
|
-
externalDbBackend: externalDbBackend,
|
|
691
|
-
dialect: dialect,
|
|
692
|
-
currentNodeId: currentNodeId,
|
|
693
|
-
endpoint: endpoint,
|
|
694
|
-
fencingToken: fencingToken,
|
|
695
|
-
requireLeader: requireLeader,
|
|
696
|
-
currentLeader: currentLeader,
|
|
697
|
-
discoveryHandler: discoveryHandler,
|
|
698
|
-
onTransition: onTransition,
|
|
699
|
-
shutdown: shutdown,
|
|
700
|
-
NotLeaderError: NotLeaderError,
|
|
701
|
-
_resetForTest: _resetForTest,
|
|
702
|
-
_heartbeatNowForTest: _heartbeatNowForTest,
|
|
703
|
-
};
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Cluster coordination — leader election + fencing tokens.
|
|
4
|
+
*
|
|
5
|
+
* Opt-in via `b.cluster.init(...)`. When init is never called, the
|
|
6
|
+
* local process behaves as a permanent single leader: `isLeader()`
|
|
7
|
+
* always returns true, `fencingToken()` returns 0, no heartbeat thread
|
|
8
|
+
* runs, no DB is touched. Single-node deployments pay zero overhead.
|
|
9
|
+
*
|
|
10
|
+
* When init IS called, the framework starts a heartbeat that renews
|
|
11
|
+
* the leader lease via the configured provider. On lease loss (network
|
|
12
|
+
* partition, takeover, lease expiry) the node transitions to follower
|
|
13
|
+
* and write-side framework primitives throw `NotLeaderError`.
|
|
14
|
+
*
|
|
15
|
+
* Threat model:
|
|
16
|
+
* - Two leaders writing simultaneously: prevented by fencing tokens.
|
|
17
|
+
* Every leader-only DB write includes the current token; a
|
|
18
|
+
* CHECK constraint on the audit-tip row rejects a stale token.
|
|
19
|
+
* The application-layer `requireLeader()` gate is just an early
|
|
20
|
+
* rejection optimisation; the DB constraint is the canonical guard.
|
|
21
|
+
* - Follower receiving a write: rejected at the framework boundary.
|
|
22
|
+
* Operators front the cluster with a load balancer that routes
|
|
23
|
+
* write paths to the current leader.
|
|
24
|
+
* - External-db unreachable: heartbeat fails; after `leaseTtl` no
|
|
25
|
+
* leader exists and writes fail closed. When the DB recovers,
|
|
26
|
+
* election resumes.
|
|
27
|
+
*
|
|
28
|
+
* Public API:
|
|
29
|
+
* await cluster.init(opts) one-time bootstrap
|
|
30
|
+
* cluster.isLeader() sync; true on leader (or single-node)
|
|
31
|
+
* cluster.currentNodeId() sync; configured nodeId
|
|
32
|
+
* cluster.endpoint() sync; this node's routable URL
|
|
33
|
+
* (operator-supplied at init), or
|
|
34
|
+
* null if unconfigured. Stored in
|
|
35
|
+
* the leader-election row so
|
|
36
|
+
* external observers can resolve
|
|
37
|
+
* "where is the current leader?"
|
|
38
|
+
* cluster.fencingToken() sync; current monotonic token
|
|
39
|
+
* cluster.requireLeader() sync; throws NotLeaderError
|
|
40
|
+
* cluster.currentLeader() async; { nodeId, leaseExpiresAt,
|
|
41
|
+
* fencingToken,
|
|
42
|
+
* endpoint } | null
|
|
43
|
+
* cluster.discoveryHandler() returns an HTTP request handler
|
|
44
|
+
* (req, res) → JSON. Mount on any
|
|
45
|
+
* route to expose the current
|
|
46
|
+
* leader for service-mesh / LB
|
|
47
|
+
* consumption. 200 with leader,
|
|
48
|
+
* 503 with `{ leader: null }`
|
|
49
|
+
* when no leader.
|
|
50
|
+
* cluster.onTransition(fn) register transition handler
|
|
51
|
+
* await cluster.shutdown() releases lease, stops heartbeat
|
|
52
|
+
*/
|
|
53
|
+
var C = require("./constants");
|
|
54
|
+
var clusterProviderDb = require("./cluster-provider-db");
|
|
55
|
+
var crypto = require("./crypto");
|
|
56
|
+
var lazyRequire = require("./lazy-require");
|
|
57
|
+
var { boot } = require("./log");
|
|
58
|
+
var safeAsync = require("./safe-async");
|
|
59
|
+
var safeJson = require("./safe-json");
|
|
60
|
+
var safeSql = require("./safe-sql");
|
|
61
|
+
var safeUrl = require("./safe-url");
|
|
62
|
+
var { FrameworkError, ClusterError } = require("./framework-error");
|
|
63
|
+
|
|
64
|
+
// Lazy: vault → db → cluster forms a load-time chain, and external-db is
|
|
65
|
+
// loaded before its init has run; both are safe to call once cluster
|
|
66
|
+
// reaches runtime, but eager require here would deadlock the load order.
|
|
67
|
+
var externalDb = lazyRequire(function () { return require("./external-db"); });
|
|
68
|
+
var vault = lazyRequire(function () { return require("./vault"); });
|
|
69
|
+
|
|
70
|
+
var DEFAULT_LEASE_TTL = C.TIME.seconds(30);
|
|
71
|
+
var DEFAULT_HEARTBEAT = C.TIME.seconds(10);
|
|
72
|
+
var MIN_LEASE_TTL = C.TIME.seconds(5);
|
|
73
|
+
var MIN_HEARTBEAT = C.TIME.seconds(1);
|
|
74
|
+
|
|
75
|
+
var initialized = false;
|
|
76
|
+
var terminated = false; // set true by shutdown() so the
|
|
77
|
+
// permanent-leader fallback isn't
|
|
78
|
+
// re-engaged after a graceful exit
|
|
79
|
+
var nodeId = null;
|
|
80
|
+
var role = null; // 'leader' | 'follower'
|
|
81
|
+
var provider = null;
|
|
82
|
+
var lease = null; // current lease (if leader)
|
|
83
|
+
var heartbeatTimer = null;
|
|
84
|
+
var heartbeatMs = null;
|
|
85
|
+
var leaseTtlMs = null;
|
|
86
|
+
var transitionHandlers = [];
|
|
87
|
+
// Backend coordinates for write-dispatch code in audit/consent/etc.
|
|
88
|
+
// These are set when cluster.init is called with `externalDbBackend`
|
|
89
|
+
// (the default DB-row provider path); operators using a custom
|
|
90
|
+
// provider can set them via init opts directly.
|
|
91
|
+
var configuredExternalDbBackend = null;
|
|
92
|
+
var configuredDialect = null;
|
|
93
|
+
// Operator-supplied routable endpoint for THIS node, used by external
|
|
94
|
+
// load balancers / service meshes to learn where to send write traffic.
|
|
95
|
+
// Stored in the leader-election row on every acquire/renew so any node
|
|
96
|
+
// (or external observer) can resolve "where is the current leader?"
|
|
97
|
+
// via cluster.currentLeader() / cluster.discoveryHandler().
|
|
98
|
+
var configuredEndpoint = null;
|
|
99
|
+
|
|
100
|
+
var log = boot("cluster");
|
|
101
|
+
|
|
102
|
+
class NotLeaderError extends FrameworkError {
|
|
103
|
+
constructor(message) {
|
|
104
|
+
super(message || "not leader: write rejected by cluster gate", "NOT_LEADER");
|
|
105
|
+
this.name = "NotLeaderError";
|
|
106
|
+
this.statusCode = 503; // operator's load balancer should retry on the leader
|
|
107
|
+
this.isClusterError = true;
|
|
108
|
+
this.isNotLeaderError = true;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
var _err = ClusterError.factory;
|
|
113
|
+
|
|
114
|
+
function _emitTransition(kind, detail) {
|
|
115
|
+
var event = Object.assign({ kind: kind, nodeId: nodeId, at: Date.now() }, detail || {});
|
|
116
|
+
for (var i = 0; i < transitionHandlers.length; i++) {
|
|
117
|
+
try { transitionHandlers[i](event); }
|
|
118
|
+
catch (e) { log.error("transition handler threw: " + e.message); }
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// ---- init ----
|
|
123
|
+
|
|
124
|
+
async function init(opts) {
|
|
125
|
+
if (initialized) {
|
|
126
|
+
throw _err("ALREADY_INITIALIZED", "cluster.init() called twice", true);
|
|
127
|
+
}
|
|
128
|
+
opts = opts || {};
|
|
129
|
+
if (!opts.nodeId) {
|
|
130
|
+
throw _err("INVALID_CONFIG", "cluster.init({ nodeId }) is required", true);
|
|
131
|
+
}
|
|
132
|
+
nodeId = String(opts.nodeId);
|
|
133
|
+
|
|
134
|
+
leaseTtlMs = opts.leaseTtl != null ? Number(opts.leaseTtl) : DEFAULT_LEASE_TTL;
|
|
135
|
+
if (leaseTtlMs < MIN_LEASE_TTL) {
|
|
136
|
+
throw _err("INVALID_TTL",
|
|
137
|
+
"leaseTtl must be >= " + MIN_LEASE_TTL + "ms (got " + leaseTtlMs + ")",
|
|
138
|
+
true);
|
|
139
|
+
}
|
|
140
|
+
heartbeatMs = opts.heartbeatInterval != null
|
|
141
|
+
? Number(opts.heartbeatInterval)
|
|
142
|
+
: DEFAULT_HEARTBEAT;
|
|
143
|
+
if (heartbeatMs < MIN_HEARTBEAT) {
|
|
144
|
+
throw _err("INVALID_HEARTBEAT",
|
|
145
|
+
"heartbeatInterval must be >= " + MIN_HEARTBEAT + "ms (got " + heartbeatMs + ")",
|
|
146
|
+
true);
|
|
147
|
+
}
|
|
148
|
+
if (heartbeatMs >= leaseTtlMs) {
|
|
149
|
+
throw _err("INVALID_HEARTBEAT",
|
|
150
|
+
"heartbeatInterval must be < leaseTtl (got heartbeat=" + heartbeatMs +
|
|
151
|
+
", leaseTtl=" + leaseTtlMs + "); recommend ~1/3 of leaseTtl",
|
|
152
|
+
true);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
role = (opts.role || "leader").toLowerCase();
|
|
156
|
+
if (role !== "leader" && role !== "follower") {
|
|
157
|
+
throw _err("INVALID_ROLE", "role must be 'leader' or 'follower'", true);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Optional endpoint. If provided, validate scheme + shape via url-safe
|
|
161
|
+
// — HTTPS-only by default since this is the URL external services use
|
|
162
|
+
// to reach the leader. Operators with internal cleartext clusters opt
|
|
163
|
+
// in via opts.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
|
|
164
|
+
if (opts.endpoint != null) {
|
|
165
|
+
try {
|
|
166
|
+
safeUrl.parse(opts.endpoint, {
|
|
167
|
+
allowedProtocols: opts.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
|
|
168
|
+
errorClass: ClusterError,
|
|
169
|
+
});
|
|
170
|
+
} catch (e) {
|
|
171
|
+
// Re-throw with a config-shaped error so operators see the cluster.init
|
|
172
|
+
// boundary, not a bare url-safe trace.
|
|
173
|
+
throw _err("INVALID_ENDPOINT",
|
|
174
|
+
"cluster.init({ endpoint }) rejected: " + e.message, true);
|
|
175
|
+
}
|
|
176
|
+
configuredEndpoint = String(opts.endpoint);
|
|
177
|
+
} else {
|
|
178
|
+
configuredEndpoint = null;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (typeof opts.onTransition === "function") {
|
|
182
|
+
transitionHandlers.push(opts.onTransition);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Provider: either operator-supplied, or build the default DB-row
|
|
186
|
+
// provider against an externalDb backend.
|
|
187
|
+
if (opts.provider) {
|
|
188
|
+
provider = opts.provider;
|
|
189
|
+
// Operator-custom provider: they may still be writing framework
|
|
190
|
+
// state to an externalDb backend, in which case they pass these
|
|
191
|
+
// separately so write-dispatch code knows where to go.
|
|
192
|
+
configuredExternalDbBackend = opts.externalDbBackend || null;
|
|
193
|
+
configuredDialect = (opts.dialect || "postgres").toLowerCase();
|
|
194
|
+
} else {
|
|
195
|
+
if (!opts.externalDbBackend) {
|
|
196
|
+
throw _err("INVALID_CONFIG",
|
|
197
|
+
"cluster.init requires either { provider } or { externalDbBackend }", true);
|
|
198
|
+
}
|
|
199
|
+
provider = clusterProviderDb.create({
|
|
200
|
+
externalDbBackend: opts.externalDbBackend,
|
|
201
|
+
dialect: opts.dialect,
|
|
202
|
+
});
|
|
203
|
+
configuredExternalDbBackend = opts.externalDbBackend;
|
|
204
|
+
configuredDialect = (opts.dialect || "postgres").toLowerCase();
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
if (typeof provider.ensureSchema === "function") {
|
|
208
|
+
await provider.ensureSchema();
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
initialized = true;
|
|
212
|
+
log("initialized as nodeId='" + nodeId + "', role='" + role + "'");
|
|
213
|
+
|
|
214
|
+
// Initial acquisition attempt (only if role === 'leader')
|
|
215
|
+
if (role === "leader") {
|
|
216
|
+
await _tryAcquire();
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// Boot-time rollback detection on the audit + consent chains. Runs
|
|
220
|
+
// regardless of role — every node should refuse to participate in a
|
|
221
|
+
// cluster whose shared chains have been rolled back (a follower
|
|
222
|
+
// would face the same chain integrity failure if it later took
|
|
223
|
+
// over). Skipped when configuredExternalDbBackend is unset, which
|
|
224
|
+
// means a custom provider is in use without externalDb-resident
|
|
225
|
+
// framework state — the operator owns rollback detection in that
|
|
226
|
+
// case.
|
|
227
|
+
if (configuredExternalDbBackend) {
|
|
228
|
+
await _checkChainTipRollback("audit", "_blamejs_audit_log", "_blamejs_audit_tip");
|
|
229
|
+
await _checkChainTipRollback("consent", "_blamejs_consent_log", "_blamejs_consent_tip");
|
|
230
|
+
// Vault-key consistency: every node in a cluster must hold the
|
|
231
|
+
// SAME vault key. A node booting with a different key would seal
|
|
232
|
+
// new writes under a key the rest of the cluster can't unseal,
|
|
233
|
+
// and (on takeover) be unable to unseal the rest of the cluster's
|
|
234
|
+
// sealed columns — silent corruption. Compare a fingerprint of
|
|
235
|
+
// this node's vault keys against the canonical one stored at
|
|
236
|
+
// first cluster boot; refuse to participate on mismatch.
|
|
237
|
+
await _checkVaultKeyConsistency();
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Start heartbeat
|
|
241
|
+
heartbeatTimer = safeAsync.repeating(_heartbeat, heartbeatMs, { name: "cluster-heartbeat" });
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Cluster-mode equivalent of db.js's single-node audit.tip-sidecar
|
|
245
|
+
// rollback check. Reads the persistent _blamejs_audit_tip row and
|
|
246
|
+
// compares to the current chain head in _blamejs_audit_log:
|
|
247
|
+
//
|
|
248
|
+
// - No tip row: first cluster boot or operator-cleared. Skip
|
|
249
|
+
// silently (matches the single-node sidecar-missing path).
|
|
250
|
+
// - Tip recorded a counter > current MAX: chain was truncated /
|
|
251
|
+
// restored from older snapshot. FATAL — refuse boot.
|
|
252
|
+
// - Tip recorded a hash that doesn't match the row at that
|
|
253
|
+
// counter: the row at that counter was substituted (different
|
|
254
|
+
// hash for same counter). FATAL — refuse boot.
|
|
255
|
+
//
|
|
256
|
+
// process.exit(1) is the framework's convention for boot-time
|
|
257
|
+
// integrity failures (audit chain, checkpoints, single-node
|
|
258
|
+
// rollback). Cluster mode keeps the same posture so operators see
|
|
259
|
+
// a single boot-time failure mode regardless of deployment shape.
|
|
260
|
+
// Generalized boot-time rollback check used by both audit and consent
|
|
261
|
+
// chains. chainName is the human-readable label included in log
|
|
262
|
+
// output ("audit" / "consent"). logTable is the chain table
|
|
263
|
+
// (_blamejs_audit_log / _blamejs_consent_log). tipTable is the
|
|
264
|
+
// single-row coordination table that records the latest counter +
|
|
265
|
+
// rowHash + fencingToken (_blamejs_audit_tip / _blamejs_consent_tip).
|
|
266
|
+
//
|
|
267
|
+
// Surfaces three outcomes:
|
|
268
|
+
// - tip table missing → operator running cluster gates-only mode
|
|
269
|
+
// (cluster wired for leader election but framework state still
|
|
270
|
+
// lives in per-node SQLite without `frameworkSchema.ensureSchema`);
|
|
271
|
+
// skip silently.
|
|
272
|
+
// - no tip row → first cluster boot or operator-cleared; skip.
|
|
273
|
+
// - currentMax < tipCounter, or tip rowHash != row-at-counter
|
|
274
|
+
// hash → FATAL via process.exit(1). Same posture as the
|
|
275
|
+
// single-node audit.tip sidecar rollback check.
|
|
276
|
+
async function _checkChainTipRollback(chainName, logTable, tipTable) {
|
|
277
|
+
// Both tables are framework-internal constants from the call sites
|
|
278
|
+
// (`_blamejs_audit_log`, `_blamejs_consent_log`, etc.). Validate +
|
|
279
|
+
// quote per the framework's identifier-quoting convention so a
|
|
280
|
+
// future rename can't silently break the query.
|
|
281
|
+
safeSql.validateIdentifier(logTable, { allowReserved: true });
|
|
282
|
+
safeSql.validateIdentifier(tipTable, { allowReserved: true });
|
|
283
|
+
var qLogTable = safeSql.quoteIdentifier(logTable);
|
|
284
|
+
var qTipTable = safeSql.quoteIdentifier(tipTable);
|
|
285
|
+
|
|
286
|
+
var tipRows;
|
|
287
|
+
try {
|
|
288
|
+
tipRows = await externalDb().query(
|
|
289
|
+
"SELECT atMonotonicCounter, rowHash FROM " + qTipTable +
|
|
290
|
+
" WHERE scope = " + (configuredDialect === "postgres" ? "$1" : "?"),
|
|
291
|
+
[chainName],
|
|
292
|
+
{ backend: configuredExternalDbBackend }
|
|
293
|
+
);
|
|
294
|
+
} catch (e) {
|
|
295
|
+
var msg = (e && e.message) || "";
|
|
296
|
+
if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
|
|
297
|
+
log(chainName + "-tip table not present — skipping rollback check (cluster gates-only mode)");
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
throw e;
|
|
301
|
+
}
|
|
302
|
+
if (!tipRows.rows || tipRows.rows.length === 0) {
|
|
303
|
+
log("no " + chainName + "-tip row — skipping rollback check (first cluster boot or operator-cleared)");
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
var tip = tipRows.rows[0];
|
|
307
|
+
var tipCounter = Number(tip.atMonotonicCounter);
|
|
308
|
+
var tipHash = tip.rowHash;
|
|
309
|
+
|
|
310
|
+
var currentRows = await externalDb().query(
|
|
311
|
+
"SELECT MAX(monotonicCounter) AS m FROM " + qLogTable,
|
|
312
|
+
[],
|
|
313
|
+
{ backend: configuredExternalDbBackend }
|
|
314
|
+
);
|
|
315
|
+
var currentMax = (currentRows.rows && currentRows.rows[0] && currentRows.rows[0].m)
|
|
316
|
+
? Number(currentRows.rows[0].m)
|
|
317
|
+
: 0;
|
|
318
|
+
|
|
319
|
+
if (currentMax < tipCounter) {
|
|
320
|
+
throw _err("ROLLBACK_DETECTED",
|
|
321
|
+
"FATAL: cluster-mode " + chainName + "-log rollback detected. " +
|
|
322
|
+
chainName + "-tip counter: " + tipCounter +
|
|
323
|
+
"; current external-db max: " + currentMax +
|
|
324
|
+
". Either external-db was restored from an older snapshot, or " +
|
|
325
|
+
logTable + " rows have been deleted. Investigate before continuing.",
|
|
326
|
+
true);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (tipHash) {
|
|
330
|
+
var hashRows = await externalDb().query(
|
|
331
|
+
"SELECT rowHash FROM " + qLogTable + " WHERE monotonicCounter = " +
|
|
332
|
+
(configuredDialect === "postgres" ? "$1" : "?"),
|
|
333
|
+
[tipCounter],
|
|
334
|
+
{ backend: configuredExternalDbBackend }
|
|
335
|
+
);
|
|
336
|
+
if (hashRows.rows && hashRows.rows.length > 0) {
|
|
337
|
+
var rowAtTip = hashRows.rows[0].rowHash;
|
|
338
|
+
if (rowAtTip !== tipHash) {
|
|
339
|
+
throw _err("ROLLBACK_DETECTED",
|
|
340
|
+
"FATAL: cluster-mode " + chainName + "-log rollback detected (row-hash mismatch). " +
|
|
341
|
+
chainName + "-tip counter: " + tipCounter +
|
|
342
|
+
"; " + chainName + "-tip rowHash: " + tipHash +
|
|
343
|
+
"; current row rowHash: " + rowAtTip +
|
|
344
|
+
". The row at the recorded tip counter has a different hash — " +
|
|
345
|
+
"indicates row substitution at the chain head. Investigate before continuing.",
|
|
346
|
+
true);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
log("cluster " + chainName + "-tip rollback check ok (tip counter " + tipCounter +
|
|
351
|
+
", current " + currentMax + ")");
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Compute a deterministic fingerprint of THIS node's vault keys.
|
|
355
|
+
// SHA3-512 of the concatenated public keys (PQC + classical halves of
|
|
356
|
+
// the hybrid encryption keypair). One-way: nothing about the private
|
|
357
|
+
// key material is recoverable from the fingerprint, so it's safe to
|
|
358
|
+
// store in the coordination table that all cluster nodes can read.
|
|
359
|
+
//
|
|
360
|
+
// Returns null if vault.init() hasn't been called — cluster gates-only
|
|
361
|
+
// mode (no sealed-column work) doesn't need this check, same defensive
|
|
362
|
+
// posture as the audit-tip rollback check skipping when there's no
|
|
363
|
+
// audit-tip table.
|
|
364
|
+
function _vaultKeyFingerprint() {
|
|
365
|
+
var keysJson;
|
|
366
|
+
try {
|
|
367
|
+
keysJson = vault().getKeysJson();
|
|
368
|
+
} catch (e) {
|
|
369
|
+
// vault.init() not called — gates-only mode. Skip silently.
|
|
370
|
+
if (/vault.init\(\) must be awaited/.test((e && e.message) || "")) {
|
|
371
|
+
return null;
|
|
372
|
+
}
|
|
373
|
+
throw e;
|
|
374
|
+
}
|
|
375
|
+
// vault.getKeysJson() returns the keys serialized as JSON (the same
|
|
376
|
+
// format vault writes to disk). Parse to extract the public halves;
|
|
377
|
+
// we never touch privateKey/ecPrivateKey here.
|
|
378
|
+
var keys = safeJson.parse(keysJson);
|
|
379
|
+
if (!keys || !keys.publicKey || !keys.ecPublicKey) return null;
|
|
380
|
+
// Domain-separation prefix so this fingerprint can't be confused
|
|
381
|
+
// with a hash of the same bytes computed elsewhere in the framework.
|
|
382
|
+
return crypto.sha3Hash("blamejs/cluster-state/v1\n" +
|
|
383
|
+
keys.publicKey + "\n" +
|
|
384
|
+
keys.ecPublicKey);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
async function _checkVaultKeyConsistency() {
|
|
388
|
+
var localFp = _vaultKeyFingerprint();
|
|
389
|
+
if (localFp === null) {
|
|
390
|
+
log("vault not initialized — skipping vault-key consistency check (cluster gates-only mode)");
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
var nowMs = Date.now();
|
|
394
|
+
var ph = configuredDialect === "postgres";
|
|
395
|
+
|
|
396
|
+
// First boot: try to record THIS node's fingerprint. ON CONFLICT DO
|
|
397
|
+
// NOTHING means the FIRST node to boot wins; subsequent nodes
|
|
398
|
+
// observe whatever's already there. Every node then SELECTs and
|
|
399
|
+
// compares — any mismatch (including ours after a losing race)
|
|
400
|
+
// surfaces the drift.
|
|
401
|
+
try {
|
|
402
|
+
await externalDb().query(
|
|
403
|
+
"INSERT INTO _blamejs_cluster_state " +
|
|
404
|
+
" (scope, vaultKeyFp, recordedAt, recordedByNode) " +
|
|
405
|
+
"VALUES ('state', " +
|
|
406
|
+
(ph ? "$1, $2, $3" : "?, ?, ?") + ") " +
|
|
407
|
+
"ON CONFLICT (scope) DO NOTHING",
|
|
408
|
+
[localFp, nowMs, nodeId],
|
|
409
|
+
{ backend: configuredExternalDbBackend }
|
|
410
|
+
);
|
|
411
|
+
} catch (e) {
|
|
412
|
+
// Table missing → the cluster-provider-db ensureSchema didn't run
|
|
413
|
+
// (custom provider that doesn't create _blamejs_cluster_state).
|
|
414
|
+
// Skip silently — same defensive posture as the audit-tip check.
|
|
415
|
+
var msg = (e && e.message) || "";
|
|
416
|
+
if (/no such table|does not exist|relation .* does not exist/i.test(msg)) {
|
|
417
|
+
log("cluster-state table not present — skipping vault-key consistency check (custom provider)");
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
throw e;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// Read whatever fingerprint is canonical (ours if first boot,
|
|
424
|
+
// someone else's if we lost the race or are joining an existing cluster).
|
|
425
|
+
var rows = await externalDb().query(
|
|
426
|
+
"SELECT vaultKeyFp, recordedByNode, recordedAt FROM _blamejs_cluster_state " +
|
|
427
|
+
"WHERE scope = 'state'",
|
|
428
|
+
[],
|
|
429
|
+
{ backend: configuredExternalDbBackend }
|
|
430
|
+
);
|
|
431
|
+
if (!rows.rows || rows.rows.length === 0) {
|
|
432
|
+
// Should never happen — we just INSERTed. Surface as fatal so the
|
|
433
|
+
// condition isn't silently ignored.
|
|
434
|
+
throw _err("CLUSTER_STATE_MISSING",
|
|
435
|
+
"FATAL: cluster-state row missing immediately after INSERT — " +
|
|
436
|
+
"external-db may not be honoring writes. Refusing boot.",
|
|
437
|
+
true);
|
|
438
|
+
}
|
|
439
|
+
var canonical = rows.rows[0];
|
|
440
|
+
if (canonical.vaultKeyFp !== localFp) {
|
|
441
|
+
var fpPrefix = C.BYTES.bytes(16);
|
|
442
|
+
throw _err("VAULT_KEY_DRIFT",
|
|
443
|
+
"FATAL: vault-key drift detected. " +
|
|
444
|
+
"local node: " + nodeId +
|
|
445
|
+
"; local fingerprint: " + localFp.slice(0, fpPrefix) + "…" +
|
|
446
|
+
"; canonical recorded by: " + canonical.recordedByNode +
|
|
447
|
+
"; canonical fingerprint: " + canonical.vaultKeyFp.slice(0, fpPrefix) + "…" +
|
|
448
|
+
". This node holds a DIFFERENT vault key than the rest of the " +
|
|
449
|
+
"cluster. Sealed-column writes from this node would be unreadable " +
|
|
450
|
+
"by the others (and vice versa). Restore the same vault key file " +
|
|
451
|
+
"before booting this node into the cluster.",
|
|
452
|
+
true);
|
|
453
|
+
}
|
|
454
|
+
log("cluster vault-key consistency ok (fingerprint " +
|
|
455
|
+
localFp.slice(0, C.BYTES.bytes(16)) + "… recorded by " + canonical.recordedByNode + ")");
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
async function _tryAcquire() {
|
|
459
|
+
if (role !== "leader") return; // pinned-follower role: never claim
|
|
460
|
+
try {
|
|
461
|
+
var got = await provider.acquireLease(nodeId, leaseTtlMs, {
|
|
462
|
+
endpoint: configuredEndpoint,
|
|
463
|
+
});
|
|
464
|
+
if (got) {
|
|
465
|
+
var wasLeader = !!lease;
|
|
466
|
+
lease = got;
|
|
467
|
+
if (!wasLeader) {
|
|
468
|
+
log("acquired lease — fencingToken=" + lease.fencingToken);
|
|
469
|
+
_emitTransition("lease-acquired", { fencingToken: lease.fencingToken });
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
} catch (e) {
|
|
473
|
+
log.error("acquire failed: " + e.message);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
async function _heartbeat() {
|
|
478
|
+
if (!initialized) return;
|
|
479
|
+
if (!lease) {
|
|
480
|
+
// Not currently leader — try to acquire (lease may have expired
|
|
481
|
+
// on the previous holder).
|
|
482
|
+
await _tryAcquire();
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
// We hold a lease — renew it. Re-supply the configured endpoint so a
|
|
486
|
+
// hot-reload of the operator's config (e.g. node moves to a new
|
|
487
|
+
// routable URL after a restart) eventually reaches the discovery row.
|
|
488
|
+
try {
|
|
489
|
+
lease = await provider.renewLease(lease, { endpoint: configuredEndpoint });
|
|
490
|
+
} catch (e) {
|
|
491
|
+
if (e.code === "LEASE_LOST") {
|
|
492
|
+
log.error("lease lost: " + e.message);
|
|
493
|
+
var lostToken = lease ? lease.fencingToken : null;
|
|
494
|
+
lease = null;
|
|
495
|
+
_emitTransition("lease-lost", { fencingToken: lostToken });
|
|
496
|
+
// Attempt to re-acquire on the next heartbeat naturally.
|
|
497
|
+
} else {
|
|
498
|
+
// Transient error — retry on next heartbeat. If it persists past
|
|
499
|
+
// leaseTtl another node will steal, and we'll detect via LEASE_LOST.
|
|
500
|
+
log.error("renew failed transiently: " + e.message);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// ---- public sync surface ----
|
|
506
|
+
|
|
507
|
+
function isLeader() {
|
|
508
|
+
if (terminated) return false; // post-shutdown: never leader
|
|
509
|
+
if (!initialized) return true; // never-initialized: permanent leader
|
|
510
|
+
return !!lease && Date.now() < lease.expiresAt;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Has cluster.init been called with a real configuration? Used by
|
|
514
|
+
// write-dispatch code (audit, consent, …) to decide whether framework
|
|
515
|
+
// state should go to local SQLite or external-db.
|
|
516
|
+
function isClusterMode() {
|
|
517
|
+
return initialized && !!configuredExternalDbBackend;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function externalDbBackend() {
|
|
521
|
+
return configuredExternalDbBackend;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function dialect() {
|
|
525
|
+
return configuredDialect;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
function currentNodeId() {
|
|
529
|
+
return initialized ? nodeId : "single-node-local";
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// This node's routable endpoint (operator-configured at cluster.init).
|
|
533
|
+
// Returns null when not configured or in single-node fallback. External
|
|
534
|
+
// observers should call discoveryHandler() / currentLeader() instead —
|
|
535
|
+
// this getter is for the local node's own self-identity.
|
|
536
|
+
function endpoint() {
|
|
537
|
+
return configuredEndpoint;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function fencingToken() {
|
|
541
|
+
if (!initialized) return 0;
|
|
542
|
+
return lease ? lease.fencingToken : 0;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function requireLeader() {
|
|
546
|
+
if (!isLeader()) {
|
|
547
|
+
throw new NotLeaderError(
|
|
548
|
+
"node '" + currentNodeId() + "' is not currently leader" +
|
|
549
|
+
(initialized ? "" : " (cluster not initialized)")
|
|
550
|
+
);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
async function currentLeader() {
|
|
555
|
+
if (!initialized) {
|
|
556
|
+
return {
|
|
557
|
+
nodeId: "single-node-local",
|
|
558
|
+
leaseExpiresAt: Infinity,
|
|
559
|
+
fencingToken: 0,
|
|
560
|
+
endpoint: null,
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
return await provider.currentLeader();
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// HTTP request handler — replies with the current cluster leader for
|
|
567
|
+
// service-mesh / load-balancer discovery. Operators mount this at
|
|
568
|
+
// whatever route they want (`/cluster/leader`, `/health/leader`, etc.).
|
|
569
|
+
//
|
|
570
|
+
// 200 application/json — leader present
|
|
571
|
+
// { leader: { nodeId, endpoint, fencingToken, leaseExpiresAt },
|
|
572
|
+
// self: { nodeId, endpoint, isLeader } }
|
|
573
|
+
//
|
|
574
|
+
// 503 application/json — no leader (no row, expired lease, DB
|
|
575
|
+
// unreachable, single-node not initialized with cluster mode)
|
|
576
|
+
// { leader: null, self: { nodeId, endpoint, isLeader } }
|
|
577
|
+
//
|
|
578
|
+
// No auth — this endpoint is intended to be called by infrastructure
|
|
579
|
+
// inside the trust boundary (LB, healthcheck, dashboard). Operators
|
|
580
|
+
// who expose it externally should layer auth via their own middleware.
|
|
581
|
+
//
|
|
582
|
+
// Handler is method-agnostic so it works behind any HTTP probe shape
|
|
583
|
+
// (GET, HEAD, etc.). Cache-Control: no-store to avoid stale-leader
|
|
584
|
+
// responses pinned by a caching proxy during a takeover.
|
|
585
|
+
function discoveryHandler() {
|
|
586
|
+
return async function (req, res) {
|
|
587
|
+
var selfInfo = {
|
|
588
|
+
nodeId: currentNodeId(),
|
|
589
|
+
endpoint: configuredEndpoint,
|
|
590
|
+
isLeader: isLeader(),
|
|
591
|
+
};
|
|
592
|
+
var body;
|
|
593
|
+
var status;
|
|
594
|
+
try {
|
|
595
|
+
var leader = await currentLeader();
|
|
596
|
+
if (leader && leader.nodeId && leader.nodeId !== "single-node-local") {
|
|
597
|
+
body = { leader: leader, self: selfInfo };
|
|
598
|
+
status = 200;
|
|
599
|
+
} else if (leader && leader.nodeId === "single-node-local") {
|
|
600
|
+
// Permanent-leader fallback (cluster.init never called). Reply
|
|
601
|
+
// 200 — the operator's app is healthy and the "leader" is this
|
|
602
|
+
// process. Useful so the discovery endpoint is never a false
|
|
603
|
+
// negative in single-node deployments.
|
|
604
|
+
body = { leader: leader, self: selfInfo };
|
|
605
|
+
status = 200;
|
|
606
|
+
} else {
|
|
607
|
+
body = { leader: null, self: selfInfo };
|
|
608
|
+
status = 503;
|
|
609
|
+
}
|
|
610
|
+
} catch (e) {
|
|
611
|
+
body = { leader: null, self: selfInfo, error: e.message };
|
|
612
|
+
status = 503;
|
|
613
|
+
}
|
|
614
|
+
var json = JSON.stringify(body);
|
|
615
|
+
res.writeHead(status, {
|
|
616
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
617
|
+
"Content-Length": Buffer.byteLength(json),
|
|
618
|
+
"Cache-Control": "no-store",
|
|
619
|
+
});
|
|
620
|
+
res.end(json);
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
function onTransition(handler) {
|
|
625
|
+
if (typeof handler !== "function") {
|
|
626
|
+
throw _err("INVALID_HANDLER", "onTransition expects a function", true);
|
|
627
|
+
}
|
|
628
|
+
transitionHandlers.push(handler);
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
async function shutdown() {
|
|
632
|
+
if (!initialized) return;
|
|
633
|
+
if (heartbeatTimer) {
|
|
634
|
+
heartbeatTimer.stop();
|
|
635
|
+
heartbeatTimer = null;
|
|
636
|
+
}
|
|
637
|
+
if (lease) {
|
|
638
|
+
try {
|
|
639
|
+
await provider.releaseLease(lease);
|
|
640
|
+
_emitTransition("lease-released", { fencingToken: lease.fencingToken });
|
|
641
|
+
log("lease released on shutdown");
|
|
642
|
+
} catch (e) {
|
|
643
|
+
log.error("release on shutdown failed: " + e.message);
|
|
644
|
+
}
|
|
645
|
+
lease = null;
|
|
646
|
+
}
|
|
647
|
+
initialized = false;
|
|
648
|
+
terminated = true;
|
|
649
|
+
provider = null;
|
|
650
|
+
role = null;
|
|
651
|
+
leaseTtlMs = null;
|
|
652
|
+
heartbeatMs = null;
|
|
653
|
+
configuredExternalDbBackend = null;
|
|
654
|
+
configuredDialect = null;
|
|
655
|
+
configuredEndpoint = null;
|
|
656
|
+
transitionHandlers = [];
|
|
657
|
+
// nodeId is preserved post-shutdown so audit metadata still reflects
|
|
658
|
+
// who this process was; cleared only by _resetForTest.
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// ---- test helpers — not part of public contract ----
|
|
662
|
+
|
|
663
|
+
function _resetForTest() {
|
|
664
|
+
if (heartbeatTimer) heartbeatTimer.stop();
|
|
665
|
+
heartbeatTimer = null;
|
|
666
|
+
initialized = false;
|
|
667
|
+
terminated = false;
|
|
668
|
+
nodeId = null;
|
|
669
|
+
role = null;
|
|
670
|
+
provider = null;
|
|
671
|
+
lease = null;
|
|
672
|
+
leaseTtlMs = null;
|
|
673
|
+
heartbeatMs = null;
|
|
674
|
+
configuredExternalDbBackend = null;
|
|
675
|
+
configuredDialect = null;
|
|
676
|
+
configuredEndpoint = null;
|
|
677
|
+
transitionHandlers = [];
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
async function _heartbeatNowForTest() {
|
|
681
|
+
// Drive one heartbeat synchronously without waiting for the timer —
|
|
682
|
+
// lets tests deterministically observe lease state transitions.
|
|
683
|
+
await _heartbeat();
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
module.exports = {
|
|
687
|
+
init: init,
|
|
688
|
+
isLeader: isLeader,
|
|
689
|
+
isClusterMode: isClusterMode,
|
|
690
|
+
externalDbBackend: externalDbBackend,
|
|
691
|
+
dialect: dialect,
|
|
692
|
+
currentNodeId: currentNodeId,
|
|
693
|
+
endpoint: endpoint,
|
|
694
|
+
fencingToken: fencingToken,
|
|
695
|
+
requireLeader: requireLeader,
|
|
696
|
+
currentLeader: currentLeader,
|
|
697
|
+
discoveryHandler: discoveryHandler,
|
|
698
|
+
onTransition: onTransition,
|
|
699
|
+
shutdown: shutdown,
|
|
700
|
+
NotLeaderError: NotLeaderError,
|
|
701
|
+
_resetForTest: _resetForTest,
|
|
702
|
+
_heartbeatNowForTest: _heartbeatNowForTest,
|
|
703
|
+
};
|