@ocap/chain-node 0.0.0-stage → 1.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -2
- package/esm/access-log.d.mts +34 -0
- package/esm/access-log.mjs +71 -0
- package/esm/config-consistency.d.mts +59 -0
- package/esm/config-consistency.mjs +220 -0
- package/esm/env.d.mts +32 -0
- package/esm/env.mjs +218 -0
- package/esm/follower-config.d.mts +25 -0
- package/esm/follower-config.mjs +57 -0
- package/esm/index.d.mts +7 -0
- package/esm/index.mjs +7 -0
- package/esm/standalone-env.d.mts +92 -0
- package/esm/standalone-env.mjs +152 -0
- package/esm/start-chain-node.d.mts +14 -0
- package/esm/start-chain-node.mjs +1107 -0
- package/esm/test-hooks-token.d.mts +12 -0
- package/esm/test-hooks-token.mjs +21 -0
- package/esm/testing.d.mts +13 -0
- package/esm/testing.mjs +23 -0
- package/esm/types.d.mts +310 -0
- package/esm/types.mjs +1 -0
- package/lib/_virtual/rolldown_runtime.cjs +29 -0
- package/lib/access-log.cjs +73 -0
- package/lib/access-log.d.cts +34 -0
- package/lib/config-consistency.cjs +227 -0
- package/lib/config-consistency.d.cts +59 -0
- package/lib/env.cjs +221 -0
- package/lib/env.d.cts +32 -0
- package/lib/follower-config.cjs +63 -0
- package/lib/follower-config.d.cts +25 -0
- package/lib/index.cjs +32 -0
- package/lib/index.d.cts +7 -0
- package/lib/standalone-env.cjs +165 -0
- package/lib/standalone-env.d.cts +92 -0
- package/lib/start-chain-node.cjs +1116 -0
- package/lib/start-chain-node.d.cts +14 -0
- package/lib/test-hooks-token.cjs +24 -0
- package/lib/test-hooks-token.d.cts +12 -0
- package/lib/testing.cjs +23 -0
- package/lib/testing.d.cts +13 -0
- package/lib/types.cjs +0 -0
- package/lib/types.d.cts +310 -0
- package/package.json +78 -4
package/README.md
CHANGED
|
@@ -1,3 +1,235 @@
|
|
|
1
|
-
#
|
|
1
|
+
# `@ocap/chain-node`
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`@ocap/chain-node` is the single canonical runtime assembly layer for OCAP blockchain nodes (`validator`, `standby`, keyless `follower`, and `hotstuff`). It owns deterministic storage boot, snapshot trust-anchor bootstrap, role runtime wiring, Express HTTP + WebSocket mounting, fail-closed `NODE_HALTED` enforcement, and ordered graceful shutdown.
|
|
4
|
+
|
|
5
|
+
For full architectural rationale and design history, see [`docs/chain-node-design.md`](../../docs/chain-node-design.md).
|
|
6
|
+
|
|
7
|
+
## Responsibilities & Boundaries
|
|
8
|
+
|
|
9
|
+
### Owned by `@ocap/chain-node`
|
|
10
|
+
|
|
11
|
+
- **Storage & single-writer lease lifecycle**: `SqliteLedgerStorage` (`SqliteSingleWriterLease`), `SqliteTrustedStorage` (`trusted.sqlite`), `SqliteStateDB` (`statedb.sqlite`), `SqliteIndexDB` (`indexdb.sqlite`), `NodePreferencesManager`, `INodeMaintenanceController`, and `Blacklist` (`filter.json`).
|
|
12
|
+
- **Role-specific consensus & replication runtimes**:
|
|
13
|
+
- Single-validator `Ledger` checkpointing (`periodicTimer` + witness quorum policy).
|
|
14
|
+
- Follower snapshot bootstrap (`prepareFollowerDataDirectory`), `FollowerSyncService`, single-hop `LeaderRouter` (`writePolicy: 'forward'`), and monotonic follower high-water mark persistence.
|
|
15
|
+
- Multi-validator `HotStuffNodeRuntime` + `SqliteHotStuffStateStore` with split consensus listener isolation (`/consensus` returns `404` on the public HTTP port).
|
|
16
|
+
- **`OCAPResolver` orchestration**: Deferred initialization (`deferInitialize: true`), keyless follower `statedb` chain-config rehydration without mutating `process.env`, and Promise-based startup validation (never calling `process.exit()` from library code).
|
|
17
|
+
- **HTTP + WebSocket server assembly**: CORS, shutdown fence (`503` + `Connection: close`), active HTTP response tracking, Socket.IO (`createSocketHandler`), REST `/api` (`createAPIHandler`), `/node` (`createNodeDashboardRouter`), optional `/dashboard` and `/node/explorer`, and `/` GraphQL (`createHttpHandler`, serving `/`, `/api`, and `/api/gql`).
|
|
18
|
+
- **Fail-closed `NODE_HALTED` gate**: Blocks all non-diagnostic GraphQL operations (`CustomError('NODE_HALTED', 'Chain node is halted fail-closed')`) and returns HTTP `503` (`{ code: 'NODE_HALTED', halted: true }`, `Cache-Control: no-store`) on `/api/ledger/status`, `/api/follower/status`, and non-diagnostic `/api/*` data endpoints whenever the underlying runtime halts.
|
|
19
|
+
- **LIFO partial-boot rollback & 9-step reverse graceful shutdown**.
|
|
20
|
+
|
|
21
|
+
### Explicitly Kept Out of `@ocap/chain-node`
|
|
22
|
+
|
|
23
|
+
- **`.env` file loading & implicit `process.env` reads**: `startChainNode` never reads `process.env` for configuration (the sole environment check is rejecting `testHooks` when `process.env.NODE_ENV === 'production'`). Callers parse env vars explicitly or pass an env dictionary into pure helpers (`roleConfigFromEnv(env)`, `leaseConfigFromEnv(env)`).
|
|
24
|
+
- **Secret-key / mnemonic derivation**: Callers construct `ChainNodeSignerConfig` (`ed25519`, `secp256k1`, or `eth`) before calling `startChainNode`.
|
|
25
|
+
- **Pre-Resolver immutable chain-config merging**: Application-specific `applyImmutable` / `checkChainImmutables` logic (for example `bridge/ocap-node/src/immutable.ts`) runs before passing `chainConfig` into `startChainNode`.
|
|
26
|
+
- **Application-specific HTTP/admin routes**: Mounted by the caller via `profile.configureApp(app, ctx)`.
|
|
27
|
+
- **Telemetry SDKs & optional index pruning packages**: `@sentry/*` stays in application entrypoints; `@ocap/indexdb-prune` is injected via `profile.maintenance.pruneIndexDbFn` rather than hard-depended on by `@ocap/chain-node`.
|
|
28
|
+
- **Global process signal handlers**: `startChainNode` does not bind `process.on('SIGINT' | 'SIGTERM')` automatically; CLI entrypoints opt in via `installProcessSignalHandlers(node, options?)`.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## Exports & Configuration Surface
|
|
33
|
+
|
|
34
|
+
### Main Entrypoint (`@ocap/chain-node`)
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
import {
|
|
38
|
+
installProcessSignalHandlers,
|
|
39
|
+
leaseConfigFromEnv,
|
|
40
|
+
roleConfigFromEnv,
|
|
41
|
+
startChainNode,
|
|
42
|
+
type ChainNodeRoleConfig,
|
|
43
|
+
type ChainProfile,
|
|
44
|
+
type RunningChainNode,
|
|
45
|
+
} from '@ocap/chain-node';
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
- `startChainNode(profile)` / `startChainNode({ profile, testHooks })`: Boots a chain node and resolves to a `RunningChainNode` handle (`endpoint`, `rpcUrl`, `wsUrl`, `port`, `host`, `chainId`, `role`, `nodeId`, `dataDir`, `proposerDid`, `proposerPublicKey`, `signatureKeyType`, `app`, `server`, `ledger`, `ledgerStorage`, `statedb`, `trustedStorage`, `indexdb`, `resolver`, `followerSync`, `hotstuffRuntime`, `preferencesManager`, `maintenanceController`, `ledgerHealth()`, `advanceClock?(seconds)`, `restart(nextProfile?)`, `stop()`).
|
|
49
|
+
- `roleConfigFromEnv(env, { signer? })`: Pure parser mapping explicit `env` keys (`CHAIN_ROLE` / `CHAIN_MODE`, `CONSENSUS_MODE`, `CHAIN_CONSENSUS_SK` + `CHAIN_CONSENSUS_PROPOSER` + `CHAIN_CONSENSUS_KEY_TYPE`, `CHAIN_WRITE_POLICY`, `ALLOW_PRIVATE_PEER_URLS`, `DISALLOW_KEYLESS_CASCADED_UPSTREAM`, `FOLLOWER_POLL_INTERVAL_MS`, `FOLLOWER_MAX_SYNC_LAG`, `FOLLOWER_MAX_UPSTREAM_STALE_MS`, `CONSENSUS_BIND_HOST`, `CONSENSUS_PORT`, `HOTSTUFF_VALIDATORS`, `HOTSTUFF_QUORUM_THRESHOLD`, witness variables) into a typed `ChainNodeRoleConfig`.
|
|
50
|
+
- `leaseConfigFromEnv(env)`: Pure parser mapping explicit `env` keys (`LEDGER_WRITER_LEASE`, `LEDGER_WRITER_LEASE_TTL_MS`, `LEDGER_WRITER_LEASE_STARTUP_WAIT_MS`, `CHAIN_LEASE_HOLDER_ID`) into `ILedgerLeaseOptions | false`.
|
|
51
|
+
- `installProcessSignalHandlers(node, options?)`: Attaches idempotent `SIGTERM`/`SIGINT` handlers that invoke `node.stop()` and returns an `uninstall()` disposer.
|
|
52
|
+
|
|
53
|
+
### `ChainProfile` & Node Roles
|
|
54
|
+
|
|
55
|
+
```ts
|
|
56
|
+
interface ChainProfileBase {
|
|
57
|
+
nodeId?: string; // defaults to persisted <dataDir>/.node-id
|
|
58
|
+
dataDir: string;
|
|
59
|
+
http: {
|
|
60
|
+
host?: string; // defaults to '127.0.0.1'
|
|
61
|
+
port: number; // 0 binds an OS-assigned ephemeral port (preserved across node.restart())
|
|
62
|
+
corsOrigins?: string | string[] | boolean;
|
|
63
|
+
drainTimeoutMs?: number; // HTTP active-response drain timeout on stop() (default: 5000ms)
|
|
64
|
+
enableDashboard?: boolean;
|
|
65
|
+
enableExplorer?: boolean;
|
|
66
|
+
explorerDistDir?: string;
|
|
67
|
+
enableAccessLog?: boolean;
|
|
68
|
+
adminEnv?: Record<string, string | undefined>;
|
|
69
|
+
onGraphqlError?: (err: Error & { originalError?: Error }) => void;
|
|
70
|
+
};
|
|
71
|
+
storage?: ChainProfileBase['storage'];
|
|
72
|
+
checkpoint?: {
|
|
73
|
+
batchSize?: number;
|
|
74
|
+
timeoutMs?: number;
|
|
75
|
+
auto?: boolean;
|
|
76
|
+
periodicTimer?: boolean;
|
|
77
|
+
timerIntervalMs?: number;
|
|
78
|
+
};
|
|
79
|
+
da?: ChainProfileBase['da'];
|
|
80
|
+
blacklist?: ChainProfileBase['blacklist'];
|
|
81
|
+
txQueue?: ChainProfileBase['txQueue'];
|
|
82
|
+
maintenance?: ChainProfileBase['maintenance'];
|
|
83
|
+
validateTokenConfig?: boolean;
|
|
84
|
+
logger?: IPipelineLogger;
|
|
85
|
+
configureApp?: (app: Express, ctx: ChainAppHookContext) => void | Promise<void>;
|
|
86
|
+
}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
| Role (`role.kind`) | Signer | `chainConfig` | Write Behavior & Runtime Semantics |
|
|
90
|
+
| --- | --- | --- | --- |
|
|
91
|
+
| `'validator'` | Required (`ed25519` \| `secp256k1` \| `eth`) | Required | Single-validator leader. Executes transactions, seals signed checkpoints (`batchSize` / `timeoutMs` + optional `periodicTimer`), and enforces `witnessQuorum` / `trustedWitnesses` when configured. |
|
|
92
|
+
| `'follower'` (keyless, default `runtimeRole: 'follower'`) | Omitted | Optional (rehydrated from `statedb.sqlite`) | Read-scaling / Sentinel verification node. Bootstraps from a signed snapshot, runs `FollowerSyncService` against `upstreamChainUrl`, and enforces `writePolicy: 'reject'` (`FOLLOWER_READ_ONLY`) or `'forward'` (single-hop `LeaderRouter` proxy to `upstreamChainUrl`). Auto-generates `<dataDir>/.follower-trusted.key` (`0600`) for local `trusted.sqlite` MAC verification. |
|
|
93
|
+
| `'follower'` (`runtimeRole: 'standby'`) | Required | Optional / Partial | Consensus standby follower. Runs `FollowerSyncService` with validator identity keys loaded; merges `.follower-trusted.key` into `legacyTrustedSecretKeys` so promoting a former keyless follower data directory preserves `trusted.sqlite` readability. |
|
|
94
|
+
| `'hotstuff'` | Required | Required | Multi-validator HotStuff BFT consensus node (`HotStuffNodeRuntime` + `SqliteHotStuffStateStore` + dedicated `consensusBindHost:consensusPort` listener). |
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Trust-Anchor & Snapshot Bootstrap Contract
|
|
99
|
+
|
|
100
|
+
Keyless followers and standby nodes never trust an unverified upstream stream:
|
|
101
|
+
|
|
102
|
+
1. **Lease-first safety**: If `<dataDir>/ledger.sqlite` already exists on disk, `startChainNode` acquires the `SqliteSingleWriterLease` **before** `prepareFollowerDataDirectory` inspects or mutates any files, ensuring a duplicate concurrent boot fails immediately with `LEDGER_LEASE_HELD`.
|
|
103
|
+
2. **Signed snapshot verification (`prepareFollowerDataDirectory`)**:
|
|
104
|
+
- Downloads `latest-snapshot.json`, signed `manifest.json`, `SHA256SUMS`, `statedb.sqlite.zst`, and `ledger.sqlite.zst` from `snapshotBaseUrl` (protected by SSRF private-IP filtering unless `allowPrivatePeers: true` is explicitly set in local harnesses).
|
|
105
|
+
- Verifies the snapshot `manifest.json` signature against the pinned `trustedProposer` / `trustedProposerPublicKey` (and optional `hotstuffValidators`) **before** decompressing archives.
|
|
106
|
+
- Verifies `.zst` SHA-256 checksums against `SHA256SUMS`, decompresses `statedb.sqlite` and `ledger.sqlite`, verifies checkpoint signatures inside `ledger.sqlite`, and writes `<dataDir>/.follower-bootstrap.json` (`0600`) recording `chainId`, `snapshotSequence`, `checkpointHeight`, `checkpointHash`, `stateCommitHash`, `proposer`, `proposerPublicKey`, `bootstrappedAt`, `lastKnownSequence`, and `lastKnownCheckpointHeight`.
|
|
107
|
+
- Cold follower boot without an existing bootstrapped DB and without `snapshotBaseUrl` fails closed; tampered manifests, `.zst` archives, or mismatched `trustedProposerPublicKey` abort startup before `OCAPResolver` opens.
|
|
108
|
+
3. **Anti-cascading upstream guard (`disallowKeylessCascadedUpstream`)**:
|
|
109
|
+
- When enabled (`true` on Sentinel verification followers), `FollowerSyncService` verifies `/api/ledger/status` upstream signatures and refuses to sync from another keyless/unauthenticated follower, requiring a cryptographic `'validator'` upstream.
|
|
110
|
+
4. **Fail-closed `NODE_HALTED`**:
|
|
111
|
+
- If `FollowerSyncService` or `LedgerEntryReplayer` detects an offline `statedb.sqlite` tamper, `stateRoot` mismatch against a verified checkpoint, or unrecoverable upstream divergence, it transitions to `halted = true`.
|
|
112
|
+
- Once halted, all non-diagnostic GraphQL queries/mutations throw `CustomError('NODE_HALTED', 'Chain node is halted fail-closed')` and `/api/ledger/status`, `/api/follower/status`, and non-diagnostic `/api/*` data endpoints return HTTP `503` with `{ code: 'NODE_HALTED', halted: true }` (`Cache-Control: no-store`).
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Deterministic Boot & Shutdown Order
|
|
117
|
+
|
|
118
|
+
### 12-Step Boot Sequence (with LIFO Rollback)
|
|
119
|
+
|
|
120
|
+
Every resource opened during `startChainNode` pushes a cleanup closure onto a LIFO rollback stack. If any step throws, all previously opened resources are closed in reverse order and `SqliteSingleWriterLease` is released before the Promise rejects:
|
|
121
|
+
|
|
122
|
+
1. Validate `testHooks` token (`FORBIDDEN_IN_PRODUCTION` in `NODE_ENV=production`), verify `dataDir` is not a symlink (`assertSafeNonSymlinkDirectory`), and validate `ChainNodeRoleConfig`.
|
|
123
|
+
2. Load or atomically persist `<dataDir>/.node-id` (`0600`).
|
|
124
|
+
3. Resolve consensus proposer identity (`ed25519`, `secp256k1`, or `eth`) purely in memory without mutating `process.env`.
|
|
125
|
+
4. For `role.kind === 'follower'`, acquire pre-existing ledger writer lease if present, run `prepareFollowerDataDirectory`, and provision `.follower-trusted.key` when keyless.
|
|
126
|
+
5. Initialize writable `SqliteLedgerStorage` (`ledger.sqlite`) and acquire `SqliteSingleWriterLease`.
|
|
127
|
+
6. Open `SqliteTrustedStorage` (`trusted.sqlite`), open `SqliteStateDB` (`statedb.sqlite`), rehydrate `IChainConfig` from `statedb` for keyless followers, and initialize `Ledger` (wiring logical clock + periodic checkpoint timer).
|
|
128
|
+
7. Initialize `SqliteIndexDB` (`indexdb.sqlite`), `NodePreferencesManager`, and `Blacklist` (`filter.json`).
|
|
129
|
+
8. Initialize `OCAPResolver` (`deferInitialize: true`, `skipInitializeStateDB: true` on followers).
|
|
130
|
+
9. Wire role runtime (`FollowerSyncService` + optional `LeaderRouter` + initial `rebuildIndex()` / orphan replay for followers; `HotStuffNodeRuntime` + `SqliteHotStuffStateStore` for HotStuff).
|
|
131
|
+
10. Start background sync/consensus loops (`followerSync.start()` or `hotstuffRuntime.start()`).
|
|
132
|
+
11. Assemble Express + Socket.IO HTTP server and bind `http.host:http.port` (**no global `express.json()`** is mounted so route-specific body limits like the 256 KB `/api/ledger/witness/attestations` endpoint remain intact).
|
|
133
|
+
12. Run optional `testHooks.onAfterBoot` and return `RunningChainNode`.
|
|
134
|
+
|
|
135
|
+
### Access log and client IP (#1922)
|
|
136
|
+
|
|
137
|
+
`http.enableAccessLog` (`CHAIN_ACCESS_LOG=1` in the apps) writes one line per finished response through the
|
|
138
|
+
profile logger, in a fixed positional format:
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
[access] <method> <path> <status> <ms>ms <bytes> <clientIp> <host> <cfRay> "<userAgent>"
|
|
142
|
+
[access] POST /api 200 12ms 1834 203.0.113.7 main.abtnetwork.io 8f1c2d3e4a5b6c7d-SJC "ArcWallet/5.10.9 (iOS 18.1)"
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
`bytes` is the response body size. Missing values are `-`. All fields except the user agent are free of whitespace
|
|
146
|
+
(whitespace, `"`, `\` and control characters are %-encoded); the user agent is last and quoted with `"`, `\` and
|
|
147
|
+
control characters escaped, so a client cannot end the line or forge a field. The app's logger may put a prefix before
|
|
148
|
+
`[access]` (`qldb-chain-api` prints `[chain] [access] …`), so count fields from the `[access]` token, not from the
|
|
149
|
+
start of the line: `sed -n 's/.*\[access\] //p' | awk '{print $6}'` is the client IP (`$3` status, `$4` duration).
|
|
150
|
+
|
|
151
|
+
The client IP is resolved by `@ocap/util/lib/client-ip` and is the same value the GraphQL context
|
|
152
|
+
(`ctx.request.ip`), the slow-request log and the admin auth failure limiter use:
|
|
153
|
+
|
|
154
|
+
1. the trusted proxy headers in `CHAIN_CLIENT_IP_HEADERS` order — **only when the TCP peer is a loopback address**,
|
|
155
|
+
and only when the value is a bare IP literal;
|
|
156
|
+
2. otherwise `req.ip` (equal to the socket address unless the app sets `trust proxy`), then the socket address.
|
|
157
|
+
A socket whose address is gone (the client reset the connection) counts as an unknown peer: headers are ignored.
|
|
158
|
+
|
|
159
|
+
`CHAIN_CLIENT_IP_HEADERS` (comma-separated, ordered; `none` or empty trusts no header):
|
|
160
|
+
|
|
161
|
+
| Deployment | Value | Why |
|
|
162
|
+
|---|---|---|
|
|
163
|
+
| Blocklet Server nginx (default) | unset = `x-real-ip` | nginx sets `X-Real-IP $remote_addr` but passes a client-supplied `CF-Connecting-IP` through, so trusting `cf-connecting-ip` there would let a client pick its own IP |
|
|
164
|
+
| Cloudflare Tunnel (cloudflared on the same host) | `cf-connecting-ip` | see below; must be set explicitly |
|
|
165
|
+
|
|
166
|
+
Why `cf-connecting-ip` can be trusted under the tunnel: the chain process listens on `127.0.0.1` only and is reached
|
|
167
|
+
through the cloudflared daemon on the same host (#1875 decision E4). Cloudflare's edge sets `cf-connecting-ip` to the
|
|
168
|
+
real client address, and a request that already carries its own `cf-connecting-ip` is rejected at the edge (403,
|
|
169
|
+
error 1000); a client's `x-real-ip` did not reach the origin either (both checked with a local quick tunnel on
|
|
170
|
+
2026-11-19). If the tunnel opt-in is forgotten, the default `x-real-ip` is absent and every request resolves to
|
|
171
|
+
`127.0.0.1`: one shared bucket and a useless log column, but nothing a client can spoof. cloudflared forwards the
|
|
172
|
+
original `Host` header unchanged (unless an ingress rule sets `originRequest.httpHostHeader`), so the same-origin
|
|
173
|
+
check of mutating admin session requests keeps working.
|
|
174
|
+
|
|
175
|
+
Limits of the loopback rule: **any process on the same host can connect over loopback and set any client IP** (a
|
|
176
|
+
builder host runs ~30 other blocklets); that is accepted by #1875 E4. A non-loopback peer talks to the process
|
|
177
|
+
directly (e.g. a node bound to `0.0.0.0`), so its headers are its own claim and are ignored.
|
|
178
|
+
|
|
179
|
+
The admin auth failure limiter (`createAdminAuthGuard`, 10 failures per minute) buckets by this client IP. A correct
|
|
180
|
+
token is compared before the limiter is consulted, so the limiter never locks out a correct token, not even from a
|
|
181
|
+
rate-limited address. What a shared bucket breaks is fairness: when every request appears as `127.0.0.1` (the tunnel
|
|
182
|
+
without the opt-in, or any proxy without a trusted header), one client's wrong guesses rate-limit everyone else's
|
|
183
|
+
mistyped attempts. Resolving the real client restores per-client buckets.
|
|
184
|
+
|
|
185
|
+
WebSocket per-IP connection limits (`@arcblock/ws`, 64 per IP) use their own setting: set `WS_TRUST_PROXY=1` with
|
|
186
|
+
`WS_TRUSTED_PROXIES=127.0.0.1/32,::1/128`, or every subscriber behind the tunnel counts against one address.
|
|
187
|
+
|
|
188
|
+
### 9-Step Reverse Graceful Shutdown (`node.stop()`)
|
|
189
|
+
|
|
190
|
+
1. Stop `followerSync` / `hotstuffRuntime` background loops and clear maintenance/HWM timers.
|
|
191
|
+
2. Flip `isShuttingDown = true` (new HTTP requests receive `503` + `Connection: close`), close idle keep-alive sockets, and wait up to `http.drainTimeoutMs` (`5000ms` default) for `activeResponses === 0` **before** calling `server.close()` (preventing Socket.IO/`WsServer.attach` `closeAllConnections()` from resetting in-flight responses).
|
|
192
|
+
3. Close the HTTP + WebSocket listener (`server.close()`).
|
|
193
|
+
4. Call `resolver.shutdown()` (drains in-flight pre-flights, transaction queue, and `flushIndexWrites()`).
|
|
194
|
+
5. Demote ledger role (`ledger.setNodeRole('readonly')`) to fence new writes and periodic checkpoint timers.
|
|
195
|
+
6. Dispose `Blacklist` file watchers (`filter.dispose()`).
|
|
196
|
+
7. Close `statedb` and `trustedStorage`.
|
|
197
|
+
8. Close `indexdb` and flush final follower high-water mark.
|
|
198
|
+
9. In `finally`, call `ledger.close()` to drain any in-flight checkpoint and release `SqliteSingleWriterLease` last.
|
|
199
|
+
|
|
200
|
+
---
|
|
201
|
+
|
|
202
|
+
## Extension & Test Injection Hooks
|
|
203
|
+
|
|
204
|
+
### Application Hook (`profile.configureApp`)
|
|
205
|
+
|
|
206
|
+
`configureApp(app, ctx)` runs after CORS and the shutdown fence, and before `/api`, `/node`, and `/` GraphQL. Handlers receive `ChainAppHookContext` (`nodeId`, `dataDir`, `role`, `config`, `statedb`, `indexdb`, `ledger`, `resolver`, `preferencesManager`, `maintenanceController`).
|
|
207
|
+
|
|
208
|
+
Because `@ocap/chain-node` intentionally avoids mounting a global `express.json()`, any custom POST/PUT route registered inside `configureApp` should attach route-scoped body parsing (for example `app.post('/admin/foo', express.json({ limit: '64kb' }), handler)`).
|
|
209
|
+
|
|
210
|
+
### Non-Production Test Hooks (`@ocap/chain-node/testing`)
|
|
211
|
+
|
|
212
|
+
Deterministic fault-injection and clock hooks live behind a dedicated subpath export:
|
|
213
|
+
|
|
214
|
+
```ts
|
|
215
|
+
import { startChainNode } from '@ocap/chain-node';
|
|
216
|
+
import { createChainNodeTestHooks } from '@ocap/chain-node/testing';
|
|
217
|
+
|
|
218
|
+
const testHooks = createChainNodeTestHooks({
|
|
219
|
+
initialClockOffsetSeconds: 0,
|
|
220
|
+
followerSync: { startPaused: true },
|
|
221
|
+
});
|
|
222
|
+
const node = await startChainNode({ profile, testHooks });
|
|
223
|
+
node.advanceClock?.(120);
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
- **Unforgeable capability**: `createChainNodeTestHooks` stamps objects with a private module-scoped `Symbol('@ocap/chain-node/test-hooks-token')` (not `Symbol.for`), so test hooks cannot be injected via JSON configs, CLI flags, or environment variables.
|
|
227
|
+
- **Production hard block**: Passing `testHooks` or calling `createChainNodeTestHooks` when `process.env.NODE_ENV === 'production'` immediately throws `CustomError('FORBIDDEN_IN_PRODUCTION', ...)`.
|
|
228
|
+
|
|
229
|
+
---
|
|
230
|
+
|
|
231
|
+
## Relationship to OCAP Packages & Applications
|
|
232
|
+
|
|
233
|
+
- **`bridge/ocap-node` (`@arcblock/bridge-ocap-node`)**: Migrated onto `@ocap/chain-node` (`#1812`). Wraps `startChainNode` with bridge rollup immutable state (`bridge/ocap-node/src/immutable.ts`) and exposes a CLI supporting both `--role validator` and keyless `--role follower` (`--write-policy reject|forward`, `--upstream`, `--snapshot-base-url`, `--trusted-proposer`, `--trusted-proposer-pk`).
|
|
234
|
+
- **`apps/qldb-chain-api` & `apps/asset-chain-api`**: Production OCAP chain applications. `@ocap/chain-node` extracts their shared storage, role runtime, HTTP/WS, and graceful shutdown stack (`roleConfigFromEnv`, `leaseConfigFromEnv`, `configureApp`, `maintenance.pruneIndexDbFn`), with full app entrypoint cutover tracked in `#1820` and `#1821` (production apps never import `@ocap/chain-node/testing`).
|
|
235
|
+
- **Retired legacy apps**: `apps/fs-chain-api` and `apps/fs-chain-manager` were removed in `#1809`.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { NextFunction, Request, Response } from "express";
|
|
2
|
+
|
|
3
|
+
//#region src/access-log.d.ts
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* `CHAIN_ACCESS_LOG=1` line (#1922). Behind Cloudflare Tunnel there is no nginx access log, so this line carries what
|
|
7
|
+
* slow-request analysis (#1874) needs. Fixed positional format, one request per line:
|
|
8
|
+
*
|
|
9
|
+
* [access] <method> <path> <status> <ms>ms <bytes> <clientIp> <host> <cfRay> "<userAgent>"
|
|
10
|
+
*
|
|
11
|
+
* - `bytes`: response body bytes written (like nginx `$body_bytes_sent`).
|
|
12
|
+
* - `clientIp`: `@ocap/util/lib/client-ip` (`CHAIN_CLIENT_IP_HEADERS`, default `x-real-ip`, read from a loopback peer only, else socket).
|
|
13
|
+
* - absent values are `-`. Every field except the user agent is whitespace-free (whitespace, `"`, `\` and control
|
|
14
|
+
* characters are %-encoded), so `awk '{print $7}'` is the client IP; the user agent is last, quoted, with `"`, `\`
|
|
15
|
+
* and control characters backslash-escaped, so a client cannot end the line or forge a field.
|
|
16
|
+
*/
|
|
17
|
+
interface AccessLogFields {
|
|
18
|
+
method: string;
|
|
19
|
+
path: string;
|
|
20
|
+
status: number;
|
|
21
|
+
durationMs: number;
|
|
22
|
+
bytes: number;
|
|
23
|
+
clientIp: string;
|
|
24
|
+
host?: string;
|
|
25
|
+
cfRay?: string;
|
|
26
|
+
userAgent?: string;
|
|
27
|
+
}
|
|
28
|
+
declare function formatAccessLogLine(f: AccessLogFields): string;
|
|
29
|
+
/** Express middleware writing one `formatAccessLogLine` per finished response through `logger.info`. */
|
|
30
|
+
declare function createAccessLogMiddleware(logger: {
|
|
31
|
+
info: (msg: string) => void;
|
|
32
|
+
}): (req: Request, res: Response, next: NextFunction) => void;
|
|
33
|
+
//#endregion
|
|
34
|
+
export { AccessLogFields, createAccessLogMiddleware, formatAccessLogLine };
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { resolveClientIp } from "@ocap/util/lib/client-ip";
|
|
2
|
+
|
|
3
|
+
//#region src/access-log.ts
|
|
4
|
+
const MAX_PATH = 2048;
|
|
5
|
+
const MAX_HOST = 255;
|
|
6
|
+
const MAX_CF_RAY = 64;
|
|
7
|
+
const MAX_UA = 256;
|
|
8
|
+
const pctByte = (c) => `%${c.charCodeAt(0).toString(16).toUpperCase().padStart(2, "0")}`;
|
|
9
|
+
const token = (value, max) => {
|
|
10
|
+
if (!value) return "-";
|
|
11
|
+
return value.replace(/[\s"\\\u0000-\u001f\u007f]/g, pctByte).slice(0, max);
|
|
12
|
+
};
|
|
13
|
+
const quoted = (value, max) => {
|
|
14
|
+
if (!value) return "\"-\"";
|
|
15
|
+
return `"${value.slice(0, max).replace(/["\\\u0000-\u001f\u007f]/g, (c) => c === "\"" || c === "\\" ? `\\${c}` : `\\x${c.charCodeAt(0).toString(16).padStart(2, "0")}`)}"`;
|
|
16
|
+
};
|
|
17
|
+
function formatAccessLogLine(f) {
|
|
18
|
+
return [
|
|
19
|
+
"[access]",
|
|
20
|
+
token(f.method, 16),
|
|
21
|
+
token(f.path, MAX_PATH),
|
|
22
|
+
String(f.status),
|
|
23
|
+
`${Math.max(0, Math.round(f.durationMs))}ms`,
|
|
24
|
+
String(f.bytes),
|
|
25
|
+
token(f.clientIp, 45),
|
|
26
|
+
token(f.host, MAX_HOST),
|
|
27
|
+
token(f.cfRay, MAX_CF_RAY),
|
|
28
|
+
quoted(f.userAgent, MAX_UA)
|
|
29
|
+
].join(" ");
|
|
30
|
+
}
|
|
31
|
+
const headerString = (v) => Array.isArray(v) ? v[0] : v;
|
|
32
|
+
const chunkBytes = (chunk, encoding) => {
|
|
33
|
+
if (chunk === void 0 || chunk === null || typeof chunk === "function") return 0;
|
|
34
|
+
if (typeof chunk === "string") return Buffer.byteLength(chunk, typeof encoding === "string" ? encoding : "utf8");
|
|
35
|
+
if (chunk instanceof Uint8Array) return chunk.byteLength;
|
|
36
|
+
return 0;
|
|
37
|
+
};
|
|
38
|
+
/** Express middleware writing one `formatAccessLogLine` per finished response through `logger.info`. */
|
|
39
|
+
function createAccessLogMiddleware(logger) {
|
|
40
|
+
return (req, res, next) => {
|
|
41
|
+
const startedAt = Date.now();
|
|
42
|
+
let bytes = 0;
|
|
43
|
+
const origWrite = res.write;
|
|
44
|
+
const origEnd = res.end;
|
|
45
|
+
res.write = function write(...args) {
|
|
46
|
+
bytes += chunkBytes(args[0], args[1]);
|
|
47
|
+
return origWrite.apply(this, args);
|
|
48
|
+
};
|
|
49
|
+
res.end = function end(...args) {
|
|
50
|
+
bytes += chunkBytes(args[0], args[1]);
|
|
51
|
+
return origEnd.apply(this, args);
|
|
52
|
+
};
|
|
53
|
+
res.once("finish", () => {
|
|
54
|
+
logger.info(formatAccessLogLine({
|
|
55
|
+
method: req.method,
|
|
56
|
+
path: req.originalUrl,
|
|
57
|
+
status: res.statusCode,
|
|
58
|
+
durationMs: Date.now() - startedAt,
|
|
59
|
+
bytes,
|
|
60
|
+
clientIp: resolveClientIp(req),
|
|
61
|
+
host: headerString(req.headers.host),
|
|
62
|
+
cfRay: headerString(req.headers["cf-ray"]),
|
|
63
|
+
userAgent: headerString(req.headers["user-agent"])
|
|
64
|
+
}));
|
|
65
|
+
});
|
|
66
|
+
next();
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
//#endregion
|
|
71
|
+
export { createAccessLogMiddleware, formatAccessLogLine };
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { TChainConfigCheckMode } from "./standalone-env.mjs";
|
|
2
|
+
import { IPipelineLogger } from "@ocap/types";
|
|
3
|
+
|
|
4
|
+
//#region src/config-consistency.d.ts
|
|
5
|
+
declare const CONSENSUS_CONFIG_KEYS: readonly ["chainId", "mode", "moderator.address", "token.address", "token.symbol", "token.decimal", "vaults", "reservedSymbols", "transaction"];
|
|
6
|
+
type Plain = Record<string, unknown>;
|
|
7
|
+
/** The consensus-relevant projection of a chain config, with defaults applied identically on both sides. */
|
|
8
|
+
declare function normalizeConsensusConfig(config: unknown): Plain;
|
|
9
|
+
interface ConfigDifference {
|
|
10
|
+
path: string;
|
|
11
|
+
local: unknown;
|
|
12
|
+
upstream: unknown;
|
|
13
|
+
}
|
|
14
|
+
/** Differences between two configs' consensus projections, by dotted path (arrays compare as a whole). */
|
|
15
|
+
declare function diffConsensusConfig(local: unknown, upstream: unknown): ConfigDifference[];
|
|
16
|
+
/** POST `{ getConfig }` to `<upstreamUrl>/api/gql` and return the parsed in-memory config of the upstream. */
|
|
17
|
+
declare function fetchUpstreamChainConfig(upstreamUrl: string, {
|
|
18
|
+
fetchImpl,
|
|
19
|
+
timeoutMs
|
|
20
|
+
}?: {
|
|
21
|
+
fetchImpl?: typeof fetch;
|
|
22
|
+
timeoutMs?: number;
|
|
23
|
+
}): Promise<unknown>;
|
|
24
|
+
declare function formatConfigDifferences(diff: ConfigDifference[]): string;
|
|
25
|
+
interface VerifyFollowerConfigOptions {
|
|
26
|
+
mode: TChainConfigCheckMode;
|
|
27
|
+
/** The config the follower will replay with (`resolver.config`). */
|
|
28
|
+
localConfig: unknown;
|
|
29
|
+
upstreamUrl: string | null | undefined;
|
|
30
|
+
/** Defaults to `fetchUpstreamChainConfig`; tests pass a stub upstream. */
|
|
31
|
+
fetchUpstreamConfig?: (upstreamUrl: string) => Promise<unknown>;
|
|
32
|
+
logger: Pick<IPipelineLogger, 'info' | 'warn'>;
|
|
33
|
+
/**
|
|
34
|
+
* The data dir already held this node's chain state before this start (not a fresh snapshot bootstrap). With
|
|
35
|
+
* `enforce`, an upstream that stays unavailable past the deadline is then a warning, not a refusal.
|
|
36
|
+
*/
|
|
37
|
+
hasLocalState?: boolean;
|
|
38
|
+
/** `enforce` only: keep retrying an unavailable upstream this long (default 60 s, CHAIN_CONFIG_CHECK_TIMEOUT_SEC). */
|
|
39
|
+
retryDeadlineMs?: number;
|
|
40
|
+
/** First backoff delay (default 1 s), doubled per attempt up to `retryMaxDelayMs` (default 10 s). */
|
|
41
|
+
retryDelayMs?: number;
|
|
42
|
+
retryMaxDelayMs?: number;
|
|
43
|
+
now?: () => number;
|
|
44
|
+
sleep?: (ms: number) => Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
interface VerifyFollowerConfigResult {
|
|
47
|
+
status: 'skipped' | 'match' | 'mismatch' | 'unavailable';
|
|
48
|
+
diff: ConfigDifference[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Compare the follower's replay config with the upstream leader's. `off`: no request. `warn`: one attempt, log
|
|
52
|
+
* any difference or failure and continue. `enforce`: a definite difference throws `CHAIN_CONFIG_MISMATCH` (lists
|
|
53
|
+
* every differing key). An unavailable upstream (down, HTTP error such as 429, bad body) is retried with backoff
|
|
54
|
+
* until `retryDeadlineMs`; then a node with existing local state starts with a warning, and only a fresh bootstrap
|
|
55
|
+
* throws `CHAIN_CONFIG_CHECK_UNAVAILABLE`.
|
|
56
|
+
*/
|
|
57
|
+
declare function verifyFollowerConfigAgainstUpstream(opts: VerifyFollowerConfigOptions): Promise<VerifyFollowerConfigResult>;
|
|
58
|
+
//#endregion
|
|
59
|
+
export { CONSENSUS_CONFIG_KEYS, ConfigDifference, VerifyFollowerConfigOptions, VerifyFollowerConfigResult, diffConsensusConfig, fetchUpstreamChainConfig, formatConfigDifferences, normalizeConsensusConfig, verifyFollowerConfigAgainstUpstream };
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import { defaultGasSecurityKnobs } from "@ocap/config";
|
|
2
|
+
import defaultConfig from "@ocap/config/lib/default";
|
|
3
|
+
import { CustomError } from "@ocap/util/lib/error";
|
|
4
|
+
|
|
5
|
+
//#region src/config-consistency.ts
|
|
6
|
+
/**
|
|
7
|
+
* Follower startup consistency check (#1961): before a follower replays anything it asks the upstream leader for
|
|
8
|
+
* its in-memory chain config (`getConfig`, core/resolver `getConfig()` returns `JSON.stringify(this.config)`) and
|
|
9
|
+
* compares every consensus-relevant key, both sides normalized with the same defaults. A follower that replays with
|
|
10
|
+
* a different config computes different state commits (e.g. `transaction.bridge.enabled` off rejects the first
|
|
11
|
+
* bridge reserve entry in `bridge-reserve-v2-gate.ts`) and halts; this check turns that into a clear startup error.
|
|
12
|
+
*
|
|
13
|
+
* Consensus-relevant keys (`CONSENSUS_CONFIG_KEYS`) are what the tx pipeline / replayer reads from `context.config`:
|
|
14
|
+
* - `chainId` every protocol (`tx.chainId` checks), replayer entry chain id
|
|
15
|
+
* - `mode` `Config.create` defaults (`ledger` zeroes fees and gas)
|
|
16
|
+
* - `moderator.address` moderator-only protocols (token / factory / rollup governance)
|
|
17
|
+
* - `token.address/symbol/decimal` official token, fee and stake units, reserved base symbol
|
|
18
|
+
* - `vaults` where fees, gas and slashed stake go (`txFee`, `txFees`, `txGas`, `slashedStake`)
|
|
19
|
+
* - `reservedSymbols` `create_token_factory` verification (compared as a case-insensitive set)
|
|
20
|
+
* - `transaction` the whole section: `txFee`, `txGas` (+ free-gas / burst knobs), `txStake`,
|
|
21
|
+
* `maxTxSize`, `maxAssetSize`, `maxListSize`, `maxMultisig`, `delegate`, `supportedTxs`,
|
|
22
|
+
* `multiSignV2Txs`, `maxWriteSetSize`, `security`, `bridge`, `rollupV1`, `exchangeV2`
|
|
23
|
+
* Not compared: `version` (software), `accounts` and token metadata/supply (genesis only: a follower never re-runs
|
|
24
|
+
* genesis, it starts from a verified snapshot), `moderator.pk` (protocols compare the address), `genesisTime`.
|
|
25
|
+
*/
|
|
26
|
+
const CONSENSUS_CONFIG_KEYS = [
|
|
27
|
+
"chainId",
|
|
28
|
+
"mode",
|
|
29
|
+
"moderator.address",
|
|
30
|
+
"token.address",
|
|
31
|
+
"token.symbol",
|
|
32
|
+
"token.decimal",
|
|
33
|
+
"vaults",
|
|
34
|
+
"reservedSymbols",
|
|
35
|
+
"transaction"
|
|
36
|
+
];
|
|
37
|
+
/** Protocol fallback of `transaction.exchangeV2.defaultTtlSeconds` (core/tx-protocols exchange-v2.ts). */
|
|
38
|
+
const DEFAULT_EXCHANGE_V2_TTL_SECONDS = 720 * 60 * 60;
|
|
39
|
+
const isPlainObject = (v) => Boolean(v) && typeof v === "object" && !Array.isArray(v);
|
|
40
|
+
/** JSON round trip: drops `undefined`, functions and frozen-ness, so both sides compare as the wire would. */
|
|
41
|
+
function toJsonValue(value) {
|
|
42
|
+
if (value === void 0) return void 0;
|
|
43
|
+
return JSON.parse(JSON.stringify(value));
|
|
44
|
+
}
|
|
45
|
+
function getPath(obj, dotted) {
|
|
46
|
+
let cur = obj;
|
|
47
|
+
for (const seg of dotted.split(".")) {
|
|
48
|
+
if (!isPlainObject(cur)) return void 0;
|
|
49
|
+
cur = cur[seg];
|
|
50
|
+
}
|
|
51
|
+
return cur;
|
|
52
|
+
}
|
|
53
|
+
function setPath(obj, dotted, value) {
|
|
54
|
+
const segs = dotted.split(".");
|
|
55
|
+
let cur = obj;
|
|
56
|
+
for (const seg of segs.slice(0, -1)) {
|
|
57
|
+
if (!isPlainObject(cur[seg])) cur[seg] = {};
|
|
58
|
+
cur = cur[seg];
|
|
59
|
+
}
|
|
60
|
+
cur[segs[segs.length - 1]] = value;
|
|
61
|
+
}
|
|
62
|
+
/** Fill only keys that are absent (`undefined`); present values, records and arrays are kept as they are. */
|
|
63
|
+
function fillAbsent(target, defaults) {
|
|
64
|
+
const out = { ...target };
|
|
65
|
+
for (const [k, v] of Object.entries(defaults)) if (out[k] === void 0) out[k] = toJsonValue(v);
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
function normalizeTransaction(raw) {
|
|
69
|
+
const tx = fillAbsent(isPlainObject(raw) ? toJsonValue(raw) : {}, defaultConfig.transaction);
|
|
70
|
+
tx.txGas = fillAbsent(isPlainObject(tx.txGas) ? tx.txGas : {}, defaultGasSecurityKnobs.txGas);
|
|
71
|
+
tx.security = fillAbsent(isPlainObject(tx.security) ? tx.security : {}, defaultGasSecurityKnobs.security);
|
|
72
|
+
tx.bridge = { enabled: isPlainObject(tx.bridge) && tx.bridge.enabled === true };
|
|
73
|
+
tx.rollupV1 = { enabled: !(isPlainObject(tx.rollupV1) && tx.rollupV1.enabled === false) };
|
|
74
|
+
const exchangeV2 = isPlainObject(tx.exchangeV2) ? tx.exchangeV2 : {};
|
|
75
|
+
tx.exchangeV2 = {
|
|
76
|
+
...exchangeV2,
|
|
77
|
+
defaultTtlSeconds: typeof exchangeV2.defaultTtlSeconds === "number" ? exchangeV2.defaultTtlSeconds : DEFAULT_EXCHANGE_V2_TTL_SECONDS
|
|
78
|
+
};
|
|
79
|
+
return tx;
|
|
80
|
+
}
|
|
81
|
+
/** The consensus-relevant projection of a chain config, with defaults applied identically on both sides. */
|
|
82
|
+
function normalizeConsensusConfig(config) {
|
|
83
|
+
const out = {};
|
|
84
|
+
for (const key of CONSENSUS_CONFIG_KEYS) {
|
|
85
|
+
const value = toJsonValue(getPath(config, key));
|
|
86
|
+
switch (key) {
|
|
87
|
+
case "mode":
|
|
88
|
+
setPath(out, key, value === "ledger" ? "ledger" : "chain");
|
|
89
|
+
break;
|
|
90
|
+
case "token.address":
|
|
91
|
+
setPath(out, key, typeof value === "string" ? value : "");
|
|
92
|
+
break;
|
|
93
|
+
case "vaults":
|
|
94
|
+
setPath(out, key, isPlainObject(value) ? value : {});
|
|
95
|
+
break;
|
|
96
|
+
case "reservedSymbols":
|
|
97
|
+
setPath(out, key, Array.from(new Set((Array.isArray(value) ? value : []).map((s) => String(s ?? "").trim().toUpperCase()).filter(Boolean))).sort());
|
|
98
|
+
break;
|
|
99
|
+
case "transaction":
|
|
100
|
+
setPath(out, key, normalizeTransaction(value));
|
|
101
|
+
break;
|
|
102
|
+
default: setPath(out, key, value === void 0 ? null : value);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
107
|
+
function collectDiff(local, upstream, prefix, out) {
|
|
108
|
+
if (isPlainObject(local) && isPlainObject(upstream)) {
|
|
109
|
+
const keys = Array.from(new Set([...Object.keys(local), ...Object.keys(upstream)])).sort();
|
|
110
|
+
for (const k of keys) collectDiff(local[k], upstream[k], prefix ? `${prefix}.${k}` : k, out);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
if (JSON.stringify(local) !== JSON.stringify(upstream)) out.push({
|
|
114
|
+
path: prefix,
|
|
115
|
+
local: local === void 0 ? null : local,
|
|
116
|
+
upstream: upstream === void 0 ? null : upstream
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
/** Differences between two configs' consensus projections, by dotted path (arrays compare as a whole). */
|
|
120
|
+
function diffConsensusConfig(local, upstream) {
|
|
121
|
+
const out = [];
|
|
122
|
+
collectDiff(normalizeConsensusConfig(local), normalizeConsensusConfig(upstream), "", out);
|
|
123
|
+
return out;
|
|
124
|
+
}
|
|
125
|
+
const MAX_CONFIG_RESPONSE_BYTES = 8 * 1024 * 1024;
|
|
126
|
+
const GET_CONFIG_QUERY = "{ getConfig { code config } }";
|
|
127
|
+
/** POST `{ getConfig }` to `<upstreamUrl>/api/gql` and return the parsed in-memory config of the upstream. */
|
|
128
|
+
async function fetchUpstreamChainConfig(upstreamUrl, { fetchImpl = fetch, timeoutMs = 1e4 } = {}) {
|
|
129
|
+
const url = `${String(upstreamUrl).trim().replace(/\/+$/, "")}/api/gql`;
|
|
130
|
+
const res = await fetchImpl(url, {
|
|
131
|
+
method: "POST",
|
|
132
|
+
headers: {
|
|
133
|
+
"content-type": "application/json",
|
|
134
|
+
accept: "application/json"
|
|
135
|
+
},
|
|
136
|
+
body: JSON.stringify({ query: GET_CONFIG_QUERY }),
|
|
137
|
+
redirect: "error",
|
|
138
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
139
|
+
});
|
|
140
|
+
const text = await res.text();
|
|
141
|
+
if (!res.ok) throw new Error(`getConfig from ${url} failed: HTTP ${res.status}`);
|
|
142
|
+
if (text.length > MAX_CONFIG_RESPONSE_BYTES) throw new Error(`getConfig from ${url} failed: response larger than ${MAX_CONFIG_RESPONSE_BYTES} bytes`);
|
|
143
|
+
const body = JSON.parse(text);
|
|
144
|
+
if (body.errors?.length) throw new Error(`getConfig from ${url} failed: ${body.errors.map((e) => e.message).join("; ")}`);
|
|
145
|
+
const raw = body.data?.getConfig?.config;
|
|
146
|
+
if (typeof raw !== "string" || !raw) throw new Error(`getConfig from ${url} failed: no config in the response (code=${body.data?.getConfig?.code})`);
|
|
147
|
+
return assertUpstreamConfigShape(JSON.parse(raw), url);
|
|
148
|
+
}
|
|
149
|
+
/** A response that is not a chain config is a bad body ("unavailable"), never a definite mismatch. */
|
|
150
|
+
function assertUpstreamConfigShape(parsed, source) {
|
|
151
|
+
if (!isPlainObject(parsed)) throw new Error(`getConfig from ${source} failed: config is not an object`);
|
|
152
|
+
if (typeof parsed.chainId !== "string" || !parsed.chainId.trim() || !isPlainObject(parsed.transaction)) throw new Error(`getConfig from ${source} failed: config has no chainId / transaction section`);
|
|
153
|
+
return parsed;
|
|
154
|
+
}
|
|
155
|
+
const formatValue = (v) => {
|
|
156
|
+
const s = JSON.stringify(v);
|
|
157
|
+
return s.length > 160 ? `${s.slice(0, 157)}...` : s;
|
|
158
|
+
};
|
|
159
|
+
function formatConfigDifferences(diff) {
|
|
160
|
+
return diff.map((d) => `${d.path} (local=${formatValue(d.local)}, upstream=${formatValue(d.upstream)})`).join("; ");
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Compare the follower's replay config with the upstream leader's. `off`: no request. `warn`: one attempt, log
|
|
164
|
+
* any difference or failure and continue. `enforce`: a definite difference throws `CHAIN_CONFIG_MISMATCH` (lists
|
|
165
|
+
* every differing key). An unavailable upstream (down, HTTP error such as 429, bad body) is retried with backoff
|
|
166
|
+
* until `retryDeadlineMs`; then a node with existing local state starts with a warning, and only a fresh bootstrap
|
|
167
|
+
* throws `CHAIN_CONFIG_CHECK_UNAVAILABLE`.
|
|
168
|
+
*/
|
|
169
|
+
async function verifyFollowerConfigAgainstUpstream(opts) {
|
|
170
|
+
if (opts.mode === "off") return {
|
|
171
|
+
status: "skipped",
|
|
172
|
+
diff: []
|
|
173
|
+
};
|
|
174
|
+
const enforce = opts.mode === "enforce";
|
|
175
|
+
const now = opts.now ?? (() => Date.now());
|
|
176
|
+
const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
177
|
+
const unavailable = (reason) => {
|
|
178
|
+
const message$1 = `Follower config check (CHAIN_CONFIG_CHECK=${opts.mode}) could not read the upstream config: ${reason}`;
|
|
179
|
+
if (enforce && !opts.hasLocalState) throw new CustomError("CHAIN_CONFIG_CHECK_UNAVAILABLE", `${message$1}; refusing to start a fresh bootstrap before replay`);
|
|
180
|
+
opts.logger.warn(`[chain-node] ${message$1}; starting without the check${enforce ? " (existing local state)" : ""}`);
|
|
181
|
+
return {
|
|
182
|
+
status: "unavailable",
|
|
183
|
+
diff: []
|
|
184
|
+
};
|
|
185
|
+
};
|
|
186
|
+
const upstreamUrl = String(opts.upstreamUrl ?? "").trim();
|
|
187
|
+
if (!upstreamUrl) return unavailable("no upstream URL (UPSTREAM_CHAIN_URL) is configured");
|
|
188
|
+
const fetchConfig = opts.fetchUpstreamConfig ?? ((u) => fetchUpstreamChainConfig(u));
|
|
189
|
+
const deadline = now() + Math.max(0, opts.retryDeadlineMs ?? 6e4);
|
|
190
|
+
const maxDelay = Math.max(0, opts.retryMaxDelayMs ?? 1e4);
|
|
191
|
+
let delay = Math.max(0, opts.retryDelayMs ?? 1e3);
|
|
192
|
+
let upstreamConfig = null;
|
|
193
|
+
while (upstreamConfig === null) try {
|
|
194
|
+
upstreamConfig = assertUpstreamConfigShape(await fetchConfig(upstreamUrl), upstreamUrl);
|
|
195
|
+
} catch (err) {
|
|
196
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
197
|
+
if (!enforce || now() + delay > deadline) return unavailable(reason);
|
|
198
|
+
opts.logger.warn(`[chain-node] follower config check: upstream unavailable (${reason}); retrying in ${delay} ms`);
|
|
199
|
+
await sleep(delay);
|
|
200
|
+
delay = Math.min(maxDelay, delay * 2);
|
|
201
|
+
}
|
|
202
|
+
const diff = diffConsensusConfig(opts.localConfig, upstreamConfig);
|
|
203
|
+
if (diff.length === 0) {
|
|
204
|
+
opts.logger.info(`[chain-node] follower consensus config matches upstream ${upstreamUrl}`);
|
|
205
|
+
return {
|
|
206
|
+
status: "match",
|
|
207
|
+
diff
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
const message = `Follower consensus config differs from upstream ${upstreamUrl} in ${diff.length} key(s): ${formatConfigDifferences(diff)}`;
|
|
211
|
+
if (enforce) throw new CustomError("CHAIN_CONFIG_MISMATCH", `${message}; refusing to start before replay (#1961)`);
|
|
212
|
+
opts.logger.warn(`[chain-node] ${message}`);
|
|
213
|
+
return {
|
|
214
|
+
status: "mismatch",
|
|
215
|
+
diff
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
//#endregion
|
|
220
|
+
export { CONSENSUS_CONFIG_KEYS, diffConsensusConfig, fetchUpstreamChainConfig, formatConfigDifferences, normalizeConsensusConfig, verifyFollowerConfigAgainstUpstream };
|