@optimystic/db-p2p 0.13.0 → 0.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/libp2p-key-network.d.ts +1 -1
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +20 -5
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +2 -1
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js +16 -3
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/protocol-client.d.ts +12 -0
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +52 -2
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +5 -1
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +9 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +65 -22
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +20 -0
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +101 -16
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/package.json +2 -2
- package/src/libp2p-key-network.ts +21 -6
- package/src/libp2p-node-base.ts +38 -3
- package/src/libp2p-node.ts +36 -21
- package/src/protocol-client.ts +56 -3
- package/src/repo/client.ts +5 -1
- package/src/repo/cluster-coordinator.ts +66 -26
- package/src/repo/coordinator-repo.ts +99 -15
package/src/libp2p-node-base.ts
CHANGED
|
@@ -5,10 +5,10 @@ import { identify } from '@libp2p/identify';
|
|
|
5
5
|
import { ping } from '@libp2p/ping';
|
|
6
6
|
import { gossipsub } from '@chainsafe/libp2p-gossipsub';
|
|
7
7
|
import { bootstrap } from '@libp2p/bootstrap';
|
|
8
|
-
import { circuitRelayServer } from '@libp2p/circuit-relay-v2';
|
|
8
|
+
import { circuitRelayServer, type CircuitRelayServerInit } from '@libp2p/circuit-relay-v2';
|
|
9
9
|
import { peerIdFromString } from '@libp2p/peer-id';
|
|
10
10
|
import { generateKeyPair } from '@libp2p/crypto/keys';
|
|
11
|
-
import type { PrivateKey } from '@libp2p/interface';
|
|
11
|
+
import type { ConnectionGater, PrivateKey } from '@libp2p/interface';
|
|
12
12
|
import { clusterService } from './cluster/service.js';
|
|
13
13
|
import { repoService } from './repo/service.js';
|
|
14
14
|
import { StorageRepo } from './storage/storage-repo.js';
|
|
@@ -51,11 +51,37 @@ export type NodeOptions = {
|
|
|
51
51
|
* For non-TCP transports (e.g. WebSockets), set `listenAddrs` explicitly.
|
|
52
52
|
*/
|
|
53
53
|
port?: number;
|
|
54
|
+
/**
|
|
55
|
+
* WebSocket listen port. When set, the Node `createLibp2pNode` defaulting
|
|
56
|
+
* branch adds `webSockets()` to the transports and `/ip4/<wsHost>/tcp/<wsPort>/ws`
|
|
57
|
+
* to the listen addrs. Browsers and other WS-only peers (RN, web) can dial here.
|
|
58
|
+
* Ignored when `transports`/`listenAddrs` are explicitly provided.
|
|
59
|
+
*/
|
|
60
|
+
wsPort?: number;
|
|
61
|
+
/** Interface to bind the WS listener to. Defaults to `0.0.0.0`. */
|
|
62
|
+
wsHost?: string;
|
|
63
|
+
/**
|
|
64
|
+
* Drop the default TCP transport and TCP listen addr. Useful for browser-only
|
|
65
|
+
* bootstraps that listen on `/ws` (typically fronted as `/wss`) only.
|
|
66
|
+
* Ignored when `transports`/`listenAddrs` are explicitly provided.
|
|
67
|
+
*/
|
|
68
|
+
disableTcp?: boolean;
|
|
54
69
|
bootstrapNodes: string[];
|
|
55
70
|
networkName: string;
|
|
56
71
|
fretProfile?: 'edge' | 'core';
|
|
57
72
|
id?: string; // optional peer id
|
|
58
73
|
relay?: boolean; // enable relay service
|
|
74
|
+
/**
|
|
75
|
+
* Init passed to `circuitRelayServer(...)` when `relay` is enabled.
|
|
76
|
+
*
|
|
77
|
+
* `@libp2p/circuit-relay-v2` defaults to `applyDefaultLimit: true`, which
|
|
78
|
+
* stamps every reservation with `Limit { data: 128 KiB, duration: 2 min }`
|
|
79
|
+
* and resets the relayed stream once either cap is hit — silently killing
|
|
80
|
+
* long-lived service↔browser circuits. Trusted local clusters (e.g. the
|
|
81
|
+
* reference-peer service nodes) should pass
|
|
82
|
+
* `{ reservations: { applyDefaultLimit: false } }` to lift the cap.
|
|
83
|
+
*/
|
|
84
|
+
relayServerInit?: CircuitRelayServerInit;
|
|
59
85
|
/** Storage provider - either an IRawStorage instance or a factory function. Defaults to MemoryRawStorage if not provided. */
|
|
60
86
|
storage?: RawStorageProvider;
|
|
61
87
|
clusterSize?: number; // desired cluster size per key
|
|
@@ -106,6 +132,14 @@ export type NodeOptions = {
|
|
|
106
132
|
* or `privateKeyFromProtobuf(...)` from `@libp2p/crypto/keys`).
|
|
107
133
|
*/
|
|
108
134
|
privateKey?: PrivateKey;
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Optional libp2p connection gater. The libp2p browser default denies
|
|
138
|
+
* dialing insecure WebSockets and private/loopback addresses; callers
|
|
139
|
+
* that need to dial local or unsecured bootstraps (web reference dev,
|
|
140
|
+
* Playwright e2e, RN simulators) supply a permissive gater here.
|
|
141
|
+
*/
|
|
142
|
+
connectionGater?: ConnectionGater;
|
|
109
143
|
};
|
|
110
144
|
|
|
111
145
|
function resolveStorage(provider: RawStorageProvider | undefined): IRawStorage {
|
|
@@ -183,6 +217,7 @@ export async function createLibp2pNodeBase(
|
|
|
183
217
|
inboundConnectionUpgradeTimeout: 10_000,
|
|
184
218
|
dialQueue: { concurrency: 2, attempts: 2 }
|
|
185
219
|
},
|
|
220
|
+
...(options.connectionGater ? { connectionGater: options.connectionGater } : {}),
|
|
186
221
|
transports,
|
|
187
222
|
connectionEncrypters: [noise()],
|
|
188
223
|
streamMuxers: [yamux()],
|
|
@@ -196,7 +231,7 @@ export async function createLibp2pNodeBase(
|
|
|
196
231
|
heartbeatInterval: 7000
|
|
197
232
|
}),
|
|
198
233
|
// Circuit relay server - enables this node to relay connections for other peers
|
|
199
|
-
...(options.relay ? { relay: circuitRelayServer() } : {}),
|
|
234
|
+
...(options.relay ? { relay: circuitRelayServer(options.relayServerInit) } : {}),
|
|
200
235
|
|
|
201
236
|
// Custom services - create wrapper factories that inject dependencies
|
|
202
237
|
cluster: (components: any) => {
|
package/src/libp2p-node.ts
CHANGED
|
@@ -1,21 +1,36 @@
|
|
|
1
|
-
import type { Libp2p } from 'libp2p';
|
|
2
|
-
import { tcp } from '@libp2p/tcp';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
type
|
|
8
|
-
type
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
1
|
+
import type { Libp2p } from 'libp2p';
|
|
2
|
+
import { tcp } from '@libp2p/tcp';
|
|
3
|
+
import { webSockets } from '@libp2p/websockets';
|
|
4
|
+
import { circuitRelayTransport } from '@libp2p/circuit-relay-v2';
|
|
5
|
+
import {
|
|
6
|
+
createLibp2pNodeBase,
|
|
7
|
+
type Libp2pTransports,
|
|
8
|
+
type NodeOptions,
|
|
9
|
+
type RawStorageProvider,
|
|
10
|
+
} from './libp2p-node-base.js';
|
|
11
|
+
|
|
12
|
+
export type { Libp2pTransports, NodeOptions, RawStorageProvider };
|
|
13
|
+
|
|
14
|
+
export async function createLibp2pNode(options: NodeOptions): Promise<Libp2p> {
|
|
15
|
+
const port = options.port ?? 0;
|
|
16
|
+
const wsHost = options.wsHost ?? '0.0.0.0';
|
|
17
|
+
|
|
18
|
+
const defaultTransports: Libp2pTransports = [];
|
|
19
|
+
const defaultListenAddrs: string[] = [];
|
|
20
|
+
|
|
21
|
+
if (!options.disableTcp) {
|
|
22
|
+
defaultTransports.push(tcp());
|
|
23
|
+
defaultListenAddrs.push(`/ip4/0.0.0.0/tcp/${port}`);
|
|
24
|
+
}
|
|
25
|
+
if (options.wsPort !== undefined) {
|
|
26
|
+
defaultTransports.push(webSockets());
|
|
27
|
+
defaultListenAddrs.push(`/ip4/${wsHost}/tcp/${options.wsPort}/ws`);
|
|
28
|
+
}
|
|
29
|
+
// Always include the relay transport so this node can dial through relays
|
|
30
|
+
defaultTransports.push(circuitRelayTransport());
|
|
31
|
+
|
|
32
|
+
return await createLibp2pNodeBase(options, {
|
|
33
|
+
listenAddrs: defaultListenAddrs,
|
|
34
|
+
transports: defaultTransports,
|
|
35
|
+
});
|
|
36
|
+
}
|
package/src/protocol-client.ts
CHANGED
|
@@ -7,6 +7,22 @@ import { createLogger } from './logger.js';
|
|
|
7
7
|
|
|
8
8
|
const log = createLogger('protocol-client');
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Thrown when the per-peer dial deadline expires before a stream is established.
|
|
12
|
+
* Distinct from a libp2p dial failure (no route, refused, etc.) so the
|
|
13
|
+
* batch-retry loop and diagnostic surfaces can identify a slow/unreachable peer
|
|
14
|
+
* specifically. `.code === DIAL_TIMEOUT_ERROR_CODE`.
|
|
15
|
+
*/
|
|
16
|
+
export const DIAL_TIMEOUT_ERROR_CODE = 'DIAL_TIMEOUT';
|
|
17
|
+
|
|
18
|
+
export class DialTimeoutError extends Error {
|
|
19
|
+
readonly code = DIAL_TIMEOUT_ERROR_CODE;
|
|
20
|
+
constructor(peer: string, protocol: string, ms: number) {
|
|
21
|
+
super(`dial timeout: peer=${peer} protocol=${protocol} after ${ms}ms`);
|
|
22
|
+
this.name = 'DialTimeoutError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
10
26
|
/** Base class for clients that communicate via a libp2p protocol */
|
|
11
27
|
export class ProtocolClient {
|
|
12
28
|
constructor(
|
|
@@ -17,23 +33,60 @@ export class ProtocolClient {
|
|
|
17
33
|
protected async processMessage<T>(
|
|
18
34
|
message: unknown,
|
|
19
35
|
protocol: string,
|
|
20
|
-
options?: { signal?: AbortSignal; correlationId?: string }
|
|
36
|
+
options?: { signal?: AbortSignal; correlationId?: string; dialTimeoutMs?: number }
|
|
21
37
|
): Promise<T> {
|
|
22
38
|
const peer = this.peerId.toString();
|
|
23
39
|
const cid = options?.correlationId;
|
|
24
40
|
log('dial peer=%s protocol=%s%s', peer, protocol, cid ? ` cid=${cid}` : '');
|
|
25
41
|
const t0 = Date.now();
|
|
26
42
|
|
|
43
|
+
// Per-peer dial deadline. When set, an unreachable peer fails fast so the
|
|
44
|
+
// caller can re-pick a different coordinator — independent of any overall
|
|
45
|
+
// transaction budget the caller may also be enforcing.
|
|
46
|
+
const dialTimeoutMs = options?.dialTimeoutMs;
|
|
47
|
+
const dialController = dialTimeoutMs && dialTimeoutMs > 0 ? new AbortController() : undefined;
|
|
48
|
+
let dialTimer: ReturnType<typeof setTimeout> | undefined;
|
|
49
|
+
const onParentAbort = () => dialController?.abort(options?.signal?.reason);
|
|
50
|
+
if (dialController) {
|
|
51
|
+
dialTimer = setTimeout(() => {
|
|
52
|
+
dialController.abort(new DialTimeoutError(peer, protocol, dialTimeoutMs!));
|
|
53
|
+
}, dialTimeoutMs);
|
|
54
|
+
if (options?.signal) {
|
|
55
|
+
if (options.signal.aborted) dialController.abort(options.signal.reason);
|
|
56
|
+
else options.signal.addEventListener('abort', onParentAbort, { once: true });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const dialSignal = dialController?.signal ?? options?.signal;
|
|
60
|
+
|
|
27
61
|
let stream: Libp2pStream;
|
|
28
62
|
try {
|
|
29
63
|
stream = await this.peerNetwork.connect(
|
|
30
64
|
this.peerId,
|
|
31
65
|
protocol,
|
|
32
|
-
{ signal:
|
|
66
|
+
{ signal: dialSignal }
|
|
33
67
|
) as unknown as Libp2pStream;
|
|
34
68
|
} catch (err) {
|
|
35
|
-
|
|
69
|
+
const elapsed = Date.now() - t0;
|
|
70
|
+
// If the dial AbortController fired due to our own timer, surface the
|
|
71
|
+
// dial-timeout error rather than the underlying AbortError so callers
|
|
72
|
+
// can distinguish "peer was slow" from "user/parent cancelled".
|
|
73
|
+
if (dialController?.signal.aborted && dialController.signal.reason instanceof DialTimeoutError) {
|
|
74
|
+
log('dial:timeout peer=%s protocol=%s ms=%d%s', peer, protocol, elapsed, cid ? ` cid=${cid}` : '');
|
|
75
|
+
throw dialController.signal.reason;
|
|
76
|
+
}
|
|
77
|
+
const errCode = (err as { code?: unknown })?.code;
|
|
78
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
79
|
+
const truncatedMsg = errMessage.length > 200 ? errMessage.slice(0, 200) + '…' : errMessage;
|
|
80
|
+
log('dial:fail peer=%s protocol=%s ms=%d code=%s msg=%s%s',
|
|
81
|
+
peer, protocol, elapsed,
|
|
82
|
+
typeof errCode === 'string' && errCode.length > 0 ? errCode : 'none',
|
|
83
|
+
truncatedMsg,
|
|
84
|
+
cid ? ` cid=${cid}` : ''
|
|
85
|
+
);
|
|
36
86
|
throw err;
|
|
87
|
+
} finally {
|
|
88
|
+
if (dialTimer) clearTimeout(dialTimer);
|
|
89
|
+
if (options?.signal) options.signal.removeEventListener('abort', onParentAbort);
|
|
37
90
|
}
|
|
38
91
|
log('dial:ok peer=%s ms=%d%s', peer, Date.now() - t0, cid ? ` cid=${cid}` : '');
|
|
39
92
|
|
package/src/repo/client.ts
CHANGED
|
@@ -73,7 +73,11 @@ export class RepoClient extends ProtocolClient implements IRepo {
|
|
|
73
73
|
}
|
|
74
74
|
let response: any
|
|
75
75
|
const preferred = (this.protocolPrefix ?? '/db-p2p') + '/repo/1.0.0'
|
|
76
|
-
response = await withTimeout(() => super.processMessage<any>(message, preferred, {
|
|
76
|
+
response = await withTimeout(() => super.processMessage<any>(message, preferred, {
|
|
77
|
+
signal: options?.signal,
|
|
78
|
+
correlationId,
|
|
79
|
+
dialTimeoutMs: options?.dialTimeoutMs,
|
|
80
|
+
}))
|
|
77
81
|
|
|
78
82
|
if (response?.redirect?.peers?.length) {
|
|
79
83
|
if (hop >= 2) {
|
|
@@ -37,10 +37,11 @@ interface ClusterTransactionState {
|
|
|
37
37
|
/** Manages distributed transactions across clusters */
|
|
38
38
|
export class ClusterCoordinator {
|
|
39
39
|
private transactions: Map<string, ClusterTransactionState> = new Map();
|
|
40
|
-
private readonly retryInitialIntervalMs
|
|
41
|
-
private readonly retryBackoffFactor
|
|
42
|
-
private readonly retryMaxIntervalMs
|
|
43
|
-
private readonly retryMaxAttempts
|
|
40
|
+
private readonly retryInitialIntervalMs: number;
|
|
41
|
+
private readonly retryBackoffFactor: number;
|
|
42
|
+
private readonly retryMaxIntervalMs: number;
|
|
43
|
+
private readonly retryMaxAttempts: number;
|
|
44
|
+
private readonly commitBroadcastImmediateRetries: number;
|
|
44
45
|
|
|
45
46
|
constructor(
|
|
46
47
|
private readonly keyNetwork: IKeyNetwork,
|
|
@@ -54,7 +55,13 @@ export class ClusterCoordinator {
|
|
|
54
55
|
private readonly fretService?: FretService,
|
|
55
56
|
private readonly reputation?: IPeerReputation,
|
|
56
57
|
private readonly stateStore?: ITransactionStateStore
|
|
57
|
-
) {
|
|
58
|
+
) {
|
|
59
|
+
this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
|
|
60
|
+
this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
|
|
61
|
+
this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
|
|
62
|
+
this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
|
|
63
|
+
this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
|
|
64
|
+
}
|
|
58
65
|
|
|
59
66
|
/**
|
|
60
67
|
* Creates a base 58 BTC string hash for a message to uniquely identify a transaction
|
|
@@ -528,27 +535,7 @@ export class ClusterCoordinator {
|
|
|
528
535
|
// so each peer can independently reach consensus and execute the operations.
|
|
529
536
|
// Without this, only the coordinator's local cluster executes — remote peers
|
|
530
537
|
// never see enough commits to reach consensus on their own.
|
|
531
|
-
const
|
|
532
|
-
peerIds.map(peerIdStr => {
|
|
533
|
-
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
534
|
-
return isLocal
|
|
535
|
-
? this.localCluster!.update(record)
|
|
536
|
-
: this.createClusterClient(peerIdFromString(peerIdStr)).update(record).catch(err => {
|
|
537
|
-
log('cluster-tx:consensus-broadcast-error', { messageHash: record.messageHash, peerId: peerIdStr, error: (err as Error).message });
|
|
538
|
-
return null;
|
|
539
|
-
});
|
|
540
|
-
})
|
|
541
|
-
);
|
|
542
|
-
|
|
543
|
-
// Check for broadcast failures (excluding local)
|
|
544
|
-
const broadcastFailures: string[] = [];
|
|
545
|
-
broadcastResults.forEach((result, idx) => {
|
|
546
|
-
const peerId = peerIds[idx]!;
|
|
547
|
-
const isLocal = this.localCluster && peerId === this.localCluster.peerId.toString();
|
|
548
|
-
if (!isLocal && (result.status === 'rejected' || result.value === null)) {
|
|
549
|
-
broadcastFailures.push(peerId);
|
|
550
|
-
}
|
|
551
|
-
});
|
|
538
|
+
const { failures: broadcastFailures } = await this.broadcastMergedRecord(record, peerIds);
|
|
552
539
|
if (broadcastFailures.length > 0) {
|
|
553
540
|
this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
|
|
554
541
|
} else {
|
|
@@ -565,6 +552,59 @@ export class ClusterCoordinator {
|
|
|
565
552
|
return record;
|
|
566
553
|
}
|
|
567
554
|
|
|
555
|
+
/**
|
|
556
|
+
* Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
|
|
557
|
+
* in-line re-attempts per peer before giving up. The libp2p connection used during
|
|
558
|
+
* the prior commit phase is typically still warm, so a single immediate retry recovers
|
|
559
|
+
* most transient stream errors without falling back to the scheduled retry timer.
|
|
560
|
+
* Local cluster is invoked exactly once — local failures are fatal, not transient.
|
|
561
|
+
*/
|
|
562
|
+
private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[] }> {
|
|
563
|
+
const maxAttempts = 1 + Math.max(0, this.commitBroadcastImmediateRetries);
|
|
564
|
+
const results = await Promise.all(peerIds.map(async peerIdStr => {
|
|
565
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
566
|
+
if (isLocal) {
|
|
567
|
+
try {
|
|
568
|
+
await this.localCluster!.update(record);
|
|
569
|
+
return { peerId: peerIdStr, success: true as const };
|
|
570
|
+
} catch (err) {
|
|
571
|
+
log('cluster-tx:consensus-broadcast-error', {
|
|
572
|
+
messageHash: record.messageHash,
|
|
573
|
+
peerId: peerIdStr,
|
|
574
|
+
error: (err as Error).message
|
|
575
|
+
});
|
|
576
|
+
return { peerId: peerIdStr, success: false as const };
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
let lastError: unknown;
|
|
580
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
581
|
+
try {
|
|
582
|
+
await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
|
|
583
|
+
return { peerId: peerIdStr, success: true as const };
|
|
584
|
+
} catch (err) {
|
|
585
|
+
lastError = err;
|
|
586
|
+
if (attempt < maxAttempts) {
|
|
587
|
+
log('cluster-tx:consensus-broadcast-retry', {
|
|
588
|
+
messageHash: record.messageHash,
|
|
589
|
+
peerId: peerIdStr,
|
|
590
|
+
attempt,
|
|
591
|
+
error: (err as Error).message
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
log('cluster-tx:consensus-broadcast-error', {
|
|
597
|
+
messageHash: record.messageHash,
|
|
598
|
+
peerId: peerIdStr,
|
|
599
|
+
attempts: maxAttempts,
|
|
600
|
+
error: lastError instanceof Error ? lastError.message : String(lastError)
|
|
601
|
+
});
|
|
602
|
+
return { peerId: peerIdStr, success: false as const };
|
|
603
|
+
}));
|
|
604
|
+
const failures = results.filter(r => !r.success).map(r => r.peerId);
|
|
605
|
+
return { failures };
|
|
606
|
+
}
|
|
607
|
+
|
|
568
608
|
private updateTransactionRecord(record: ClusterRecord, stage: string): void {
|
|
569
609
|
const state = this.transactions.get(record.messageHash);
|
|
570
610
|
if (!state) {
|
|
@@ -65,6 +65,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
65
65
|
private readonly localPeerId?: PeerId;
|
|
66
66
|
private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
|
|
67
67
|
private static readonly RESPONSIBILITY_TTL_MS = 60_000;
|
|
68
|
+
private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
|
|
69
|
+
private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
|
|
70
|
+
private readonly readRepairWindowMs: number;
|
|
71
|
+
private readonly readRepairSampleRate: number;
|
|
72
|
+
/** Test seam: overridable clock for window-based read-repair gating. */
|
|
73
|
+
now: () => number = () => Date.now();
|
|
74
|
+
/** Test seam: overridable RNG (0..1) for sample-rate gating. */
|
|
75
|
+
rand: () => number = () => Math.random();
|
|
68
76
|
|
|
69
77
|
constructor(
|
|
70
78
|
readonly keyNetwork: IKeyNetwork,
|
|
@@ -86,8 +94,19 @@ export class CoordinatorRepo implements IRepo {
|
|
|
86
94
|
minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
|
|
87
95
|
allowClusterDownsize: cfg?.allowClusterDownsize ?? true,
|
|
88
96
|
clusterSizeTolerance: cfg?.clusterSizeTolerance ?? 0.5,
|
|
89
|
-
partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000
|
|
97
|
+
partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000,
|
|
98
|
+
commitBroadcastRetryInitialMs: cfg?.commitBroadcastRetryInitialMs ?? 250,
|
|
99
|
+
commitBroadcastRetryBackoffFactor: cfg?.commitBroadcastRetryBackoffFactor ?? 2,
|
|
100
|
+
commitBroadcastRetryMaxIntervalMs: cfg?.commitBroadcastRetryMaxIntervalMs ?? 8000,
|
|
101
|
+
commitBroadcastRetryMaxAttempts: cfg?.commitBroadcastRetryMaxAttempts ?? 5,
|
|
102
|
+
commitBroadcastImmediateRetries: cfg?.commitBroadcastImmediateRetries ?? 1,
|
|
103
|
+
readRepairMode: cfg?.readRepairMode ?? 'lazy',
|
|
104
|
+
readRepairWindowMs: cfg?.readRepairWindowMs ?? 10000,
|
|
105
|
+
readRepairSampleRate: cfg?.readRepairSampleRate ?? 0
|
|
90
106
|
};
|
|
107
|
+
this.readRepairMode = policy.readRepairMode!;
|
|
108
|
+
this.readRepairWindowMs = policy.readRepairWindowMs!;
|
|
109
|
+
this.readRepairSampleRate = policy.readRepairSampleRate!;
|
|
91
110
|
const localClusterRef = localCluster && localPeerId ? {
|
|
92
111
|
update: localCluster.update.bind(localCluster),
|
|
93
112
|
peerId: localPeerId,
|
|
@@ -159,24 +178,44 @@ export class CoordinatorRepo implements IRepo {
|
|
|
159
178
|
// First try local storage
|
|
160
179
|
const localResult = await this.storageRepo.get(blockGets, options);
|
|
161
180
|
|
|
162
|
-
//
|
|
163
|
-
//
|
|
181
|
+
// Decide per-block whether to consult cluster peers. Two triggers:
|
|
182
|
+
// (a) Missing — block isn't present locally at all (legacy behavior).
|
|
183
|
+
// (b) Stale-by-policy — block is present but read-repair policy says verify.
|
|
184
|
+
// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
|
|
164
185
|
const skipClusterFetch = (options as any)?.skipClusterFetch;
|
|
165
186
|
if (this.clusterLatestCallback && !skipClusterFetch) {
|
|
166
187
|
for (const blockId of blockGets.blockIds) {
|
|
167
188
|
const localEntry = localResult[blockId];
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
189
|
+
const localRev = localEntry?.state?.latest?.rev;
|
|
190
|
+
const isMissing = !localEntry?.state?.latest;
|
|
191
|
+
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
192
|
+
if (!isMissing && !isStale) continue;
|
|
193
|
+
|
|
194
|
+
if (isStale) {
|
|
195
|
+
log('cluster-tx:read-repair-triggered', {
|
|
196
|
+
blockId,
|
|
197
|
+
mode: this.readRepairMode,
|
|
198
|
+
ageMs: this.ageMs(blockId),
|
|
199
|
+
localRev
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
try {
|
|
204
|
+
await this.fetchBlockFromCluster(blockId, blockGets.context);
|
|
205
|
+
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
206
|
+
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
207
|
+
if (refreshed[blockId]) {
|
|
208
|
+
localResult[blockId] = refreshed[blockId];
|
|
209
|
+
}
|
|
210
|
+
if (isStale) {
|
|
211
|
+
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
212
|
+
log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
213
|
+
} else {
|
|
214
|
+
log('cluster-tx:read-repair-noop', { blockId });
|
|
176
215
|
}
|
|
177
|
-
} catch (err) {
|
|
178
|
-
log('cluster-fetch:error', { blockId, error: (err as Error).message });
|
|
179
216
|
}
|
|
217
|
+
} catch (err) {
|
|
218
|
+
log('cluster-fetch:error', { blockId, error: (err as Error).message });
|
|
180
219
|
}
|
|
181
220
|
}
|
|
182
221
|
}
|
|
@@ -184,6 +223,44 @@ export class CoordinatorRepo implements IRepo {
|
|
|
184
223
|
return localResult;
|
|
185
224
|
}
|
|
186
225
|
|
|
226
|
+
/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
|
|
227
|
+
private shouldReadRepair(blockId: BlockId): boolean {
|
|
228
|
+
switch (this.readRepairMode) {
|
|
229
|
+
case 'off': return false;
|
|
230
|
+
case 'paranoid': return true;
|
|
231
|
+
case 'lazy': {
|
|
232
|
+
const lastSeen = this.lastSeenCommitMs.get(blockId);
|
|
233
|
+
if (lastSeen == null) return true;
|
|
234
|
+
if (this.now() - lastSeen > this.readRepairWindowMs) return true;
|
|
235
|
+
if (this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate) return true;
|
|
236
|
+
return false;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** Milliseconds since we last marked this block fresh, or undefined if never. */
|
|
242
|
+
private ageMs(blockId: BlockId): number | undefined {
|
|
243
|
+
const lastSeen = this.lastSeenCommitMs.get(blockId);
|
|
244
|
+
return lastSeen == null ? undefined : this.now() - lastSeen;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
|
|
248
|
+
private markBlocksSeen(blockIds: BlockId[]): void {
|
|
249
|
+
const now = this.now();
|
|
250
|
+
for (const id of blockIds) {
|
|
251
|
+
this.lastSeenCommitMs.set(id, now);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Test seam: directly set the last-seen timestamp for a block. Used by read-repair
|
|
257
|
+
* specs to simulate "the local commit happened at time T" without needing to drive
|
|
258
|
+
* a full pend/commit cycle through the cluster coordinator.
|
|
259
|
+
*/
|
|
260
|
+
setLastSeenForTest(blockId: BlockId, ts: number): void {
|
|
261
|
+
this.lastSeenCommitMs.set(blockId, ts);
|
|
262
|
+
}
|
|
263
|
+
|
|
187
264
|
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext): Promise<void> {
|
|
188
265
|
if (!this.clusterLatestCallback) return;
|
|
189
266
|
|
|
@@ -210,6 +287,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
210
287
|
// Found on cluster - trigger restoration to sync the block
|
|
211
288
|
await this.storageRepo.get({ blockIds: [blockId], context: { committed: [clusterLatest], rev: clusterLatest.rev } });
|
|
212
289
|
log('cluster-fetch:synced', { blockId, rev: clusterLatest.rev });
|
|
290
|
+
this.markBlocksSeen([blockId]);
|
|
213
291
|
}
|
|
214
292
|
}
|
|
215
293
|
|
|
@@ -327,7 +405,9 @@ export class CoordinatorRepo implements IRepo {
|
|
|
327
405
|
|
|
328
406
|
const peerCount = await this.coordinator.getClusterSize(blockIds[0]!);
|
|
329
407
|
if (peerCount <= 1) {
|
|
330
|
-
|
|
408
|
+
const result = await this.storageRepo.commit(request, options);
|
|
409
|
+
if (result.success) this.markBlocksSeen(blockIds);
|
|
410
|
+
return result;
|
|
331
411
|
}
|
|
332
412
|
|
|
333
413
|
const message: RepoMessage = {
|
|
@@ -338,6 +418,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
338
418
|
try {
|
|
339
419
|
const { record, localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
|
|
340
420
|
if (localExecuted) {
|
|
421
|
+
this.markBlocksSeen(blockIds);
|
|
341
422
|
return { success: true };
|
|
342
423
|
}
|
|
343
424
|
// Local cluster didn't execute during consensus. Attempt a local commit,
|
|
@@ -346,13 +427,16 @@ export class CoordinatorRepo implements IRepo {
|
|
|
346
427
|
// after missing the pend phase (unreachable during pend, fresh join, etc.).
|
|
347
428
|
// The cluster's majority is authoritative; this peer will catch up via sync.
|
|
348
429
|
try {
|
|
349
|
-
|
|
430
|
+
const result = await this.storageRepo.commit(request, options);
|
|
431
|
+
if (result.success) this.markBlocksSeen(blockIds);
|
|
432
|
+
return result;
|
|
350
433
|
} catch (err) {
|
|
351
434
|
if (clusterReachedCommitConsensus(record)) {
|
|
352
435
|
log('coordinator-repo:commit-local-failed-cluster-succeeded', {
|
|
353
436
|
actionId: request.actionId,
|
|
354
437
|
error: (err as Error).message
|
|
355
438
|
});
|
|
439
|
+
this.markBlocksSeen(blockIds);
|
|
356
440
|
return { success: true };
|
|
357
441
|
}
|
|
358
442
|
throw err;
|