@agentplat/mesh-sim 0.3.0-alpha.1 → 0.3.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -2
- package/dist/index.d.ts +119 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1817 -44
- package/dist/index.js.map +1 -1
- package/dist/reducer-scenario.d.ts +129 -0
- package/dist/reducer-scenario.d.ts.map +1 -0
- package/dist/reducer-scenario.js +774 -0
- package/dist/reducer-scenario.js.map +1 -0
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,10 +1,51 @@
|
|
|
1
1
|
import { processMeshEnvelope, reduceMeshPeer, } from '@agentplat/mesh';
|
|
2
|
-
import { MESH_PROTOCOL, MESH_SIGNATURE_ALGORITHM, MESH_WIRE_VERSION, canonicalizeMeshJsonBytes, } from '@agentplat/mesh-protocol';
|
|
2
|
+
import { MESH_PROTOCOL, MESH_SIGNATURE_ALGORITHM, MESH_WIRE_VERSION, canonicalizeMeshJsonBytes, validateSignedMeshEnvelope, } from '@agentplat/mesh-protocol';
|
|
3
|
+
export const MESH_SIMULATION_FAULT_LIMITS = Object.freeze({
|
|
4
|
+
maximumFaults: 4_096,
|
|
5
|
+
maximumDuplicateCopies: 16,
|
|
6
|
+
maximumClockOffset: 24 * 60 * 60 * 1_000,
|
|
7
|
+
maximumLinksPerFault: 4_096,
|
|
8
|
+
});
|
|
9
|
+
const metricDeltaKeys = Object.freeze([
|
|
10
|
+
'clockOffsetChanges',
|
|
11
|
+
'crashSuppressedEvents',
|
|
12
|
+
'delayedMessages',
|
|
13
|
+
'deliveredMessages',
|
|
14
|
+
'droppedMessages',
|
|
15
|
+
'duplicatedMessages',
|
|
16
|
+
'emittedEffects',
|
|
17
|
+
'faultEvents',
|
|
18
|
+
'heals',
|
|
19
|
+
'partitionSuppressedMessages',
|
|
20
|
+
'partitions',
|
|
21
|
+
'peerCrashes',
|
|
22
|
+
'peerResumes',
|
|
23
|
+
'processedEvents',
|
|
24
|
+
'rejectedMessages',
|
|
25
|
+
'reorderedMessages',
|
|
26
|
+
]);
|
|
3
27
|
/** Creates a bounded kernel after hashing its serializable configuration. */
|
|
4
28
|
export async function createMeshSimulationKernel(config) {
|
|
5
29
|
validateConfig(config);
|
|
6
|
-
const
|
|
7
|
-
|
|
30
|
+
const faultPlan = normalizeFaultPlan(config.faultPlan, config);
|
|
31
|
+
const frozenConfig = freezeSimulationConfig(config, faultPlan);
|
|
32
|
+
const [configurationDigest, faultPlanDigest] = await Promise.all([
|
|
33
|
+
digest(configurationProjection(frozenConfig, faultPlan)),
|
|
34
|
+
digest(faultPlan),
|
|
35
|
+
]);
|
|
36
|
+
return new DeterministicMeshSimulationKernel(frozenConfig, configurationDigest, faultPlan, faultPlanDigest);
|
|
37
|
+
}
|
|
38
|
+
/** Restores a strict v2 snapshot using the runtime handles from the same config. */
|
|
39
|
+
export async function restoreMeshSimulationKernel(config, snapshot) {
|
|
40
|
+
validateConfig(config);
|
|
41
|
+
const faultPlan = normalizeFaultPlan(config.faultPlan, config);
|
|
42
|
+
const frozenConfig = freezeSimulationConfig(config, faultPlan);
|
|
43
|
+
const [configurationDigest, faultPlanDigest] = await Promise.all([
|
|
44
|
+
digest(configurationProjection(frozenConfig, faultPlan)),
|
|
45
|
+
digest(faultPlan),
|
|
46
|
+
]);
|
|
47
|
+
const restored = await restoreSnapshot(snapshot, frozenConfig, configurationDigest, faultPlanDigest, faultPlan);
|
|
48
|
+
return new DeterministicMeshSimulationKernel(frozenConfig, configurationDigest, faultPlan, faultPlanDigest, restored);
|
|
8
49
|
}
|
|
9
50
|
export async function runMeshSimulation(config, events) {
|
|
10
51
|
const kernel = await createMeshSimulationKernel(config);
|
|
@@ -19,8 +60,7 @@ export async function replayMeshSimulation(config, events, expected) {
|
|
|
19
60
|
let firstDivergence;
|
|
20
61
|
const count = Math.max(expectedRecords.length, actualRecords.length);
|
|
21
62
|
for (let index = 0; index < count; index += 1) {
|
|
22
|
-
if (expectedRecords[index]?.chainDigest !==
|
|
23
|
-
actualRecords[index]?.chainDigest) {
|
|
63
|
+
if (expectedRecords[index]?.chainDigest !== actualRecords[index]?.chainDigest) {
|
|
24
64
|
firstDivergence = index;
|
|
25
65
|
break;
|
|
26
66
|
}
|
|
@@ -31,42 +71,82 @@ export async function replayMeshSimulation(config, events, expected) {
|
|
|
31
71
|
matches,
|
|
32
72
|
expectedChainDigest: expected.chainDigest,
|
|
33
73
|
actualChainDigest: actual.chainDigest,
|
|
34
|
-
...(matches
|
|
35
|
-
? {}
|
|
36
|
-
: { firstDivergence: firstDivergence ?? Math.min(count, 0) }),
|
|
74
|
+
...(matches || firstDivergence === undefined ? {} : { firstDivergence }),
|
|
37
75
|
});
|
|
38
76
|
}
|
|
39
77
|
class DeterministicMeshSimulationKernel {
|
|
40
78
|
config;
|
|
41
79
|
configurationDigest;
|
|
80
|
+
#faultPlan;
|
|
81
|
+
#faultPlanDigest;
|
|
42
82
|
#peers = new Map();
|
|
83
|
+
#topology = new Map();
|
|
84
|
+
#clockOffsets = new Map();
|
|
43
85
|
#queue = [];
|
|
44
86
|
#eventIds = new Set();
|
|
45
87
|
#prngStates = new Map();
|
|
46
88
|
#records = [];
|
|
89
|
+
#faultRecords = [];
|
|
47
90
|
#insertionSequence = 0;
|
|
91
|
+
#faultCursor = 0;
|
|
48
92
|
#logicalTime = 0;
|
|
49
93
|
#processedEvents = 0;
|
|
50
94
|
#emittedEffects = 0;
|
|
51
95
|
#deliveredMessages = 0;
|
|
52
96
|
#rejectedMessages = 0;
|
|
97
|
+
#peerCrashes = 0;
|
|
98
|
+
#peerResumes = 0;
|
|
99
|
+
#droppedMessages = 0;
|
|
100
|
+
#duplicatedMessages = 0;
|
|
101
|
+
#delayedMessages = 0;
|
|
102
|
+
#reorderedMessages = 0;
|
|
103
|
+
#partitions = 0;
|
|
104
|
+
#heals = 0;
|
|
105
|
+
#clockOffsetChanges = 0;
|
|
106
|
+
#crashSuppressedEvents = 0;
|
|
107
|
+
#partitionSuppressedMessages = 0;
|
|
53
108
|
#chainDigest;
|
|
54
|
-
constructor(config, configurationDigest) {
|
|
109
|
+
constructor(config, configurationDigest, faultPlan, faultPlanDigest, snapshot) {
|
|
55
110
|
this.config = config;
|
|
56
111
|
this.configurationDigest = configurationDigest;
|
|
112
|
+
this.#faultPlan = faultPlan;
|
|
113
|
+
this.#faultPlanDigest = faultPlanDigest;
|
|
57
114
|
this.#chainDigest = configurationDigest;
|
|
58
115
|
for (const peer of config.peers) {
|
|
59
116
|
this.#peers.set(peer.peerId, {
|
|
60
117
|
config: peer,
|
|
61
118
|
state: peer.state,
|
|
62
119
|
outboundSequence: peer.outboundSequence ?? 0,
|
|
120
|
+
available: true,
|
|
63
121
|
});
|
|
122
|
+
this.#clockOffsets.set(peer.peerId, 0);
|
|
123
|
+
}
|
|
124
|
+
for (const link of config.links)
|
|
125
|
+
this.#topology.set(linkKey(link.fromPeerId, link.toPeerId), link);
|
|
126
|
+
if (snapshot === undefined) {
|
|
127
|
+
for (const fault of faultPlan.faults)
|
|
128
|
+
this.#enqueueFault(fault);
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
this.#restore(snapshot);
|
|
64
132
|
}
|
|
65
133
|
}
|
|
66
134
|
enqueue(input) {
|
|
67
|
-
|
|
135
|
+
const internal = input;
|
|
136
|
+
if (typeof internal?.eventId === 'string' &&
|
|
137
|
+
internal.eventId.startsWith('fault:'))
|
|
138
|
+
throw new TypeError('Mesh simulation eventId uses a reserved namespace');
|
|
139
|
+
if (internal?.action?.kind === 'fault.apply')
|
|
140
|
+
throw new TypeError('Mesh simulation faults must come from the configured fault plan');
|
|
141
|
+
this.#enqueue(internal);
|
|
142
|
+
}
|
|
143
|
+
#enqueue(input) {
|
|
144
|
+
if (!input ||
|
|
145
|
+
typeof input !== 'object' ||
|
|
146
|
+
this.#eventIds.has(input.eventId)) {
|
|
68
147
|
throw new TypeError('Invalid or duplicate Mesh simulation event');
|
|
69
148
|
}
|
|
149
|
+
assertBoundedString(input.eventId, 'eventId', 768);
|
|
70
150
|
if (!this.#peers.has(input.targetPeerId) ||
|
|
71
151
|
!Number.isSafeInteger(input.logicalTime) ||
|
|
72
152
|
input.logicalTime < this.#logicalTime ||
|
|
@@ -77,6 +157,9 @@ class DeterministicMeshSimulationKernel {
|
|
|
77
157
|
if (this.#queue.length >= this.config.limits.maximumQueuedEvents) {
|
|
78
158
|
throw new RangeError('Mesh simulation queue limit exceeded');
|
|
79
159
|
}
|
|
160
|
+
if (this.#insertionSequence >= maximumIssuedEvents(this.config))
|
|
161
|
+
throw new RangeError('Mesh simulation event issuance limit exceeded');
|
|
162
|
+
const action = freezeAction(input.action);
|
|
80
163
|
this.#insertionSequence += 1;
|
|
81
164
|
const event = Object.freeze({
|
|
82
165
|
eventId: input.eventId,
|
|
@@ -86,18 +169,17 @@ class DeterministicMeshSimulationKernel {
|
|
|
86
169
|
priority: input.priority,
|
|
87
170
|
insertionSequence: this.#insertionSequence,
|
|
88
171
|
}),
|
|
89
|
-
action
|
|
172
|
+
action,
|
|
90
173
|
});
|
|
91
174
|
this.#eventIds.add(input.eventId);
|
|
92
175
|
this.#queue.push(event);
|
|
93
176
|
this.#queue.sort(compareEvents);
|
|
94
177
|
}
|
|
95
178
|
random(scope) {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
let state = this.#prngStates.get(scope) ??
|
|
100
|
-
mixSeed(this.config.seed >>> 0, scope);
|
|
179
|
+
assertBoundedString(scope, 'random scope', 768);
|
|
180
|
+
if (!this.#prngStates.has(scope) && this.#prngStates.size >= 4096)
|
|
181
|
+
throw new RangeError('Mesh simulation PRNG stream limit exceeded');
|
|
182
|
+
let state = this.#prngStates.get(scope) ?? mixSeed(this.config.seed >>> 0, scope);
|
|
101
183
|
state = xorshift32(state);
|
|
102
184
|
this.#prngStates.set(scope, state);
|
|
103
185
|
return state / 0x1_0000_0000;
|
|
@@ -115,18 +197,54 @@ class DeterministicMeshSimulationKernel {
|
|
|
115
197
|
const peer = this.#peers.get(event.targetPeerId);
|
|
116
198
|
if (!peer)
|
|
117
199
|
throw new TypeError('Mesh simulation target disappeared');
|
|
200
|
+
const metricsBefore = this.#metrics();
|
|
118
201
|
let effects = Object.freeze([]);
|
|
202
|
+
let transportOutcome = emptyTransportOutcome();
|
|
203
|
+
const transportDeliveries = [];
|
|
119
204
|
let accepted;
|
|
120
205
|
let rejectionCode;
|
|
121
|
-
|
|
206
|
+
let faultResult;
|
|
207
|
+
if (event.action.kind === 'fault.apply') {
|
|
208
|
+
faultResult = this.#applyFault(event.action.fault);
|
|
209
|
+
accepted = faultResult.applied;
|
|
210
|
+
this.#faultCursor += 1;
|
|
211
|
+
this.#faultRecords.push(Object.freeze({
|
|
212
|
+
faultId: event.action.fault.faultId,
|
|
213
|
+
kind: event.action.fault.kind,
|
|
214
|
+
order: event.order,
|
|
215
|
+
applied: faultResult.applied,
|
|
216
|
+
affectedEventIds: faultResult.affectedEventIds,
|
|
217
|
+
affectedLinkIds: faultResult.affectedLinkIds,
|
|
218
|
+
affectedDeliveries: faultResult.affectedDeliveries,
|
|
219
|
+
}));
|
|
220
|
+
}
|
|
221
|
+
else if (!peer.available) {
|
|
222
|
+
accepted = false;
|
|
223
|
+
rejectionCode = 'simulation_peer_crashed';
|
|
224
|
+
this.#crashSuppressedEvents += 1;
|
|
225
|
+
if (event.action.kind === 'message.delivery') {
|
|
226
|
+
this.#rejectedMessages += 1;
|
|
227
|
+
this.#droppedMessages += 1;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
else if (event.action.kind === 'message.delivery' &&
|
|
231
|
+
!this.#linkEnabled(event.action.envelope.sender.peerId, event.targetPeerId)) {
|
|
232
|
+
accepted = false;
|
|
233
|
+
rejectionCode = 'simulation_partitioned';
|
|
234
|
+
this.#rejectedMessages += 1;
|
|
235
|
+
this.#droppedMessages += 1;
|
|
236
|
+
this.#partitionSuppressedMessages += 1;
|
|
237
|
+
}
|
|
238
|
+
else if (event.action.kind === 'peer.input') {
|
|
122
239
|
const transition = reduceMeshPeer(peer.state, event.action.input, this.#logicalTime);
|
|
123
240
|
peer.state = transition.state;
|
|
124
|
-
effects = await this.#interpret(peer, transition.effects, event);
|
|
241
|
+
effects = await this.#interpret(peer, transition.effects, event, transportDeliveries);
|
|
242
|
+
transportOutcome = transportOutcomeFromDeliveries(transportDeliveries);
|
|
125
243
|
}
|
|
126
244
|
else {
|
|
127
245
|
const decision = await processMeshEnvelope(peer.state, {
|
|
128
246
|
envelope: event.action.envelope,
|
|
129
|
-
verifiedAt: timestampAt(this.config.startTime, this.#logicalTime),
|
|
247
|
+
verifiedAt: timestampAt(this.config.startTime, this.#logicalTime + (this.#clockOffsets.get(peer.config.peerId) ?? 0)),
|
|
130
248
|
receivedAt: this.#logicalTime,
|
|
131
249
|
verifier: peer.config.verifier,
|
|
132
250
|
resolver: peer.config.resolver,
|
|
@@ -138,7 +256,8 @@ class DeterministicMeshSimulationKernel {
|
|
|
138
256
|
accepted = decision.accepted;
|
|
139
257
|
if (decision.accepted) {
|
|
140
258
|
peer.state = decision.state;
|
|
141
|
-
effects = await this.#interpret(peer, decision.effects, event);
|
|
259
|
+
effects = await this.#interpret(peer, decision.effects, event, transportDeliveries);
|
|
260
|
+
transportOutcome = transportOutcomeFromDeliveries(transportDeliveries);
|
|
142
261
|
}
|
|
143
262
|
else {
|
|
144
263
|
rejectionCode = decision.code;
|
|
@@ -165,10 +284,24 @@ class DeterministicMeshSimulationKernel {
|
|
|
165
284
|
peerId: event.targetPeerId,
|
|
166
285
|
order: event.order,
|
|
167
286
|
inputKind: event.action.kind,
|
|
287
|
+
...(event.action.kind === 'message.delivery'
|
|
288
|
+
? {
|
|
289
|
+
inputDeliverySourcePeerId: event.action.envelope.sender.peerId,
|
|
290
|
+
}
|
|
291
|
+
: {}),
|
|
168
292
|
actionDigest,
|
|
169
293
|
...(accepted === undefined ? {} : { accepted }),
|
|
170
294
|
...(rejectionCode === undefined ? {} : { rejectionCode }),
|
|
295
|
+
...(event.action.kind === 'fault.apply'
|
|
296
|
+
? {
|
|
297
|
+
faultId: event.action.fault.faultId,
|
|
298
|
+
faultKind: event.action.fault.kind,
|
|
299
|
+
faultApplied: faultResult?.applied ?? false,
|
|
300
|
+
}
|
|
301
|
+
: {}),
|
|
171
302
|
effectKinds: Object.freeze(effects.map((effect) => effect.kind)),
|
|
303
|
+
metricsDelta: metricDelta(metricsBefore, this.#metrics()),
|
|
304
|
+
transportOutcome,
|
|
172
305
|
effectsDigest,
|
|
173
306
|
stateDigest,
|
|
174
307
|
};
|
|
@@ -180,8 +313,7 @@ class DeterministicMeshSimulationKernel {
|
|
|
180
313
|
...recordBase,
|
|
181
314
|
chainDigest: this.#chainDigest,
|
|
182
315
|
});
|
|
183
|
-
|
|
184
|
-
this.#records.push(record);
|
|
316
|
+
this.#records.push(record);
|
|
185
317
|
return record;
|
|
186
318
|
}
|
|
187
319
|
async runUntilIdle() {
|
|
@@ -191,14 +323,11 @@ class DeterministicMeshSimulationKernel {
|
|
|
191
323
|
seed: this.config.seed,
|
|
192
324
|
prngVersion: this.config.prngVersion,
|
|
193
325
|
configurationDigest: this.configurationDigest,
|
|
326
|
+
faultPlanDigest: this.#faultPlanDigest,
|
|
327
|
+
faultPlan: this.#faultPlan,
|
|
194
328
|
chainDigest: this.#chainDigest,
|
|
195
|
-
metrics:
|
|
196
|
-
|
|
197
|
-
emittedEffects: this.#emittedEffects,
|
|
198
|
-
deliveredMessages: this.#deliveredMessages,
|
|
199
|
-
rejectedMessages: this.#rejectedMessages,
|
|
200
|
-
finalLogicalTime: this.#logicalTime,
|
|
201
|
-
}),
|
|
329
|
+
metrics: this.#metrics(),
|
|
330
|
+
faults: Object.freeze([...this.#faultRecords]),
|
|
202
331
|
...(this.config.recordingMode === 'full'
|
|
203
332
|
? { records: Object.freeze([...this.#records]) }
|
|
204
333
|
: {}),
|
|
@@ -207,20 +336,30 @@ class DeterministicMeshSimulationKernel {
|
|
|
207
336
|
}
|
|
208
337
|
snapshot() {
|
|
209
338
|
return Object.freeze({
|
|
210
|
-
schemaVersion:
|
|
339
|
+
schemaVersion: 2,
|
|
340
|
+
configurationDigest: this.configurationDigest,
|
|
341
|
+
faultPlanDigest: this.#faultPlanDigest,
|
|
211
342
|
logicalTime: this.#logicalTime,
|
|
212
343
|
insertionSequence: this.#insertionSequence,
|
|
344
|
+
faultCursor: this.#faultCursor,
|
|
213
345
|
peerStates: this.#peerStates(),
|
|
214
346
|
outboundSequences: frozenRecord([...this.#peers].map(([peerId, peer]) => [
|
|
215
347
|
peerId,
|
|
216
348
|
peer.outboundSequence,
|
|
217
349
|
])),
|
|
218
350
|
prngStates: frozenRecord([...this.#prngStates]),
|
|
351
|
+
peerAvailability: frozenRecord([...this.#peers].map(([peerId, peer]) => [peerId, peer.available])),
|
|
352
|
+
clockOffsets: frozenRecord([...this.#clockOffsets]),
|
|
353
|
+
topology: Object.freeze([...this.#topology.values()].map((link) => Object.freeze({ ...link }))),
|
|
219
354
|
queuedEvents: Object.freeze([...this.#queue]),
|
|
355
|
+
eventIds: Object.freeze([...this.#eventIds].sort()),
|
|
356
|
+
metrics: this.#metrics(),
|
|
357
|
+
records: Object.freeze([...this.#records]),
|
|
358
|
+
faults: Object.freeze([...this.#faultRecords]),
|
|
220
359
|
chainDigest: this.#chainDigest,
|
|
221
360
|
});
|
|
222
361
|
}
|
|
223
|
-
async #interpret(peer, initial, root) {
|
|
362
|
+
async #interpret(peer, initial, root, transportDeliveries) {
|
|
224
363
|
const effects = [...initial];
|
|
225
364
|
const prepared = new Map();
|
|
226
365
|
for (let index = 0; index < effects.length; index += 1) {
|
|
@@ -229,6 +368,8 @@ class DeterministicMeshSimulationKernel {
|
|
|
229
368
|
}
|
|
230
369
|
const effect = effects[index];
|
|
231
370
|
if (effect.kind === 'message.prepare') {
|
|
371
|
+
if (peer.outboundSequence >= Number.MAX_SAFE_INTEGER)
|
|
372
|
+
throw new RangeError('Mesh simulation outbound sequence exhausted');
|
|
232
373
|
peer.outboundSequence += 1;
|
|
233
374
|
const envelope = await this.#prepare(peer, effect);
|
|
234
375
|
prepared.set(effect.effectId, envelope);
|
|
@@ -251,10 +392,36 @@ class DeterministicMeshSimulationKernel {
|
|
|
251
392
|
if (!envelope || envelope.messageId !== effect.messageId) {
|
|
252
393
|
throw new TypeError('Mesh simulation lacks prepared delivery');
|
|
253
394
|
}
|
|
254
|
-
const link = this.
|
|
255
|
-
|
|
256
|
-
if (!
|
|
395
|
+
const link = this.#topology.get(linkKey(peer.config.peerId, effect.audiencePeerId));
|
|
396
|
+
const recipient = this.#peers.get(effect.audiencePeerId);
|
|
397
|
+
if (!recipient) {
|
|
398
|
+
this.#rejectedMessages += 1;
|
|
399
|
+
this.#droppedMessages += 1;
|
|
400
|
+
transportDeliveries.push(Object.freeze({
|
|
401
|
+
eventId: `delivery:${envelope.messageId}`,
|
|
402
|
+
fromPeerId: peer.config.peerId,
|
|
403
|
+
toPeerId: effect.audiencePeerId,
|
|
404
|
+
outcome: 'destination_missing',
|
|
405
|
+
}));
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
if (!link?.enabled || !recipient?.available) {
|
|
257
409
|
this.#rejectedMessages += 1;
|
|
410
|
+
this.#droppedMessages += 1;
|
|
411
|
+
if (!link?.enabled)
|
|
412
|
+
this.#partitionSuppressedMessages += 1;
|
|
413
|
+
if (!recipient.available)
|
|
414
|
+
this.#crashSuppressedEvents += 1;
|
|
415
|
+
transportDeliveries.push(Object.freeze({
|
|
416
|
+
eventId: `delivery:${envelope.messageId}`,
|
|
417
|
+
fromPeerId: peer.config.peerId,
|
|
418
|
+
toPeerId: effect.audiencePeerId,
|
|
419
|
+
outcome: !link?.enabled && !recipient.available
|
|
420
|
+
? 'crash_partition'
|
|
421
|
+
: !link?.enabled
|
|
422
|
+
? 'partition'
|
|
423
|
+
: 'crash',
|
|
424
|
+
}));
|
|
258
425
|
continue;
|
|
259
426
|
}
|
|
260
427
|
this.enqueue({
|
|
@@ -265,6 +432,12 @@ class DeterministicMeshSimulationKernel {
|
|
|
265
432
|
action: { kind: 'message.delivery', envelope },
|
|
266
433
|
});
|
|
267
434
|
this.#deliveredMessages += 1;
|
|
435
|
+
transportDeliveries.push(Object.freeze({
|
|
436
|
+
eventId: `delivery:${envelope.messageId}`,
|
|
437
|
+
fromPeerId: peer.config.peerId,
|
|
438
|
+
toPeerId: effect.audiencePeerId,
|
|
439
|
+
outcome: 'delivered',
|
|
440
|
+
}));
|
|
268
441
|
}
|
|
269
442
|
}
|
|
270
443
|
return Object.freeze(effects);
|
|
@@ -311,27 +484,323 @@ class DeterministicMeshSimulationKernel {
|
|
|
311
484
|
#peerStates() {
|
|
312
485
|
return frozenRecord([...this.#peers].map(([peerId, peer]) => [peerId, peer.state]));
|
|
313
486
|
}
|
|
487
|
+
#metrics() {
|
|
488
|
+
return Object.freeze({
|
|
489
|
+
processedEvents: this.#processedEvents,
|
|
490
|
+
emittedEffects: this.#emittedEffects,
|
|
491
|
+
deliveredMessages: this.#deliveredMessages,
|
|
492
|
+
rejectedMessages: this.#rejectedMessages,
|
|
493
|
+
faultEvents: this.#faultCursor,
|
|
494
|
+
peerCrashes: this.#peerCrashes,
|
|
495
|
+
peerResumes: this.#peerResumes,
|
|
496
|
+
droppedMessages: this.#droppedMessages,
|
|
497
|
+
duplicatedMessages: this.#duplicatedMessages,
|
|
498
|
+
delayedMessages: this.#delayedMessages,
|
|
499
|
+
reorderedMessages: this.#reorderedMessages,
|
|
500
|
+
partitions: this.#partitions,
|
|
501
|
+
heals: this.#heals,
|
|
502
|
+
clockOffsetChanges: this.#clockOffsetChanges,
|
|
503
|
+
crashSuppressedEvents: this.#crashSuppressedEvents,
|
|
504
|
+
partitionSuppressedMessages: this.#partitionSuppressedMessages,
|
|
505
|
+
finalLogicalTime: this.#logicalTime,
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
#linkEnabled(fromPeerId, toPeerId) {
|
|
509
|
+
return this.#topology.get(linkKey(fromPeerId, toPeerId))?.enabled === true;
|
|
510
|
+
}
|
|
511
|
+
#enqueueFault(fault) {
|
|
512
|
+
this.#enqueue({
|
|
513
|
+
eventId: `fault:${fault.faultId}`,
|
|
514
|
+
targetPeerId: faultTargetPeerId(fault, this.config),
|
|
515
|
+
logicalTime: fault.logicalTime,
|
|
516
|
+
priority: fault.priority,
|
|
517
|
+
action: { kind: 'fault.apply', fault },
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
#applyFault(fault) {
|
|
521
|
+
if (fault.kind === 'peer.crash') {
|
|
522
|
+
const peer = this.#peers.get(fault.peerId);
|
|
523
|
+
if (!peer || !peer.available)
|
|
524
|
+
return emptyFaultResult();
|
|
525
|
+
peer.available = false;
|
|
526
|
+
this.#peerCrashes += 1;
|
|
527
|
+
const dropped = this.#queue.filter((event) => event.targetPeerId === fault.peerId &&
|
|
528
|
+
event.action.kind === 'message.delivery');
|
|
529
|
+
for (const event of dropped) {
|
|
530
|
+
this.#removeQueuedEvent(event.eventId);
|
|
531
|
+
this.#droppedMessages += 1;
|
|
532
|
+
this.#crashSuppressedEvents += 1;
|
|
533
|
+
}
|
|
534
|
+
return {
|
|
535
|
+
applied: true,
|
|
536
|
+
affectedEventIds: Object.freeze(dropped.map((event) => event.eventId)),
|
|
537
|
+
affectedLinkIds: Object.freeze([]),
|
|
538
|
+
affectedDeliveries: Object.freeze(dropped.map(affectedDeliveryMetadata)),
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
if (fault.kind === 'peer.resume') {
|
|
542
|
+
const peer = this.#peers.get(fault.peerId);
|
|
543
|
+
if (!peer || peer.available)
|
|
544
|
+
return emptyFaultResult();
|
|
545
|
+
peer.available = true;
|
|
546
|
+
this.#peerResumes += 1;
|
|
547
|
+
return {
|
|
548
|
+
applied: true,
|
|
549
|
+
affectedEventIds: Object.freeze([fault.peerId]),
|
|
550
|
+
affectedLinkIds: Object.freeze([]),
|
|
551
|
+
affectedDeliveries: Object.freeze([]),
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
if (fault.kind === 'network.partition' || fault.kind === 'network.heal') {
|
|
555
|
+
const enabled = fault.kind === 'network.heal';
|
|
556
|
+
const affectedLinkIds = [];
|
|
557
|
+
const droppedDeliveries = [];
|
|
558
|
+
const requestedKeys = new Set();
|
|
559
|
+
for (const requested of fault.links) {
|
|
560
|
+
const key = linkKey(requested.fromPeerId, requested.toPeerId);
|
|
561
|
+
requestedKeys.add(key);
|
|
562
|
+
const link = this.#topology.get(key);
|
|
563
|
+
if (!link || link.enabled === enabled)
|
|
564
|
+
continue;
|
|
565
|
+
this.#topology.set(key, Object.freeze({ ...link, enabled }));
|
|
566
|
+
affectedLinkIds.push(key);
|
|
567
|
+
}
|
|
568
|
+
if (!enabled) {
|
|
569
|
+
const dropped = this.#queue.filter((event) => event.action.kind === 'message.delivery' &&
|
|
570
|
+
requestedKeys.has(linkKey(event.action.envelope.sender.peerId, event.targetPeerId)));
|
|
571
|
+
for (const event of dropped) {
|
|
572
|
+
this.#removeQueuedEvent(event.eventId);
|
|
573
|
+
this.#droppedMessages += 1;
|
|
574
|
+
this.#partitionSuppressedMessages += 1;
|
|
575
|
+
droppedDeliveries.push(event);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
if (affectedLinkIds.length > 0 || droppedDeliveries.length > 0) {
|
|
579
|
+
if (enabled)
|
|
580
|
+
this.#heals += 1;
|
|
581
|
+
else
|
|
582
|
+
this.#partitions += 1;
|
|
583
|
+
}
|
|
584
|
+
return {
|
|
585
|
+
applied: affectedLinkIds.length > 0 || droppedDeliveries.length > 0,
|
|
586
|
+
affectedEventIds: Object.freeze(droppedDeliveries.map((event) => event.eventId)),
|
|
587
|
+
affectedLinkIds: Object.freeze(affectedLinkIds),
|
|
588
|
+
affectedDeliveries: Object.freeze(droppedDeliveries.map(affectedDeliveryMetadata)),
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
if (fault.kind === 'clock.offset') {
|
|
592
|
+
const current = this.#clockOffsets.get(fault.peerId);
|
|
593
|
+
if (current === undefined || current === fault.offset)
|
|
594
|
+
return emptyFaultResult();
|
|
595
|
+
this.#clockOffsets.set(fault.peerId, fault.offset);
|
|
596
|
+
this.#clockOffsetChanges += 1;
|
|
597
|
+
return {
|
|
598
|
+
applied: true,
|
|
599
|
+
affectedEventIds: Object.freeze([fault.peerId]),
|
|
600
|
+
affectedLinkIds: Object.freeze([]),
|
|
601
|
+
affectedDeliveries: Object.freeze([]),
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
if (!('deliveryEventId' in fault))
|
|
605
|
+
return emptyFaultResult();
|
|
606
|
+
const index = this.#queue.findIndex((event) => event.eventId === fault.deliveryEventId &&
|
|
607
|
+
event.action.kind === 'message.delivery');
|
|
608
|
+
if (index < 0)
|
|
609
|
+
return emptyFaultResult();
|
|
610
|
+
const delivery = this.#queue[index];
|
|
611
|
+
if (delivery.action.kind !== 'message.delivery')
|
|
612
|
+
throw new TypeError('Mesh simulation delivery queue became inconsistent');
|
|
613
|
+
if (fault.kind === 'message.drop') {
|
|
614
|
+
this.#queue.splice(index, 1);
|
|
615
|
+
this.#droppedMessages += 1;
|
|
616
|
+
return {
|
|
617
|
+
applied: true,
|
|
618
|
+
affectedEventIds: Object.freeze([delivery.eventId]),
|
|
619
|
+
affectedLinkIds: Object.freeze([]),
|
|
620
|
+
affectedDeliveries: Object.freeze([affectedDeliveryMetadata(delivery)]),
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
if (fault.kind === 'message.duplicate') {
|
|
624
|
+
if (this.#queue.length + fault.copies >
|
|
625
|
+
this.config.limits.maximumQueuedEvents ||
|
|
626
|
+
this.#insertionSequence + fault.copies >
|
|
627
|
+
maximumIssuedEvents(this.config))
|
|
628
|
+
throw new RangeError('Mesh simulation queue limit exceeded');
|
|
629
|
+
const duplicateIds = Array.from({ length: fault.copies }, (_, index) => `${delivery.eventId}:duplicate:${fault.faultId}:${index + 1}`);
|
|
630
|
+
for (const eventId of duplicateIds)
|
|
631
|
+
assertBoundedString(eventId, 'duplicate eventId', 768);
|
|
632
|
+
if (duplicateIds.some((eventId) => this.#eventIds.has(eventId)))
|
|
633
|
+
throw new TypeError('Duplicate Mesh simulation fault delivery');
|
|
634
|
+
const affected = [];
|
|
635
|
+
for (let copy = 1; copy <= fault.copies; copy += 1) {
|
|
636
|
+
const eventId = duplicateIds[copy - 1];
|
|
637
|
+
this.#enqueueAction(eventId, delivery.targetPeerId, delivery.order.logicalTime, delivery.order.priority, delivery.action);
|
|
638
|
+
affected.push(eventId);
|
|
639
|
+
}
|
|
640
|
+
this.#duplicatedMessages += affected.length;
|
|
641
|
+
this.#deliveredMessages += affected.length;
|
|
642
|
+
return {
|
|
643
|
+
applied: true,
|
|
644
|
+
affectedEventIds: Object.freeze(affected),
|
|
645
|
+
affectedLinkIds: Object.freeze([]),
|
|
646
|
+
affectedDeliveries: Object.freeze([]),
|
|
647
|
+
};
|
|
648
|
+
}
|
|
649
|
+
if (fault.kind === 'message.delay') {
|
|
650
|
+
const logicalTime = delivery.order.logicalTime + fault.delay;
|
|
651
|
+
if (!Number.isSafeInteger(logicalTime) ||
|
|
652
|
+
logicalTime > this.config.limits.maximumLogicalTime)
|
|
653
|
+
throw new RangeError('Mesh simulation delayed event exceeds time limit');
|
|
654
|
+
this.#queue[index] = Object.freeze({
|
|
655
|
+
...delivery,
|
|
656
|
+
order: Object.freeze({ ...delivery.order, logicalTime }),
|
|
657
|
+
});
|
|
658
|
+
this.#queue.sort(compareEvents);
|
|
659
|
+
this.#delayedMessages += 1;
|
|
660
|
+
return {
|
|
661
|
+
applied: true,
|
|
662
|
+
affectedEventIds: Object.freeze([delivery.eventId]),
|
|
663
|
+
affectedLinkIds: Object.freeze([]),
|
|
664
|
+
affectedDeliveries: Object.freeze([]),
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
if (fault.kind !== 'message.reorder')
|
|
668
|
+
return emptyFaultResult();
|
|
669
|
+
this.#queue[index] = Object.freeze({
|
|
670
|
+
...delivery,
|
|
671
|
+
order: Object.freeze({
|
|
672
|
+
...delivery.order,
|
|
673
|
+
logicalTime: fault.newLogicalTime,
|
|
674
|
+
priority: fault.newPriority,
|
|
675
|
+
}),
|
|
676
|
+
});
|
|
677
|
+
this.#queue.sort(compareEvents);
|
|
678
|
+
this.#reorderedMessages += 1;
|
|
679
|
+
return {
|
|
680
|
+
applied: true,
|
|
681
|
+
affectedEventIds: Object.freeze([delivery.eventId]),
|
|
682
|
+
affectedLinkIds: Object.freeze([]),
|
|
683
|
+
affectedDeliveries: Object.freeze([]),
|
|
684
|
+
};
|
|
685
|
+
}
|
|
686
|
+
#removeQueuedEvent(eventId) {
|
|
687
|
+
const index = this.#queue.findIndex((event) => event.eventId === eventId);
|
|
688
|
+
if (index >= 0)
|
|
689
|
+
this.#queue.splice(index, 1);
|
|
690
|
+
}
|
|
691
|
+
#enqueueAction(eventId, targetPeerId, logicalTime, priority, action) {
|
|
692
|
+
this.enqueue({ eventId, targetPeerId, logicalTime, priority, action });
|
|
693
|
+
}
|
|
694
|
+
#restore(snapshot) {
|
|
695
|
+
this.#logicalTime = snapshot.logicalTime;
|
|
696
|
+
this.#insertionSequence = snapshot.insertionSequence;
|
|
697
|
+
this.#faultCursor = snapshot.faultCursor;
|
|
698
|
+
this.#chainDigest = snapshot.chainDigest;
|
|
699
|
+
this.#processedEvents = snapshot.metrics.processedEvents;
|
|
700
|
+
this.#emittedEffects = snapshot.metrics.emittedEffects;
|
|
701
|
+
this.#deliveredMessages = snapshot.metrics.deliveredMessages;
|
|
702
|
+
this.#rejectedMessages = snapshot.metrics.rejectedMessages;
|
|
703
|
+
this.#peerCrashes = snapshot.metrics.peerCrashes;
|
|
704
|
+
this.#peerResumes = snapshot.metrics.peerResumes;
|
|
705
|
+
this.#droppedMessages = snapshot.metrics.droppedMessages;
|
|
706
|
+
this.#duplicatedMessages = snapshot.metrics.duplicatedMessages;
|
|
707
|
+
this.#delayedMessages = snapshot.metrics.delayedMessages;
|
|
708
|
+
this.#reorderedMessages = snapshot.metrics.reorderedMessages;
|
|
709
|
+
this.#partitions = snapshot.metrics.partitions;
|
|
710
|
+
this.#heals = snapshot.metrics.heals;
|
|
711
|
+
this.#clockOffsetChanges = snapshot.metrics.clockOffsetChanges;
|
|
712
|
+
this.#crashSuppressedEvents = snapshot.metrics.crashSuppressedEvents;
|
|
713
|
+
this.#partitionSuppressedMessages =
|
|
714
|
+
snapshot.metrics.partitionSuppressedMessages;
|
|
715
|
+
this.#queue.push(...snapshot.queuedEvents);
|
|
716
|
+
for (const eventId of snapshot.eventIds)
|
|
717
|
+
this.#eventIds.add(eventId);
|
|
718
|
+
for (const [scope, state] of Object.entries(snapshot.prngStates))
|
|
719
|
+
this.#prngStates.set(scope, state);
|
|
720
|
+
for (const [peerId, state] of Object.entries(snapshot.peerStates)) {
|
|
721
|
+
const peer = this.#peers.get(peerId);
|
|
722
|
+
if (!peer)
|
|
723
|
+
throw new TypeError('Mesh simulation restored peer is unknown');
|
|
724
|
+
peer.state = state;
|
|
725
|
+
peer.outboundSequence = snapshot.outboundSequences[peerId];
|
|
726
|
+
peer.available = snapshot.peerAvailability[peerId];
|
|
727
|
+
}
|
|
728
|
+
this.#clockOffsets.clear();
|
|
729
|
+
for (const [peerId, offset] of Object.entries(snapshot.clockOffsets))
|
|
730
|
+
this.#clockOffsets.set(peerId, offset);
|
|
731
|
+
this.#topology.clear();
|
|
732
|
+
for (const link of snapshot.topology)
|
|
733
|
+
this.#topology.set(linkKey(link.fromPeerId, link.toPeerId), link);
|
|
734
|
+
this.#records.push(...snapshot.records);
|
|
735
|
+
this.#faultRecords.push(...snapshot.faults);
|
|
736
|
+
}
|
|
314
737
|
}
|
|
315
738
|
function validateConfig(config) {
|
|
316
|
-
|
|
317
|
-
|
|
739
|
+
assertPlainData(config, 'configuration');
|
|
740
|
+
if (config.prngVersion !== 'xorshift32-v1' ||
|
|
741
|
+
!['full', 'digest', 'metrics'].includes(config.recordingMode) ||
|
|
318
742
|
!Number.isSafeInteger(config.seed) ||
|
|
743
|
+
typeof config.startTime !== 'string' ||
|
|
319
744
|
!Number.isFinite(Date.parse(config.startTime)) ||
|
|
320
745
|
!Array.isArray(config.peers) ||
|
|
321
|
-
config.peers.length < 1
|
|
746
|
+
config.peers.length < 1 ||
|
|
747
|
+
config.peers.length > 256 ||
|
|
748
|
+
!isDenseArray(config.peers) ||
|
|
749
|
+
!Array.isArray(config.links) ||
|
|
750
|
+
config.links.length > 256 * 255 ||
|
|
751
|
+
!isDenseArray(config.links) ||
|
|
752
|
+
!config.limits ||
|
|
753
|
+
typeof config.limits !== 'object') {
|
|
322
754
|
throw new TypeError('Invalid Mesh simulation configuration');
|
|
323
755
|
}
|
|
324
|
-
const
|
|
325
|
-
|
|
326
|
-
|
|
756
|
+
const peerIds = new Set();
|
|
757
|
+
for (const peer of config.peers) {
|
|
758
|
+
assertPlainData(peer, 'peer');
|
|
759
|
+
assertBoundedString(peer.peerId, 'peerId', 256);
|
|
760
|
+
assertPlainData(peer.state, 'peer state');
|
|
761
|
+
assertPlainData(peer.state.identity, 'peer identity');
|
|
762
|
+
if (peer.state.identity.peerId !== peer.peerId ||
|
|
763
|
+
(peer.outboundSequence !== undefined &&
|
|
764
|
+
(!Number.isSafeInteger(peer.outboundSequence) ||
|
|
765
|
+
peer.outboundSequence < 0)) ||
|
|
766
|
+
peerIds.has(peer.peerId))
|
|
767
|
+
throw new TypeError('Invalid or duplicate Mesh simulation peer');
|
|
768
|
+
peerIds.add(peer.peerId);
|
|
769
|
+
}
|
|
770
|
+
const linkKeys = new Set();
|
|
771
|
+
for (const link of config.links) {
|
|
772
|
+
assertPlainData(link, 'link');
|
|
773
|
+
assertBoundedString(link.fromPeerId, 'link source peerId', 256);
|
|
774
|
+
assertBoundedString(link.toPeerId, 'link target peerId', 256);
|
|
775
|
+
if (!peerIds.has(link.fromPeerId) ||
|
|
776
|
+
!peerIds.has(link.toPeerId) ||
|
|
777
|
+
link.fromPeerId === link.toPeerId ||
|
|
778
|
+
!Number.isSafeInteger(link.latency) ||
|
|
779
|
+
link.latency < 0 ||
|
|
780
|
+
typeof link.enabled !== 'boolean')
|
|
781
|
+
throw new TypeError('Invalid Mesh simulation link');
|
|
782
|
+
const key = linkKey(link.fromPeerId, link.toPeerId);
|
|
783
|
+
if (linkKeys.has(key))
|
|
784
|
+
throw new TypeError('Duplicate Mesh simulation link');
|
|
785
|
+
linkKeys.add(key);
|
|
327
786
|
}
|
|
787
|
+
assertPlainData(config.limits, 'limits');
|
|
328
788
|
for (const value of Object.values(config.limits)) {
|
|
329
|
-
if (
|
|
789
|
+
if (typeof value !== 'number' ||
|
|
790
|
+
!Number.isSafeInteger(value) ||
|
|
791
|
+
value < 1) {
|
|
330
792
|
throw new RangeError('Invalid Mesh simulation limit');
|
|
331
793
|
}
|
|
332
794
|
}
|
|
795
|
+
if (config.limits.maximumEvents >
|
|
796
|
+
Math.floor(Number.MAX_SAFE_INTEGER / config.limits.maximumInternalSteps))
|
|
797
|
+
throw new RangeError('Mesh simulation outbound sequence capacity exceeded');
|
|
798
|
+
const maximumSequenceIncrements = config.limits.maximumEvents * config.limits.maximumInternalSteps;
|
|
799
|
+
if (config.peers.some((peer) => (peer.outboundSequence ?? 0) >
|
|
800
|
+
Number.MAX_SAFE_INTEGER - maximumSequenceIncrements))
|
|
801
|
+
throw new RangeError('Mesh simulation outbound sequence capacity exceeded');
|
|
333
802
|
}
|
|
334
|
-
function configurationProjection(config) {
|
|
803
|
+
function configurationProjection(config, faultPlan) {
|
|
335
804
|
return {
|
|
336
805
|
seed: config.seed,
|
|
337
806
|
prngVersion: config.prngVersion,
|
|
@@ -344,21 +813,1223 @@ function configurationProjection(config) {
|
|
|
344
813
|
})),
|
|
345
814
|
links: config.links,
|
|
346
815
|
limits: config.limits,
|
|
816
|
+
faultPlan,
|
|
347
817
|
invariants: (config.invariants ?? []).map((invariant) => invariant.name),
|
|
348
818
|
};
|
|
349
819
|
}
|
|
820
|
+
function metricDelta(before, after) {
|
|
821
|
+
return Object.freeze(Object.fromEntries(metricDeltaKeys.map((key) => [key, after[key] - before[key]])));
|
|
822
|
+
}
|
|
823
|
+
function emptyFaultResult() {
|
|
824
|
+
return {
|
|
825
|
+
applied: false,
|
|
826
|
+
affectedEventIds: Object.freeze([]),
|
|
827
|
+
affectedLinkIds: Object.freeze([]),
|
|
828
|
+
affectedDeliveries: Object.freeze([]),
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
function affectedDeliveryMetadata(event) {
|
|
832
|
+
if (event.action.kind !== 'message.delivery')
|
|
833
|
+
throw new TypeError('Mesh simulation affected event is not a delivery');
|
|
834
|
+
return Object.freeze({
|
|
835
|
+
eventId: event.eventId,
|
|
836
|
+
fromPeerId: event.action.envelope.sender.peerId,
|
|
837
|
+
toPeerId: event.targetPeerId,
|
|
838
|
+
order: event.order,
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
function emptyTransportOutcome() {
|
|
842
|
+
return Object.freeze({
|
|
843
|
+
delivered: 0,
|
|
844
|
+
droppedByCrash: 0,
|
|
845
|
+
droppedByPartition: 0,
|
|
846
|
+
droppedByCrashAndPartition: 0,
|
|
847
|
+
droppedByDestinationMissing: 0,
|
|
848
|
+
deliveries: Object.freeze([]),
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
function transportOutcomeFromDeliveries(deliveries) {
|
|
852
|
+
return Object.freeze({
|
|
853
|
+
delivered: deliveries.filter(({ outcome }) => outcome === 'delivered')
|
|
854
|
+
.length,
|
|
855
|
+
droppedByCrash: deliveries.filter(({ outcome }) => outcome === 'crash')
|
|
856
|
+
.length,
|
|
857
|
+
droppedByPartition: deliveries.filter(({ outcome }) => outcome === 'partition').length,
|
|
858
|
+
droppedByCrashAndPartition: deliveries.filter(({ outcome }) => outcome === 'crash_partition').length,
|
|
859
|
+
droppedByDestinationMissing: deliveries.filter(({ outcome }) => outcome === 'destination_missing').length,
|
|
860
|
+
deliveries: Object.freeze([...deliveries]),
|
|
861
|
+
});
|
|
862
|
+
}
|
|
863
|
+
function freezeSimulationConfig(config, faultPlan) {
|
|
864
|
+
return Object.freeze({
|
|
865
|
+
seed: config.seed,
|
|
866
|
+
prngVersion: config.prngVersion,
|
|
867
|
+
recordingMode: config.recordingMode,
|
|
868
|
+
startTime: config.startTime,
|
|
869
|
+
peers: Object.freeze(config.peers.map((peer) => Object.freeze({
|
|
870
|
+
peerId: peer.peerId,
|
|
871
|
+
state: restorePeerState(peer.state),
|
|
872
|
+
signer: peer.signer,
|
|
873
|
+
verifier: peer.verifier,
|
|
874
|
+
resolver: peer.resolver,
|
|
875
|
+
cryptoPolicy: deepFreezeCopy(peer.cryptoPolicy),
|
|
876
|
+
admissionPolicy: peer.admissionPolicy,
|
|
877
|
+
privateKey: peer.privateKey,
|
|
878
|
+
...(peer.outboundSequence === undefined
|
|
879
|
+
? {}
|
|
880
|
+
: { outboundSequence: peer.outboundSequence }),
|
|
881
|
+
...(peer.crypto === undefined ? {} : { crypto: peer.crypto }),
|
|
882
|
+
...(peer.protocolOptions === undefined
|
|
883
|
+
? {}
|
|
884
|
+
: {
|
|
885
|
+
protocolOptions: deepFreezeCopy(peer.protocolOptions),
|
|
886
|
+
}),
|
|
887
|
+
}))),
|
|
888
|
+
links: Object.freeze(config.links.map((link) => Object.freeze({ ...link }))),
|
|
889
|
+
limits: Object.freeze({ ...config.limits }),
|
|
890
|
+
faultPlan,
|
|
891
|
+
...(config.invariants === undefined
|
|
892
|
+
? {}
|
|
893
|
+
: {
|
|
894
|
+
invariants: Object.freeze(config.invariants.map((invariant) => Object.freeze({
|
|
895
|
+
name: invariant.name,
|
|
896
|
+
evaluate: invariant.evaluate,
|
|
897
|
+
}))),
|
|
898
|
+
}),
|
|
899
|
+
});
|
|
900
|
+
}
|
|
350
901
|
function freezeAction(action) {
|
|
351
902
|
if (action?.kind === 'peer.input') {
|
|
352
903
|
return Object.freeze({
|
|
353
904
|
kind: 'peer.input',
|
|
354
|
-
input:
|
|
905
|
+
input: deepFreezeCopy(action.input),
|
|
355
906
|
});
|
|
356
907
|
}
|
|
357
908
|
if (action?.kind === 'message.delivery') {
|
|
358
|
-
return Object.freeze({
|
|
909
|
+
return Object.freeze({
|
|
910
|
+
kind: 'message.delivery',
|
|
911
|
+
envelope: deepFreezeCopy(action.envelope),
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
if (action?.kind === 'fault.apply') {
|
|
915
|
+
return Object.freeze({
|
|
916
|
+
kind: 'fault.apply',
|
|
917
|
+
fault: deepFreezeCopy(action.fault),
|
|
918
|
+
});
|
|
359
919
|
}
|
|
360
920
|
throw new TypeError('Invalid Mesh simulation action');
|
|
361
921
|
}
|
|
922
|
+
function normalizeFaultPlan(input, config) {
|
|
923
|
+
if (input === undefined)
|
|
924
|
+
return Object.freeze({ schemaVersion: 1, faults: Object.freeze([]) });
|
|
925
|
+
assertPlainData(input, 'fault plan');
|
|
926
|
+
assertExactKeys(input, ['faults', 'schemaVersion'], ['faults', 'schemaVersion']);
|
|
927
|
+
if (input.schemaVersion !== 1 ||
|
|
928
|
+
!Array.isArray(input.faults) ||
|
|
929
|
+
input.faults.length > MESH_SIMULATION_FAULT_LIMITS.maximumFaults ||
|
|
930
|
+
!isDenseArray(input.faults))
|
|
931
|
+
throw new TypeError('Invalid Mesh simulation fault plan');
|
|
932
|
+
const peerIds = new Set(config.peers.map((peer) => peer.peerId));
|
|
933
|
+
const topology = new Set(config.links.map((link) => linkKey(link.fromPeerId, link.toPeerId)));
|
|
934
|
+
const faultIds = new Set();
|
|
935
|
+
const faults = input.faults.map((fault) => {
|
|
936
|
+
assertPlainData(fault, 'fault');
|
|
937
|
+
const common = ['faultId', 'kind', 'logicalTime', 'priority'];
|
|
938
|
+
const required = fault.kind === 'peer.crash' || fault.kind === 'peer.resume'
|
|
939
|
+
? [...common, 'peerId']
|
|
940
|
+
: fault.kind === 'message.drop'
|
|
941
|
+
? [...common, 'deliveryEventId']
|
|
942
|
+
: fault.kind === 'message.duplicate'
|
|
943
|
+
? [...common, 'copies', 'deliveryEventId']
|
|
944
|
+
: fault.kind === 'message.delay'
|
|
945
|
+
? [...common, 'delay', 'deliveryEventId']
|
|
946
|
+
: fault.kind === 'message.reorder'
|
|
947
|
+
? [
|
|
948
|
+
...common,
|
|
949
|
+
'deliveryEventId',
|
|
950
|
+
'newLogicalTime',
|
|
951
|
+
'newPriority',
|
|
952
|
+
]
|
|
953
|
+
: fault.kind === 'network.partition' ||
|
|
954
|
+
fault.kind === 'network.heal'
|
|
955
|
+
? [...common, 'links']
|
|
956
|
+
: fault.kind === 'clock.offset'
|
|
957
|
+
? [...common, 'offset', 'peerId']
|
|
958
|
+
: undefined;
|
|
959
|
+
if (required === undefined)
|
|
960
|
+
throw new TypeError('Unsupported Mesh simulation fault');
|
|
961
|
+
assertExactKeys(fault, required, required);
|
|
962
|
+
assertBoundedString(fault.faultId, 'faultId');
|
|
963
|
+
if (faultIds.has(fault.faultId) ||
|
|
964
|
+
!Number.isSafeInteger(fault.logicalTime) ||
|
|
965
|
+
fault.logicalTime < 0 ||
|
|
966
|
+
fault.logicalTime > config.limits.maximumLogicalTime ||
|
|
967
|
+
!Number.isSafeInteger(fault.priority))
|
|
968
|
+
throw new TypeError('Invalid Mesh simulation fault identity or time');
|
|
969
|
+
faultIds.add(fault.faultId);
|
|
970
|
+
if ('peerId' in fault &&
|
|
971
|
+
(typeof fault.peerId !== 'string' || !peerIds.has(fault.peerId)))
|
|
972
|
+
throw new TypeError('Mesh simulation fault peer is unknown');
|
|
973
|
+
if ('peerId' in fault)
|
|
974
|
+
assertBoundedString(fault.peerId, 'fault peerId', 256);
|
|
975
|
+
if ('deliveryEventId' in fault)
|
|
976
|
+
assertBoundedString(fault.deliveryEventId, 'deliveryEventId', 768);
|
|
977
|
+
if (fault.kind === 'message.duplicate' &&
|
|
978
|
+
(!Number.isSafeInteger(fault.copies) ||
|
|
979
|
+
fault.copies < 1 ||
|
|
980
|
+
fault.copies > MESH_SIMULATION_FAULT_LIMITS.maximumDuplicateCopies))
|
|
981
|
+
throw new RangeError('Mesh simulation duplicate bound exceeded');
|
|
982
|
+
if (fault.kind === 'message.delay' &&
|
|
983
|
+
(!Number.isSafeInteger(fault.delay) ||
|
|
984
|
+
fault.delay < 1 ||
|
|
985
|
+
fault.delay > config.limits.maximumLogicalTime))
|
|
986
|
+
throw new RangeError('Mesh simulation delay is invalid');
|
|
987
|
+
if (fault.kind === 'message.reorder' &&
|
|
988
|
+
(!Number.isSafeInteger(fault.newLogicalTime) ||
|
|
989
|
+
fault.newLogicalTime < fault.logicalTime ||
|
|
990
|
+
fault.newLogicalTime > config.limits.maximumLogicalTime ||
|
|
991
|
+
!Number.isSafeInteger(fault.newPriority)))
|
|
992
|
+
throw new RangeError('Mesh simulation reorder is invalid');
|
|
993
|
+
if (fault.kind === 'clock.offset' &&
|
|
994
|
+
(!Number.isSafeInteger(fault.offset) ||
|
|
995
|
+
Math.abs(fault.offset) >
|
|
996
|
+
MESH_SIMULATION_FAULT_LIMITS.maximumClockOffset ||
|
|
997
|
+
Date.parse(config.startTime) + fault.logicalTime + fault.offset < 0))
|
|
998
|
+
throw new RangeError('Mesh simulation clock offset is invalid');
|
|
999
|
+
if (fault.kind === 'network.partition' || fault.kind === 'network.heal') {
|
|
1000
|
+
if (!Array.isArray(fault.links) ||
|
|
1001
|
+
fault.links.length < 1 ||
|
|
1002
|
+
fault.links.length >
|
|
1003
|
+
MESH_SIMULATION_FAULT_LIMITS.maximumLinksPerFault ||
|
|
1004
|
+
!isDenseArray(fault.links))
|
|
1005
|
+
throw new RangeError('Mesh simulation fault link bound exceeded');
|
|
1006
|
+
const seen = new Set();
|
|
1007
|
+
for (const link of fault.links) {
|
|
1008
|
+
assertPlainData(link, 'fault link');
|
|
1009
|
+
assertExactKeys(link, ['fromPeerId', 'toPeerId'], ['fromPeerId', 'toPeerId']);
|
|
1010
|
+
assertBoundedString(link.fromPeerId, 'fault link source peerId', 256);
|
|
1011
|
+
assertBoundedString(link.toPeerId, 'fault link target peerId', 256);
|
|
1012
|
+
const key = linkKey(link.fromPeerId, link.toPeerId);
|
|
1013
|
+
if (!topology.has(key) || seen.has(key))
|
|
1014
|
+
throw new TypeError('Invalid Mesh simulation fault link');
|
|
1015
|
+
seen.add(key);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
return deepFreezeCopy(fault);
|
|
1019
|
+
});
|
|
1020
|
+
return Object.freeze({ schemaVersion: 1, faults: Object.freeze(faults) });
|
|
1021
|
+
}
|
|
1022
|
+
async function restoreSnapshot(input, config, configurationDigest, faultPlanDigest, faultPlan) {
|
|
1023
|
+
assertPlainData(input, 'snapshot');
|
|
1024
|
+
assertExactKeys(input, [
|
|
1025
|
+
'chainDigest',
|
|
1026
|
+
'clockOffsets',
|
|
1027
|
+
'configurationDigest',
|
|
1028
|
+
'eventIds',
|
|
1029
|
+
'faultCursor',
|
|
1030
|
+
'faultPlanDigest',
|
|
1031
|
+
'faults',
|
|
1032
|
+
'insertionSequence',
|
|
1033
|
+
'logicalTime',
|
|
1034
|
+
'metrics',
|
|
1035
|
+
'outboundSequences',
|
|
1036
|
+
'peerAvailability',
|
|
1037
|
+
'peerStates',
|
|
1038
|
+
'prngStates',
|
|
1039
|
+
'queuedEvents',
|
|
1040
|
+
'records',
|
|
1041
|
+
'schemaVersion',
|
|
1042
|
+
'topology',
|
|
1043
|
+
], [
|
|
1044
|
+
'chainDigest',
|
|
1045
|
+
'clockOffsets',
|
|
1046
|
+
'configurationDigest',
|
|
1047
|
+
'eventIds',
|
|
1048
|
+
'faultCursor',
|
|
1049
|
+
'faultPlanDigest',
|
|
1050
|
+
'faults',
|
|
1051
|
+
'insertionSequence',
|
|
1052
|
+
'logicalTime',
|
|
1053
|
+
'metrics',
|
|
1054
|
+
'outboundSequences',
|
|
1055
|
+
'peerAvailability',
|
|
1056
|
+
'peerStates',
|
|
1057
|
+
'prngStates',
|
|
1058
|
+
'queuedEvents',
|
|
1059
|
+
'records',
|
|
1060
|
+
'schemaVersion',
|
|
1061
|
+
'topology',
|
|
1062
|
+
]);
|
|
1063
|
+
const snapshot = input;
|
|
1064
|
+
if (snapshot.schemaVersion !== 2 ||
|
|
1065
|
+
snapshot.configurationDigest !== configurationDigest ||
|
|
1066
|
+
snapshot.faultPlanDigest !== faultPlanDigest ||
|
|
1067
|
+
!isDigest(snapshot.chainDigest) ||
|
|
1068
|
+
!Number.isSafeInteger(snapshot.logicalTime) ||
|
|
1069
|
+
snapshot.logicalTime < 0 ||
|
|
1070
|
+
snapshot.logicalTime > config.limits.maximumLogicalTime ||
|
|
1071
|
+
!Number.isSafeInteger(snapshot.insertionSequence) ||
|
|
1072
|
+
snapshot.insertionSequence < 0 ||
|
|
1073
|
+
snapshot.insertionSequence > maximumIssuedEvents(config) ||
|
|
1074
|
+
!Number.isSafeInteger(snapshot.faultCursor) ||
|
|
1075
|
+
snapshot.faultCursor < 0 ||
|
|
1076
|
+
snapshot.faultCursor > faultPlan.faults.length ||
|
|
1077
|
+
!Array.isArray(snapshot.queuedEvents) ||
|
|
1078
|
+
snapshot.queuedEvents.length > config.limits.maximumQueuedEvents ||
|
|
1079
|
+
!isDenseArray(snapshot.queuedEvents) ||
|
|
1080
|
+
!Array.isArray(snapshot.records) ||
|
|
1081
|
+
snapshot.records.length > config.limits.maximumEvents ||
|
|
1082
|
+
!isDenseArray(snapshot.records) ||
|
|
1083
|
+
!Array.isArray(snapshot.faults) ||
|
|
1084
|
+
snapshot.faults.length > faultPlan.faults.length ||
|
|
1085
|
+
!isDenseArray(snapshot.faults) ||
|
|
1086
|
+
!Array.isArray(snapshot.eventIds) ||
|
|
1087
|
+
snapshot.eventIds.length > snapshot.insertionSequence ||
|
|
1088
|
+
!isDenseArray(snapshot.eventIds))
|
|
1089
|
+
throw new TypeError('Invalid Mesh simulation snapshot');
|
|
1090
|
+
const peerIds = config.peers.map((peer) => peer.peerId).sort();
|
|
1091
|
+
const peerIdSet = new Set(peerIds);
|
|
1092
|
+
for (const [name, record] of Object.entries({
|
|
1093
|
+
peerStates: snapshot.peerStates,
|
|
1094
|
+
outboundSequences: snapshot.outboundSequences,
|
|
1095
|
+
prngStates: snapshot.prngStates,
|
|
1096
|
+
peerAvailability: snapshot.peerAvailability,
|
|
1097
|
+
clockOffsets: snapshot.clockOffsets,
|
|
1098
|
+
}))
|
|
1099
|
+
assertDataRecord(record, `snapshot ${name}`);
|
|
1100
|
+
assertExactRecordKeys(snapshot.peerStates, peerIds, 'peerStates');
|
|
1101
|
+
assertExactRecordKeys(snapshot.outboundSequences, peerIds, 'outboundSequences');
|
|
1102
|
+
assertExactRecordKeys(snapshot.peerAvailability, peerIds, 'peerAvailability');
|
|
1103
|
+
assertExactRecordKeys(snapshot.clockOffsets, peerIds, 'clockOffsets');
|
|
1104
|
+
const restoredPeerStates = new Map();
|
|
1105
|
+
for (const peer of config.peers) {
|
|
1106
|
+
const state = restorePeerState(snapshot.peerStates[peer.peerId]);
|
|
1107
|
+
restoredPeerStates.set(peer.peerId, state);
|
|
1108
|
+
if (!state ||
|
|
1109
|
+
state.identity?.peerId !== peer.peerId ||
|
|
1110
|
+
state.identity.tenantId !== peer.state.identity.tenantId ||
|
|
1111
|
+
state.identity.meshId !== peer.state.identity.meshId ||
|
|
1112
|
+
state.lastLogicalTime > snapshot.logicalTime ||
|
|
1113
|
+
!Number.isSafeInteger(snapshot.outboundSequences[peer.peerId]) ||
|
|
1114
|
+
snapshot.outboundSequences[peer.peerId] < 0 ||
|
|
1115
|
+
typeof snapshot.peerAvailability[peer.peerId] !== 'boolean' ||
|
|
1116
|
+
!Number.isSafeInteger(snapshot.clockOffsets[peer.peerId]) ||
|
|
1117
|
+
Math.abs(snapshot.clockOffsets[peer.peerId]) >
|
|
1118
|
+
MESH_SIMULATION_FAULT_LIMITS.maximumClockOffset)
|
|
1119
|
+
throw new TypeError('Invalid Mesh simulation snapshot peer');
|
|
1120
|
+
}
|
|
1121
|
+
const prngEntries = Object.entries(snapshot.prngStates);
|
|
1122
|
+
if (prngEntries.length > 4096)
|
|
1123
|
+
throw new RangeError('Mesh simulation PRNG stream limit exceeded');
|
|
1124
|
+
for (const [scope, state] of prngEntries) {
|
|
1125
|
+
assertBoundedString(scope, 'random scope', 768);
|
|
1126
|
+
if (!Number.isSafeInteger(state) || state < 0 || state > 0xffff_ffff)
|
|
1127
|
+
throw new TypeError('Invalid Mesh simulation PRNG snapshot');
|
|
1128
|
+
}
|
|
1129
|
+
const topology = validateSnapshotTopology(snapshot.topology, config);
|
|
1130
|
+
const queuedEvents = snapshot.queuedEvents.map((event) => validateSnapshotEvent(event, snapshot, config, peerIdSet));
|
|
1131
|
+
const queuedInsertionSequences = new Set(queuedEvents.map((event) => event.order.insertionSequence));
|
|
1132
|
+
if (queuedInsertionSequences.size !== queuedEvents.length)
|
|
1133
|
+
throw new TypeError('Mesh simulation snapshot queue insertion sequence is not unique');
|
|
1134
|
+
for (let index = 1; index < queuedEvents.length; index += 1)
|
|
1135
|
+
if (compareEvents(queuedEvents[index - 1], queuedEvents[index]) >= 0)
|
|
1136
|
+
throw new TypeError('Mesh simulation snapshot queue is not ordered');
|
|
1137
|
+
const eventIds = [...snapshot.eventIds];
|
|
1138
|
+
if (eventIds.some((eventId) => typeof eventId !== 'string' ||
|
|
1139
|
+
eventId.length < 1 ||
|
|
1140
|
+
new TextEncoder().encode(eventId).byteLength > 768) ||
|
|
1141
|
+
eventIds.length !== snapshot.insertionSequence ||
|
|
1142
|
+
new Set(eventIds).size !== eventIds.length ||
|
|
1143
|
+
queuedEvents.some((event) => !eventIds.includes(event.eventId)) ||
|
|
1144
|
+
faultPlan.faults.some((fault) => !eventIds.includes(`fault:${fault.faultId}`)))
|
|
1145
|
+
throw new TypeError('Invalid Mesh simulation snapshot event IDs');
|
|
1146
|
+
const metrics = validateMetrics(snapshot.metrics, snapshot);
|
|
1147
|
+
const records = snapshot.records.map((record) => {
|
|
1148
|
+
validateRecord(record, config, peerIdSet, snapshot.insertionSequence);
|
|
1149
|
+
return deepFreezeCopy(record);
|
|
1150
|
+
});
|
|
1151
|
+
let chain = configurationDigest;
|
|
1152
|
+
const issuedInsertionSequences = new Set(queuedInsertionSequences);
|
|
1153
|
+
for (const record of records) {
|
|
1154
|
+
if (issuedInsertionSequences.has(record.order.insertionSequence))
|
|
1155
|
+
throw new TypeError('Mesh simulation snapshot insertion sequence is not unique');
|
|
1156
|
+
issuedInsertionSequences.add(record.order.insertionSequence);
|
|
1157
|
+
const { chainDigest, ...base } = record;
|
|
1158
|
+
chain = await digest({ previous: chain, record: base });
|
|
1159
|
+
if (chain !== chainDigest)
|
|
1160
|
+
throw new TypeError('Mesh simulation snapshot record chain is invalid');
|
|
1161
|
+
}
|
|
1162
|
+
if (chain !== snapshot.chainDigest ||
|
|
1163
|
+
records.length !== metrics.processedEvents)
|
|
1164
|
+
throw new TypeError('Mesh simulation snapshot digest is invalid');
|
|
1165
|
+
const recordedMetricTotals = Object.fromEntries(metricDeltaKeys.map((key) => [key, 0]));
|
|
1166
|
+
for (const record of records)
|
|
1167
|
+
for (const key of metricDeltaKeys) {
|
|
1168
|
+
const total = recordedMetricTotals[key] + record.metricsDelta[key];
|
|
1169
|
+
if (!Number.isSafeInteger(total))
|
|
1170
|
+
throw new TypeError('Mesh simulation snapshot metrics overflow');
|
|
1171
|
+
recordedMetricTotals[key] = total;
|
|
1172
|
+
}
|
|
1173
|
+
if (metricDeltaKeys.some((key) => metrics[key] !== recordedMetricTotals[key]))
|
|
1174
|
+
throw new TypeError('Mesh simulation snapshot metrics do not match records');
|
|
1175
|
+
const peerStatesDigest = await digest(snapshot.peerStates);
|
|
1176
|
+
const expectedPeerStatesDigest = records.length > 0
|
|
1177
|
+
? records.at(-1).stateDigest
|
|
1178
|
+
: await digest(frozenRecord(config.peers.map((peer) => [peer.peerId, peer.state])));
|
|
1179
|
+
if (peerStatesDigest !== expectedPeerStatesDigest)
|
|
1180
|
+
throw new TypeError('Mesh simulation snapshot peer digest is invalid');
|
|
1181
|
+
const plannedFaults = new Map(faultPlan.faults.map((fault) => [fault.faultId, fault]));
|
|
1182
|
+
const faults = snapshot.faults.map((fault) => validateFaultRecord(fault, records, plannedFaults, eventIds, queuedEvents, config));
|
|
1183
|
+
const removedDeliveryIds = new Set();
|
|
1184
|
+
for (const fault of faults)
|
|
1185
|
+
for (const delivery of fault.affectedDeliveries) {
|
|
1186
|
+
if (removedDeliveryIds.has(delivery.eventId))
|
|
1187
|
+
throw new TypeError('Mesh simulation snapshot delivery is attributed to multiple faults');
|
|
1188
|
+
removedDeliveryIds.add(delivery.eventId);
|
|
1189
|
+
}
|
|
1190
|
+
const historicalAvailability = new Map(config.peers.map((peer) => [peer.peerId, true]));
|
|
1191
|
+
const historicalTopology = new Map(config.links.map((link) => [
|
|
1192
|
+
linkKey(link.fromPeerId, link.toPeerId),
|
|
1193
|
+
link.enabled,
|
|
1194
|
+
]));
|
|
1195
|
+
for (const record of records) {
|
|
1196
|
+
if (record.inputKind === 'fault.apply') {
|
|
1197
|
+
const planned = plannedFaults.get(record.faultId);
|
|
1198
|
+
if (!planned)
|
|
1199
|
+
throw new TypeError('Unknown Mesh simulation historical fault');
|
|
1200
|
+
if (planned.kind === 'peer.crash' && record.faultApplied)
|
|
1201
|
+
historicalAvailability.set(planned.peerId, false);
|
|
1202
|
+
else if (planned.kind === 'peer.resume' && record.faultApplied)
|
|
1203
|
+
historicalAvailability.set(planned.peerId, true);
|
|
1204
|
+
else if ((planned.kind === 'network.partition' ||
|
|
1205
|
+
planned.kind === 'network.heal') &&
|
|
1206
|
+
record.faultApplied)
|
|
1207
|
+
for (const link of planned.links)
|
|
1208
|
+
historicalTopology.set(linkKey(link.fromPeerId, link.toPeerId), planned.kind === 'network.heal');
|
|
1209
|
+
continue;
|
|
1210
|
+
}
|
|
1211
|
+
const targetAvailable = historicalAvailability.get(record.peerId) === true;
|
|
1212
|
+
const inputLinkEnabled = record.inputKind !== 'message.delivery'
|
|
1213
|
+
? true
|
|
1214
|
+
: historicalTopology.get(linkKey(record.inputDeliverySourcePeerId, record.peerId)) === true;
|
|
1215
|
+
const expectedSuppression = !targetAvailable
|
|
1216
|
+
? 'simulation_peer_crashed'
|
|
1217
|
+
: !inputLinkEnabled
|
|
1218
|
+
? 'simulation_partitioned'
|
|
1219
|
+
: undefined;
|
|
1220
|
+
if ((expectedSuppression !== undefined &&
|
|
1221
|
+
record.rejectionCode !== expectedSuppression) ||
|
|
1222
|
+
(expectedSuppression === undefined &&
|
|
1223
|
+
(record.rejectionCode === 'simulation_peer_crashed' ||
|
|
1224
|
+
record.rejectionCode === 'simulation_partitioned')))
|
|
1225
|
+
throw new TypeError('Mesh simulation snapshot input transport outcome is inconsistent');
|
|
1226
|
+
for (const delivery of record.transportOutcome.deliveries) {
|
|
1227
|
+
const recipientConfigured = historicalAvailability.has(delivery.toPeerId);
|
|
1228
|
+
const recipientAvailable = historicalAvailability.get(delivery.toPeerId) === true;
|
|
1229
|
+
const linkEnabled = historicalTopology.get(linkKey(delivery.fromPeerId, delivery.toPeerId)) === true;
|
|
1230
|
+
const expectedOutcome = !recipientConfigured
|
|
1231
|
+
? 'destination_missing'
|
|
1232
|
+
: !linkEnabled && !recipientAvailable
|
|
1233
|
+
? 'crash_partition'
|
|
1234
|
+
: !linkEnabled
|
|
1235
|
+
? 'partition'
|
|
1236
|
+
: !recipientAvailable
|
|
1237
|
+
? 'crash'
|
|
1238
|
+
: 'delivered';
|
|
1239
|
+
if (delivery.fromPeerId !== record.peerId ||
|
|
1240
|
+
delivery.outcome !== expectedOutcome)
|
|
1241
|
+
throw new TypeError('Mesh simulation snapshot effect transport outcome is inconsistent');
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
if (faults.length !== snapshot.faultCursor ||
|
|
1245
|
+
faults.length !== metrics.faultEvents)
|
|
1246
|
+
throw new TypeError('Mesh simulation snapshot fault cursor is invalid');
|
|
1247
|
+
const recordedFaults = records.filter((record) => record.faultId !== undefined);
|
|
1248
|
+
if (recordedFaults.length !== faults.length ||
|
|
1249
|
+
faults.some((fault, index) => recordedFaults[index]?.faultId !== fault.faultId ||
|
|
1250
|
+
recordedFaults[index]?.order.insertionSequence !==
|
|
1251
|
+
fault.order.insertionSequence))
|
|
1252
|
+
throw new TypeError('Mesh simulation snapshot fault ledger is not ordered');
|
|
1253
|
+
const processedFaultIds = new Set();
|
|
1254
|
+
const expectedAvailability = new Map(config.peers.map((peer) => [peer.peerId, true]));
|
|
1255
|
+
const expectedClockOffsets = new Map(config.peers.map((peer) => [peer.peerId, 0]));
|
|
1256
|
+
const expectedTopology = new Map(config.links.map((link) => [
|
|
1257
|
+
linkKey(link.fromPeerId, link.toPeerId),
|
|
1258
|
+
link.enabled,
|
|
1259
|
+
]));
|
|
1260
|
+
const expectedFaultMetrics = {
|
|
1261
|
+
peerCrashes: 0,
|
|
1262
|
+
peerResumes: 0,
|
|
1263
|
+
duplicatedMessages: 0,
|
|
1264
|
+
delayedMessages: 0,
|
|
1265
|
+
reorderedMessages: 0,
|
|
1266
|
+
partitions: 0,
|
|
1267
|
+
heals: 0,
|
|
1268
|
+
clockOffsetChanges: 0,
|
|
1269
|
+
};
|
|
1270
|
+
let faultDroppedMessages = 0;
|
|
1271
|
+
let crashFaultDrops = 0;
|
|
1272
|
+
let partitionFaultDrops = 0;
|
|
1273
|
+
for (const fault of faults) {
|
|
1274
|
+
const planned = plannedFaults.get(fault.faultId);
|
|
1275
|
+
const record = records.find((candidate) => candidate.faultId === fault.faultId &&
|
|
1276
|
+
candidate.order.insertionSequence === fault.order.insertionSequence);
|
|
1277
|
+
if (!planned ||
|
|
1278
|
+
planned.kind !== fault.kind ||
|
|
1279
|
+
!record ||
|
|
1280
|
+
record.actionDigest !==
|
|
1281
|
+
(await digest({ kind: 'fault.apply', fault: planned })) ||
|
|
1282
|
+
processedFaultIds.has(fault.faultId))
|
|
1283
|
+
throw new TypeError('Mesh simulation snapshot fault is not configured');
|
|
1284
|
+
const partitionDrops = planned.kind === 'network.partition' ? fault.affectedEventIds.length : 0;
|
|
1285
|
+
if ((planned.kind === 'message.duplicate' &&
|
|
1286
|
+
record.metricsDelta.duplicatedMessages !==
|
|
1287
|
+
(fault.applied ? fault.affectedEventIds.length : 0)) ||
|
|
1288
|
+
(planned.kind === 'peer.crash' &&
|
|
1289
|
+
(record.metricsDelta.droppedMessages !==
|
|
1290
|
+
(fault.applied ? fault.affectedEventIds.length : 0) ||
|
|
1291
|
+
record.metricsDelta.crashSuppressedEvents !==
|
|
1292
|
+
record.metricsDelta.droppedMessages)) ||
|
|
1293
|
+
(planned.kind === 'network.partition' &&
|
|
1294
|
+
(record.metricsDelta.droppedMessages !== partitionDrops ||
|
|
1295
|
+
record.metricsDelta.partitionSuppressedMessages !== partitionDrops)))
|
|
1296
|
+
throw new TypeError('Mesh simulation snapshot fault metrics mismatch');
|
|
1297
|
+
processedFaultIds.add(fault.faultId);
|
|
1298
|
+
if (planned.kind === 'peer.crash') {
|
|
1299
|
+
expectedAvailability.set(planned.peerId, false);
|
|
1300
|
+
if (fault.applied) {
|
|
1301
|
+
expectedFaultMetrics.peerCrashes += 1;
|
|
1302
|
+
crashFaultDrops += fault.affectedEventIds.length;
|
|
1303
|
+
faultDroppedMessages += fault.affectedEventIds.length;
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
else if (planned.kind === 'peer.resume') {
|
|
1307
|
+
expectedAvailability.set(planned.peerId, true);
|
|
1308
|
+
if (fault.applied)
|
|
1309
|
+
expectedFaultMetrics.peerResumes += 1;
|
|
1310
|
+
}
|
|
1311
|
+
else if (planned.kind === 'network.partition' ||
|
|
1312
|
+
planned.kind === 'network.heal') {
|
|
1313
|
+
const enabled = planned.kind === 'network.heal';
|
|
1314
|
+
for (const link of planned.links)
|
|
1315
|
+
expectedTopology.set(linkKey(link.fromPeerId, link.toPeerId), enabled);
|
|
1316
|
+
if (fault.applied)
|
|
1317
|
+
expectedFaultMetrics[planned.kind === 'network.heal' ? 'heals' : 'partitions'] += 1;
|
|
1318
|
+
if (fault.applied && planned.kind === 'network.partition') {
|
|
1319
|
+
const drops = fault.affectedEventIds.length;
|
|
1320
|
+
partitionFaultDrops += drops;
|
|
1321
|
+
faultDroppedMessages += drops;
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
else if (planned.kind === 'clock.offset') {
|
|
1325
|
+
expectedClockOffsets.set(planned.peerId, planned.offset);
|
|
1326
|
+
if (fault.applied)
|
|
1327
|
+
expectedFaultMetrics.clockOffsetChanges += 1;
|
|
1328
|
+
}
|
|
1329
|
+
else if (planned.kind === 'message.duplicate' && fault.applied) {
|
|
1330
|
+
expectedFaultMetrics.duplicatedMessages += fault.affectedEventIds.length;
|
|
1331
|
+
}
|
|
1332
|
+
else if (planned.kind === 'message.delay' && fault.applied) {
|
|
1333
|
+
expectedFaultMetrics.delayedMessages += 1;
|
|
1334
|
+
}
|
|
1335
|
+
else if (planned.kind === 'message.reorder' && fault.applied) {
|
|
1336
|
+
expectedFaultMetrics.reorderedMessages += 1;
|
|
1337
|
+
}
|
|
1338
|
+
else if (planned.kind === 'message.drop' && fault.applied) {
|
|
1339
|
+
faultDroppedMessages += 1;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
const emittedEffects = records.reduce((total, record) => total + record.effectKinds.length, 0);
|
|
1343
|
+
const messageDeliverEffects = records.reduce((total, record) => total +
|
|
1344
|
+
record.effectKinds.filter((kind) => kind === 'message.deliver').length, 0);
|
|
1345
|
+
const rejectedMessageRecords = records.filter((record) => record.inputKind === 'message.delivery' && record.accepted === false).length;
|
|
1346
|
+
const processedCrashSuppressions = records.filter((record) => record.rejectionCode === 'simulation_peer_crashed').length;
|
|
1347
|
+
const processedCrashDeliveryDrops = records.filter((record) => record.inputKind === 'message.delivery' &&
|
|
1348
|
+
record.rejectionCode === 'simulation_peer_crashed').length;
|
|
1349
|
+
const processedPartitionDrops = records.filter((record) => record.inputKind === 'message.delivery' &&
|
|
1350
|
+
record.rejectionCode === 'simulation_partitioned').length;
|
|
1351
|
+
const successfulDeliveries = metrics.deliveredMessages - expectedFaultMetrics.duplicatedMessages;
|
|
1352
|
+
const immediateDeliveryDrops = messageDeliverEffects - successfulDeliveries;
|
|
1353
|
+
const immediateCrashSuppressions = records.reduce((total, record) => total +
|
|
1354
|
+
record.transportOutcome.droppedByCrash +
|
|
1355
|
+
record.transportOutcome.droppedByCrashAndPartition, 0);
|
|
1356
|
+
const immediatePartitionSuppressions = records.reduce((total, record) => total +
|
|
1357
|
+
record.transportOutcome.droppedByPartition +
|
|
1358
|
+
record.transportOutcome.droppedByCrashAndPartition, 0);
|
|
1359
|
+
const expectedOutboundSequences = new Map(config.peers.map((peer) => [
|
|
1360
|
+
peer.peerId,
|
|
1361
|
+
(peer.outboundSequence ?? 0) +
|
|
1362
|
+
records.reduce((total, record) => total +
|
|
1363
|
+
(record.peerId === peer.peerId
|
|
1364
|
+
? record.effectKinds.filter((kind) => kind === 'message.prepare')
|
|
1365
|
+
.length
|
|
1366
|
+
: 0), 0),
|
|
1367
|
+
]));
|
|
1368
|
+
const crashSuppressionBase = processedCrashSuppressions + crashFaultDrops;
|
|
1369
|
+
const partitionSuppressionBase = processedPartitionDrops + partitionFaultDrops;
|
|
1370
|
+
if (metrics.processedEvents !== records.length ||
|
|
1371
|
+
metrics.emittedEffects !== emittedEffects ||
|
|
1372
|
+
metrics.deliveredMessages < expectedFaultMetrics.duplicatedMessages ||
|
|
1373
|
+
successfulDeliveries > messageDeliverEffects ||
|
|
1374
|
+
immediateDeliveryDrops < 0 ||
|
|
1375
|
+
metrics.rejectedMessages !==
|
|
1376
|
+
rejectedMessageRecords + immediateDeliveryDrops ||
|
|
1377
|
+
metrics.droppedMessages !==
|
|
1378
|
+
immediateDeliveryDrops +
|
|
1379
|
+
processedCrashDeliveryDrops +
|
|
1380
|
+
processedPartitionDrops +
|
|
1381
|
+
faultDroppedMessages ||
|
|
1382
|
+
metrics.crashSuppressedEvents !==
|
|
1383
|
+
crashSuppressionBase + immediateCrashSuppressions ||
|
|
1384
|
+
metrics.partitionSuppressedMessages !==
|
|
1385
|
+
partitionSuppressionBase + immediatePartitionSuppressions ||
|
|
1386
|
+
config.peers.some((peer) => snapshot.outboundSequences[peer.peerId] !==
|
|
1387
|
+
expectedOutboundSequences.get(peer.peerId)))
|
|
1388
|
+
throw new TypeError('Mesh simulation snapshot metrics are inconsistent');
|
|
1389
|
+
if (config.peers.some((peer) => snapshot.peerAvailability[peer.peerId] !==
|
|
1390
|
+
expectedAvailability.get(peer.peerId) ||
|
|
1391
|
+
snapshot.clockOffsets[peer.peerId] !==
|
|
1392
|
+
expectedClockOffsets.get(peer.peerId)) ||
|
|
1393
|
+
topology.some((link) => link.enabled !==
|
|
1394
|
+
expectedTopology.get(linkKey(link.fromPeerId, link.toPeerId))) ||
|
|
1395
|
+
Object.entries(expectedFaultMetrics).some(([name, value]) => metrics[name] !== value))
|
|
1396
|
+
throw new TypeError('Mesh simulation snapshot fault state is invalid');
|
|
1397
|
+
const queuedFaults = queuedEvents.filter((event) => event.action.kind === 'fault.apply');
|
|
1398
|
+
const queuedFaultIds = new Set();
|
|
1399
|
+
for (const event of queuedFaults) {
|
|
1400
|
+
const planned = plannedFaults.get(event.action.fault.faultId);
|
|
1401
|
+
if (!planned ||
|
|
1402
|
+
processedFaultIds.has(planned.faultId) ||
|
|
1403
|
+
queuedFaultIds.has(planned.faultId) ||
|
|
1404
|
+
event.eventId !== `fault:${planned.faultId}` ||
|
|
1405
|
+
event.targetPeerId !== faultTargetPeerId(planned, config) ||
|
|
1406
|
+
event.order.logicalTime !== planned.logicalTime ||
|
|
1407
|
+
event.order.priority !== planned.priority ||
|
|
1408
|
+
(await digest(event.action.fault)) !== (await digest(planned)))
|
|
1409
|
+
throw new TypeError('Mesh simulation snapshot fault queue is invalid');
|
|
1410
|
+
queuedFaultIds.add(planned.faultId);
|
|
1411
|
+
}
|
|
1412
|
+
if (processedFaultIds.size + queuedFaultIds.size !== faultPlan.faults.length)
|
|
1413
|
+
throw new TypeError('Mesh simulation snapshot fault plan is incomplete');
|
|
1414
|
+
return Object.freeze({
|
|
1415
|
+
schemaVersion: 2,
|
|
1416
|
+
configurationDigest,
|
|
1417
|
+
faultPlanDigest,
|
|
1418
|
+
logicalTime: snapshot.logicalTime,
|
|
1419
|
+
insertionSequence: snapshot.insertionSequence,
|
|
1420
|
+
faultCursor: snapshot.faultCursor,
|
|
1421
|
+
peerStates: frozenRecord(peerIds.map((peerId) => [peerId, restoredPeerStates.get(peerId)])),
|
|
1422
|
+
outboundSequences: frozenRecord(peerIds.map((peerId) => [peerId, snapshot.outboundSequences[peerId]])),
|
|
1423
|
+
prngStates: frozenRecord(Object.entries(snapshot.prngStates).map(([key, value]) => [key, value])),
|
|
1424
|
+
peerAvailability: frozenRecord(peerIds.map((peerId) => [peerId, snapshot.peerAvailability[peerId]])),
|
|
1425
|
+
clockOffsets: frozenRecord(peerIds.map((peerId) => [peerId, snapshot.clockOffsets[peerId]])),
|
|
1426
|
+
topology,
|
|
1427
|
+
queuedEvents: Object.freeze(queuedEvents),
|
|
1428
|
+
eventIds: Object.freeze(eventIds.sort()),
|
|
1429
|
+
metrics,
|
|
1430
|
+
records: Object.freeze(records),
|
|
1431
|
+
faults: Object.freeze(faults),
|
|
1432
|
+
chainDigest: snapshot.chainDigest,
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
function validateSnapshotTopology(input, config) {
|
|
1436
|
+
if (!Array.isArray(input) ||
|
|
1437
|
+
input.length !== config.links.length ||
|
|
1438
|
+
!isDenseArray(input))
|
|
1439
|
+
throw new TypeError('Invalid Mesh simulation snapshot topology');
|
|
1440
|
+
const expected = new Map(config.links.map((link) => [linkKey(link.fromPeerId, link.toPeerId), link]));
|
|
1441
|
+
const result = input.map((link) => {
|
|
1442
|
+
assertPlainData(link, 'snapshot link');
|
|
1443
|
+
assertExactKeys(link, ['enabled', 'fromPeerId', 'latency', 'toPeerId'], ['enabled', 'fromPeerId', 'latency', 'toPeerId']);
|
|
1444
|
+
assertBoundedString(link.fromPeerId, 'snapshot link source peerId', 256);
|
|
1445
|
+
assertBoundedString(link.toPeerId, 'snapshot link target peerId', 256);
|
|
1446
|
+
const original = expected.get(linkKey(link.fromPeerId, link.toPeerId));
|
|
1447
|
+
if (!original ||
|
|
1448
|
+
link.latency !== original.latency ||
|
|
1449
|
+
typeof link.enabled !== 'boolean')
|
|
1450
|
+
throw new TypeError('Invalid Mesh simulation snapshot link');
|
|
1451
|
+
expected.delete(linkKey(link.fromPeerId, link.toPeerId));
|
|
1452
|
+
return Object.freeze({ ...link });
|
|
1453
|
+
});
|
|
1454
|
+
if (expected.size !== 0)
|
|
1455
|
+
throw new TypeError('Incomplete Mesh simulation snapshot topology');
|
|
1456
|
+
return Object.freeze(result);
|
|
1457
|
+
}
|
|
1458
|
+
function validateSnapshotEvent(input, snapshot, config, peerIds) {
|
|
1459
|
+
assertPlainData(input, 'snapshot event');
|
|
1460
|
+
assertExactKeys(input, ['action', 'eventId', 'order', 'targetPeerId'], ['action', 'eventId', 'order', 'targetPeerId']);
|
|
1461
|
+
assertBoundedString(input.eventId, 'eventId', 768);
|
|
1462
|
+
assertBoundedString(input.targetPeerId, 'snapshot event peerId', 256);
|
|
1463
|
+
if (!peerIds.has(input.targetPeerId))
|
|
1464
|
+
throw new TypeError('Mesh simulation snapshot event peer is unknown');
|
|
1465
|
+
assertPlainData(input.order, 'snapshot event order');
|
|
1466
|
+
assertExactKeys(input.order, ['insertionSequence', 'logicalTime', 'priority'], ['insertionSequence', 'logicalTime', 'priority']);
|
|
1467
|
+
if (!Number.isSafeInteger(input.order.logicalTime) ||
|
|
1468
|
+
input.order.logicalTime < snapshot.logicalTime ||
|
|
1469
|
+
input.order.logicalTime > config.limits.maximumLogicalTime ||
|
|
1470
|
+
!Number.isSafeInteger(input.order.priority) ||
|
|
1471
|
+
!Number.isSafeInteger(input.order.insertionSequence) ||
|
|
1472
|
+
input.order.insertionSequence < 1 ||
|
|
1473
|
+
input.order.insertionSequence > snapshot.insertionSequence)
|
|
1474
|
+
throw new TypeError('Invalid Mesh simulation snapshot event order');
|
|
1475
|
+
assertPlainData(input.action, 'snapshot action');
|
|
1476
|
+
if (input.action.kind === 'peer.input' ||
|
|
1477
|
+
input.action.kind === 'message.delivery')
|
|
1478
|
+
assertExactKeys(input.action, input.action.kind === 'peer.input'
|
|
1479
|
+
? ['input', 'kind']
|
|
1480
|
+
: ['envelope', 'kind'], input.action.kind === 'peer.input'
|
|
1481
|
+
? ['input', 'kind']
|
|
1482
|
+
: ['envelope', 'kind']);
|
|
1483
|
+
else if (input.action.kind === 'fault.apply')
|
|
1484
|
+
assertExactKeys(input.action, ['fault', 'kind'], ['fault', 'kind']);
|
|
1485
|
+
else
|
|
1486
|
+
throw new TypeError('Invalid Mesh simulation snapshot action');
|
|
1487
|
+
let action;
|
|
1488
|
+
if (input.action.kind === 'peer.input') {
|
|
1489
|
+
action = Object.freeze({
|
|
1490
|
+
kind: 'peer.input',
|
|
1491
|
+
input: validateSnapshotPeerInput(input.action.input, peerIds),
|
|
1492
|
+
});
|
|
1493
|
+
}
|
|
1494
|
+
else if (input.action.kind === 'message.delivery') {
|
|
1495
|
+
const envelopeCopy = deepFreezeCopy(input.action.envelope);
|
|
1496
|
+
const target = config.peers.find((peer) => peer.peerId === input.targetPeerId);
|
|
1497
|
+
const validated = validateSignedMeshEnvelope(envelopeCopy, {
|
|
1498
|
+
limits: target.protocolOptions?.limits,
|
|
1499
|
+
});
|
|
1500
|
+
if (!validated.ok)
|
|
1501
|
+
throw new TypeError('Invalid Mesh simulation snapshot envelope');
|
|
1502
|
+
action = Object.freeze({
|
|
1503
|
+
kind: 'message.delivery',
|
|
1504
|
+
envelope: validated.value,
|
|
1505
|
+
});
|
|
1506
|
+
}
|
|
1507
|
+
else {
|
|
1508
|
+
const normalized = normalizeFaultPlan({ schemaVersion: 1, faults: [input.action.fault] }, config);
|
|
1509
|
+
action = Object.freeze({
|
|
1510
|
+
kind: 'fault.apply',
|
|
1511
|
+
fault: normalized.faults[0],
|
|
1512
|
+
});
|
|
1513
|
+
}
|
|
1514
|
+
if (!canonicalizeMeshJsonBytes(action).ok)
|
|
1515
|
+
throw new TypeError('Invalid Mesh simulation snapshot action');
|
|
1516
|
+
return Object.freeze({
|
|
1517
|
+
eventId: input.eventId,
|
|
1518
|
+
targetPeerId: input.targetPeerId,
|
|
1519
|
+
order: Object.freeze({ ...input.order }),
|
|
1520
|
+
action,
|
|
1521
|
+
});
|
|
1522
|
+
}
|
|
1523
|
+
function validateSnapshotPeerInput(input, peerIds) {
|
|
1524
|
+
assertPlainData(input, 'snapshot peer input');
|
|
1525
|
+
if (input.kind === 'peer.start') {
|
|
1526
|
+
assertExactKeys(input, ['kind'], ['kind']);
|
|
1527
|
+
}
|
|
1528
|
+
else if (input.kind === 'peer.stop') {
|
|
1529
|
+
assertExactKeys(input, ['kind', 'reason'], ['kind', 'reason']);
|
|
1530
|
+
if (!['policy', 'requested', 'resource_limit'].includes(input.reason))
|
|
1531
|
+
throw new TypeError('Invalid Mesh simulation snapshot peer input');
|
|
1532
|
+
}
|
|
1533
|
+
else if (input.kind === 'peer.ping') {
|
|
1534
|
+
assertExactKeys(input, ['kind', 'peerId'], ['kind', 'peerId']);
|
|
1535
|
+
assertBoundedString(input.peerId, 'snapshot ping peerId', 256);
|
|
1536
|
+
if (!peerIds.has(input.peerId))
|
|
1537
|
+
throw new TypeError('Invalid Mesh simulation snapshot ping peer');
|
|
1538
|
+
}
|
|
1539
|
+
else {
|
|
1540
|
+
throw new TypeError('Invalid Mesh simulation snapshot peer input');
|
|
1541
|
+
}
|
|
1542
|
+
return deepFreezeCopy(input);
|
|
1543
|
+
}
|
|
1544
|
+
function validateMetrics(input, snapshot) {
|
|
1545
|
+
assertPlainData(input, 'snapshot metrics');
|
|
1546
|
+
const keys = [
|
|
1547
|
+
'clockOffsetChanges',
|
|
1548
|
+
'crashSuppressedEvents',
|
|
1549
|
+
'delayedMessages',
|
|
1550
|
+
'deliveredMessages',
|
|
1551
|
+
'droppedMessages',
|
|
1552
|
+
'duplicatedMessages',
|
|
1553
|
+
'emittedEffects',
|
|
1554
|
+
'faultEvents',
|
|
1555
|
+
'finalLogicalTime',
|
|
1556
|
+
'heals',
|
|
1557
|
+
'partitionSuppressedMessages',
|
|
1558
|
+
'partitions',
|
|
1559
|
+
'peerCrashes',
|
|
1560
|
+
'peerResumes',
|
|
1561
|
+
'processedEvents',
|
|
1562
|
+
'rejectedMessages',
|
|
1563
|
+
'reorderedMessages',
|
|
1564
|
+
];
|
|
1565
|
+
assertExactKeys(input, keys, keys);
|
|
1566
|
+
if (Object.values(input).some((value) => typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) ||
|
|
1567
|
+
input.processedEvents > snapshot.records.length ||
|
|
1568
|
+
input.finalLogicalTime !== snapshot.logicalTime)
|
|
1569
|
+
throw new TypeError('Invalid Mesh simulation snapshot metrics');
|
|
1570
|
+
return Object.freeze({ ...input });
|
|
1571
|
+
}
|
|
1572
|
+
function validateRecord(record, config, peerIds, insertionSequence) {
|
|
1573
|
+
assertPlainData(record, 'snapshot record');
|
|
1574
|
+
const supported = [
|
|
1575
|
+
'accepted',
|
|
1576
|
+
'actionDigest',
|
|
1577
|
+
'chainDigest',
|
|
1578
|
+
'effectKinds',
|
|
1579
|
+
'effectsDigest',
|
|
1580
|
+
'eventId',
|
|
1581
|
+
'faultApplied',
|
|
1582
|
+
'faultId',
|
|
1583
|
+
'faultKind',
|
|
1584
|
+
'inputKind',
|
|
1585
|
+
'inputDeliverySourcePeerId',
|
|
1586
|
+
'metricsDelta',
|
|
1587
|
+
'order',
|
|
1588
|
+
'peerId',
|
|
1589
|
+
'rejectionCode',
|
|
1590
|
+
'stateDigest',
|
|
1591
|
+
'transportOutcome',
|
|
1592
|
+
];
|
|
1593
|
+
assertExactKeys(record, supported, [
|
|
1594
|
+
'actionDigest',
|
|
1595
|
+
'chainDigest',
|
|
1596
|
+
'effectKinds',
|
|
1597
|
+
'effectsDigest',
|
|
1598
|
+
'eventId',
|
|
1599
|
+
'inputKind',
|
|
1600
|
+
'metricsDelta',
|
|
1601
|
+
'order',
|
|
1602
|
+
'peerId',
|
|
1603
|
+
'stateDigest',
|
|
1604
|
+
'transportOutcome',
|
|
1605
|
+
]);
|
|
1606
|
+
const effectKinds = new Set([
|
|
1607
|
+
'event.emit',
|
|
1608
|
+
'intake.backpressure',
|
|
1609
|
+
'message.deliver',
|
|
1610
|
+
'message.prepare',
|
|
1611
|
+
'timer.schedule',
|
|
1612
|
+
]);
|
|
1613
|
+
const faultKinds = new Set([
|
|
1614
|
+
'peer.crash',
|
|
1615
|
+
'peer.resume',
|
|
1616
|
+
'message.drop',
|
|
1617
|
+
'message.duplicate',
|
|
1618
|
+
'message.delay',
|
|
1619
|
+
'message.reorder',
|
|
1620
|
+
'network.partition',
|
|
1621
|
+
'network.heal',
|
|
1622
|
+
'clock.offset',
|
|
1623
|
+
]);
|
|
1624
|
+
assertBoundedString(record.peerId, 'snapshot record peerId', 256);
|
|
1625
|
+
if (record.inputKind === 'message.delivery')
|
|
1626
|
+
assertBoundedString(record.inputDeliverySourcePeerId, 'snapshot input delivery source peerId', 256);
|
|
1627
|
+
assertPlainData(record.metricsDelta, 'snapshot record metrics delta');
|
|
1628
|
+
assertExactKeys(record.metricsDelta, metricDeltaKeys, metricDeltaKeys);
|
|
1629
|
+
assertPlainData(record.transportOutcome, 'snapshot transport outcome');
|
|
1630
|
+
assertExactKeys(record.transportOutcome, [
|
|
1631
|
+
'delivered',
|
|
1632
|
+
'deliveries',
|
|
1633
|
+
'droppedByCrash',
|
|
1634
|
+
'droppedByCrashAndPartition',
|
|
1635
|
+
'droppedByDestinationMissing',
|
|
1636
|
+
'droppedByPartition',
|
|
1637
|
+
], [
|
|
1638
|
+
'delivered',
|
|
1639
|
+
'deliveries',
|
|
1640
|
+
'droppedByCrash',
|
|
1641
|
+
'droppedByCrashAndPartition',
|
|
1642
|
+
'droppedByDestinationMissing',
|
|
1643
|
+
'droppedByPartition',
|
|
1644
|
+
]);
|
|
1645
|
+
if (!Array.isArray(record.transportOutcome.deliveries) ||
|
|
1646
|
+
record.transportOutcome.deliveries.length >
|
|
1647
|
+
config.limits.maximumInternalSteps ||
|
|
1648
|
+
!isDenseArray(record.transportOutcome.deliveries))
|
|
1649
|
+
throw new TypeError('Invalid Mesh simulation transport outcomes');
|
|
1650
|
+
const transportDeliveries = record.transportOutcome.deliveries.map((delivery) => {
|
|
1651
|
+
assertPlainData(delivery, 'snapshot transport delivery');
|
|
1652
|
+
assertExactKeys(delivery, ['eventId', 'fromPeerId', 'outcome', 'toPeerId'], ['eventId', 'fromPeerId', 'outcome', 'toPeerId']);
|
|
1653
|
+
assertBoundedString(delivery.eventId, 'transport eventId', 768);
|
|
1654
|
+
assertBoundedString(delivery.fromPeerId, 'transport source peerId', 256);
|
|
1655
|
+
assertBoundedString(delivery.toPeerId, 'transport target peerId', 256);
|
|
1656
|
+
if (!peerIds.has(delivery.fromPeerId) ||
|
|
1657
|
+
(delivery.outcome === 'destination_missing'
|
|
1658
|
+
? peerIds.has(delivery.toPeerId)
|
|
1659
|
+
: !peerIds.has(delivery.toPeerId)) ||
|
|
1660
|
+
![
|
|
1661
|
+
'crash',
|
|
1662
|
+
'crash_partition',
|
|
1663
|
+
'delivered',
|
|
1664
|
+
'destination_missing',
|
|
1665
|
+
'partition',
|
|
1666
|
+
].includes(delivery.outcome))
|
|
1667
|
+
throw new TypeError('Invalid Mesh simulation transport outcome');
|
|
1668
|
+
return delivery;
|
|
1669
|
+
});
|
|
1670
|
+
const transportCounts = {
|
|
1671
|
+
delivered: transportDeliveries.filter(({ outcome }) => outcome === 'delivered').length,
|
|
1672
|
+
droppedByCrash: transportDeliveries.filter(({ outcome }) => outcome === 'crash').length,
|
|
1673
|
+
droppedByPartition: transportDeliveries.filter(({ outcome }) => outcome === 'partition').length,
|
|
1674
|
+
droppedByCrashAndPartition: transportDeliveries.filter(({ outcome }) => outcome === 'crash_partition').length,
|
|
1675
|
+
droppedByDestinationMissing: transportDeliveries.filter(({ outcome }) => outcome === 'destination_missing').length,
|
|
1676
|
+
};
|
|
1677
|
+
if (!peerIds.has(record.peerId) ||
|
|
1678
|
+
(record.inputKind === 'message.delivery'
|
|
1679
|
+
? typeof record.inputDeliverySourcePeerId !== 'string'
|
|
1680
|
+
: record.inputDeliverySourcePeerId !== undefined) ||
|
|
1681
|
+
!['peer.input', 'message.delivery', 'fault.apply'].includes(record.inputKind) ||
|
|
1682
|
+
!isDigest(record.actionDigest) ||
|
|
1683
|
+
!isDigest(record.effectsDigest) ||
|
|
1684
|
+
!isDigest(record.stateDigest) ||
|
|
1685
|
+
!isDigest(record.chainDigest) ||
|
|
1686
|
+
!Array.isArray(record.effectKinds) ||
|
|
1687
|
+
record.effectKinds.length > config.limits.maximumInternalSteps ||
|
|
1688
|
+
!isDenseArray(record.effectKinds) ||
|
|
1689
|
+
record.effectKinds.some((kind) => typeof kind !== 'string' ||
|
|
1690
|
+
!effectKinds.has(kind)) ||
|
|
1691
|
+
(record.accepted !== undefined && typeof record.accepted !== 'boolean') ||
|
|
1692
|
+
(record.rejectionCode !== undefined &&
|
|
1693
|
+
(typeof record.rejectionCode !== 'string' ||
|
|
1694
|
+
new TextEncoder().encode(record.rejectionCode).byteLength > 256)) ||
|
|
1695
|
+
(record.faultId !== undefined &&
|
|
1696
|
+
(typeof record.faultId !== 'string' ||
|
|
1697
|
+
new TextEncoder().encode(record.faultId).byteLength > 256)) ||
|
|
1698
|
+
(record.faultKind !== undefined && !faultKinds.has(record.faultKind)) ||
|
|
1699
|
+
(record.faultApplied !== undefined &&
|
|
1700
|
+
typeof record.faultApplied !== 'boolean') ||
|
|
1701
|
+
(record.inputKind === 'peer.input' &&
|
|
1702
|
+
(record.accepted !== undefined || record.rejectionCode !== undefined) &&
|
|
1703
|
+
!(record.accepted === false &&
|
|
1704
|
+
record.rejectionCode === 'simulation_peer_crashed')) ||
|
|
1705
|
+
(record.inputKind === 'message.delivery' &&
|
|
1706
|
+
typeof record.accepted !== 'boolean') ||
|
|
1707
|
+
(record.rejectionCode !== undefined && record.accepted !== false) ||
|
|
1708
|
+
(record.inputKind === 'fault.apply'
|
|
1709
|
+
? record.faultId === undefined ||
|
|
1710
|
+
record.faultKind === undefined ||
|
|
1711
|
+
typeof record.faultApplied !== 'boolean' ||
|
|
1712
|
+
typeof record.accepted !== 'boolean' ||
|
|
1713
|
+
record.rejectionCode !== undefined
|
|
1714
|
+
: record.faultId !== undefined ||
|
|
1715
|
+
record.faultKind !== undefined ||
|
|
1716
|
+
record.faultApplied !== undefined) ||
|
|
1717
|
+
Object.values(record.metricsDelta).some((value) => typeof value !== 'number' ||
|
|
1718
|
+
!Number.isSafeInteger(value) ||
|
|
1719
|
+
value < 0 ||
|
|
1720
|
+
value > maximumIssuedEvents(config)) ||
|
|
1721
|
+
record.metricsDelta.processedEvents !== 1 ||
|
|
1722
|
+
record.metricsDelta.emittedEffects !== record.effectKinds.length ||
|
|
1723
|
+
record.metricsDelta.faultEvents !==
|
|
1724
|
+
(record.inputKind === 'fault.apply' ? 1 : 0) ||
|
|
1725
|
+
Object.entries(transportCounts).some(([key, value]) => record.transportOutcome[key] !== value) ||
|
|
1726
|
+
transportDeliveries.length !==
|
|
1727
|
+
record.effectKinds.filter((kind) => kind === 'message.deliver').length ||
|
|
1728
|
+
new Set(transportDeliveries.map(({ eventId }) => eventId)).size !==
|
|
1729
|
+
transportDeliveries.length)
|
|
1730
|
+
throw new TypeError('Invalid Mesh simulation snapshot record');
|
|
1731
|
+
const faultOnlyMetricKeys = [
|
|
1732
|
+
'clockOffsetChanges',
|
|
1733
|
+
'delayedMessages',
|
|
1734
|
+
'duplicatedMessages',
|
|
1735
|
+
'heals',
|
|
1736
|
+
'partitions',
|
|
1737
|
+
'peerCrashes',
|
|
1738
|
+
'peerResumes',
|
|
1739
|
+
'reorderedMessages',
|
|
1740
|
+
];
|
|
1741
|
+
if (record.inputKind !== 'fault.apply' &&
|
|
1742
|
+
faultOnlyMetricKeys.some((key) => record.metricsDelta[key] !== 0))
|
|
1743
|
+
throw new TypeError('Invalid Mesh simulation snapshot record metrics');
|
|
1744
|
+
if (record.inputKind === 'fault.apply') {
|
|
1745
|
+
const applied = record.faultApplied === true;
|
|
1746
|
+
const expectedUnit = (kind) => applied && record.faultKind === kind ? 1 : 0;
|
|
1747
|
+
if (record.effectKinds.length !== 0 ||
|
|
1748
|
+
record.metricsDelta.peerCrashes !== expectedUnit('peer.crash') ||
|
|
1749
|
+
record.metricsDelta.peerResumes !== expectedUnit('peer.resume') ||
|
|
1750
|
+
record.metricsDelta.delayedMessages !== expectedUnit('message.delay') ||
|
|
1751
|
+
record.metricsDelta.reorderedMessages !==
|
|
1752
|
+
expectedUnit('message.reorder') ||
|
|
1753
|
+
record.metricsDelta.partitions !== expectedUnit('network.partition') ||
|
|
1754
|
+
record.metricsDelta.heals !== expectedUnit('network.heal') ||
|
|
1755
|
+
record.metricsDelta.clockOffsetChanges !== expectedUnit('clock.offset') ||
|
|
1756
|
+
record.metricsDelta.rejectedMessages !== 0 ||
|
|
1757
|
+
(record.faultKind === 'message.duplicate'
|
|
1758
|
+
? record.metricsDelta.duplicatedMessages < (applied ? 1 : 0) ||
|
|
1759
|
+
record.metricsDelta.deliveredMessages !==
|
|
1760
|
+
record.metricsDelta.duplicatedMessages
|
|
1761
|
+
: record.metricsDelta.duplicatedMessages !== 0 ||
|
|
1762
|
+
record.metricsDelta.deliveredMessages !== 0) ||
|
|
1763
|
+
(record.faultKind === 'message.drop'
|
|
1764
|
+
? record.metricsDelta.droppedMessages !== (applied ? 1 : 0)
|
|
1765
|
+
: record.faultKind === 'peer.crash'
|
|
1766
|
+
? record.metricsDelta.droppedMessages !==
|
|
1767
|
+
record.metricsDelta.crashSuppressedEvents ||
|
|
1768
|
+
record.metricsDelta.partitionSuppressedMessages !== 0
|
|
1769
|
+
: record.faultKind === 'network.partition'
|
|
1770
|
+
? record.metricsDelta.droppedMessages !==
|
|
1771
|
+
record.metricsDelta.partitionSuppressedMessages ||
|
|
1772
|
+
record.metricsDelta.crashSuppressedEvents !== 0
|
|
1773
|
+
: record.metricsDelta.droppedMessages !== 0 ||
|
|
1774
|
+
record.metricsDelta.crashSuppressedEvents !== 0 ||
|
|
1775
|
+
record.metricsDelta.partitionSuppressedMessages !== 0))
|
|
1776
|
+
throw new TypeError('Invalid Mesh simulation snapshot fault metrics');
|
|
1777
|
+
}
|
|
1778
|
+
else {
|
|
1779
|
+
const effectDrops = transportCounts.droppedByCrash +
|
|
1780
|
+
transportCounts.droppedByPartition +
|
|
1781
|
+
transportCounts.droppedByCrashAndPartition +
|
|
1782
|
+
transportCounts.droppedByDestinationMissing;
|
|
1783
|
+
const inputRejected = record.inputKind === 'message.delivery' && record.accepted === false
|
|
1784
|
+
? 1
|
|
1785
|
+
: 0;
|
|
1786
|
+
const inputDropped = record.inputKind === 'message.delivery' &&
|
|
1787
|
+
(record.rejectionCode === 'simulation_peer_crashed' ||
|
|
1788
|
+
record.rejectionCode === 'simulation_partitioned')
|
|
1789
|
+
? 1
|
|
1790
|
+
: 0;
|
|
1791
|
+
if (record.metricsDelta.deliveredMessages !== transportCounts.delivered ||
|
|
1792
|
+
record.metricsDelta.rejectedMessages !== inputRejected + effectDrops ||
|
|
1793
|
+
record.metricsDelta.droppedMessages !== inputDropped + effectDrops ||
|
|
1794
|
+
record.metricsDelta.crashSuppressedEvents !==
|
|
1795
|
+
(record.rejectionCode === 'simulation_peer_crashed' ? 1 : 0) +
|
|
1796
|
+
transportCounts.droppedByCrash +
|
|
1797
|
+
transportCounts.droppedByCrashAndPartition ||
|
|
1798
|
+
record.metricsDelta.partitionSuppressedMessages !==
|
|
1799
|
+
(record.rejectionCode === 'simulation_partitioned' ? 1 : 0) +
|
|
1800
|
+
transportCounts.droppedByPartition +
|
|
1801
|
+
transportCounts.droppedByCrashAndPartition)
|
|
1802
|
+
throw new TypeError('Invalid Mesh simulation snapshot transport metrics');
|
|
1803
|
+
}
|
|
1804
|
+
assertBoundedString(record.eventId, 'snapshot record eventId', 768);
|
|
1805
|
+
assertPlainData(record.order, 'snapshot record order');
|
|
1806
|
+
assertExactKeys(record.order, ['insertionSequence', 'logicalTime', 'priority'], ['insertionSequence', 'logicalTime', 'priority']);
|
|
1807
|
+
if (!Number.isSafeInteger(record.order.logicalTime) ||
|
|
1808
|
+
record.order.logicalTime < 0 ||
|
|
1809
|
+
record.order.logicalTime > config.limits.maximumLogicalTime ||
|
|
1810
|
+
!Number.isSafeInteger(record.order.priority) ||
|
|
1811
|
+
!Number.isSafeInteger(record.order.insertionSequence) ||
|
|
1812
|
+
record.order.insertionSequence < 1 ||
|
|
1813
|
+
record.order.insertionSequence > insertionSequence)
|
|
1814
|
+
throw new TypeError('Invalid Mesh simulation snapshot record order');
|
|
1815
|
+
}
|
|
1816
|
+
function validateFaultRecord(input, records, plannedFaults, eventIds, queuedEvents, config) {
|
|
1817
|
+
assertPlainData(input, 'snapshot fault record');
|
|
1818
|
+
assertExactKeys(input, [
|
|
1819
|
+
'affectedDeliveries',
|
|
1820
|
+
'affectedEventIds',
|
|
1821
|
+
'affectedLinkIds',
|
|
1822
|
+
'applied',
|
|
1823
|
+
'faultId',
|
|
1824
|
+
'kind',
|
|
1825
|
+
'order',
|
|
1826
|
+
], [
|
|
1827
|
+
'affectedDeliveries',
|
|
1828
|
+
'affectedEventIds',
|
|
1829
|
+
'affectedLinkIds',
|
|
1830
|
+
'applied',
|
|
1831
|
+
'faultId',
|
|
1832
|
+
'kind',
|
|
1833
|
+
'order',
|
|
1834
|
+
]);
|
|
1835
|
+
assertBoundedString(input.faultId, 'snapshot faultId');
|
|
1836
|
+
assertPlainData(input.order, 'snapshot fault order');
|
|
1837
|
+
assertExactKeys(input.order, ['insertionSequence', 'logicalTime', 'priority'], ['insertionSequence', 'logicalTime', 'priority']);
|
|
1838
|
+
const planned = plannedFaults.get(input.faultId);
|
|
1839
|
+
const maximumAffected = planned?.kind === 'message.duplicate'
|
|
1840
|
+
? planned.copies
|
|
1841
|
+
: planned?.kind === 'network.partition'
|
|
1842
|
+
? config.limits.maximumQueuedEvents
|
|
1843
|
+
: planned?.kind === 'network.heal'
|
|
1844
|
+
? 0
|
|
1845
|
+
: planned?.kind === 'peer.crash'
|
|
1846
|
+
? config.limits.maximumQueuedEvents
|
|
1847
|
+
: 1;
|
|
1848
|
+
const maximumAffectedLinks = planned?.kind === 'network.partition' || planned?.kind === 'network.heal'
|
|
1849
|
+
? planned.links.length
|
|
1850
|
+
: 0;
|
|
1851
|
+
if (!planned ||
|
|
1852
|
+
planned.kind !== input.kind ||
|
|
1853
|
+
typeof input.applied !== 'boolean' ||
|
|
1854
|
+
!Number.isSafeInteger(input.order.logicalTime) ||
|
|
1855
|
+
!Number.isSafeInteger(input.order.priority) ||
|
|
1856
|
+
!Number.isSafeInteger(input.order.insertionSequence) ||
|
|
1857
|
+
!Array.isArray(input.affectedEventIds) ||
|
|
1858
|
+
input.affectedEventIds.length > maximumAffected ||
|
|
1859
|
+
!isDenseArray(input.affectedEventIds) ||
|
|
1860
|
+
!Array.isArray(input.affectedLinkIds) ||
|
|
1861
|
+
input.affectedLinkIds.length > maximumAffectedLinks ||
|
|
1862
|
+
!isDenseArray(input.affectedLinkIds) ||
|
|
1863
|
+
!Array.isArray(input.affectedDeliveries) ||
|
|
1864
|
+
input.affectedDeliveries.length > input.affectedEventIds.length ||
|
|
1865
|
+
!isDenseArray(input.affectedDeliveries))
|
|
1866
|
+
throw new TypeError('Invalid Mesh simulation snapshot fault record');
|
|
1867
|
+
const affectedEventIds = [...input.affectedEventIds];
|
|
1868
|
+
const affectedLinkIds = [...input.affectedLinkIds];
|
|
1869
|
+
for (const eventId of affectedEventIds)
|
|
1870
|
+
assertBoundedString(eventId, 'snapshot affected eventId', 768);
|
|
1871
|
+
for (const linkId of affectedLinkIds)
|
|
1872
|
+
assertBoundedString(linkId, 'snapshot affected linkId', 768);
|
|
1873
|
+
if (new Set(affectedEventIds).size !== affectedEventIds.length ||
|
|
1874
|
+
new Set(affectedLinkIds).size !== affectedLinkIds.length ||
|
|
1875
|
+
(!input.applied &&
|
|
1876
|
+
(affectedEventIds.length !== 0 || affectedLinkIds.length !== 0)))
|
|
1877
|
+
throw new TypeError('Invalid Mesh simulation snapshot affected events');
|
|
1878
|
+
const affectedSet = new Set(affectedEventIds);
|
|
1879
|
+
const affectedDeliveries = input.affectedDeliveries.map((delivery) => {
|
|
1880
|
+
assertPlainData(delivery, 'snapshot affected delivery');
|
|
1881
|
+
assertExactKeys(delivery, ['eventId', 'fromPeerId', 'order', 'toPeerId'], ['eventId', 'fromPeerId', 'order', 'toPeerId']);
|
|
1882
|
+
assertBoundedString(delivery.eventId, 'affected delivery eventId', 768);
|
|
1883
|
+
assertBoundedString(delivery.fromPeerId, 'affected delivery source', 256);
|
|
1884
|
+
assertBoundedString(delivery.toPeerId, 'affected delivery target', 256);
|
|
1885
|
+
assertPlainData(delivery.order, 'affected delivery order');
|
|
1886
|
+
assertExactKeys(delivery.order, ['insertionSequence', 'logicalTime', 'priority'], ['insertionSequence', 'logicalTime', 'priority']);
|
|
1887
|
+
if (!affectedSet.has(delivery.eventId) ||
|
|
1888
|
+
!config.peers.some((peer) => peer.peerId === delivery.toPeerId) ||
|
|
1889
|
+
!Number.isSafeInteger(delivery.order.logicalTime) ||
|
|
1890
|
+
delivery.order.logicalTime < input.order.logicalTime ||
|
|
1891
|
+
delivery.order.logicalTime > config.limits.maximumLogicalTime ||
|
|
1892
|
+
!Number.isSafeInteger(delivery.order.priority) ||
|
|
1893
|
+
!Number.isSafeInteger(delivery.order.insertionSequence) ||
|
|
1894
|
+
delivery.order.insertionSequence < 1 ||
|
|
1895
|
+
delivery.order.insertionSequence > maximumIssuedEvents(config))
|
|
1896
|
+
throw new TypeError('Invalid Mesh simulation affected delivery');
|
|
1897
|
+
return Object.freeze({
|
|
1898
|
+
eventId: delivery.eventId,
|
|
1899
|
+
fromPeerId: delivery.fromPeerId,
|
|
1900
|
+
toPeerId: delivery.toPeerId,
|
|
1901
|
+
order: Object.freeze({ ...delivery.order }),
|
|
1902
|
+
});
|
|
1903
|
+
});
|
|
1904
|
+
if (new Set(affectedDeliveries.map(({ eventId }) => eventId)).size !==
|
|
1905
|
+
affectedDeliveries.length)
|
|
1906
|
+
throw new TypeError('Duplicate Mesh simulation affected delivery');
|
|
1907
|
+
const record = records.find((candidate) => candidate.faultId === input.faultId &&
|
|
1908
|
+
candidate.faultKind === input.kind &&
|
|
1909
|
+
candidate.order.insertionSequence === input.order.insertionSequence);
|
|
1910
|
+
if (!record ||
|
|
1911
|
+
record.accepted !== input.applied ||
|
|
1912
|
+
record.faultApplied !== input.applied ||
|
|
1913
|
+
input.order.logicalTime !== planned.logicalTime ||
|
|
1914
|
+
input.order.priority !== planned.priority ||
|
|
1915
|
+
record.order.logicalTime !== input.order.logicalTime ||
|
|
1916
|
+
record.order.priority !== input.order.priority)
|
|
1917
|
+
throw new TypeError('Invalid Mesh simulation snapshot fault record');
|
|
1918
|
+
const expectedSingle = planned.kind === 'message.drop' ||
|
|
1919
|
+
planned.kind === 'message.delay' ||
|
|
1920
|
+
planned.kind === 'message.reorder'
|
|
1921
|
+
? planned.deliveryEventId
|
|
1922
|
+
: planned.kind === 'peer.resume' || planned.kind === 'clock.offset'
|
|
1923
|
+
? planned.peerId
|
|
1924
|
+
: undefined;
|
|
1925
|
+
if (expectedSingle !== undefined &&
|
|
1926
|
+
input.applied &&
|
|
1927
|
+
(affectedEventIds.length !== 1 || affectedEventIds[0] !== expectedSingle))
|
|
1928
|
+
throw new TypeError('Invalid Mesh simulation snapshot affected events');
|
|
1929
|
+
if (planned.kind === 'message.duplicate' && input.applied) {
|
|
1930
|
+
const expected = Array.from({ length: planned.copies }, (_, index) => `${planned.deliveryEventId}:duplicate:${planned.faultId}:${index + 1}`);
|
|
1931
|
+
if (affectedEventIds.length !== expected.length ||
|
|
1932
|
+
expected.some((eventId) => !affectedSet.has(eventId)))
|
|
1933
|
+
throw new TypeError('Invalid Mesh simulation snapshot affected events');
|
|
1934
|
+
}
|
|
1935
|
+
if (input.applied &&
|
|
1936
|
+
(planned.kind === 'message.drop' ||
|
|
1937
|
+
planned.kind === 'message.duplicate' ||
|
|
1938
|
+
planned.kind === 'message.delay' ||
|
|
1939
|
+
planned.kind === 'message.reorder') &&
|
|
1940
|
+
affectedEventIds.some((eventId) => !eventIds.includes(eventId)))
|
|
1941
|
+
throw new TypeError('Invalid Mesh simulation snapshot affected events');
|
|
1942
|
+
if ((planned.kind === 'network.partition' || planned.kind === 'network.heal') &&
|
|
1943
|
+
input.applied) {
|
|
1944
|
+
const configured = new Set(planned.links.map((link) => linkKey(link.fromPeerId, link.toPeerId)));
|
|
1945
|
+
const queuedIds = new Set(queuedEvents.map((event) => event.eventId));
|
|
1946
|
+
const recordedIds = new Set(records.map((candidate) => candidate.eventId));
|
|
1947
|
+
if (affectedEventIds.length + affectedLinkIds.length === 0 ||
|
|
1948
|
+
affectedLinkIds.some((linkId) => !configured.has(linkId)) ||
|
|
1949
|
+
(planned.kind === 'network.heal' && affectedEventIds.length > 0) ||
|
|
1950
|
+
affectedEventIds.some((eventId) => !eventIds.includes(eventId) ||
|
|
1951
|
+
queuedIds.has(eventId) ||
|
|
1952
|
+
recordedIds.has(eventId) ||
|
|
1953
|
+
eventId.startsWith('fault:')))
|
|
1954
|
+
throw new TypeError('Invalid Mesh simulation snapshot affected links');
|
|
1955
|
+
}
|
|
1956
|
+
if (planned.kind === 'peer.crash') {
|
|
1957
|
+
const queuedIds = new Set(queuedEvents.map((event) => event.eventId));
|
|
1958
|
+
const recordedIds = new Set(records.map((candidate) => candidate.eventId));
|
|
1959
|
+
if (affectedEventIds.some((eventId) => !eventIds.includes(eventId) ||
|
|
1960
|
+
queuedIds.has(eventId) ||
|
|
1961
|
+
recordedIds.has(eventId) ||
|
|
1962
|
+
eventId.startsWith('fault:')))
|
|
1963
|
+
throw new TypeError('Invalid Mesh simulation snapshot crash drops');
|
|
1964
|
+
}
|
|
1965
|
+
const affectedDeliveryIds = new Set(affectedDeliveries.map(({ eventId }) => eventId));
|
|
1966
|
+
const expectedRemovedDeliveryIds = planned.kind === 'peer.crash'
|
|
1967
|
+
? affectedEventIds
|
|
1968
|
+
: planned.kind === 'message.drop' && input.applied
|
|
1969
|
+
? [planned.deliveryEventId]
|
|
1970
|
+
: planned.kind === 'network.partition' && input.applied
|
|
1971
|
+
? affectedEventIds
|
|
1972
|
+
: [];
|
|
1973
|
+
if (affectedDeliveryIds.size !== expectedRemovedDeliveryIds.length ||
|
|
1974
|
+
expectedRemovedDeliveryIds.some((eventId) => !affectedDeliveryIds.has(eventId)) ||
|
|
1975
|
+
affectedDeliveries.some((delivery) => (planned.kind === 'peer.crash' &&
|
|
1976
|
+
delivery.toPeerId !== planned.peerId) ||
|
|
1977
|
+
(planned.kind === 'message.drop' &&
|
|
1978
|
+
delivery.eventId !== planned.deliveryEventId) ||
|
|
1979
|
+
(planned.kind === 'network.partition' &&
|
|
1980
|
+
!planned.links.some((link) => link.fromPeerId === delivery.fromPeerId &&
|
|
1981
|
+
link.toPeerId === delivery.toPeerId))))
|
|
1982
|
+
throw new TypeError('Invalid Mesh simulation snapshot affected delivery causality');
|
|
1983
|
+
return Object.freeze({
|
|
1984
|
+
faultId: input.faultId,
|
|
1985
|
+
kind: input.kind,
|
|
1986
|
+
order: Object.freeze({ ...input.order }),
|
|
1987
|
+
applied: input.applied,
|
|
1988
|
+
affectedEventIds: Object.freeze(affectedEventIds),
|
|
1989
|
+
affectedLinkIds: Object.freeze(affectedLinkIds),
|
|
1990
|
+
affectedDeliveries: Object.freeze(affectedDeliveries),
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
function restorePeerState(value) {
|
|
1994
|
+
assertPlainData(value, 'snapshot peer state');
|
|
1995
|
+
const keys = [
|
|
1996
|
+
'admittedPeers',
|
|
1997
|
+
'identity',
|
|
1998
|
+
'lastLogicalTime',
|
|
1999
|
+
'limits',
|
|
2000
|
+
'localEventSequence',
|
|
2001
|
+
'messageIds',
|
|
2002
|
+
'peers',
|
|
2003
|
+
'pendingPings',
|
|
2004
|
+
'pendingPreparations',
|
|
2005
|
+
'replay',
|
|
2006
|
+
'status',
|
|
2007
|
+
];
|
|
2008
|
+
assertExactKeys(value, keys, keys);
|
|
2009
|
+
const recordNames = [
|
|
2010
|
+
'admittedPeers',
|
|
2011
|
+
'messageIds',
|
|
2012
|
+
'peers',
|
|
2013
|
+
'pendingPings',
|
|
2014
|
+
'pendingPreparations',
|
|
2015
|
+
'replay',
|
|
2016
|
+
];
|
|
2017
|
+
for (const name of recordNames)
|
|
2018
|
+
assertDataRecord(value[name], `snapshot peer ${name}`);
|
|
2019
|
+
const copy = deepFreezeCopy(value);
|
|
2020
|
+
return Object.freeze({
|
|
2021
|
+
...copy,
|
|
2022
|
+
admittedPeers: frozenRecord(Object.entries(copy.admittedPeers).map(([key, entry]) => [key, entry])),
|
|
2023
|
+
peers: frozenRecord(Object.entries(copy.peers).map(([key, entry]) => [key, entry])),
|
|
2024
|
+
replay: frozenRecord(Object.entries(copy.replay).map(([key, entry]) => [key, entry])),
|
|
2025
|
+
messageIds: frozenRecord(Object.entries(copy.messageIds).map(([key, entry]) => [key, entry])),
|
|
2026
|
+
pendingPings: frozenRecord(Object.entries(copy.pendingPings).map(([key, entry]) => [key, entry])),
|
|
2027
|
+
pendingPreparations: frozenRecord(Object.entries(copy.pendingPreparations).map(([key, entry]) => [
|
|
2028
|
+
key,
|
|
2029
|
+
entry,
|
|
2030
|
+
])),
|
|
2031
|
+
});
|
|
2032
|
+
}
|
|
362
2033
|
function compareEvents(left, right) {
|
|
363
2034
|
return (left.order.logicalTime - right.order.logicalTime ||
|
|
364
2035
|
left.order.priority - right.order.priority ||
|
|
@@ -367,6 +2038,107 @@ function compareEvents(left, right) {
|
|
|
367
2038
|
function timestampAt(startTime, logicalTime) {
|
|
368
2039
|
return new Date(Date.parse(startTime) + logicalTime).toISOString();
|
|
369
2040
|
}
|
|
2041
|
+
function linkKey(fromPeerId, toPeerId) {
|
|
2042
|
+
return JSON.stringify([fromPeerId, toPeerId]);
|
|
2043
|
+
}
|
|
2044
|
+
function faultTargetPeerId(fault, config) {
|
|
2045
|
+
const targetPeerId = 'peerId' in fault
|
|
2046
|
+
? fault.peerId
|
|
2047
|
+
: 'links' in fault
|
|
2048
|
+
? fault.links[0]?.fromPeerId
|
|
2049
|
+
: config.peers[0]?.peerId;
|
|
2050
|
+
if (targetPeerId === undefined)
|
|
2051
|
+
throw new TypeError('Mesh simulation fault has no target');
|
|
2052
|
+
return targetPeerId;
|
|
2053
|
+
}
|
|
2054
|
+
function maximumIssuedEvents(config) {
|
|
2055
|
+
const queueChurn = config.limits.maximumQueuedEvents * (config.limits.maximumEvents + 1);
|
|
2056
|
+
const effectEvents = config.limits.maximumEvents * config.limits.maximumInternalSteps;
|
|
2057
|
+
return Math.min(Number.MAX_SAFE_INTEGER, queueChurn + effectEvents);
|
|
2058
|
+
}
|
|
2059
|
+
function isDigest(value) {
|
|
2060
|
+
return typeof value === 'string' && /^[0-9a-f]{64}$/u.test(value);
|
|
2061
|
+
}
|
|
2062
|
+
function assertBoundedString(value, name, maximumBytes = 256) {
|
|
2063
|
+
if (typeof value !== 'string' ||
|
|
2064
|
+
value.length < 1 ||
|
|
2065
|
+
new TextEncoder().encode(value).byteLength > maximumBytes)
|
|
2066
|
+
throw new TypeError(`Invalid Mesh simulation ${name}`);
|
|
2067
|
+
}
|
|
2068
|
+
function isDenseArray(value) {
|
|
2069
|
+
if (!Array.isArray(value) ||
|
|
2070
|
+
Object.getPrototypeOf(value) !== Array.prototype ||
|
|
2071
|
+
Object.getOwnPropertySymbols(value).length > 0)
|
|
2072
|
+
return false;
|
|
2073
|
+
const names = Object.getOwnPropertyNames(value);
|
|
2074
|
+
if (names.length !== value.length + 1)
|
|
2075
|
+
return false;
|
|
2076
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
2077
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
|
|
2078
|
+
if (descriptor === undefined ||
|
|
2079
|
+
!('value' in descriptor) ||
|
|
2080
|
+
descriptor.enumerable !== true)
|
|
2081
|
+
return false;
|
|
2082
|
+
}
|
|
2083
|
+
return true;
|
|
2084
|
+
}
|
|
2085
|
+
function assertPlainData(value, name) {
|
|
2086
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
2087
|
+
throw new TypeError(`Mesh simulation ${name} must be a plain record`);
|
|
2088
|
+
const prototype = Object.getPrototypeOf(value);
|
|
2089
|
+
const descriptors = Object.getOwnPropertyDescriptors(value);
|
|
2090
|
+
if ((prototype !== null && prototype !== Object.prototype) ||
|
|
2091
|
+
Object.getOwnPropertySymbols(value).length > 0 ||
|
|
2092
|
+
Object.values(descriptors).some((descriptor) => !descriptor.enumerable || !Object.hasOwn(descriptor, 'value')))
|
|
2093
|
+
throw new TypeError(`Mesh simulation ${name} must contain data only`);
|
|
2094
|
+
}
|
|
2095
|
+
function assertDataRecord(value, name) {
|
|
2096
|
+
assertPlainData(value, name);
|
|
2097
|
+
}
|
|
2098
|
+
function assertExactKeys(value, supportedKeys, requiredKeys) {
|
|
2099
|
+
const supported = new Set(supportedKeys);
|
|
2100
|
+
const keys = Object.keys(value);
|
|
2101
|
+
if (keys.some((key) => !supported.has(key)) ||
|
|
2102
|
+
requiredKeys.some((key) => !Object.hasOwn(value, key)))
|
|
2103
|
+
throw new TypeError('Mesh simulation value contains unsupported fields');
|
|
2104
|
+
}
|
|
2105
|
+
function assertExactRecordKeys(value, expectedKeys, name) {
|
|
2106
|
+
const actual = Object.keys(value).sort();
|
|
2107
|
+
if (actual.length !== expectedKeys.length ||
|
|
2108
|
+
actual.some((key, index) => key !== expectedKeys[index]))
|
|
2109
|
+
throw new TypeError(`Invalid Mesh simulation snapshot ${name}`);
|
|
2110
|
+
}
|
|
2111
|
+
function deepFreezeCopy(value, context = {
|
|
2112
|
+
seen: new WeakSet(),
|
|
2113
|
+
nodes: 0,
|
|
2114
|
+
}, depth = 0) {
|
|
2115
|
+
if (value === null ||
|
|
2116
|
+
typeof value === 'string' ||
|
|
2117
|
+
typeof value === 'boolean' ||
|
|
2118
|
+
(typeof value === 'number' && Number.isFinite(value)))
|
|
2119
|
+
return value;
|
|
2120
|
+
if (typeof value !== 'object')
|
|
2121
|
+
throw new TypeError('Mesh simulation snapshot value must contain data only');
|
|
2122
|
+
if (depth > 128 || context.nodes >= 100_000 || context.seen.has(value))
|
|
2123
|
+
throw new RangeError('Mesh simulation snapshot value exceeds data limits');
|
|
2124
|
+
context.seen.add(value);
|
|
2125
|
+
context.nodes += 1;
|
|
2126
|
+
if (Array.isArray(value)) {
|
|
2127
|
+
if (value.length > 100_000 || !isDenseArray(value))
|
|
2128
|
+
throw new RangeError('Mesh simulation snapshot array exceeds data limits');
|
|
2129
|
+
const copy = value.map((entry) => deepFreezeCopy(entry, context, depth + 1));
|
|
2130
|
+
context.seen.delete(value);
|
|
2131
|
+
return Object.freeze(copy);
|
|
2132
|
+
}
|
|
2133
|
+
assertPlainData(value, 'snapshot value');
|
|
2134
|
+
const record = Object.getPrototypeOf(value) === null
|
|
2135
|
+
? Object.create(null)
|
|
2136
|
+
: {};
|
|
2137
|
+
for (const [key, entry] of Object.entries(value))
|
|
2138
|
+
record[key] = deepFreezeCopy(entry, context, depth + 1);
|
|
2139
|
+
context.seen.delete(value);
|
|
2140
|
+
return Object.freeze(record);
|
|
2141
|
+
}
|
|
370
2142
|
function xorshift32(input) {
|
|
371
2143
|
let value = input || 0x6d2b79f5;
|
|
372
2144
|
value ^= value << 13;
|
|
@@ -424,4 +2196,5 @@ export const THREE_PEER_SCENARIO_IDS = Object.freeze([
|
|
|
424
2196
|
'peer-b',
|
|
425
2197
|
'peer-c',
|
|
426
2198
|
]);
|
|
2199
|
+
export * from './reducer-scenario.js';
|
|
427
2200
|
//# sourceMappingURL=index.js.map
|