@homeostate/tool-simulator 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +154 -0
- package/README.md +14 -3
- package/dist/compare.d.ts +0 -5
- package/dist/compare.d.ts.map +1 -1
- package/dist/compare.js +0 -13
- package/dist/compare.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/lost-keys.d.ts +35 -0
- package/dist/lost-keys.d.ts.map +1 -0
- package/dist/lost-keys.js +139 -0
- package/dist/lost-keys.js.map +1 -0
- package/dist/network.d.ts +35 -5
- package/dist/network.d.ts.map +1 -1
- package/dist/network.js +47 -12
- package/dist/network.js.map +1 -1
- package/dist/randomized.d.ts +10 -4
- package/dist/randomized.d.ts.map +1 -1
- package/dist/randomized.js +27 -6
- package/dist/randomized.js.map +1 -1
- package/package.json +7 -7
- package/src/compare.ts +0 -18
- package/src/index.ts +1 -0
- package/src/lost-keys.ts +164 -0
- package/src/network.ts +83 -26
- package/src/randomized.ts +29 -7
package/src/network.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createSyncEngine,
|
|
3
|
-
defaultSyncFilter,
|
|
4
3
|
type CrdtBackend,
|
|
5
4
|
type PersistableDoc,
|
|
6
5
|
type StoreAdapter,
|
|
@@ -8,13 +7,8 @@ import {
|
|
|
8
7
|
type SyncEngineConfig,
|
|
9
8
|
type Unsubscribe,
|
|
10
9
|
} from "@homeostate/core";
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
firstDifference,
|
|
14
|
-
formatPath,
|
|
15
|
-
formatValue,
|
|
16
|
-
syncedJson,
|
|
17
|
-
} from "./compare.js";
|
|
10
|
+
import { asJson, firstDifference, formatPath, formatValue } from "./compare.js";
|
|
11
|
+
import { createLostKeyCheck } from "./lost-keys.js";
|
|
18
12
|
import { createRandom, randomSeed } from "./random.js";
|
|
19
13
|
|
|
20
14
|
/**
|
|
@@ -28,8 +22,11 @@ export interface PeerSetup<S extends object = any> {
|
|
|
28
22
|
adapter: StoreAdapter<S>;
|
|
29
23
|
backend: CrdtBackend;
|
|
30
24
|
doc: PersistableDoc;
|
|
31
|
-
/**
|
|
32
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Passed to `createSyncEngine`, typed by the adapter's state; its `filter`, `keys` and `omit`
|
|
27
|
+
* also decide what convergence compares.
|
|
28
|
+
*/
|
|
29
|
+
config?: NoInfer<SyncEngineConfig<S>>;
|
|
33
30
|
/** Names the peer in convergence errors. Defaults to `peer-<index>`. */
|
|
34
31
|
name?: string;
|
|
35
32
|
/**
|
|
@@ -74,6 +71,17 @@ export interface NetworkOptions {
|
|
|
74
71
|
* as two peers writing in a loop. Defaults to 100 000.
|
|
75
72
|
*/
|
|
76
73
|
maxDeliveries?: number;
|
|
74
|
+
/**
|
|
75
|
+
* Whether `advance` and `settle` reject with a `LostKeyError` when a peer's store and
|
|
76
|
+
* document both lose a synced top-level key that no peer deleted. Defaults to `true`.
|
|
77
|
+
*
|
|
78
|
+
* A peer holds a key once a checkpoint sees it in both its store's synced state and its
|
|
79
|
+
* document. A key is deleted for good once it leaves a store or document outside a delivery:
|
|
80
|
+
* in a step, or in a test's own change, between `advance` and `settle` calls. Checkpoints
|
|
81
|
+
* come after each delivery and when `advance` or `settle` returns. A key deleted during a
|
|
82
|
+
* delivery, such as by a store subscriber or a timer, is reported as lost: pass `false`.
|
|
83
|
+
*/
|
|
84
|
+
checkLostKeys?: boolean;
|
|
77
85
|
}
|
|
78
86
|
|
|
79
87
|
export interface NetworkStats {
|
|
@@ -93,6 +101,9 @@ export interface NetworkStats {
|
|
|
93
101
|
* document goes to every peer it is linked to, after the latency, unless it is dropped or its
|
|
94
102
|
* link is cut first. Linking two peers, as `connect` and `heal` do, makes them exchange their
|
|
95
103
|
* whole documents, as providers do on reconnecting, so what was lost on the way arrives.
|
|
104
|
+
*
|
|
105
|
+
* Unless `checkLostKeys` is `false`, `advance` and `settle` also check after each delivery
|
|
106
|
+
* that no peer's store and document both lost a synced key that no peer deleted.
|
|
96
107
|
*/
|
|
97
108
|
export interface Network {
|
|
98
109
|
/** The seed of this network's random choices. */
|
|
@@ -122,11 +133,21 @@ export interface Network {
|
|
|
122
133
|
/**
|
|
123
134
|
* Link every pair of peers and have each exchange documents, as if every peer reconnected,
|
|
124
135
|
* so that updates lost to drops and cut links arrive too.
|
|
136
|
+
*
|
|
137
|
+
* With a `dropRate`, the updates stores write in answer to that exchange, such as a key one
|
|
138
|
+
* derives from synced state, can be dropped too. Repeat `heal()` and `settle()` until
|
|
139
|
+
* `stats().dropped` stops growing before checking convergence.
|
|
125
140
|
*/
|
|
126
141
|
heal: () => void;
|
|
127
|
-
/**
|
|
142
|
+
/**
|
|
143
|
+
* Deliver the messages due within the next `ms` of virtual time, and move time on. Rejects
|
|
144
|
+
* with a `LostKeyError` when a peer lost a synced key; see `checkLostKeys`.
|
|
145
|
+
*/
|
|
128
146
|
advance: (ms: number) => Promise<void>;
|
|
129
|
-
/**
|
|
147
|
+
/**
|
|
148
|
+
* Deliver every message on its way, and those they lead to, until none are left. Rejects
|
|
149
|
+
* with a `LostKeyError` when a peer lost a synced key; see `checkLostKeys`.
|
|
150
|
+
*/
|
|
130
151
|
settle: () => Promise<void>;
|
|
131
152
|
stats: () => NetworkStats;
|
|
132
153
|
/**
|
|
@@ -135,7 +156,10 @@ export interface Network {
|
|
|
135
156
|
* document. Empty once the peers have converged.
|
|
136
157
|
*/
|
|
137
158
|
differences: () => string[];
|
|
138
|
-
/**
|
|
159
|
+
/**
|
|
160
|
+
* Throws a `ConvergenceError` unless `differences()` is empty. Its message also says how many
|
|
161
|
+
* messages are still on their way, and how many updates were dropped since the last `heal()`.
|
|
162
|
+
*/
|
|
139
163
|
expectConverged: () => void;
|
|
140
164
|
/** Disconnect every engine, stop listening to every document and drop what is on its way. */
|
|
141
165
|
destroy: () => void;
|
|
@@ -203,8 +227,12 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
203
227
|
dropRate = 0,
|
|
204
228
|
reorder = false,
|
|
205
229
|
maxDeliveries = 100_000,
|
|
230
|
+
checkLostKeys = true,
|
|
206
231
|
} = options;
|
|
207
232
|
const random = createRandom(seed);
|
|
233
|
+
const lostKeys = checkLostKeys ? createLostKeyCheck(seed) : undefined;
|
|
234
|
+
/** Whether `advance` or `settle` is delivering, so a document's change is not the test's. */
|
|
235
|
+
let delivering = false;
|
|
208
236
|
const peers: Peer[] = [];
|
|
209
237
|
const unsubscribers: Unsubscribe[] = [];
|
|
210
238
|
/** Pairs of peers whose link is cut; every other pair is linked. */
|
|
@@ -215,6 +243,8 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
215
243
|
let now = 0;
|
|
216
244
|
let order = 0;
|
|
217
245
|
const stats = { sent: 0, delivered: 0, dropped: 0, cut: 0 };
|
|
246
|
+
/** `stats.dropped` at the last `heal()`, or `undefined` before the first. */
|
|
247
|
+
let droppedAtHeal: number | undefined;
|
|
218
248
|
|
|
219
249
|
const delay = (): number => {
|
|
220
250
|
if (typeof latency === "number") return latency;
|
|
@@ -279,11 +309,12 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
279
309
|
const deliver = (message: Message): void => {
|
|
280
310
|
now = Math.max(now, message.at);
|
|
281
311
|
stats.delivered++;
|
|
312
|
+
lostKeys?.changed(message.to, false);
|
|
282
313
|
message.to.doc.apply(message.update);
|
|
283
314
|
};
|
|
284
315
|
|
|
285
316
|
/** Deliver messages due by `until`, letting each delivery's work finish before the next. */
|
|
286
|
-
const
|
|
317
|
+
const deliverUntil = async (until: number): Promise<void> => {
|
|
287
318
|
let deliveries = 0;
|
|
288
319
|
for (;;) {
|
|
289
320
|
const next = queue[0];
|
|
@@ -295,6 +326,7 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
295
326
|
queue.shift();
|
|
296
327
|
deliver(next);
|
|
297
328
|
await Promise.resolve();
|
|
329
|
+
lostKeys?.afterDelivery(next.to, now, next.from);
|
|
298
330
|
continue;
|
|
299
331
|
}
|
|
300
332
|
// Asynchronous stores may still send; only stop once a full turn sends nothing.
|
|
@@ -304,6 +336,18 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
304
336
|
}
|
|
305
337
|
};
|
|
306
338
|
|
|
339
|
+
/** `deliverUntil`, with the lost key check's checkpoints before and after. */
|
|
340
|
+
const run = async (until: number): Promise<void> => {
|
|
341
|
+
lostKeys?.beforeDeliveries(peers);
|
|
342
|
+
delivering = true;
|
|
343
|
+
try {
|
|
344
|
+
await deliverUntil(until);
|
|
345
|
+
} finally {
|
|
346
|
+
delivering = false;
|
|
347
|
+
}
|
|
348
|
+
for (const peer of peers) lostKeys?.afterDelivery(peer, now);
|
|
349
|
+
};
|
|
350
|
+
|
|
307
351
|
const view = (peer: Peer) => ({
|
|
308
352
|
store: peer.syncedState(),
|
|
309
353
|
document: asJson(peer.backend.read()),
|
|
@@ -321,7 +365,7 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
321
365
|
|
|
322
366
|
const views = peers.map(view);
|
|
323
367
|
peers.forEach((peer, i) => {
|
|
324
|
-
if (peer.engine.
|
|
368
|
+
if (peer.engine.getStatus().connected)
|
|
325
369
|
describe(
|
|
326
370
|
`${peer.name}'s store differs from its document`,
|
|
327
371
|
views[i].store,
|
|
@@ -351,12 +395,11 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
351
395
|
|
|
352
396
|
addPeer: <S extends object>(setup: PeerSetup<S>): Peer<S> => {
|
|
353
397
|
const index = peers.length;
|
|
354
|
-
const engine = createSyncEngine(
|
|
355
|
-
setup.
|
|
356
|
-
setup.
|
|
357
|
-
setup.
|
|
358
|
-
);
|
|
359
|
-
const filter = setup.config?.filter ?? defaultSyncFilter;
|
|
398
|
+
const engine = createSyncEngine({
|
|
399
|
+
...setup.config,
|
|
400
|
+
backend: setup.backend,
|
|
401
|
+
adapter: setup.adapter,
|
|
402
|
+
});
|
|
360
403
|
const peer: Peer<S> = {
|
|
361
404
|
index,
|
|
362
405
|
name: setup.name ?? `peer-${index}`,
|
|
@@ -364,11 +407,15 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
364
407
|
backend: setup.backend,
|
|
365
408
|
doc: setup.doc,
|
|
366
409
|
engine,
|
|
367
|
-
syncedState: () =>
|
|
410
|
+
syncedState: () => asJson(engine.getSyncedState()),
|
|
368
411
|
};
|
|
369
412
|
peers.push(peer);
|
|
413
|
+
lostKeys?.addPeer(peer);
|
|
370
414
|
unsubscribers.push(
|
|
371
|
-
setup.doc.subscribe((update) =>
|
|
415
|
+
setup.doc.subscribe((update) => {
|
|
416
|
+
lostKeys?.changed(peer, !delivering);
|
|
417
|
+
broadcast(peer, update);
|
|
418
|
+
}),
|
|
372
419
|
);
|
|
373
420
|
for (const other of peers) link(peer, other);
|
|
374
421
|
if (setup.connect !== false) engine.connect();
|
|
@@ -398,6 +445,7 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
398
445
|
},
|
|
399
446
|
|
|
400
447
|
heal: () => {
|
|
448
|
+
droppedAtHeal = stats.dropped;
|
|
401
449
|
for (const a of peers)
|
|
402
450
|
for (const b of peers) if (a.index < b.index) link(a, b);
|
|
403
451
|
},
|
|
@@ -418,13 +466,22 @@ export function createNetwork(options: NetworkOptions = {}): Network {
|
|
|
418
466
|
const lines = differences();
|
|
419
467
|
if (lines.length === 0) return;
|
|
420
468
|
const inFlight =
|
|
421
|
-
queue.length
|
|
422
|
-
?
|
|
469
|
+
queue.length === 1
|
|
470
|
+
? "\n1 message is still on its way: settle() delivers it."
|
|
471
|
+
: queue.length > 1
|
|
472
|
+
? `\n${queue.length} messages are still on their way: settle() delivers them.`
|
|
473
|
+
: "";
|
|
474
|
+
const dropped = stats.dropped - (droppedAtHeal ?? 0);
|
|
475
|
+
const droppedHint =
|
|
476
|
+
dropped > 0
|
|
477
|
+
? `\n${dropped === 1 ? "1 update was" : `${dropped} updates were`} dropped${
|
|
478
|
+
droppedAtHeal === undefined ? "" : " since the last heal()"
|
|
479
|
+
}: heal() and settle() until stats().dropped stops growing.`
|
|
423
480
|
: "";
|
|
424
481
|
throw new ConvergenceError(
|
|
425
482
|
`${peers.length} peers did not converge (seed ${seed}):\n${lines
|
|
426
483
|
.map((line) => `- ${line}`)
|
|
427
|
-
.join("\n")}${inFlight}`,
|
|
484
|
+
.join("\n")}${inFlight}${droppedHint}`,
|
|
428
485
|
lines,
|
|
429
486
|
seed,
|
|
430
487
|
);
|
package/src/randomized.ts
CHANGED
|
@@ -14,6 +14,9 @@ export type RandomAction<S extends object = any> = (
|
|
|
14
14
|
random: Random,
|
|
15
15
|
) => void;
|
|
16
16
|
|
|
17
|
+
/** Most rounds of `heal()` and `settle()` at the end of a run. */
|
|
18
|
+
const HEAL_ROUNDS = 10;
|
|
19
|
+
|
|
17
20
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
18
21
|
export interface RandomizedOptions<S extends object = any> {
|
|
19
22
|
/** Makes peer `index`; called once per peer as the run starts. */
|
|
@@ -26,7 +29,7 @@ export interface RandomizedOptions<S extends object = any> {
|
|
|
26
29
|
steps?: number;
|
|
27
30
|
/** What a step may do to a random peer's store, one picked at random. Defaults to `[randomEdit]`. */
|
|
28
31
|
actions?: readonly RandomAction<S>[];
|
|
29
|
-
/** Latency, drops and
|
|
32
|
+
/** Latency, drops, reordering and the lost key check, as for `createNetwork`. */
|
|
30
33
|
network?: Omit<NetworkOptions, "seed">;
|
|
31
34
|
/** Chance that a step cuts or restores a random link instead of changing a store. Defaults to 0.1. */
|
|
32
35
|
linkChangeRate?: number;
|
|
@@ -38,10 +41,16 @@ export interface RandomizedOptions<S extends object = any> {
|
|
|
38
41
|
* Runs random steps against a fresh network, then heals it, settles it and checks that every
|
|
39
42
|
* peer converged. Each step changes a random peer's store with a random action, or cuts or
|
|
40
43
|
* restores a random link, and lets some virtual time pass, so changes cross each other on
|
|
41
|
-
* their way.
|
|
44
|
+
* their way. Throughout, the network checks that no peer's store and document both lose a
|
|
45
|
+
* synced key that no peer deleted, unless `network.checkLostKeys` is `false`.
|
|
46
|
+
*
|
|
47
|
+
* With a `dropRate`, the updates stores write in answer to the healing exchange can be dropped
|
|
48
|
+
* too, so the run heals and settles again until a round drops nothing, at most 10 rounds, as a
|
|
49
|
+
* provider that resyncs would.
|
|
42
50
|
*
|
|
43
|
-
* Rejects with an error that names the seed and step when a step throws
|
|
44
|
-
* converge; pass that `seed` to replay the same run.
|
|
51
|
+
* Rejects with an error that names the seed and step when a step throws, a peer loses a synced
|
|
52
|
+
* key or the peers do not converge; pass that `seed` to replay the same run. When the last of
|
|
53
|
+
* those rounds still dropped updates, the error says so. Give each document a fixed ID for runs
|
|
45
54
|
* that replay exactly, such as `doc.clientID = index + 1` for Yjs.
|
|
46
55
|
*/
|
|
47
56
|
export async function runRandomized<S extends object>(
|
|
@@ -61,6 +70,8 @@ export async function runRandomized<S extends object>(
|
|
|
61
70
|
const random = createRandom(seed ^ 0x5bd1e995);
|
|
62
71
|
const network = createNetwork({ ...networkOptions, seed });
|
|
63
72
|
let step = 0;
|
|
73
|
+
/** Whether every round of healing dropped updates, the last included. */
|
|
74
|
+
let stillDropping = false;
|
|
64
75
|
|
|
65
76
|
try {
|
|
66
77
|
const peers = Array.from({ length: peerCount }, (_, index) =>
|
|
@@ -79,16 +90,27 @@ export async function runRandomized<S extends object>(
|
|
|
79
90
|
await network.advance(random.int(stepInterval + 1));
|
|
80
91
|
}
|
|
81
92
|
|
|
82
|
-
|
|
83
|
-
|
|
93
|
+
// Stores may write in answer to the documents a heal exchanges, and those updates can be
|
|
94
|
+
// dropped too: heal again until a round drops nothing, as a provider that resyncs would.
|
|
95
|
+
let dropping = true;
|
|
96
|
+
for (let round = 0; dropping && round < HEAL_ROUNDS; round++) {
|
|
97
|
+
const dropped = network.stats().dropped;
|
|
98
|
+
network.heal();
|
|
99
|
+
await network.settle();
|
|
100
|
+
dropping = network.stats().dropped !== dropped;
|
|
101
|
+
}
|
|
102
|
+
stillDropping = dropping;
|
|
84
103
|
network.expectConverged();
|
|
85
104
|
} catch (error) {
|
|
86
105
|
const when =
|
|
87
106
|
step < steps ? `at step ${step + 1} of ${steps}` : `after ${steps} steps`;
|
|
88
107
|
const message = error instanceof Error ? error.message : String(error);
|
|
108
|
+
const dropping = stillDropping
|
|
109
|
+
? `\nUpdates were still being dropped after ${HEAL_ROUNDS} rounds of heal() and settle(): does a store write in answer to every exchange?`
|
|
110
|
+
: "";
|
|
89
111
|
throw Object.assign(
|
|
90
112
|
new Error(
|
|
91
|
-
`Randomized run failed ${when}; pass \`seed: ${seed}\` to replay it.\n${message}`,
|
|
113
|
+
`Randomized run failed ${when}; pass \`seed: ${seed}\` to replay it.\n${message}${dropping}`,
|
|
92
114
|
),
|
|
93
115
|
{ cause: error },
|
|
94
116
|
);
|