@homeostate/tool-simulator 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/network.ts CHANGED
@@ -1,6 +1,5 @@
1
1
  import {
2
2
  createSyncEngine,
3
- defaultSyncFilter,
4
3
  type CrdtBackend,
5
4
  type PersistableDoc,
6
5
  type StoreAdapter,
@@ -8,13 +7,8 @@ import {
8
7
  type SyncEngineConfig,
9
8
  type Unsubscribe,
10
9
  } from "@homeostate/core";
11
- import {
12
- asJson,
13
- firstDifference,
14
- formatPath,
15
- formatValue,
16
- syncedJson,
17
- } from "./compare.js";
10
+ import { asJson, firstDifference, formatPath, formatValue } from "./compare.js";
11
+ import { createLostKeyCheck } from "./lost-keys.js";
18
12
  import { createRandom, randomSeed } from "./random.js";
19
13
 
20
14
  /**
@@ -28,8 +22,11 @@ export interface PeerSetup<S extends object = any> {
28
22
  adapter: StoreAdapter<S>;
29
23
  backend: CrdtBackend;
30
24
  doc: PersistableDoc;
31
- /** Passed to `createSyncEngine`; its `filter` also decides what convergence compares. */
32
- config?: SyncEngineConfig;
25
+ /**
26
+ * Passed to `createSyncEngine`, typed by the adapter's state; its `filter`, `keys` and `omit`
27
+ * also decide what convergence compares.
28
+ */
29
+ config?: NoInfer<SyncEngineConfig<S>>;
33
30
  /** Names the peer in convergence errors. Defaults to `peer-<index>`. */
34
31
  name?: string;
35
32
  /**
@@ -74,6 +71,17 @@ export interface NetworkOptions {
74
71
  * as two peers writing in a loop. Defaults to 100 000.
75
72
  */
76
73
  maxDeliveries?: number;
74
+ /**
75
+ * Whether `advance` and `settle` reject with a `LostKeyError` when a peer's store and
76
+ * document both lose a synced top-level key that no peer deleted. Defaults to `true`.
77
+ *
78
+ * A peer holds a key once a checkpoint sees it in both its store's synced state and its
79
+ * document. A key is deleted for good once it leaves a store or document outside a delivery:
80
+ * in a step, or in a test's own change, between `advance` and `settle` calls. Checkpoints
81
+ * come after each delivery and when `advance` or `settle` returns. A key deleted during a
82
+ * delivery, such as by a store subscriber or a timer, is reported as lost: pass `false`.
83
+ */
84
+ checkLostKeys?: boolean;
77
85
  }
78
86
 
79
87
  export interface NetworkStats {
@@ -93,6 +101,9 @@ export interface NetworkStats {
93
101
  * document goes to every peer it is linked to, after the latency, unless it is dropped or its
94
102
  * link is cut first. Linking two peers, as `connect` and `heal` do, makes them exchange their
95
103
  * whole documents, as providers do on reconnecting, so what was lost on the way arrives.
104
+ *
105
+ * Unless `checkLostKeys` is `false`, `advance` and `settle` also check after each delivery
106
+ * that no peer's store and document both lost a synced key that no peer deleted.
96
107
  */
97
108
  export interface Network {
98
109
  /** The seed of this network's random choices. */
@@ -122,11 +133,21 @@ export interface Network {
122
133
  /**
123
134
  * Link every pair of peers and have each exchange documents, as if every peer reconnected,
124
135
  * so that updates lost to drops and cut links arrive too.
136
+ *
137
+ * With a `dropRate`, the updates stores write in answer to that exchange, such as a key one
138
+ * derives from synced state, can be dropped too. Repeat `heal()` and `settle()` until
139
+ * `stats().dropped` stops growing before checking convergence.
125
140
  */
126
141
  heal: () => void;
127
- /** Deliver the messages due within the next `ms` of virtual time, and move time on. */
142
+ /**
143
+ * Deliver the messages due within the next `ms` of virtual time, and move time on. Rejects
144
+ * with a `LostKeyError` when a peer lost a synced key; see `checkLostKeys`.
145
+ */
128
146
  advance: (ms: number) => Promise<void>;
129
- /** Deliver every message on its way, and those they lead to, until none are left. */
147
+ /**
148
+ * Deliver every message on its way, and those they lead to, until none are left. Rejects
149
+ * with a `LostKeyError` when a peer lost a synced key; see `checkLostKeys`.
150
+ */
130
151
  settle: () => Promise<void>;
131
152
  stats: () => NetworkStats;
132
153
  /**
@@ -135,7 +156,10 @@ export interface Network {
135
156
  * document. Empty once the peers have converged.
136
157
  */
137
158
  differences: () => string[];
138
- /** Throws a `ConvergenceError` unless `differences()` is empty. */
159
+ /**
160
+ * Throws a `ConvergenceError` unless `differences()` is empty. Its message also says how many
161
+ * messages are still on their way, and how many updates were dropped since the last `heal()`.
162
+ */
139
163
  expectConverged: () => void;
140
164
  /** Disconnect every engine, stop listening to every document and drop what is on its way. */
141
165
  destroy: () => void;
@@ -203,8 +227,12 @@ export function createNetwork(options: NetworkOptions = {}): Network {
203
227
  dropRate = 0,
204
228
  reorder = false,
205
229
  maxDeliveries = 100_000,
230
+ checkLostKeys = true,
206
231
  } = options;
207
232
  const random = createRandom(seed);
233
+ const lostKeys = checkLostKeys ? createLostKeyCheck(seed) : undefined;
234
+ /** Whether `advance` or `settle` is delivering, so a document's change is not the test's. */
235
+ let delivering = false;
208
236
  const peers: Peer[] = [];
209
237
  const unsubscribers: Unsubscribe[] = [];
210
238
  /** Pairs of peers whose link is cut; every other pair is linked. */
@@ -215,6 +243,8 @@ export function createNetwork(options: NetworkOptions = {}): Network {
215
243
  let now = 0;
216
244
  let order = 0;
217
245
  const stats = { sent: 0, delivered: 0, dropped: 0, cut: 0 };
246
+ /** `stats.dropped` at the last `heal()`, or `undefined` before the first. */
247
+ let droppedAtHeal: number | undefined;
218
248
 
219
249
  const delay = (): number => {
220
250
  if (typeof latency === "number") return latency;
@@ -279,11 +309,12 @@ export function createNetwork(options: NetworkOptions = {}): Network {
279
309
  const deliver = (message: Message): void => {
280
310
  now = Math.max(now, message.at);
281
311
  stats.delivered++;
312
+ lostKeys?.changed(message.to, false);
282
313
  message.to.doc.apply(message.update);
283
314
  };
284
315
 
285
316
  /** Deliver messages due by `until`, letting each delivery's work finish before the next. */
286
- const run = async (until: number): Promise<void> => {
317
+ const deliverUntil = async (until: number): Promise<void> => {
287
318
  let deliveries = 0;
288
319
  for (;;) {
289
320
  const next = queue[0];
@@ -295,6 +326,7 @@ export function createNetwork(options: NetworkOptions = {}): Network {
295
326
  queue.shift();
296
327
  deliver(next);
297
328
  await Promise.resolve();
329
+ lostKeys?.afterDelivery(next.to, now, next.from);
298
330
  continue;
299
331
  }
300
332
  // Asynchronous stores may still send; only stop once a full turn sends nothing.
@@ -304,6 +336,18 @@ export function createNetwork(options: NetworkOptions = {}): Network {
304
336
  }
305
337
  };
306
338
 
339
+ /** `deliverUntil`, with the lost key check's checkpoints before and after. */
340
+ const run = async (until: number): Promise<void> => {
341
+ lostKeys?.beforeDeliveries(peers);
342
+ delivering = true;
343
+ try {
344
+ await deliverUntil(until);
345
+ } finally {
346
+ delivering = false;
347
+ }
348
+ for (const peer of peers) lostKeys?.afterDelivery(peer, now);
349
+ };
350
+
307
351
  const view = (peer: Peer) => ({
308
352
  store: peer.syncedState(),
309
353
  document: asJson(peer.backend.read()),
@@ -321,7 +365,7 @@ export function createNetwork(options: NetworkOptions = {}): Network {
321
365
 
322
366
  const views = peers.map(view);
323
367
  peers.forEach((peer, i) => {
324
- if (peer.engine.isConnected())
368
+ if (peer.engine.getStatus().connected)
325
369
  describe(
326
370
  `${peer.name}'s store differs from its document`,
327
371
  views[i].store,
@@ -351,12 +395,11 @@ export function createNetwork(options: NetworkOptions = {}): Network {
351
395
 
352
396
  addPeer: <S extends object>(setup: PeerSetup<S>): Peer<S> => {
353
397
  const index = peers.length;
354
- const engine = createSyncEngine(
355
- setup.backend,
356
- setup.adapter,
357
- setup.config,
358
- );
359
- const filter = setup.config?.filter ?? defaultSyncFilter;
398
+ const engine = createSyncEngine({
399
+ ...setup.config,
400
+ backend: setup.backend,
401
+ adapter: setup.adapter,
402
+ });
360
403
  const peer: Peer<S> = {
361
404
  index,
362
405
  name: setup.name ?? `peer-${index}`,
@@ -364,11 +407,15 @@ export function createNetwork(options: NetworkOptions = {}): Network {
364
407
  backend: setup.backend,
365
408
  doc: setup.doc,
366
409
  engine,
367
- syncedState: () => syncedJson(setup.adapter.getState(), filter),
410
+ syncedState: () => asJson(engine.getSyncedState()),
368
411
  };
369
412
  peers.push(peer);
413
+ lostKeys?.addPeer(peer);
370
414
  unsubscribers.push(
371
- setup.doc.subscribe((update) => broadcast(peer, update)),
415
+ setup.doc.subscribe((update) => {
416
+ lostKeys?.changed(peer, !delivering);
417
+ broadcast(peer, update);
418
+ }),
372
419
  );
373
420
  for (const other of peers) link(peer, other);
374
421
  if (setup.connect !== false) engine.connect();
@@ -398,6 +445,7 @@ export function createNetwork(options: NetworkOptions = {}): Network {
398
445
  },
399
446
 
400
447
  heal: () => {
448
+ droppedAtHeal = stats.dropped;
401
449
  for (const a of peers)
402
450
  for (const b of peers) if (a.index < b.index) link(a, b);
403
451
  },
@@ -418,13 +466,22 @@ export function createNetwork(options: NetworkOptions = {}): Network {
418
466
  const lines = differences();
419
467
  if (lines.length === 0) return;
420
468
  const inFlight =
421
- queue.length > 0
422
- ? `\n${queue.length} messages are still on their way: settle() delivers them.`
469
+ queue.length === 1
470
+ ? "\n1 message is still on its way: settle() delivers it."
471
+ : queue.length > 1
472
+ ? `\n${queue.length} messages are still on their way: settle() delivers them.`
473
+ : "";
474
+ const dropped = stats.dropped - (droppedAtHeal ?? 0);
475
+ const droppedHint =
476
+ dropped > 0
477
+ ? `\n${dropped === 1 ? "1 update was" : `${dropped} updates were`} dropped${
478
+ droppedAtHeal === undefined ? "" : " since the last heal()"
479
+ }: heal() and settle() until stats().dropped stops growing.`
423
480
  : "";
424
481
  throw new ConvergenceError(
425
482
  `${peers.length} peers did not converge (seed ${seed}):\n${lines
426
483
  .map((line) => `- ${line}`)
427
- .join("\n")}${inFlight}`,
484
+ .join("\n")}${inFlight}${droppedHint}`,
428
485
  lines,
429
486
  seed,
430
487
  );
package/src/randomized.ts CHANGED
@@ -14,6 +14,9 @@ export type RandomAction<S extends object = any> = (
14
14
  random: Random,
15
15
  ) => void;
16
16
 
17
+ /** Most rounds of `heal()` and `settle()` at the end of a run. */
18
+ const HEAL_ROUNDS = 10;
19
+
17
20
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
18
21
  export interface RandomizedOptions<S extends object = any> {
19
22
  /** Makes peer `index`; called once per peer as the run starts. */
@@ -26,7 +29,7 @@ export interface RandomizedOptions<S extends object = any> {
26
29
  steps?: number;
27
30
  /** What a step may do to a random peer's store, one picked at random. Defaults to `[randomEdit]`. */
28
31
  actions?: readonly RandomAction<S>[];
29
- /** Latency, drops and reordering, as for `createNetwork`. */
32
+ /** Latency, drops, reordering and the lost key check, as for `createNetwork`. */
30
33
  network?: Omit<NetworkOptions, "seed">;
31
34
  /** Chance that a step cuts or restores a random link instead of changing a store. Defaults to 0.1. */
32
35
  linkChangeRate?: number;
@@ -38,10 +41,16 @@ export interface RandomizedOptions<S extends object = any> {
38
41
  * Runs random steps against a fresh network, then heals it, settles it and checks that every
39
42
  * peer converged. Each step changes a random peer's store with a random action, or cuts or
40
43
  * restores a random link, and lets some virtual time pass, so changes cross each other on
41
- * their way.
44
+ * their way. Throughout, the network checks that no peer's store and document both lose a
45
+ * synced key that no peer deleted, unless `network.checkLostKeys` is `false`.
46
+ *
47
+ * With a `dropRate`, the updates stores write in answer to the healing exchange can be dropped
48
+ * too, so the run heals and settles again until a round drops nothing, at most 10 rounds, as a
49
+ * provider that resyncs would.
42
50
  *
43
- * Rejects with an error that names the seed and step when a step throws or the peers do not
44
- * converge; pass that `seed` to replay the same run. Give each document a fixed ID for runs
51
+ * Rejects with an error that names the seed and step when a step throws, a peer loses a synced
52
+ * key or the peers do not converge; pass that `seed` to replay the same run. When the last of
53
+ * those rounds still dropped updates, the error says so. Give each document a fixed ID for runs
45
54
  * that replay exactly, such as `doc.clientID = index + 1` for Yjs.
46
55
  */
47
56
  export async function runRandomized<S extends object>(
@@ -61,6 +70,8 @@ export async function runRandomized<S extends object>(
61
70
  const random = createRandom(seed ^ 0x5bd1e995);
62
71
  const network = createNetwork({ ...networkOptions, seed });
63
72
  let step = 0;
73
+ /** Whether every round of healing dropped updates, the last included. */
74
+ let stillDropping = false;
64
75
 
65
76
  try {
66
77
  const peers = Array.from({ length: peerCount }, (_, index) =>
@@ -79,16 +90,27 @@ export async function runRandomized<S extends object>(
79
90
  await network.advance(random.int(stepInterval + 1));
80
91
  }
81
92
 
82
- network.heal();
83
- await network.settle();
93
+ // Stores may write in answer to the documents a heal exchanges, and those updates can be
94
+ // dropped too: heal again until a round drops nothing, as a provider that resyncs would.
95
+ let dropping = true;
96
+ for (let round = 0; dropping && round < HEAL_ROUNDS; round++) {
97
+ const dropped = network.stats().dropped;
98
+ network.heal();
99
+ await network.settle();
100
+ dropping = network.stats().dropped !== dropped;
101
+ }
102
+ stillDropping = dropping;
84
103
  network.expectConverged();
85
104
  } catch (error) {
86
105
  const when =
87
106
  step < steps ? `at step ${step + 1} of ${steps}` : `after ${steps} steps`;
88
107
  const message = error instanceof Error ? error.message : String(error);
108
+ const dropping = stillDropping
109
+ ? `\nUpdates were still being dropped after ${HEAL_ROUNDS} rounds of heal() and settle(): does a store write in answer to every exchange?`
110
+ : "";
89
111
  throw Object.assign(
90
112
  new Error(
91
- `Randomized run failed ${when}; pass \`seed: ${seed}\` to replay it.\n${message}`,
113
+ `Randomized run failed ${when}; pass \`seed: ${seed}\` to replay it.\n${message}${dropping}`,
92
114
  ),
93
115
  { cause: error },
94
116
  );