@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc674ce

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/lib/browser/{chunk-WJJXJTNS.mjs → chunk-FJPXA75J.mjs} +7 -12
  2. package/dist/lib/browser/chunk-FJPXA75J.mjs.map +7 -0
  3. package/dist/lib/browser/{chunk-FR6GJ4S4.mjs → chunk-WIQUYO7H.mjs} +39 -29
  4. package/dist/lib/browser/chunk-WIQUYO7H.mjs.map +7 -0
  5. package/dist/lib/browser/filter/index.mjs +1 -2
  6. package/dist/lib/browser/index.mjs +1933 -620
  7. package/dist/lib/browser/index.mjs.map +4 -4
  8. package/dist/lib/browser/meta.json +1 -1
  9. package/dist/lib/browser/testing/index.mjs +28 -201
  10. package/dist/lib/browser/testing/index.mjs.map +4 -4
  11. package/dist/lib/node-esm/{chunk-JQCCVDLT.mjs → chunk-7PL6UK4B.mjs} +39 -29
  12. package/dist/lib/node-esm/chunk-7PL6UK4B.mjs.map +7 -0
  13. package/dist/lib/node-esm/{chunk-LEQ77KAT.mjs → chunk-S5KGVQFZ.mjs} +7 -12
  14. package/dist/lib/node-esm/chunk-S5KGVQFZ.mjs.map +7 -0
  15. package/dist/lib/node-esm/filter/index.mjs +1 -2
  16. package/dist/lib/node-esm/index.mjs +1933 -620
  17. package/dist/lib/node-esm/index.mjs.map +4 -4
  18. package/dist/lib/node-esm/meta.json +1 -1
  19. package/dist/lib/node-esm/testing/index.mjs +28 -201
  20. package/dist/lib/node-esm/testing/index.mjs.map +4 -4
  21. package/dist/types/src/automerge/automerge-host.d.ts +36 -9
  22. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  23. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  24. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  25. package/dist/types/src/automerge/collection-synchronizer.d.ts +3 -2
  26. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  27. package/dist/types/src/automerge/echo-network-adapter.d.ts +11 -6
  28. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  29. package/dist/types/src/automerge/echo-replicator.d.ts +9 -6
  30. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  31. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  32. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  33. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +4 -4
  34. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  35. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +7 -7
  36. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  37. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  38. package/dist/types/src/db-host/automerge-data-source.d.ts +34 -0
  39. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  40. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  41. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  42. package/dist/types/src/db-host/data-service.d.ts +4 -3
  43. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  44. package/dist/types/src/db-host/documents-iterator.d.ts.map +1 -1
  45. package/dist/types/src/db-host/documents-synchronizer.d.ts +6 -5
  46. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  47. package/dist/types/src/db-host/echo-host.d.ts +43 -11
  48. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  49. package/dist/types/src/db-host/index.d.ts +1 -0
  50. package/dist/types/src/db-host/index.d.ts.map +1 -1
  51. package/dist/types/src/db-host/local-queue-service.d.ts +15 -0
  52. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  53. package/dist/types/src/db-host/query-service.d.ts +7 -2
  54. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  55. package/dist/types/src/db-host/queue-data-source.d.ts +28 -0
  56. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  57. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  58. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  59. package/dist/types/src/db-host/space-state-manager.d.ts +1 -0
  60. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  61. package/dist/types/src/db-host/stub.d.ts +10 -0
  62. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  63. package/dist/types/src/edge/echo-edge-replicator.d.ts +2 -2
  64. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  65. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  66. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  67. package/dist/types/src/pipeline/pipeline.d.ts +2 -2
  68. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  69. package/dist/types/src/query/errors.d.ts +11 -15
  70. package/dist/types/src/query/errors.d.ts.map +1 -1
  71. package/dist/types/src/query/plan.d.ts +37 -3
  72. package/dist/types/src/query/plan.d.ts.map +1 -1
  73. package/dist/types/src/query/query-executor.d.ts +39 -3
  74. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  75. package/dist/types/src/query/query-planner.d.ts +6 -0
  76. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  77. package/dist/types/src/space/auth.d.ts +3 -3
  78. package/dist/types/src/space/auth.d.ts.map +1 -1
  79. package/dist/types/src/space/control-pipeline.d.ts +2 -2
  80. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  81. package/dist/types/src/space/space-manager.d.ts +6 -6
  82. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  83. package/dist/types/src/space/space-protocol.d.ts +5 -5
  84. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  85. package/dist/types/src/space/space.d.ts +3 -3
  86. package/dist/types/src/space/space.d.ts.map +1 -1
  87. package/dist/types/src/testing/index.d.ts +0 -2
  88. package/dist/types/src/testing/index.d.ts.map +1 -1
  89. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  90. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  91. package/dist/types/src/testing/test-replicator.d.ts +6 -6
  92. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  93. package/dist/types/src/util.d.ts.map +1 -1
  94. package/dist/types/tsconfig.tsbuildinfo +1 -1
  95. package/package.json +48 -38
  96. package/src/automerge/automerge-host.test.ts +37 -14
  97. package/src/automerge/automerge-host.ts +130 -36
  98. package/src/automerge/automerge-repo.test.ts +147 -30
  99. package/src/automerge/collection-synchronizer.test.ts +16 -10
  100. package/src/automerge/collection-synchronizer.ts +41 -11
  101. package/src/automerge/echo-network-adapter.ts +23 -11
  102. package/src/automerge/echo-replicator.ts +10 -6
  103. package/src/automerge/heads-store.ts +14 -2
  104. package/src/automerge/leveldb-storage-adapter.ts +4 -4
  105. package/src/automerge/mesh-echo-replicator-connection.ts +7 -7
  106. package/src/automerge/mesh-echo-replicator.ts +2 -2
  107. package/src/db-host/automerge-data-source.test.ts +294 -0
  108. package/src/db-host/automerge-data-source.ts +144 -0
  109. package/src/db-host/data-service.ts +10 -3
  110. package/src/db-host/documents-iterator.ts +3 -1
  111. package/src/db-host/documents-synchronizer.test.ts +77 -13
  112. package/src/db-host/documents-synchronizer.ts +51 -50
  113. package/src/db-host/echo-host.ts +191 -28
  114. package/src/db-host/index.ts +1 -0
  115. package/src/db-host/local-queue-service.ts +127 -0
  116. package/src/db-host/query-service.ts +17 -4
  117. package/src/db-host/queue-data-source.ts +132 -0
  118. package/src/db-host/queue-service.test.ts +141 -0
  119. package/src/db-host/space-state-manager.ts +4 -0
  120. package/src/db-host/stub.ts +28 -0
  121. package/src/edge/echo-edge-replicator.ts +12 -8
  122. package/src/filter/filter-match.test.ts +31 -25
  123. package/src/filter/filter-match.ts +8 -14
  124. package/src/metadata/metadata-store.ts +5 -2
  125. package/src/pipeline/pipeline.ts +2 -2
  126. package/src/query/errors.ts +2 -2
  127. package/src/query/plan.ts +48 -3
  128. package/src/query/query-executor.ts +612 -131
  129. package/src/query/query-planner.test.ts +509 -35
  130. package/src/query/query-planner.ts +187 -13
  131. package/src/space/admission-discovery-extension.ts +1 -1
  132. package/src/space/auth.ts +6 -6
  133. package/src/space/control-pipeline.ts +2 -2
  134. package/src/space/space-manager.ts +6 -6
  135. package/src/space/space-protocol.ts +11 -11
  136. package/src/space/space.ts +3 -3
  137. package/src/testing/index.ts +0 -2
  138. package/src/testing/test-network-adapter.ts +16 -3
  139. package/src/testing/test-replicator.ts +8 -7
  140. package/src/util.ts +7 -3
  141. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  142. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  143. package/dist/lib/browser/chunk-FR6GJ4S4.mjs.map +0 -7
  144. package/dist/lib/browser/chunk-WJJXJTNS.mjs.map +0 -7
  145. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  146. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  147. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs.map +0 -7
  148. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs.map +0 -7
  149. package/dist/types/src/testing/test-data.d.ts +0 -18
  150. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  151. package/dist/types/src/testing/test-schema.d.ts +0 -37
  152. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  153. package/src/testing/test-data.ts +0 -127
  154. package/src/testing/test-schema.ts +0 -53
@@ -19,6 +19,7 @@ import {
19
19
  type DocHandle,
20
20
  type DocumentId,
21
21
  type HandleState,
22
+ type Message,
22
23
  type PeerId,
23
24
  Repo,
24
25
  type SharePolicy,
@@ -32,6 +33,7 @@ import { asyncTimeout, sleep } from '@dxos/async';
32
33
  import { randomBytes } from '@dxos/crypto';
33
34
  import { PublicKey } from '@dxos/keys';
34
35
  import { createTestLevel } from '@dxos/kv-store/testing';
36
+ import { log } from '@dxos/log';
35
37
  import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/teleport/testing';
36
38
  import { openAndClose } from '@dxos/test-utils';
37
39
  import { isNonNullable, range } from '@dxos/util';
@@ -162,32 +164,6 @@ describe('AutomergeRepo', () => {
162
164
  expect((await asyncTimeout(client.find<any>(handle.url), 1000))!.doc().text).to.equal(text);
163
165
  });
164
166
 
165
- test('share policy gets enabled afterwards', async () => {
166
- let sharePolicy = false;
167
-
168
- const { repos, adapters } = await createHostClientRepoTopology({
169
- sharePolicy: async () => sharePolicy,
170
- });
171
- const [host, client] = repos;
172
- await connectAdapters(adapters);
173
-
174
- const handle = host.create();
175
- const text = 'Hello world';
176
- handle.change((doc: any) => {
177
- doc.text = text;
178
- });
179
-
180
- {
181
- await client.find(handle.url, { allowableStates: ['unavailable'] });
182
- }
183
-
184
- sharePolicy = true;
185
-
186
- {
187
- await client.find(handle.url, { allowableStates: ['unavailable'] });
188
- }
189
- });
190
-
191
167
  test('two documents and share policy switching', async () => {
192
168
  const allowedDocs: DocumentId[] = [];
193
169
 
@@ -510,6 +486,139 @@ describe('AutomergeRepo', () => {
510
486
  await asyncTimeout(handleB.whenReady(), 1000);
511
487
  expect(handleB.doc()!.text).to.equal(text);
512
488
  });
489
+
490
+ test('retry share config', async () => {
491
+ let announce = false;
492
+
493
+ const { repos, adapters } = await createRepoTopology({
494
+ peers: ['A', 'B'],
495
+ connections: [['A', 'B']],
496
+ options: {
497
+ shareConfig: {
498
+ access: async () => true,
499
+ announce: async () => announce,
500
+ },
501
+ },
502
+ onMessage: (message) => {
503
+ console.log(`${message.senderId} -> ${message.targetId}: ${message.type} ${message.documentId ?? ''}`);
504
+ },
505
+ });
506
+ const [repoA, repoB] = repos;
507
+ await connectAdapters(adapters);
508
+ console.log({ peersA: repoA.peers, peersB: repoB.peers });
509
+
510
+ const docA = repoA.create({ text: 'Hello world' });
511
+
512
+ const docB = await repoB.find(docA.url, { allowableStates: ['ready', 'unavailable'] });
513
+ expect(docB.state).to.equal('unavailable');
514
+ log.info('unavailable');
515
+
516
+ announce = true;
517
+ repoB.shareConfigChanged();
518
+ await docB.whenReady();
519
+ });
520
+ });
521
+
522
+ describe('create2 tests', () => {
523
+ test('basic document creation', async () => {
524
+ const repo = new Repo({ network: [] });
525
+
526
+ const handle = await repo.create2<{ text: string }>();
527
+ expect(handle.doc()).to.deep.equal({});
528
+
529
+ handle.change((doc: any) => {
530
+ doc.text = 'Hello World';
531
+ });
532
+
533
+ expect(handle.doc()?.text).to.equal('Hello World');
534
+ });
535
+
536
+ test('document mutation', async () => {
537
+ const repo = new Repo({ network: [] });
538
+
539
+ const handle = await repo.create2<{ counter: number }>({ counter: 0 });
540
+
541
+ for (let i = 1; i <= 10; i++) {
542
+ handle.change((doc: any) => {
543
+ doc.counter = i;
544
+ });
545
+ }
546
+
547
+ expect(handle.doc()?.counter).to.equal(10);
548
+ });
549
+
550
+ test('import document and mutate', async () => {
551
+ const repo = new Repo({ network: [] });
552
+
553
+ const original = await repo.create2<{ text: string }>({ text: 'original' });
554
+ const blob = A.save(original.doc()!);
555
+
556
+ const imported = repo.import<{ text: string }>(blob);
557
+ expect(imported.doc()?.text).to.equal('original');
558
+
559
+ imported.change((doc: any) => {
560
+ doc.text = 'mutated';
561
+ });
562
+ expect(imported.doc()?.text).to.equal('mutated');
563
+ });
564
+
565
+ test('reload document with flush', async () => {
566
+ const path = createTmpPath();
567
+ const text = 'Hello World!';
568
+ let url: AutomergeUrl;
569
+
570
+ {
571
+ const level = createTestLevel(path);
572
+ const storage = await createLevelAdapter(level);
573
+ const repo = new Repo({ network: [], storage });
574
+ const handle = await repo.create2<{ text: string }>();
575
+ url = handle.url;
576
+ handle.change((doc: any) => {
577
+ doc.text = text;
578
+ });
579
+ await repo.flush([handle.documentId]);
580
+ await level.close();
581
+ }
582
+
583
+ {
584
+ const level = createTestLevel(path);
585
+ const storage = await createLevelAdapter(level);
586
+ const repo = new Repo({ network: [], storage });
587
+ const handle = await repo.find<{ text: string }>(url);
588
+ await handle.whenReady();
589
+ expect(handle.doc()?.text).to.equal(text);
590
+ await level.close();
591
+ }
592
+ });
593
+
594
+ test('reload document without flush', async () => {
595
+ const path = createTmpPath();
596
+ const text = 'Hello World!';
597
+ let url: AutomergeUrl;
598
+
599
+ {
600
+ const level = createTestLevel(path);
601
+ const storage = await createLevelAdapter(level);
602
+ const repo = new Repo({ network: [], storage });
603
+ const handle = await repo.create2<{ text: string }>();
604
+ url = handle.url;
605
+ handle.change((doc: any) => {
606
+ doc.text = text;
607
+ });
608
+ // No explicit flush - rely on auto-save.
609
+ await sleep(200);
610
+ await level.close();
611
+ }
612
+
613
+ {
614
+ const level = createTestLevel(path);
615
+ const storage = await createLevelAdapter(level);
616
+ const repo = new Repo({ network: [], storage });
617
+ const handle = await repo.find<{ text: string }>(url);
618
+ await handle.whenReady();
619
+ expect(handle.doc()?.text).to.equal(text);
620
+ }
621
+ });
513
622
  });
514
623
 
515
624
  describe('teleport', () => {
@@ -730,8 +839,7 @@ describe('AutomergeRepo', () => {
730
839
  });
731
840
  });
732
841
 
733
- const createLevelAdapter = async () => {
734
- const level = createTestLevel();
842
+ const createLevelAdapter = async (level = createTestLevel()) => {
735
843
  const storage = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
736
844
  await openAndClose(level, storage);
737
845
  return storage;
@@ -753,19 +861,23 @@ describe('AutomergeRepo', () => {
753
861
  };
754
862
  });
755
863
 
864
+ type ShareConfig = Exclude<ConstructorParameters<typeof Repo>[0], undefined>['shareConfig'];
865
+
756
866
  type ConnectedRepoOptions = {
757
867
  storages?: StorageAdapterInterface[];
758
868
  connectionStateProvider?: TestConnectionStateProvider;
759
869
  sharePolicy?: SharePolicy;
870
+ shareConfig?: ShareConfig;
760
871
  };
761
872
 
762
873
  const createRepoTopology = async <Peers extends string[], Peer extends string = Peers[number]>(args: {
763
874
  peers: Peers;
764
875
  connections: [Peer, Peer][];
765
876
  options?: ConnectedRepoOptions;
877
+ onMessage?: (message: Message) => void;
766
878
  }) => {
767
879
  const adapters = args.connections.map(
768
- () => TestAdapter.createPair(args.options?.connectionStateProvider) as [TestAdapter, TestAdapter],
880
+ () => TestAdapter.createPair(args.options?.connectionStateProvider, args.onMessage) as [TestAdapter, TestAdapter],
769
881
  );
770
882
  const repos = args.peers.map((peerId, peerIndex) => {
771
883
  const network = adapters
@@ -781,7 +893,8 @@ const createRepoTopology = async <Peers extends string[], Peer extends string =
781
893
  peerId: peerId as PeerId,
782
894
  storage: args.options?.storages?.[peerIndex],
783
895
  network,
784
- sharePolicy: args.options?.sharePolicy ?? (async () => true),
896
+ sharePolicy: (args.options?.sharePolicy ?? args.options?.shareConfig) ? undefined : async () => true,
897
+ shareConfig: args.options?.shareConfig,
785
898
  });
786
899
  });
787
900
  return { repos, adapters };
@@ -867,3 +980,7 @@ const connectPeers = async (
867
980
  };
868
981
 
869
982
  type TeleportTestPeer = { repo: Repo; meshAdapter: MeshEchoReplicator; teleport: TeleportPeer };
983
+
984
+ export const createTmpPath = (): string => {
985
+ return `/tmp/dxos-${PublicKey.random().toHex()}`;
986
+ };
@@ -3,10 +3,12 @@
3
3
  //
4
4
 
5
5
  import { type Heads } from '@automerge/automerge';
6
+ import * as Automerge from '@automerge/automerge';
6
7
  import type { DocumentId, PeerId } from '@automerge/automerge-repo';
7
8
  import { describe, expect, onTestFinished, test } from 'vitest';
8
9
 
9
10
  import { sleep } from '@dxos/async';
11
+ import { range } from '@dxos/util';
10
12
 
11
13
  import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
12
14
 
@@ -54,16 +56,18 @@ describe('CollectionSynchronizer', () => {
54
56
  peer1.onConnectionOpen(peerId2);
55
57
  peer2.onConnectionOpen(peerId1);
56
58
 
59
+ const updated = Promise.all([
60
+ peer1.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId2),
61
+ peer2.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId1),
62
+ ]);
63
+
57
64
  peer1.setLocalCollectionState(collectionId, STATE_1);
58
65
  peer2.setLocalCollectionState(collectionId, STATE_2);
59
66
 
60
67
  peer1.refreshCollection(collectionId);
61
68
  peer2.refreshCollection(collectionId);
62
69
 
63
- await Promise.all([
64
- peer1.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId2),
65
- peer2.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId1),
66
- ]);
70
+ await updated;
67
71
 
68
72
  expect(peer1.getRemoteCollectionStates(collectionId).get(peerId2)).to.deep.equal(STATE_2);
69
73
  expect(peer2.getRemoteCollectionStates(collectionId).get(peerId1)).to.deep.equal(STATE_1);
@@ -80,18 +84,20 @@ describe('CollectionSynchronizer', () => {
80
84
  });
81
85
  });
82
86
 
87
+ const TEST_HEADS = range(4).map((i) => Automerge.getHeads(Automerge.from({ i: i.toString() })));
88
+
83
89
  const STATE_1: CollectionState = {
84
90
  documents: {
85
- a: ['1'],
86
- b: ['2'],
87
- c: ['3'],
91
+ a: TEST_HEADS[0],
92
+ b: TEST_HEADS[1],
93
+ c: TEST_HEADS[2],
88
94
  } as Record<DocumentId, Heads>,
89
95
  };
90
96
 
91
97
  const STATE_2: CollectionState = {
92
98
  documents: {
93
- a: ['1'],
94
- b: ['4'],
95
- d: ['3'],
99
+ a: TEST_HEADS[0],
100
+ b: TEST_HEADS[3],
101
+ d: TEST_HEADS[2],
96
102
  } as Record<DocumentId, Heads>,
97
103
  };
@@ -15,9 +15,9 @@ import { defaultMap } from '@dxos/util';
15
15
 
16
16
  const MIN_QUERY_INTERVAL = 5_000;
17
17
 
18
- const POLL_INTERVAL = 30_000;
18
+ const POLL_INTERVAL = 10_000;
19
19
 
20
- export type CollectionSynchronizerParams = {
20
+ export type CollectionSynchronizerProps = {
21
21
  sendCollectionState: (collectionId: string, peerId: PeerId, state: CollectionState) => void;
22
22
  queryCollectionState: (collectionId: string, peerId: PeerId) => void;
23
23
  shouldSyncCollection: (collectionId: string, peerId: PeerId) => boolean;
@@ -28,9 +28,9 @@ export type CollectionSynchronizerParams = {
28
28
  */
29
29
  @trace.resource()
30
30
  export class CollectionSynchronizer extends Resource {
31
- private readonly _sendCollectionState: CollectionSynchronizerParams['sendCollectionState'];
32
- private readonly _queryCollectionState: CollectionSynchronizerParams['queryCollectionState'];
33
- private readonly _shouldSyncCollection: CollectionSynchronizerParams['shouldSyncCollection'];
31
+ private readonly _sendCollectionState: CollectionSynchronizerProps['sendCollectionState'];
32
+ private readonly _queryCollectionState: CollectionSynchronizerProps['queryCollectionState'];
33
+ private readonly _shouldSyncCollection: CollectionSynchronizerProps['shouldSyncCollection'];
34
34
 
35
35
  /**
36
36
  * CollectionId -> State.
@@ -42,7 +42,7 @@ export class CollectionSynchronizer extends Resource {
42
42
 
43
43
  public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId; newDocsAppeared: boolean }>();
44
44
 
45
- constructor(params: CollectionSynchronizerParams) {
45
+ constructor(params: CollectionSynchronizerProps) {
46
46
  super();
47
47
  this._sendCollectionState = params.sendCollectionState;
48
48
  this._queryCollectionState = params.queryCollectionState;
@@ -78,6 +78,10 @@ export class CollectionSynchronizer extends Resource {
78
78
  log('setLocalCollectionState', { collectionId, state });
79
79
  this._getOrCreatePerCollectionState(collectionId).localState = state;
80
80
 
81
+ for (const peerId of this._connectedPeers) {
82
+ this._diffCollectionState(collectionId, peerId);
83
+ }
84
+
81
85
  queueMicrotask(async () => {
82
86
  if (!this._ctx.disposed && this._activeCollections.has(collectionId)) {
83
87
  this._refreshInterestedPeers(collectionId);
@@ -119,6 +123,7 @@ export class CollectionSynchronizer extends Resource {
119
123
  * Callback when a connection to a peer is established.
120
124
  */
121
125
  onConnectionOpen(peerId: PeerId): void {
126
+ log('onConnectionOpen', { peerId });
122
127
  const spanId = getSpanName(peerId);
123
128
  trace.spanStart({
124
129
  id: spanId,
@@ -148,6 +153,8 @@ export class CollectionSynchronizer extends Resource {
148
153
  * Callback when a connection to a peer is closed.
149
154
  */
150
155
  onConnectionClosed(peerId: PeerId): void {
156
+ log('onConnectionClosed', { peerId });
157
+
151
158
  this._connectedPeers.delete(peerId);
152
159
 
153
160
  for (const perCollectionState of this._perCollectionStates.values()) {
@@ -173,8 +180,20 @@ export class CollectionSynchronizer extends Resource {
173
180
  log('onRemoteStateReceived', { collectionId, peerId, state });
174
181
  validateCollectionState(state);
175
182
  const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
176
- const existingState = perCollectionState.remoteStates.get(peerId) ?? { documents: {} };
177
- const diff = diffCollectionState(existingState, state);
183
+ perCollectionState.remoteStates.set(peerId, state);
184
+ this._diffCollectionState(collectionId, peerId);
185
+ }
186
+
187
+ private _diffCollectionState(collectionId: string, peerId: PeerId) {
188
+ const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
189
+ const remoteState = perCollectionState.remoteStates.get(peerId);
190
+ if (!remoteState) {
191
+ return;
192
+ }
193
+
194
+ log('diffCollectionState', { collectionId, peerId });
195
+ const localState = perCollectionState.localState ?? { documents: {} };
196
+ const diff = diffCollectionState(localState, remoteState);
178
197
  const spanId = getSpanName(peerId);
179
198
  if (diff.different.length === 0) {
180
199
  trace.spanEnd(spanId);
@@ -188,9 +207,20 @@ export class CollectionSynchronizer extends Resource {
188
207
  attributes: { peerId },
189
208
  });
190
209
  }
191
- if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
192
- perCollectionState.remoteStates.set(peerId, state);
193
- this.remoteStateUpdated.emit({ peerId, collectionId, newDocsAppeared: diff.missingOnLocal.length > 0 });
210
+ log('diff', {
211
+ localState: localState.documents,
212
+ remoteState: remoteState.documents,
213
+ missingOnLocal: diff.missingOnLocal,
214
+ missingOnRemote: diff.missingOnRemote,
215
+ different: diff.different,
216
+ });
217
+ if (diff.missingOnLocal.length > 0 || diff.different.length > 0 || diff.missingOnRemote.length > 0) {
218
+ log('emit remote state update');
219
+ this.remoteStateUpdated.emit({
220
+ peerId,
221
+ collectionId,
222
+ newDocsAppeared: diff.missingOnLocal.length > 0,
223
+ });
194
224
  }
195
225
  }
196
226
 
@@ -11,7 +11,7 @@ import {
11
11
  type PeerMetadata,
12
12
  } from '@automerge/automerge-repo';
13
13
 
14
- import { Trigger, synchronized } from '@dxos/async';
14
+ import { Event, Trigger, synchronized } from '@dxos/async';
15
15
  import { LifecycleState } from '@dxos/context';
16
16
  import { invariant } from '@dxos/invariant';
17
17
  import { type PublicKey } from '@dxos/keys';
@@ -23,10 +23,10 @@ import { createIdFromSpaceKey } from '../common/space-id';
23
23
 
24
24
  import {
25
25
  type EchoReplicator,
26
- type RemoteDocumentExistenceCheckParams,
26
+ type RemoteDocumentExistenceCheckProps,
27
27
  type ReplicatorConnection,
28
- type ShouldAdvertiseParams,
29
- type ShouldSyncCollectionParams,
28
+ type ShouldAdvertiseProps,
29
+ type ShouldSyncCollectionProps,
30
30
  } from './echo-replicator';
31
31
  import {
32
32
  type CollectionQueryMessage,
@@ -43,9 +43,9 @@ export interface NetworkDataMonitor {
43
43
  recordMessageSendingFailed(message: Message): void;
44
44
  }
45
45
 
46
- export type EchoNetworkAdapterParams = {
46
+ export type EchoNetworkAdapterProps = {
47
47
  getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
48
- isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckParams) => Promise<boolean>;
48
+ isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckProps) => Promise<boolean>;
49
49
  onCollectionStateQueried: (collectionId: string, peerId: PeerId) => void;
50
50
  onCollectionStateReceived: (collectionId: string, peerId: PeerId, state: unknown) => void;
51
51
  monitor?: NetworkDataMonitor;
@@ -56,6 +56,7 @@ type ConnectionEntry = {
56
56
  connection: ReplicatorConnection;
57
57
  reader: ReadableStreamDefaultReader<AutomergeProtocolMessage>;
58
58
  writer: WritableStreamDefaultWriter<AutomergeProtocolMessage>;
59
+ requestedDocuments: Set<DocumentId>;
59
60
  };
60
61
 
61
62
  /**
@@ -71,7 +72,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
71
72
  private readonly _connected = new Trigger();
72
73
  private readonly _ready = new Trigger();
73
74
 
74
- constructor(private readonly _params: EchoNetworkAdapterParams) {
75
+ public readonly documentRequested = new Event<{ documentId: DocumentId; peerId: PeerId }>();
76
+
77
+ constructor(private readonly _params: EchoNetworkAdapterProps) {
75
78
  super();
76
79
  }
77
80
 
@@ -161,7 +164,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
161
164
  this._replicators.delete(replicator);
162
165
  }
163
166
 
164
- async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseParams): Promise<boolean> {
167
+ async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseProps): Promise<boolean> {
165
168
  const connection = this._connections.get(peerId);
166
169
  if (!connection) {
167
170
  return false;
@@ -170,7 +173,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
170
173
  return connection.connection.shouldAdvertise(params);
171
174
  }
172
175
 
173
- shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionParams): boolean {
176
+ shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionProps): boolean {
174
177
  const connection = this._connections.get(peerId);
175
178
  if (!connection) {
176
179
  return false;
@@ -265,6 +268,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
265
268
  connection,
266
269
  reader: connection.readable.getReader(),
267
270
  writer: connection.writable.getWriter(),
271
+ requestedDocuments: new Set(),
268
272
  };
269
273
 
270
274
  this._connections.set(connection.peerId as PeerId, connectionEntry);
@@ -279,7 +283,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
279
283
  break;
280
284
  }
281
285
 
282
- this._onMessage(value as Message);
286
+ this._onMessage(connectionEntry, value as Message);
283
287
  }
284
288
  } catch (err) {
285
289
  if (connectionEntry.isOpen) {
@@ -293,7 +297,15 @@ export class EchoNetworkAdapter extends NetworkAdapter {
293
297
  this._params.monitor?.recordPeerConnected(connection.peerId);
294
298
  }
295
299
 
296
- private _onMessage(message: Message): void {
300
+ private _onMessage(connectionEntry: ConnectionEntry, message: Message): void {
301
+ const amMessage = message as AutomergeProtocolMessage;
302
+ if (amMessage.type === 'request') {
303
+ this.documentRequested.emit({
304
+ documentId: amMessage.documentId as DocumentId,
305
+ peerId: connectionEntry.connection.peerId as PeerId,
306
+ });
307
+ }
308
+
297
309
  if (isCollectionQueryMessage(message)) {
298
310
  this._params.onCollectionStateQueried(message.collectionId, message.senderId);
299
311
  } else if (isCollectionStateMessage(message)) {
@@ -32,7 +32,11 @@ export interface EchoReplicatorContext {
32
32
  */
33
33
  getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
34
34
  getContainingSpaceIdForDocument(documentId: string): Promise<SpaceId | null>;
35
- isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckParams): Promise<boolean>;
35
+
36
+ /**
37
+ * Returns false if collection sync hasn't happened yet.
38
+ */
39
+ isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckProps): Promise<boolean>;
36
40
 
37
41
  onConnectionOpen(connection: ReplicatorConnection): void;
38
42
  onConnectionClosed(connection: ReplicatorConnection): void;
@@ -59,12 +63,12 @@ export interface ReplicatorConnection {
59
63
  * @returns true if the document should be advertised to this peer.
60
64
  * The remote peer can still request the document by its id bypassing this check.
61
65
  */
62
- shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean>;
66
+ shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean>;
63
67
 
64
68
  /**
65
69
  * @returns true if the collection should be synced to this peer.
66
70
  */
67
- shouldSyncCollection(params: ShouldSyncCollectionParams): boolean;
71
+ shouldSyncCollection(params: ShouldSyncCollectionProps): boolean;
68
72
 
69
73
  /**
70
74
  * Batch syncing considered enabled if ReplicatorConnection implements `pushBatch` and `pullBatch` methods.
@@ -84,15 +88,15 @@ export interface ReplicatorConnection {
84
88
  pullBundle?(docHeads: Record<DocumentId, Heads>): Promise<Record<DocumentId, Uint8Array>>;
85
89
  }
86
90
 
87
- export type ShouldAdvertiseParams = {
91
+ export type ShouldAdvertiseProps = {
88
92
  documentId: string;
89
93
  };
90
94
 
91
- export type ShouldSyncCollectionParams = {
95
+ export type ShouldSyncCollectionProps = {
92
96
  collectionId: string;
93
97
  };
94
98
 
95
- export type RemoteDocumentExistenceCheckParams = {
99
+ export type RemoteDocumentExistenceCheckProps = {
96
100
  peerId: string;
97
101
  documentId: string;
98
102
  };
@@ -8,14 +8,14 @@ import type { DocumentId } from '@automerge/automerge-repo';
8
8
  import { headsEncoding } from '@dxos/indexing';
9
9
  import type { BatchLevel, SublevelDB } from '@dxos/kv-store';
10
10
 
11
- export type HeadsStoreParams = {
11
+ export type HeadsStoreProps = {
12
12
  db: SublevelDB;
13
13
  };
14
14
 
15
15
  export class HeadsStore {
16
16
  private readonly _db: SublevelDB;
17
17
 
18
- constructor({ db }: HeadsStoreParams) {
18
+ constructor({ db }: HeadsStoreProps) {
19
19
  this._db = db;
20
20
  }
21
21
 
@@ -34,4 +34,16 @@ export class HeadsStore {
34
34
  valueEncoding: headsEncoding,
35
35
  });
36
36
  }
37
+
38
+ /**
39
+ * Iterate over all document IDs and their heads.
40
+ */
41
+ async *iterateAll(): AsyncGenerator<{ documentId: DocumentId; heads: Heads }> {
42
+ for await (const [documentId, heads] of this._db.iterator<DocumentId, Heads>({
43
+ keyEncoding: 'utf8',
44
+ valueEncoding: headsEncoding,
45
+ })) {
46
+ yield { documentId, heads };
47
+ }
48
+ }
37
49
  }
@@ -16,21 +16,21 @@ export interface StorageAdapterDataMonitor {
16
16
  recordStoreDuration(durationMs: number): void;
17
17
  }
18
18
 
19
- export type LevelDBStorageAdapterParams = {
19
+ export type LevelDBStorageAdapterProps = {
20
20
  db: SublevelDB;
21
21
  callbacks?: StorageCallbacks;
22
22
  monitor?: StorageAdapterDataMonitor;
23
23
  };
24
24
 
25
- export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
25
+ export type BeforeSaveProps = { path: StorageKey; batch: BatchLevel };
26
26
 
27
27
  export interface StorageCallbacks {
28
- beforeSave(params: BeforeSaveParams): MaybePromise<void>;
28
+ beforeSave(params: BeforeSaveProps): MaybePromise<void>;
29
29
  afterSave(path: StorageKey): MaybePromise<void>;
30
30
  }
31
31
 
32
32
  export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
33
- constructor(private readonly _params: LevelDBStorageAdapterParams) {
33
+ constructor(private readonly _params: LevelDBStorageAdapterProps) {
34
34
  super();
35
35
  }
36
36
 
@@ -12,16 +12,16 @@ import { log } from '@dxos/log';
12
12
  import type { AutomergeProtocolMessage } from '@dxos/protocols';
13
13
  import { AutomergeReplicator, type AutomergeReplicatorFactory } from '@dxos/teleport-extension-automerge-replicator';
14
14
 
15
- import type { ReplicatorConnection, ShouldAdvertiseParams, ShouldSyncCollectionParams } from './echo-replicator';
15
+ import type { ReplicatorConnection, ShouldAdvertiseProps, ShouldSyncCollectionProps } from './echo-replicator';
16
16
 
17
17
  const DEFAULT_FACTORY: AutomergeReplicatorFactory = (params) => new AutomergeReplicator(...params);
18
18
 
19
- export type MeshReplicatorConnectionParams = {
19
+ export type MeshReplicatorConnectionProps = {
20
20
  ownPeerId: string;
21
21
  onRemoteConnected: () => void;
22
22
  onRemoteDisconnected: () => void;
23
- shouldAdvertise: (params: ShouldAdvertiseParams) => Promise<boolean>;
24
- shouldSyncCollection: (params: ShouldSyncCollectionParams) => boolean;
23
+ shouldAdvertise: (params: ShouldAdvertiseProps) => Promise<boolean>;
24
+ shouldSyncCollection: (params: ShouldSyncCollectionProps) => boolean;
25
25
  replicatorFactory?: AutomergeReplicatorFactory;
26
26
  };
27
27
 
@@ -35,7 +35,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
35
35
  private _remotePeerId: string | null = null;
36
36
  private _isEnabled = false;
37
37
 
38
- constructor(private readonly _params: MeshReplicatorConnectionParams) {
38
+ constructor(private readonly _params: MeshReplicatorConnectionProps) {
39
39
  super();
40
40
 
41
41
  let readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
@@ -115,11 +115,11 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
115
115
  return false;
116
116
  }
117
117
 
118
- async shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean> {
118
+ async shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean> {
119
119
  return this._params.shouldAdvertise(params);
120
120
  }
121
121
 
122
- shouldSyncCollection(params: ShouldSyncCollectionParams): boolean {
122
+ shouldSyncCollection(params: ShouldSyncCollectionProps): boolean {
123
123
  return this._params.shouldSyncCollection(params);
124
124
  }
125
125
 
@@ -14,7 +14,7 @@ import { ComplexSet, defaultMap } from '@dxos/util';
14
14
 
15
15
  import { createIdFromSpaceKey } from '../common/space-id';
16
16
 
17
- import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseParams } from './echo-replicator';
17
+ import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseProps } from './echo-replicator';
18
18
  import { MeshReplicatorConnection } from './mesh-echo-replicator-connection';
19
19
  import { getSpaceIdFromCollectionId } from './space-collection';
20
20
 
@@ -111,7 +111,7 @@ export class MeshEchoReplicator implements EchoReplicator {
111
111
  }
112
112
  }
113
113
  },
114
- shouldAdvertise: async (params: ShouldAdvertiseParams) => {
114
+ shouldAdvertise: async (params: ShouldAdvertiseProps) => {
115
115
  log('shouldAdvertise', { peerId: connection.peerId, documentId: params.documentId });
116
116
  invariant(this._context);
117
117
  try {