@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1da679c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/dist/lib/browser/{chunk-TQJTKNMS.mjs → chunk-KQYT6ADL.mjs} +109 -3
  2. package/dist/lib/browser/chunk-KQYT6ADL.mjs.map +7 -0
  3. package/dist/lib/browser/{chunk-35I6ERLG.mjs → chunk-XGG76KKU.mjs} +513 -350
  4. package/dist/lib/browser/chunk-XGG76KKU.mjs.map +7 -0
  5. package/dist/lib/browser/filter/index.mjs +3 -1
  6. package/dist/lib/browser/index.mjs +1371 -601
  7. package/dist/lib/browser/index.mjs.map +4 -4
  8. package/dist/lib/browser/meta.json +1 -1
  9. package/dist/lib/browser/testing/index.mjs +119 -56
  10. package/dist/lib/browser/testing/index.mjs.map +3 -3
  11. package/dist/lib/node-esm/{chunk-5BHLPT24.mjs → chunk-CHMJJ4DG.mjs} +513 -350
  12. package/dist/lib/node-esm/chunk-CHMJJ4DG.mjs.map +7 -0
  13. package/dist/lib/node-esm/{chunk-RVK35BS7.mjs → chunk-W4ACY3YC.mjs} +109 -3
  14. package/dist/lib/node-esm/chunk-W4ACY3YC.mjs.map +7 -0
  15. package/dist/lib/node-esm/filter/index.mjs +3 -1
  16. package/dist/lib/node-esm/index.mjs +1371 -601
  17. package/dist/lib/node-esm/index.mjs.map +4 -4
  18. package/dist/lib/node-esm/meta.json +1 -1
  19. package/dist/lib/node-esm/testing/index.mjs +119 -56
  20. package/dist/lib/node-esm/testing/index.mjs.map +3 -3
  21. package/dist/types/src/automerge/automerge-host.d.ts +15 -28
  22. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  23. package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -1
  24. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  25. package/dist/types/src/automerge/echo-network-adapter.d.ts +8 -1
  26. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  27. package/dist/types/src/automerge/echo-replicator.d.ts +21 -2
  28. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  29. package/dist/types/src/automerge/index.d.ts +1 -1
  30. package/dist/types/src/automerge/index.d.ts.map +1 -1
  31. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
  32. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  33. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +1 -0
  34. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  35. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  36. package/dist/types/src/common/codec.d.ts +1 -1
  37. package/dist/types/src/common/codec.d.ts.map +1 -1
  38. package/dist/types/src/db-host/data-service.d.ts +2 -2
  39. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  40. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  41. package/dist/types/src/db-host/documents-synchronizer.d.ts +2 -2
  42. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  43. package/dist/types/src/db-host/echo-host.d.ts +2 -2
  44. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  45. package/dist/types/src/db-host/query-service.d.ts +1 -1
  46. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  47. package/dist/types/src/db-host/space-state-manager.d.ts +1 -1
  48. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  49. package/dist/types/src/edge/echo-edge-replicator.d.ts +4 -2
  50. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  51. package/dist/types/src/filter/filter-match.d.ts +4 -1
  52. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  53. package/dist/types/src/metadata/metadata-store.d.ts +1 -1
  54. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  55. package/dist/types/src/pipeline/pipeline.d.ts +1 -1
  56. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  57. package/dist/types/src/query/errors.d.ts +24 -8
  58. package/dist/types/src/query/errors.d.ts.map +1 -1
  59. package/dist/types/src/query/plan.d.ts +8 -1
  60. package/dist/types/src/query/plan.d.ts.map +1 -1
  61. package/dist/types/src/query/query-executor.d.ts +4 -1
  62. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  63. package/dist/types/src/query/query-planner.d.ts +2 -0
  64. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  65. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  66. package/dist/types/src/space/control-pipeline.d.ts +1 -1
  67. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  68. package/dist/types/src/space/space-manager.d.ts +1 -1
  69. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  70. package/dist/types/src/space/space-protocol.d.ts +1 -1
  71. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  72. package/dist/types/src/space/space.d.ts +1 -1
  73. package/dist/types/src/space/space.d.ts.map +1 -1
  74. package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
  75. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  76. package/dist/types/src/testing/test-replicator.d.ts +1 -0
  77. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  78. package/dist/types/src/util.d.ts +1 -1
  79. package/dist/types/src/util.d.ts.map +1 -1
  80. package/dist/types/tsconfig.tsbuildinfo +1 -1
  81. package/package.json +42 -38
  82. package/src/automerge/automerge-host.test.ts +18 -8
  83. package/src/automerge/automerge-host.ts +251 -65
  84. package/src/automerge/automerge-repo.test.ts +67 -16
  85. package/src/automerge/collection-synchronizer.test.ts +2 -2
  86. package/src/automerge/collection-synchronizer.ts +4 -4
  87. package/src/automerge/echo-data-monitor.ts +1 -1
  88. package/src/automerge/echo-network-adapter.test.ts +3 -3
  89. package/src/automerge/echo-network-adapter.ts +40 -7
  90. package/src/automerge/echo-replicator.ts +23 -2
  91. package/src/automerge/index.ts +1 -1
  92. package/src/automerge/leveldb-storage-adapter.ts +7 -7
  93. package/src/automerge/mesh-echo-replicator-connection.ts +4 -0
  94. package/src/automerge/mesh-echo-replicator.ts +2 -1
  95. package/src/automerge/storage-adapter.test.ts +1 -1
  96. package/src/common/space-id.ts +1 -1
  97. package/src/db-host/data-service.ts +9 -17
  98. package/src/db-host/database-root.ts +2 -2
  99. package/src/db-host/documents-synchronizer.test.ts +1 -1
  100. package/src/db-host/documents-synchronizer.ts +39 -26
  101. package/src/db-host/echo-host.ts +13 -14
  102. package/src/db-host/query-service.ts +8 -1
  103. package/src/db-host/space-state-manager.ts +2 -2
  104. package/src/edge/echo-edge-replicator.test.ts +5 -3
  105. package/src/edge/echo-edge-replicator.ts +75 -18
  106. package/src/filter/filter-match.test.ts +23 -3
  107. package/src/filter/filter-match.ts +148 -3
  108. package/src/metadata/metadata-store.ts +3 -3
  109. package/src/pipeline/pipeline-stress.test.ts +4 -2
  110. package/src/pipeline/pipeline.test.ts +3 -2
  111. package/src/pipeline/pipeline.ts +8 -5
  112. package/src/query/errors.ts +2 -0
  113. package/src/query/plan.ts +12 -1
  114. package/src/query/query-executor.ts +66 -11
  115. package/src/query/query-planner.test.ts +146 -2
  116. package/src/query/query-planner.ts +52 -8
  117. package/src/space/admission-discovery-extension.ts +2 -2
  118. package/src/space/control-pipeline.test.ts +4 -3
  119. package/src/space/control-pipeline.ts +9 -6
  120. package/src/space/space-manager.browser.test.ts +1 -1
  121. package/src/space/space-manager.ts +5 -4
  122. package/src/space/space-protocol.browser.test.ts +2 -2
  123. package/src/space/space-protocol.test.ts +3 -2
  124. package/src/space/space-protocol.ts +6 -3
  125. package/src/space/space.test.ts +1 -1
  126. package/src/space/space.ts +3 -2
  127. package/src/testing/test-agent-builder.ts +4 -3
  128. package/src/testing/test-replicator.ts +4 -0
  129. package/src/util.ts +1 -1
  130. package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
  131. package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
  132. package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
  133. package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
  134. package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
  135. package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
  136. package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
  137. package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
  138. package/dist/lib/node/filter/index.cjs +0 -32
  139. package/dist/lib/node/filter/index.cjs.map +0 -7
  140. package/dist/lib/node/index.cjs +0 -4699
  141. package/dist/lib/node/index.cjs.map +0 -7
  142. package/dist/lib/node/meta.json +0 -1
  143. package/dist/lib/node/testing/index.cjs +0 -753
  144. package/dist/lib/node/testing/index.cjs.map +0 -7
  145. package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
  146. package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +0 -7
@@ -3,48 +3,50 @@
3
3
  //
4
4
 
5
5
  import {
6
+ type Doc,
7
+ type Heads,
6
8
  getBackend,
7
9
  getHeads,
8
- isAutomerge,
9
10
  equals as headsEquals,
11
+ isAutomerge,
10
12
  save,
11
- type Doc,
12
- type Heads,
13
13
  } from '@automerge/automerge';
14
14
  import {
15
- type DocHandleChangePayload,
16
- Repo,
17
15
  type AnyDocumentId,
18
16
  type DocHandle,
17
+ type DocHandleChangePayload,
19
18
  type DocumentId,
19
+ type HandleState,
20
20
  type PeerCandidatePayload,
21
21
  type PeerDisconnectedPayload,
22
22
  type PeerId,
23
+ Repo,
23
24
  type StorageAdapterInterface,
24
25
  type StorageKey,
25
26
  interpretAsDocumentId,
26
- type HandleState,
27
27
  } from '@automerge/automerge-repo';
28
+ import { exportBundle } from '@automerge/automerge-repo-bundles';
28
29
 
29
- import { Event, asyncTimeout } from '@dxos/async';
30
- import { Context, Resource, cancelWithContext, type Lifecycle } from '@dxos/context';
31
- import { DatabaseDirectory, type CollectionId } from '@dxos/echo-protocol';
30
+ import { DeferredTask, Event, asyncTimeout } from '@dxos/async';
31
+ import { Context, type Lifecycle, Resource, cancelWithContext } from '@dxos/context';
32
+ import { type CollectionId, DatabaseDirectory } from '@dxos/echo-protocol';
32
33
  import { type IndexMetadataStore } from '@dxos/indexing';
33
34
  import { invariant } from '@dxos/invariant';
34
35
  import { PublicKey } from '@dxos/keys';
35
36
  import { type LevelDB } from '@dxos/kv-store';
36
37
  import { log } from '@dxos/log';
37
38
  import { objectPointerCodec } from '@dxos/protocols';
39
+ import { type SpaceSyncState } from '@dxos/protocols/proto/dxos/echo/service';
38
40
  import { type DocHeadsList, type FlushRequest } from '@dxos/protocols/proto/dxos/echo/service';
39
41
  import { trace } from '@dxos/tracing';
40
- import { bufferToArray } from '@dxos/util';
42
+ import { ComplexSet, bufferToArray, range } from '@dxos/util';
41
43
 
42
- import { CollectionSynchronizer, diffCollectionState, type CollectionState } from './collection-synchronizer';
44
+ import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
43
45
  import { type EchoDataMonitor } from './echo-data-monitor';
44
46
  import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
45
47
  import { type EchoReplicator, type RemoteDocumentExistenceCheckParams } from './echo-replicator';
46
48
  import { HeadsStore } from './heads-store';
47
- import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-adapter';
49
+ import { type BeforeSaveParams, LevelDBStorageAdapter } from './leveldb-storage-adapter';
48
50
 
49
51
  export type PeerIdProvider = () => string | undefined;
50
52
 
@@ -52,7 +54,6 @@ export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | u
52
54
 
53
55
  export type AutomergeHostParams = {
54
56
  db: LevelDB;
55
-
56
57
  indexMetadataStore: IndexMetadataStore;
57
58
  dataMonitor?: EchoDataMonitor;
58
59
 
@@ -78,6 +79,19 @@ export const FIND_PARAMS = {
78
79
  allowableStates: ['ready', 'requesting'] satisfies HandleState[],
79
80
  };
80
81
 
82
+ /**
83
+ * Maximum amount of documents to sync in a single bundle.
84
+ */
85
+ const BUNDLE_SIZE = 100;
86
+ /**
87
+ * Maximum amount of concurrent tasks to run when pushing or pulling bundles.
88
+ */
89
+ const BUNDLE_SYNC_CONCURRENCY = 2;
90
+ /**
91
+ * If the number of documents to sync is greater than this threshold, we will use bundles.
92
+ */
93
+ const BUNDLE_SYNC_THRESHOLD = 50;
94
+
81
95
  /**
82
96
  * Abstracts over the AutomergeRepo.
83
97
  */
@@ -97,6 +111,14 @@ export class AutomergeHost extends Resource {
97
111
  private _storage!: StorageAdapterInterface & Lifecycle;
98
112
  private readonly _headsStore: HeadsStore;
99
113
 
114
+ private _syncTask: DeferredTask | undefined = undefined;
115
+ /**
116
+ * Cache of collections that would be synced on next sync task run.
117
+ */
118
+ private readonly _collectionsToSync = new ComplexSet<{ collectionId: string; peerId: PeerId }>(
119
+ ({ collectionId, peerId }) => `${collectionId}|${peerId}`,
120
+ );
121
+
100
122
  @trace.info()
101
123
  private _peerId!: PeerId;
102
124
 
@@ -110,6 +132,9 @@ export class AutomergeHost extends Resource {
110
132
  */
111
133
  public readonly documentsSaved = new Event();
112
134
 
135
+ private readonly _headsUpdates = new Map<DocumentId, Heads>();
136
+ private _onHeadsChangedTask?: DeferredTask | undefined;
137
+
113
138
  constructor({
114
139
  db,
115
140
  indexMetadataStore,
@@ -143,6 +168,12 @@ export class AutomergeHost extends Resource {
143
168
  protected override async _open(): Promise<void> {
144
169
  this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}` as PeerId;
145
170
 
171
+ this._onHeadsChangedTask = new DeferredTask(this._ctx, async () => {
172
+ const docHeads = Array.from(this._headsUpdates.entries());
173
+ this._headsUpdates.clear();
174
+ this._onHeadsChanged(docHeads);
175
+ });
176
+
146
177
  await this._storage.open?.();
147
178
 
148
179
  // Construct the automerge repo.
@@ -180,6 +211,22 @@ export class AutomergeHost extends Resource {
180
211
  }
181
212
  });
182
213
 
214
+ this._syncTask = new DeferredTask(this._ctx, async () => {
215
+ const collectionToSync = Array.from(this._collectionsToSync.values());
216
+ if (collectionToSync.length === 0) {
217
+ return;
218
+ }
219
+ await Promise.all(
220
+ collectionToSync.map(async ({ collectionId, peerId }) => {
221
+ try {
222
+ await this._handleCollectionSync(collectionId, peerId);
223
+ } catch (err) {
224
+ log.error('failed to sync collection', { collectionId, peerId, err });
225
+ }
226
+ }),
227
+ );
228
+ });
229
+
183
230
  await this._echoNetworkAdapter.open();
184
231
  await this._collectionSynchronizer.open();
185
232
  await this._echoNetworkAdapter.open();
@@ -190,7 +237,8 @@ export class AutomergeHost extends Resource {
190
237
  await this._collectionSynchronizer.close();
191
238
  await this._storage.close?.();
192
239
  await this._echoNetworkAdapter.close();
193
- await this._ctx.dispose();
240
+ this._syncTask = undefined;
241
+ this._onHeadsChangedTask = undefined;
194
242
  }
195
243
 
196
244
  /**
@@ -209,10 +257,12 @@ export class AutomergeHost extends Resource {
209
257
  }
210
258
 
211
259
  async addReplicator(replicator: EchoReplicator): Promise<void> {
260
+ invariant(this.isOpen, 'AutomergeHost is not open');
212
261
  await this._echoNetworkAdapter.addReplicator(replicator);
213
262
  }
214
263
 
215
264
  async removeReplicator(replicator: EchoReplicator): Promise<void> {
265
+ invariant(this.isOpen, 'AutomergeHost is not open');
216
266
  await this._echoNetworkAdapter.removeReplicator(replicator);
217
267
  }
218
268
 
@@ -220,6 +270,7 @@ export class AutomergeHost extends Resource {
220
270
  * Loads the document handle from the repo and waits for it to be ready.
221
271
  */
222
272
  async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T>> {
273
+ invariant(this.isOpen, 'AutomergeHost is not open');
223
274
  let handle: DocHandle<T> | undefined;
224
275
  if (typeof documentId === 'string') {
225
276
  // NOTE: documentId might also be a URL, in which case this lookup will fail.
@@ -242,6 +293,7 @@ export class AutomergeHost extends Resource {
242
293
  }
243
294
 
244
295
  async exportDoc(ctx: Context, id: AnyDocumentId): Promise<Uint8Array> {
296
+ invariant(this.isOpen, 'AutomergeHost is not open');
245
297
  const documentId = interpretAsDocumentId(id);
246
298
 
247
299
  const chunks = await this._storage.loadRange([documentId]);
@@ -252,6 +304,7 @@ export class AutomergeHost extends Resource {
252
304
  * Create new persisted document.
253
305
  */
254
306
  createDoc<T>(initialValue?: T | Doc<T> | Uint8Array, opts?: CreateDocOptions): DocHandle<T> {
307
+ invariant(this.isOpen, 'AutomergeHost is not open');
255
308
  if (opts?.preserveHistory) {
256
309
  if (initialValue instanceof Uint8Array) {
257
310
  return this._repo.import(initialValue);
@@ -273,6 +326,7 @@ export class AutomergeHost extends Resource {
273
326
  }
274
327
 
275
328
  async waitUntilHeadsReplicated(heads: DocHeadsList): Promise<void> {
329
+ invariant(this.isOpen, 'AutomergeHost is not open');
276
330
  const entries = heads.entries;
277
331
  if (!entries?.length) {
278
332
  return;
@@ -303,6 +357,7 @@ export class AutomergeHost extends Resource {
303
357
  }
304
358
 
305
359
  async reIndexHeads(documentIds: DocumentId[]): Promise<void> {
360
+ invariant(this.isOpen, 'AutomergeHost is not open');
306
361
  for (const documentId of documentIds) {
307
362
  log('re-indexing heads for document', { documentId });
308
363
  const handle = await this._repo.find(documentId, FIND_PARAMS);
@@ -375,14 +430,25 @@ export class AutomergeHost extends Resource {
375
430
  * Called by AutomergeStorageAdapter after levelDB batch commit.
376
431
  */
377
432
  private async _afterSave(path: StorageKey): Promise<void> {
378
- this._indexMetadataStore.notifyMarkedDirty();
433
+ if (!this.isOpen) {
434
+ return undefined;
435
+ }
379
436
 
437
+ this._indexMetadataStore.notifyMarkedDirty();
380
438
  const documentId = path[0] as DocumentId;
381
- const document = this._repo.handles[documentId]?.doc();
382
- if (document) {
383
- const heads = getHeads(document);
384
- this._onHeadsChanged(documentId, heads);
439
+ const handle = this._repo.handles[documentId];
440
+ if (!handle || !handle.isReady()) {
441
+ return;
442
+ }
443
+ const document = handle.doc();
444
+ if (!document) {
445
+ return;
385
446
  }
447
+
448
+ const heads = getHeads(document);
449
+ this._headsUpdates.set(documentId, heads);
450
+ invariant(this._onHeadsChangedTask, 'onHeadsChangedTask is not initialized');
451
+ this._onHeadsChangedTask.schedule();
386
452
  this.documentsSaved.emit();
387
453
  }
388
454
 
@@ -479,8 +545,8 @@ export class AutomergeHost extends Resource {
479
545
  this._collectionSynchronizer.refreshCollection(collectionId);
480
546
  }
481
547
 
482
- async getCollectionSyncState(collectionId: string): Promise<CollectionSyncState> {
483
- const result: CollectionSyncState = {
548
+ async getCollectionSyncState(collectionId: string): Promise<SpaceSyncState> {
549
+ const result: SpaceSyncState = {
484
550
  peers: [],
485
551
  };
486
552
 
@@ -493,13 +559,16 @@ export class AutomergeHost extends Resource {
493
559
 
494
560
  for (const [peerId, state] of remoteState) {
495
561
  const diff = diffCollectionState(localState, state);
496
- result.peers.push({
562
+ result.peers!.push({
497
563
  peerId,
498
564
  missingOnRemote: diff.missingOnRemote.length,
499
565
  missingOnLocal: diff.missingOnLocal.length,
500
566
  differentDocuments: diff.different.length,
501
- localDocumentCount: Object.keys(localState.documents).length,
502
- remoteDocumentCount: Object.keys(state.documents).length,
567
+ localDocumentCount: Object.entries(localState.documents).filter(([_, heads]) => heads.length > 0).length,
568
+ remoteDocumentCount: Object.entries(state.documents).filter(([_, heads]) => heads.length > 0).length,
569
+
570
+ totalDocumentCount: new Set([...Object.keys(localState.documents), ...Object.keys(state.documents)]).size,
571
+ unsyncedDocumentCount: new Set([...diff.missingOnLocal, ...diff.missingOnRemote, ...diff.different]).size,
503
572
  });
504
573
  }
505
574
 
@@ -515,6 +584,16 @@ export class AutomergeHost extends Resource {
515
584
  heads.map((heads, index) => [documentIds[index], heads ?? []]),
516
585
  );
517
586
  this._collectionSynchronizer.setLocalCollectionState(collectionId, { documents });
587
+
588
+ // Proactively push our updated local state to peers that are interested in this collection.
589
+ // This reduces reliance on the next periodic query and prevents replication stalls in fast paths
590
+ // where the remote queries before our local state is ready.
591
+ const interestedPeers = this._echoNetworkAdapter.getPeersInterestedInCollection(collectionId);
592
+ if (interestedPeers.length > 0) {
593
+ for (const peerId of interestedPeers) {
594
+ this._sendCollectionState(collectionId, peerId, { documents });
595
+ }
596
+ }
518
597
  }
519
598
 
520
599
  async clearLocalCollectionState(collectionId: string): Promise<void> {
@@ -546,6 +625,11 @@ export class AutomergeHost extends Resource {
546
625
  }
547
626
 
548
627
  private _onRemoteCollectionStateUpdated(collectionId: string, peerId: PeerId): void {
628
+ this._collectionsToSync.add({ collectionId, peerId });
629
+ this._syncTask?.schedule();
630
+ }
631
+
632
+ private async _handleCollectionSync(collectionId: string, peerId: PeerId) {
549
633
  const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
550
634
  const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
551
635
 
@@ -554,36 +638,170 @@ export class AutomergeHost extends Resource {
554
638
  }
555
639
 
556
640
  const { different, missingOnLocal, missingOnRemote } = diffCollectionState(localState, remoteState);
557
- const toReplicate = [...missingOnLocal, ...missingOnRemote, ...different];
558
641
 
559
- if (toReplicate.length === 0) {
642
+ if (different.length === 0 && missingOnLocal.length === 0 && missingOnRemote.length === 0) {
643
+ return;
644
+ }
645
+
646
+ const toReplicateWithoutBatching = [...different];
647
+ const bundleSyncEnabled = this._echoNetworkAdapter.bundleSyncEnabledForPeer(peerId);
648
+ if (bundleSyncEnabled && missingOnRemote.length >= BUNDLE_SYNC_THRESHOLD) {
649
+ log('pushing bundle', { amount: missingOnRemote.length });
650
+ const { syncInteractively } = await this._pushInBundles(peerId, missingOnRemote);
651
+ toReplicateWithoutBatching.push(...syncInteractively);
652
+ } else {
653
+ log.verbose('failed to push bundle, replicating interactively', {
654
+ collectionId,
655
+ peerId,
656
+ amount: missingOnRemote.length,
657
+ });
658
+ toReplicateWithoutBatching.push(...missingOnRemote);
659
+ }
660
+ if (bundleSyncEnabled && missingOnLocal.length >= BUNDLE_SYNC_THRESHOLD) {
661
+ log('pulling bundle', { amount: missingOnLocal.length });
662
+ const { syncInteractively } = await this._pullInBundles(peerId, missingOnLocal);
663
+ toReplicateWithoutBatching.push(...syncInteractively);
664
+ } else {
665
+ log.verbose('failed to pull bundle, replicating interactively', {
666
+ collectionId,
667
+ peerId,
668
+ amount: missingOnLocal.length,
669
+ });
670
+ toReplicateWithoutBatching.push(...missingOnLocal);
671
+ }
672
+
673
+ if (toReplicateWithoutBatching.length === 0) {
560
674
  return;
561
675
  }
562
676
 
563
677
  log('replicating documents after collection sync', {
564
678
  collectionId,
565
679
  peerId,
566
- toReplicate,
567
- count: toReplicate.length,
680
+ toReplicateWithoutBatching,
681
+ count: toReplicateWithoutBatching.length,
568
682
  });
569
683
 
570
684
  // Load the documents so they will start syncing.
571
- for (const documentId of toReplicate) {
685
+ for (const documentId of toReplicateWithoutBatching) {
572
686
  this._repo.findWithProgress(documentId);
573
687
  }
574
688
  }
575
689
 
576
- private _onHeadsChanged(documentId: DocumentId, heads: Heads): void {
690
+ // TODO(mykola): Add retries of batches https://gist.github.com/mykola-vrmchk/fde270259e9209fcbf1331e5abbf12cf
691
+ // TODO(mykola): Use effect to retry batches.
692
+ private async _pushInBundles(
693
+ peerId: PeerId,
694
+ documentIds: DocumentId[],
695
+ ): Promise<{ syncInteractively: DocumentId[] }> {
696
+ const documentsToPush = [...documentIds];
697
+ const syncInteractively: DocumentId[] = [];
698
+
699
+ // Push bundles in parallel with BUNDLE_SYNC_CONCURRENCY max concurrent tasks.
700
+ while (documentsToPush.length > 0) {
701
+ await Promise.all(
702
+ range(BUNDLE_SYNC_CONCURRENCY).map(async () => {
703
+ const bundle = documentsToPush.splice(0, BUNDLE_SIZE);
704
+ if (bundle.length === 0) {
705
+ return;
706
+ }
707
+ await this._pushBundle(peerId, bundle).catch((err) => {
708
+ log.warn('failed to push bundle, replicating interactively', { peerId, bundle, err });
709
+ syncInteractively.push(...bundle);
710
+ });
711
+ }),
712
+ );
713
+ }
714
+
715
+ return { syncInteractively };
716
+ }
717
+
718
+ private async _pushBundle(peerId: PeerId, documentIds: DocumentId[]): Promise<void> {
719
+ if (this._ctx.disposed) {
720
+ return;
721
+ }
722
+
723
+ const handles = documentIds.map((documentId) => this._repo.handles[documentId]);
724
+ const bundle = exportBundle(this._repo, handles);
725
+ await this._echoNetworkAdapter.pushBundle(
726
+ peerId,
727
+ Array.from(bundle.docs.entries()).map(([documentId, doc]) => ({
728
+ documentId,
729
+ data: doc.data,
730
+ heads: doc.heads,
731
+ })),
732
+ );
733
+ }
734
+
735
+ private async _pullInBundles(
736
+ peerId: PeerId,
737
+ documentIds: DocumentId[],
738
+ ): Promise<{ syncInteractively: DocumentId[] }> {
739
+ const documentsToPull = [...documentIds];
740
+ const syncInteractively: DocumentId[] = [];
741
+ const docsToImport: Record<DocumentId, Uint8Array> = {};
742
+
743
+ // Pull bundles in parallel with BUNDLE_SYNC_CONCURRENCY max concurrent tasks.
744
+ while (documentsToPull.length > 0) {
745
+ await Promise.all(
746
+ range(BUNDLE_SYNC_CONCURRENCY).map(async () => {
747
+ const bundle = documentsToPull.splice(0, BUNDLE_SIZE);
748
+ if (bundle.length === 0) {
749
+ return;
750
+ }
751
+ const result = await this._pullBundle(peerId, bundle).catch((err) => {
752
+ log.warn('failed to pull bundle, replicating interactively', { peerId, bundle, err });
753
+ syncInteractively.push(...bundle);
754
+ });
755
+ if (result) {
756
+ Object.assign(docsToImport, result.docsToImport);
757
+ }
758
+ }),
759
+ );
760
+ }
761
+
762
+ for (const [documentId, data] of Object.entries(docsToImport)) {
763
+ this._repo.import(data, { docId: documentId as DocumentId });
764
+ }
765
+ await this._repo.flush(Object.keys(docsToImport) as DocumentId[]);
766
+
767
+ return { syncInteractively };
768
+ }
769
+
770
+ private async _pullBundle(
771
+ peerId: PeerId,
772
+ documentIds: DocumentId[],
773
+ ): Promise<{ docsToImport: Record<DocumentId, Uint8Array> } | undefined> {
774
+ if (this._ctx.disposed) {
775
+ return;
776
+ }
777
+ // NOTE: We are expecting that documents that are being pulled are not present locally, so we are pulling all changes.
778
+ const docHeads = Object.fromEntries(documentIds.map((documentId) => [documentId, []]));
779
+ const bundle = await this._echoNetworkAdapter.pullBundle(peerId, docHeads);
780
+ return { docsToImport: bundle };
781
+ }
782
+
783
+ private _onHeadsChanged(docHeads: [DocumentId, Heads][]): void {
577
784
  const collectionsChanged = new Set<CollectionId>();
785
+
578
786
  for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
579
787
  const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
580
- if (state?.documents[documentId]) {
581
- const newState = structuredClone(state);
582
- newState.documents[documentId] = heads;
788
+ let newState: CollectionState | undefined;
789
+
790
+ for (const [documentId, heads] of docHeads) {
791
+ if (state?.documents[documentId]) {
792
+ if (!newState) {
793
+ newState = structuredClone(state);
794
+ }
795
+ newState.documents[documentId] = heads;
796
+ }
797
+ }
798
+
799
+ if (newState) {
583
800
  this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
584
801
  collectionsChanged.add(collectionId as CollectionId);
585
802
  }
586
803
  }
804
+
587
805
  for (const collectionId of collectionsChanged) {
588
806
  this.collectionStateUpdated.emit({ collectionId });
589
807
  }
@@ -619,35 +837,3 @@ const decodeCollectionState = (state: unknown): CollectionState => {
619
837
  const encodeCollectionState = (state: CollectionState): unknown => {
620
838
  return state;
621
839
  };
622
-
623
- export type CollectionSyncState = {
624
- peers: PeerSyncState[];
625
- };
626
-
627
- export type PeerSyncState = {
628
- peerId: PeerId;
629
- /**
630
- * Documents that are present locally but not on the remote peer.
631
- */
632
- missingOnRemote: number;
633
-
634
- /**
635
- * Documents that are present on the remote peer but not locally.
636
- */
637
- missingOnLocal: number;
638
-
639
- /**
640
- * Documents that are present on both peers but have different heads.
641
- */
642
- differentDocuments: number;
643
-
644
- /**
645
- * Total number of documents locally.
646
- */
647
- localDocumentCount: number;
648
-
649
- /**
650
- * Total number of documents on the remote peer.
651
- */
652
- remoteDocumentCount: number;
653
- };
@@ -3,6 +3,7 @@
3
3
  //
4
4
 
5
5
  import {
6
+ next as A,
6
7
  type Heads,
7
8
  change,
8
9
  clone,
@@ -10,7 +11,6 @@ import {
10
11
  from,
11
12
  getBackend,
12
13
  getHeads,
13
- next as A,
14
14
  save,
15
15
  saveSince,
16
16
  } from '@automerge/automerge';
@@ -19,14 +19,14 @@ import {
19
19
  type DocHandle,
20
20
  type DocumentId,
21
21
  type HandleState,
22
- type StorageAdapterInterface,
23
22
  type PeerId,
24
23
  Repo,
25
24
  type SharePolicy,
25
+ type StorageAdapterInterface,
26
26
  generateAutomergeUrl,
27
27
  parseAutomergeUrl,
28
28
  } from '@automerge/automerge-repo';
29
- import { onTestFinished, describe, expect, test } from 'vitest';
29
+ import { describe, expect, onTestFinished, test } from 'vitest';
30
30
 
31
31
  import { asyncTimeout, sleep } from '@dxos/async';
32
32
  import { randomBytes } from '@dxos/crypto';
@@ -36,11 +36,12 @@ import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/
36
36
  import { openAndClose } from '@dxos/test-utils';
37
37
  import { isNonNullable, range } from '@dxos/util';
38
38
 
39
+ import { TestAdapter, type TestConnectionStateProvider } from '../testing';
40
+
39
41
  import { FIND_PARAMS } from './automerge-host';
40
42
  import { EchoNetworkAdapter } from './echo-network-adapter';
41
43
  import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
42
44
  import { MeshEchoReplicator } from './mesh-echo-replicator';
43
- import { TestAdapter, type TestConnectionStateProvider } from '../testing';
44
45
 
45
46
  const HOST_AND_CLIENT: [string, string] = ['host', 'client'];
46
47
 
@@ -420,13 +421,7 @@ describe('AutomergeRepo', () => {
420
421
 
421
422
  test('client creates doc and syncs with a Repo', async () => {
422
423
  const repo = new Repo({ network: [] });
423
- const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
424
- const serverHandle = await repo.find(docId, FIND_PARAMS);
425
- serverHandle.update((doc) => {
426
- return A.loadIncremental(doc, blob);
427
- });
428
- };
429
-
424
+ const receiveByServer = (blob: Uint8Array, docId: DocumentId) => repo.import<any>(blob, { docId });
430
425
  let clientDoc = A.from<{ field?: string }>({});
431
426
  const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
432
427
  // Sync handshake.
@@ -434,7 +429,7 @@ describe('AutomergeRepo', () => {
434
429
 
435
430
  // Sync protocol.
436
431
  const sendDoc = async (doc: A.Doc<any>) => {
437
- await receiveByServer(saveSince(doc, sentHeads), documentId);
432
+ receiveByServer(saveSince(doc, sentHeads), documentId);
438
433
  sentHeads = getHeads(doc);
439
434
  };
440
435
 
@@ -456,10 +451,9 @@ describe('AutomergeRepo', () => {
456
451
 
457
452
  const repo = new Repo({ network: [], storage });
458
453
  const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
459
- const serverHandle = await repo.find(docId, FIND_PARAMS);
460
- serverHandle.update((doc) => {
461
- return A.loadIncremental(doc, blob);
462
- });
454
+ repo.import<any>(blob, { docId });
455
+ // TODO(mykola): This should not be required. Document is not persisted without it.
456
+ await repo.flush([docId]);
463
457
  };
464
458
 
465
459
  let clientDoc = A.from<{ field?: string }>({ field: 'foo' });
@@ -677,6 +671,63 @@ describe('AutomergeRepo', () => {
677
671
  await doc.whenReady();
678
672
  expect(doc.doc()).to.deep.eq(document.doc());
679
673
  });
674
+
675
+ test('document is passively replicated to connected peers', async () => {
676
+ const [spaceKey] = PublicKey.randomSequence();
677
+
678
+ const teleportBuilder = new TeleportBuilder();
679
+ onTestFinished(() => teleportBuilder.destroy());
680
+
681
+ const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
682
+ const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
683
+
684
+ const handle = peer1.repo.create();
685
+ handle.change((doc: any) => (doc.text = 'hello'));
686
+ await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
687
+
688
+ await expect
689
+ .poll(async () => {
690
+ const doc = peer2.repo.handles[handle.documentId];
691
+ await doc.whenReady();
692
+ return doc.doc()!.text;
693
+ })
694
+ .toEqual('hello');
695
+ });
696
+
697
+ test('imported document is passively replicated to connected peers', async () => {
698
+ let blob: Uint8Array;
699
+ let documentId: DocumentId;
700
+ {
701
+ const repo = new Repo();
702
+ const handle = repo.create();
703
+ handle.change((doc: any) => (doc.text = 'hello'));
704
+ blob = A.save(handle.doc()!);
705
+ documentId = handle.documentId;
706
+ }
707
+
708
+ const [spaceKey] = PublicKey.randomSequence();
709
+
710
+ const teleportBuilder = new TeleportBuilder();
711
+ onTestFinished(() => teleportBuilder.destroy());
712
+
713
+ const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
714
+ const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
715
+ await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
716
+
717
+ const handle = peer1.repo.import(blob, { docId: documentId });
718
+ await handle.whenReady();
719
+
720
+ await expect
721
+ .poll(
722
+ async () => {
723
+ const doc = peer2.repo.handles[handle.documentId];
724
+ await doc.whenReady();
725
+ return doc.doc()!.text;
726
+ },
727
+ { timeout: 1_000 },
728
+ )
729
+ .toEqual('hello');
730
+ });
680
731
  });
681
732
 
682
733
  const createLevelAdapter = async () => {
@@ -3,11 +3,11 @@
3
3
  //
4
4
 
5
5
  import type { PeerId } from '@automerge/automerge-repo';
6
- import { onTestFinished, describe, expect, test } from 'vitest';
6
+ import { describe, expect, onTestFinished, test } from 'vitest';
7
7
 
8
8
  import { sleep } from '@dxos/async';
9
9
 
10
- import { CollectionSynchronizer, diffCollectionState, type CollectionState } from './collection-synchronizer';
10
+ import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
11
11
 
12
12
  describe('CollectionSynchronizer', () => {
13
13
  test('sync two peers', async () => {