@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1da679c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-TQJTKNMS.mjs → chunk-KQYT6ADL.mjs} +109 -3
- package/dist/lib/browser/chunk-KQYT6ADL.mjs.map +7 -0
- package/dist/lib/browser/{chunk-35I6ERLG.mjs → chunk-XGG76KKU.mjs} +513 -350
- package/dist/lib/browser/chunk-XGG76KKU.mjs.map +7 -0
- package/dist/lib/browser/filter/index.mjs +3 -1
- package/dist/lib/browser/index.mjs +1371 -601
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +119 -56
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node-esm/{chunk-5BHLPT24.mjs → chunk-CHMJJ4DG.mjs} +513 -350
- package/dist/lib/node-esm/chunk-CHMJJ4DG.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-RVK35BS7.mjs → chunk-W4ACY3YC.mjs} +109 -3
- package/dist/lib/node-esm/chunk-W4ACY3YC.mjs.map +7 -0
- package/dist/lib/node-esm/filter/index.mjs +3 -1
- package/dist/lib/node-esm/index.mjs +1371 -601
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +119 -56
- package/dist/lib/node-esm/testing/index.mjs.map +3 -3
- package/dist/types/src/automerge/automerge-host.d.ts +15 -28
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +8 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts +21 -2
- package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/index.d.ts +1 -1
- package/dist/types/src/automerge/index.d.ts.map +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +1 -0
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/common/codec.d.ts +1 -1
- package/dist/types/src/common/codec.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +2 -2
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-root.d.ts.map +1 -1
- package/dist/types/src/db-host/documents-synchronizer.d.ts +2 -2
- package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
- package/dist/types/src/db-host/echo-host.d.ts +2 -2
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
- package/dist/types/src/db-host/query-service.d.ts +1 -1
- package/dist/types/src/db-host/query-service.d.ts.map +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
- package/dist/types/src/edge/echo-edge-replicator.d.ts +4 -2
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
- package/dist/types/src/filter/filter-match.d.ts +4 -1
- package/dist/types/src/filter/filter-match.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/query/errors.d.ts +24 -8
- package/dist/types/src/query/errors.d.ts.map +1 -1
- package/dist/types/src/query/plan.d.ts +8 -1
- package/dist/types/src/query/plan.d.ts.map +1 -1
- package/dist/types/src/query/query-executor.d.ts +4 -1
- package/dist/types/src/query/query-executor.d.ts.map +1 -1
- package/dist/types/src/query/query-planner.d.ts +2 -0
- package/dist/types/src/query/query-planner.d.ts.map +1 -1
- package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +1 -1
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/test-replicator.d.ts +1 -0
- package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
- package/dist/types/src/util.d.ts +1 -1
- package/dist/types/src/util.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +42 -38
- package/src/automerge/automerge-host.test.ts +18 -8
- package/src/automerge/automerge-host.ts +251 -65
- package/src/automerge/automerge-repo.test.ts +67 -16
- package/src/automerge/collection-synchronizer.test.ts +2 -2
- package/src/automerge/collection-synchronizer.ts +4 -4
- package/src/automerge/echo-data-monitor.ts +1 -1
- package/src/automerge/echo-network-adapter.test.ts +3 -3
- package/src/automerge/echo-network-adapter.ts +40 -7
- package/src/automerge/echo-replicator.ts +23 -2
- package/src/automerge/index.ts +1 -1
- package/src/automerge/leveldb-storage-adapter.ts +7 -7
- package/src/automerge/mesh-echo-replicator-connection.ts +4 -0
- package/src/automerge/mesh-echo-replicator.ts +2 -1
- package/src/automerge/storage-adapter.test.ts +1 -1
- package/src/common/space-id.ts +1 -1
- package/src/db-host/data-service.ts +9 -17
- package/src/db-host/database-root.ts +2 -2
- package/src/db-host/documents-synchronizer.test.ts +1 -1
- package/src/db-host/documents-synchronizer.ts +39 -26
- package/src/db-host/echo-host.ts +13 -14
- package/src/db-host/query-service.ts +8 -1
- package/src/db-host/space-state-manager.ts +2 -2
- package/src/edge/echo-edge-replicator.test.ts +5 -3
- package/src/edge/echo-edge-replicator.ts +75 -18
- package/src/filter/filter-match.test.ts +23 -3
- package/src/filter/filter-match.ts +148 -3
- package/src/metadata/metadata-store.ts +3 -3
- package/src/pipeline/pipeline-stress.test.ts +4 -2
- package/src/pipeline/pipeline.test.ts +3 -2
- package/src/pipeline/pipeline.ts +8 -5
- package/src/query/errors.ts +2 -0
- package/src/query/plan.ts +12 -1
- package/src/query/query-executor.ts +66 -11
- package/src/query/query-planner.test.ts +146 -2
- package/src/query/query-planner.ts +52 -8
- package/src/space/admission-discovery-extension.ts +2 -2
- package/src/space/control-pipeline.test.ts +4 -3
- package/src/space/control-pipeline.ts +9 -6
- package/src/space/space-manager.browser.test.ts +1 -1
- package/src/space/space-manager.ts +5 -4
- package/src/space/space-protocol.browser.test.ts +2 -2
- package/src/space/space-protocol.test.ts +3 -2
- package/src/space/space-protocol.ts +6 -3
- package/src/space/space.test.ts +1 -1
- package/src/space/space.ts +3 -2
- package/src/testing/test-agent-builder.ts +4 -3
- package/src/testing/test-replicator.ts +4 -0
- package/src/util.ts +1 -1
- package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
- package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
- package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
- package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
- package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
- package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
- package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
- package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
- package/dist/lib/node/filter/index.cjs +0 -32
- package/dist/lib/node/filter/index.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -4699
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/testing/index.cjs +0 -753
- package/dist/lib/node/testing/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +0 -7
|
@@ -3,48 +3,50 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
|
+
type Doc,
|
|
7
|
+
type Heads,
|
|
6
8
|
getBackend,
|
|
7
9
|
getHeads,
|
|
8
|
-
isAutomerge,
|
|
9
10
|
equals as headsEquals,
|
|
11
|
+
isAutomerge,
|
|
10
12
|
save,
|
|
11
|
-
type Doc,
|
|
12
|
-
type Heads,
|
|
13
13
|
} from '@automerge/automerge';
|
|
14
14
|
import {
|
|
15
|
-
type DocHandleChangePayload,
|
|
16
|
-
Repo,
|
|
17
15
|
type AnyDocumentId,
|
|
18
16
|
type DocHandle,
|
|
17
|
+
type DocHandleChangePayload,
|
|
19
18
|
type DocumentId,
|
|
19
|
+
type HandleState,
|
|
20
20
|
type PeerCandidatePayload,
|
|
21
21
|
type PeerDisconnectedPayload,
|
|
22
22
|
type PeerId,
|
|
23
|
+
Repo,
|
|
23
24
|
type StorageAdapterInterface,
|
|
24
25
|
type StorageKey,
|
|
25
26
|
interpretAsDocumentId,
|
|
26
|
-
type HandleState,
|
|
27
27
|
} from '@automerge/automerge-repo';
|
|
28
|
+
import { exportBundle } from '@automerge/automerge-repo-bundles';
|
|
28
29
|
|
|
29
|
-
import { Event, asyncTimeout } from '@dxos/async';
|
|
30
|
-
import { Context, Resource, cancelWithContext
|
|
31
|
-
import {
|
|
30
|
+
import { DeferredTask, Event, asyncTimeout } from '@dxos/async';
|
|
31
|
+
import { Context, type Lifecycle, Resource, cancelWithContext } from '@dxos/context';
|
|
32
|
+
import { type CollectionId, DatabaseDirectory } from '@dxos/echo-protocol';
|
|
32
33
|
import { type IndexMetadataStore } from '@dxos/indexing';
|
|
33
34
|
import { invariant } from '@dxos/invariant';
|
|
34
35
|
import { PublicKey } from '@dxos/keys';
|
|
35
36
|
import { type LevelDB } from '@dxos/kv-store';
|
|
36
37
|
import { log } from '@dxos/log';
|
|
37
38
|
import { objectPointerCodec } from '@dxos/protocols';
|
|
39
|
+
import { type SpaceSyncState } from '@dxos/protocols/proto/dxos/echo/service';
|
|
38
40
|
import { type DocHeadsList, type FlushRequest } from '@dxos/protocols/proto/dxos/echo/service';
|
|
39
41
|
import { trace } from '@dxos/tracing';
|
|
40
|
-
import { bufferToArray } from '@dxos/util';
|
|
42
|
+
import { ComplexSet, bufferToArray, range } from '@dxos/util';
|
|
41
43
|
|
|
42
|
-
import { CollectionSynchronizer, diffCollectionState
|
|
44
|
+
import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
|
|
43
45
|
import { type EchoDataMonitor } from './echo-data-monitor';
|
|
44
46
|
import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
|
|
45
47
|
import { type EchoReplicator, type RemoteDocumentExistenceCheckParams } from './echo-replicator';
|
|
46
48
|
import { HeadsStore } from './heads-store';
|
|
47
|
-
import {
|
|
49
|
+
import { type BeforeSaveParams, LevelDBStorageAdapter } from './leveldb-storage-adapter';
|
|
48
50
|
|
|
49
51
|
export type PeerIdProvider = () => string | undefined;
|
|
50
52
|
|
|
@@ -52,7 +54,6 @@ export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | u
|
|
|
52
54
|
|
|
53
55
|
export type AutomergeHostParams = {
|
|
54
56
|
db: LevelDB;
|
|
55
|
-
|
|
56
57
|
indexMetadataStore: IndexMetadataStore;
|
|
57
58
|
dataMonitor?: EchoDataMonitor;
|
|
58
59
|
|
|
@@ -78,6 +79,19 @@ export const FIND_PARAMS = {
|
|
|
78
79
|
allowableStates: ['ready', 'requesting'] satisfies HandleState[],
|
|
79
80
|
};
|
|
80
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Maximum amount of documents to sync in a single bundle.
|
|
84
|
+
*/
|
|
85
|
+
const BUNDLE_SIZE = 100;
|
|
86
|
+
/**
|
|
87
|
+
* Maximum amount of concurrent tasks to run when pushing or pulling bundles.
|
|
88
|
+
*/
|
|
89
|
+
const BUNDLE_SYNC_CONCURRENCY = 2;
|
|
90
|
+
/**
|
|
91
|
+
* If the number of documents to sync is greater than this threshold, we will use bundles.
|
|
92
|
+
*/
|
|
93
|
+
const BUNDLE_SYNC_THRESHOLD = 50;
|
|
94
|
+
|
|
81
95
|
/**
|
|
82
96
|
* Abstracts over the AutomergeRepo.
|
|
83
97
|
*/
|
|
@@ -97,6 +111,14 @@ export class AutomergeHost extends Resource {
|
|
|
97
111
|
private _storage!: StorageAdapterInterface & Lifecycle;
|
|
98
112
|
private readonly _headsStore: HeadsStore;
|
|
99
113
|
|
|
114
|
+
private _syncTask: DeferredTask | undefined = undefined;
|
|
115
|
+
/**
|
|
116
|
+
* Cache of collections that would be synced on next sync task run.
|
|
117
|
+
*/
|
|
118
|
+
private readonly _collectionsToSync = new ComplexSet<{ collectionId: string; peerId: PeerId }>(
|
|
119
|
+
({ collectionId, peerId }) => `${collectionId}|${peerId}`,
|
|
120
|
+
);
|
|
121
|
+
|
|
100
122
|
@trace.info()
|
|
101
123
|
private _peerId!: PeerId;
|
|
102
124
|
|
|
@@ -110,6 +132,9 @@ export class AutomergeHost extends Resource {
|
|
|
110
132
|
*/
|
|
111
133
|
public readonly documentsSaved = new Event();
|
|
112
134
|
|
|
135
|
+
private readonly _headsUpdates = new Map<DocumentId, Heads>();
|
|
136
|
+
private _onHeadsChangedTask?: DeferredTask | undefined;
|
|
137
|
+
|
|
113
138
|
constructor({
|
|
114
139
|
db,
|
|
115
140
|
indexMetadataStore,
|
|
@@ -143,6 +168,12 @@ export class AutomergeHost extends Resource {
|
|
|
143
168
|
protected override async _open(): Promise<void> {
|
|
144
169
|
this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}` as PeerId;
|
|
145
170
|
|
|
171
|
+
this._onHeadsChangedTask = new DeferredTask(this._ctx, async () => {
|
|
172
|
+
const docHeads = Array.from(this._headsUpdates.entries());
|
|
173
|
+
this._headsUpdates.clear();
|
|
174
|
+
this._onHeadsChanged(docHeads);
|
|
175
|
+
});
|
|
176
|
+
|
|
146
177
|
await this._storage.open?.();
|
|
147
178
|
|
|
148
179
|
// Construct the automerge repo.
|
|
@@ -180,6 +211,22 @@ export class AutomergeHost extends Resource {
|
|
|
180
211
|
}
|
|
181
212
|
});
|
|
182
213
|
|
|
214
|
+
this._syncTask = new DeferredTask(this._ctx, async () => {
|
|
215
|
+
const collectionToSync = Array.from(this._collectionsToSync.values());
|
|
216
|
+
if (collectionToSync.length === 0) {
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
await Promise.all(
|
|
220
|
+
collectionToSync.map(async ({ collectionId, peerId }) => {
|
|
221
|
+
try {
|
|
222
|
+
await this._handleCollectionSync(collectionId, peerId);
|
|
223
|
+
} catch (err) {
|
|
224
|
+
log.error('failed to sync collection', { collectionId, peerId, err });
|
|
225
|
+
}
|
|
226
|
+
}),
|
|
227
|
+
);
|
|
228
|
+
});
|
|
229
|
+
|
|
183
230
|
await this._echoNetworkAdapter.open();
|
|
184
231
|
await this._collectionSynchronizer.open();
|
|
185
232
|
await this._echoNetworkAdapter.open();
|
|
@@ -190,7 +237,8 @@ export class AutomergeHost extends Resource {
|
|
|
190
237
|
await this._collectionSynchronizer.close();
|
|
191
238
|
await this._storage.close?.();
|
|
192
239
|
await this._echoNetworkAdapter.close();
|
|
193
|
-
|
|
240
|
+
this._syncTask = undefined;
|
|
241
|
+
this._onHeadsChangedTask = undefined;
|
|
194
242
|
}
|
|
195
243
|
|
|
196
244
|
/**
|
|
@@ -209,10 +257,12 @@ export class AutomergeHost extends Resource {
|
|
|
209
257
|
}
|
|
210
258
|
|
|
211
259
|
async addReplicator(replicator: EchoReplicator): Promise<void> {
|
|
260
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
212
261
|
await this._echoNetworkAdapter.addReplicator(replicator);
|
|
213
262
|
}
|
|
214
263
|
|
|
215
264
|
async removeReplicator(replicator: EchoReplicator): Promise<void> {
|
|
265
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
216
266
|
await this._echoNetworkAdapter.removeReplicator(replicator);
|
|
217
267
|
}
|
|
218
268
|
|
|
@@ -220,6 +270,7 @@ export class AutomergeHost extends Resource {
|
|
|
220
270
|
* Loads the document handle from the repo and waits for it to be ready.
|
|
221
271
|
*/
|
|
222
272
|
async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T>> {
|
|
273
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
223
274
|
let handle: DocHandle<T> | undefined;
|
|
224
275
|
if (typeof documentId === 'string') {
|
|
225
276
|
// NOTE: documentId might also be a URL, in which case this lookup will fail.
|
|
@@ -242,6 +293,7 @@ export class AutomergeHost extends Resource {
|
|
|
242
293
|
}
|
|
243
294
|
|
|
244
295
|
async exportDoc(ctx: Context, id: AnyDocumentId): Promise<Uint8Array> {
|
|
296
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
245
297
|
const documentId = interpretAsDocumentId(id);
|
|
246
298
|
|
|
247
299
|
const chunks = await this._storage.loadRange([documentId]);
|
|
@@ -252,6 +304,7 @@ export class AutomergeHost extends Resource {
|
|
|
252
304
|
* Create new persisted document.
|
|
253
305
|
*/
|
|
254
306
|
createDoc<T>(initialValue?: T | Doc<T> | Uint8Array, opts?: CreateDocOptions): DocHandle<T> {
|
|
307
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
255
308
|
if (opts?.preserveHistory) {
|
|
256
309
|
if (initialValue instanceof Uint8Array) {
|
|
257
310
|
return this._repo.import(initialValue);
|
|
@@ -273,6 +326,7 @@ export class AutomergeHost extends Resource {
|
|
|
273
326
|
}
|
|
274
327
|
|
|
275
328
|
async waitUntilHeadsReplicated(heads: DocHeadsList): Promise<void> {
|
|
329
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
276
330
|
const entries = heads.entries;
|
|
277
331
|
if (!entries?.length) {
|
|
278
332
|
return;
|
|
@@ -303,6 +357,7 @@ export class AutomergeHost extends Resource {
|
|
|
303
357
|
}
|
|
304
358
|
|
|
305
359
|
async reIndexHeads(documentIds: DocumentId[]): Promise<void> {
|
|
360
|
+
invariant(this.isOpen, 'AutomergeHost is not open');
|
|
306
361
|
for (const documentId of documentIds) {
|
|
307
362
|
log('re-indexing heads for document', { documentId });
|
|
308
363
|
const handle = await this._repo.find(documentId, FIND_PARAMS);
|
|
@@ -375,14 +430,25 @@ export class AutomergeHost extends Resource {
|
|
|
375
430
|
* Called by AutomergeStorageAdapter after levelDB batch commit.
|
|
376
431
|
*/
|
|
377
432
|
private async _afterSave(path: StorageKey): Promise<void> {
|
|
378
|
-
this.
|
|
433
|
+
if (!this.isOpen) {
|
|
434
|
+
return undefined;
|
|
435
|
+
}
|
|
379
436
|
|
|
437
|
+
this._indexMetadataStore.notifyMarkedDirty();
|
|
380
438
|
const documentId = path[0] as DocumentId;
|
|
381
|
-
const
|
|
382
|
-
if (
|
|
383
|
-
|
|
384
|
-
|
|
439
|
+
const handle = this._repo.handles[documentId];
|
|
440
|
+
if (!handle || !handle.isReady()) {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
const document = handle.doc();
|
|
444
|
+
if (!document) {
|
|
445
|
+
return;
|
|
385
446
|
}
|
|
447
|
+
|
|
448
|
+
const heads = getHeads(document);
|
|
449
|
+
this._headsUpdates.set(documentId, heads);
|
|
450
|
+
invariant(this._onHeadsChangedTask, 'onHeadsChangedTask is not initialized');
|
|
451
|
+
this._onHeadsChangedTask.schedule();
|
|
386
452
|
this.documentsSaved.emit();
|
|
387
453
|
}
|
|
388
454
|
|
|
@@ -479,8 +545,8 @@ export class AutomergeHost extends Resource {
|
|
|
479
545
|
this._collectionSynchronizer.refreshCollection(collectionId);
|
|
480
546
|
}
|
|
481
547
|
|
|
482
|
-
async getCollectionSyncState(collectionId: string): Promise<
|
|
483
|
-
const result:
|
|
548
|
+
async getCollectionSyncState(collectionId: string): Promise<SpaceSyncState> {
|
|
549
|
+
const result: SpaceSyncState = {
|
|
484
550
|
peers: [],
|
|
485
551
|
};
|
|
486
552
|
|
|
@@ -493,13 +559,16 @@ export class AutomergeHost extends Resource {
|
|
|
493
559
|
|
|
494
560
|
for (const [peerId, state] of remoteState) {
|
|
495
561
|
const diff = diffCollectionState(localState, state);
|
|
496
|
-
result.peers
|
|
562
|
+
result.peers!.push({
|
|
497
563
|
peerId,
|
|
498
564
|
missingOnRemote: diff.missingOnRemote.length,
|
|
499
565
|
missingOnLocal: diff.missingOnLocal.length,
|
|
500
566
|
differentDocuments: diff.different.length,
|
|
501
|
-
localDocumentCount: Object.
|
|
502
|
-
remoteDocumentCount: Object.
|
|
567
|
+
localDocumentCount: Object.entries(localState.documents).filter(([_, heads]) => heads.length > 0).length,
|
|
568
|
+
remoteDocumentCount: Object.entries(state.documents).filter(([_, heads]) => heads.length > 0).length,
|
|
569
|
+
|
|
570
|
+
totalDocumentCount: new Set([...Object.keys(localState.documents), ...Object.keys(state.documents)]).size,
|
|
571
|
+
unsyncedDocumentCount: new Set([...diff.missingOnLocal, ...diff.missingOnRemote, ...diff.different]).size,
|
|
503
572
|
});
|
|
504
573
|
}
|
|
505
574
|
|
|
@@ -515,6 +584,16 @@ export class AutomergeHost extends Resource {
|
|
|
515
584
|
heads.map((heads, index) => [documentIds[index], heads ?? []]),
|
|
516
585
|
);
|
|
517
586
|
this._collectionSynchronizer.setLocalCollectionState(collectionId, { documents });
|
|
587
|
+
|
|
588
|
+
// Proactively push our updated local state to peers that are interested in this collection.
|
|
589
|
+
// This reduces reliance on the next periodic query and prevents replication stalls in fast paths
|
|
590
|
+
// where the remote queries before our local state is ready.
|
|
591
|
+
const interestedPeers = this._echoNetworkAdapter.getPeersInterestedInCollection(collectionId);
|
|
592
|
+
if (interestedPeers.length > 0) {
|
|
593
|
+
for (const peerId of interestedPeers) {
|
|
594
|
+
this._sendCollectionState(collectionId, peerId, { documents });
|
|
595
|
+
}
|
|
596
|
+
}
|
|
518
597
|
}
|
|
519
598
|
|
|
520
599
|
async clearLocalCollectionState(collectionId: string): Promise<void> {
|
|
@@ -546,6 +625,11 @@ export class AutomergeHost extends Resource {
|
|
|
546
625
|
}
|
|
547
626
|
|
|
548
627
|
private _onRemoteCollectionStateUpdated(collectionId: string, peerId: PeerId): void {
|
|
628
|
+
this._collectionsToSync.add({ collectionId, peerId });
|
|
629
|
+
this._syncTask?.schedule();
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
private async _handleCollectionSync(collectionId: string, peerId: PeerId) {
|
|
549
633
|
const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
550
634
|
const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
|
|
551
635
|
|
|
@@ -554,36 +638,170 @@ export class AutomergeHost extends Resource {
|
|
|
554
638
|
}
|
|
555
639
|
|
|
556
640
|
const { different, missingOnLocal, missingOnRemote } = diffCollectionState(localState, remoteState);
|
|
557
|
-
const toReplicate = [...missingOnLocal, ...missingOnRemote, ...different];
|
|
558
641
|
|
|
559
|
-
if (
|
|
642
|
+
if (different.length === 0 && missingOnLocal.length === 0 && missingOnRemote.length === 0) {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
const toReplicateWithoutBatching = [...different];
|
|
647
|
+
const bundleSyncEnabled = this._echoNetworkAdapter.bundleSyncEnabledForPeer(peerId);
|
|
648
|
+
if (bundleSyncEnabled && missingOnRemote.length >= BUNDLE_SYNC_THRESHOLD) {
|
|
649
|
+
log('pushing bundle', { amount: missingOnRemote.length });
|
|
650
|
+
const { syncInteractively } = await this._pushInBundles(peerId, missingOnRemote);
|
|
651
|
+
toReplicateWithoutBatching.push(...syncInteractively);
|
|
652
|
+
} else {
|
|
653
|
+
log.verbose('failed to push bundle, replicating interactively', {
|
|
654
|
+
collectionId,
|
|
655
|
+
peerId,
|
|
656
|
+
amount: missingOnRemote.length,
|
|
657
|
+
});
|
|
658
|
+
toReplicateWithoutBatching.push(...missingOnRemote);
|
|
659
|
+
}
|
|
660
|
+
if (bundleSyncEnabled && missingOnLocal.length >= BUNDLE_SYNC_THRESHOLD) {
|
|
661
|
+
log('pulling bundle', { amount: missingOnLocal.length });
|
|
662
|
+
const { syncInteractively } = await this._pullInBundles(peerId, missingOnLocal);
|
|
663
|
+
toReplicateWithoutBatching.push(...syncInteractively);
|
|
664
|
+
} else {
|
|
665
|
+
log.verbose('failed to pull bundle, replicating interactively', {
|
|
666
|
+
collectionId,
|
|
667
|
+
peerId,
|
|
668
|
+
amount: missingOnLocal.length,
|
|
669
|
+
});
|
|
670
|
+
toReplicateWithoutBatching.push(...missingOnLocal);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
if (toReplicateWithoutBatching.length === 0) {
|
|
560
674
|
return;
|
|
561
675
|
}
|
|
562
676
|
|
|
563
677
|
log('replicating documents after collection sync', {
|
|
564
678
|
collectionId,
|
|
565
679
|
peerId,
|
|
566
|
-
|
|
567
|
-
count:
|
|
680
|
+
toReplicateWithoutBatching,
|
|
681
|
+
count: toReplicateWithoutBatching.length,
|
|
568
682
|
});
|
|
569
683
|
|
|
570
684
|
// Load the documents so they will start syncing.
|
|
571
|
-
for (const documentId of
|
|
685
|
+
for (const documentId of toReplicateWithoutBatching) {
|
|
572
686
|
this._repo.findWithProgress(documentId);
|
|
573
687
|
}
|
|
574
688
|
}
|
|
575
689
|
|
|
576
|
-
|
|
690
|
+
// TODO(mykola): Add retries of batches https://gist.github.com/mykola-vrmchk/fde270259e9209fcbf1331e5abbf12cf
|
|
691
|
+
// TODO(mykola): Use effect to retry batches.
|
|
692
|
+
private async _pushInBundles(
|
|
693
|
+
peerId: PeerId,
|
|
694
|
+
documentIds: DocumentId[],
|
|
695
|
+
): Promise<{ syncInteractively: DocumentId[] }> {
|
|
696
|
+
const documentsToPush = [...documentIds];
|
|
697
|
+
const syncInteractively: DocumentId[] = [];
|
|
698
|
+
|
|
699
|
+
// Push bundles in parallel with BUNDLE_SYNC_CONCURRENCY max concurrent tasks.
|
|
700
|
+
while (documentsToPush.length > 0) {
|
|
701
|
+
await Promise.all(
|
|
702
|
+
range(BUNDLE_SYNC_CONCURRENCY).map(async () => {
|
|
703
|
+
const bundle = documentsToPush.splice(0, BUNDLE_SIZE);
|
|
704
|
+
if (bundle.length === 0) {
|
|
705
|
+
return;
|
|
706
|
+
}
|
|
707
|
+
await this._pushBundle(peerId, bundle).catch((err) => {
|
|
708
|
+
log.warn('failed to push bundle, replicating interactively', { peerId, bundle, err });
|
|
709
|
+
syncInteractively.push(...bundle);
|
|
710
|
+
});
|
|
711
|
+
}),
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
return { syncInteractively };
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
private async _pushBundle(peerId: PeerId, documentIds: DocumentId[]): Promise<void> {
|
|
719
|
+
if (this._ctx.disposed) {
|
|
720
|
+
return;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
const handles = documentIds.map((documentId) => this._repo.handles[documentId]);
|
|
724
|
+
const bundle = exportBundle(this._repo, handles);
|
|
725
|
+
await this._echoNetworkAdapter.pushBundle(
|
|
726
|
+
peerId,
|
|
727
|
+
Array.from(bundle.docs.entries()).map(([documentId, doc]) => ({
|
|
728
|
+
documentId,
|
|
729
|
+
data: doc.data,
|
|
730
|
+
heads: doc.heads,
|
|
731
|
+
})),
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
private async _pullInBundles(
|
|
736
|
+
peerId: PeerId,
|
|
737
|
+
documentIds: DocumentId[],
|
|
738
|
+
): Promise<{ syncInteractively: DocumentId[] }> {
|
|
739
|
+
const documentsToPull = [...documentIds];
|
|
740
|
+
const syncInteractively: DocumentId[] = [];
|
|
741
|
+
const docsToImport: Record<DocumentId, Uint8Array> = {};
|
|
742
|
+
|
|
743
|
+
// Pull bundles in parallel with BUNDLE_SYNC_CONCURRENCY max concurrent tasks.
|
|
744
|
+
while (documentsToPull.length > 0) {
|
|
745
|
+
await Promise.all(
|
|
746
|
+
range(BUNDLE_SYNC_CONCURRENCY).map(async () => {
|
|
747
|
+
const bundle = documentsToPull.splice(0, BUNDLE_SIZE);
|
|
748
|
+
if (bundle.length === 0) {
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
const result = await this._pullBundle(peerId, bundle).catch((err) => {
|
|
752
|
+
log.warn('failed to pull bundle, replicating interactively', { peerId, bundle, err });
|
|
753
|
+
syncInteractively.push(...bundle);
|
|
754
|
+
});
|
|
755
|
+
if (result) {
|
|
756
|
+
Object.assign(docsToImport, result.docsToImport);
|
|
757
|
+
}
|
|
758
|
+
}),
|
|
759
|
+
);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
for (const [documentId, data] of Object.entries(docsToImport)) {
|
|
763
|
+
this._repo.import(data, { docId: documentId as DocumentId });
|
|
764
|
+
}
|
|
765
|
+
await this._repo.flush(Object.keys(docsToImport) as DocumentId[]);
|
|
766
|
+
|
|
767
|
+
return { syncInteractively };
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
private async _pullBundle(
|
|
771
|
+
peerId: PeerId,
|
|
772
|
+
documentIds: DocumentId[],
|
|
773
|
+
): Promise<{ docsToImport: Record<DocumentId, Uint8Array> } | undefined> {
|
|
774
|
+
if (this._ctx.disposed) {
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
// NOTE: We are expecting that documents that are being pulled are not present locally, so we are pulling all changes.
|
|
778
|
+
const docHeads = Object.fromEntries(documentIds.map((documentId) => [documentId, []]));
|
|
779
|
+
const bundle = await this._echoNetworkAdapter.pullBundle(peerId, docHeads);
|
|
780
|
+
return { docsToImport: bundle };
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
private _onHeadsChanged(docHeads: [DocumentId, Heads][]): void {
|
|
577
784
|
const collectionsChanged = new Set<CollectionId>();
|
|
785
|
+
|
|
578
786
|
for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
|
|
579
787
|
const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
788
|
+
let newState: CollectionState | undefined;
|
|
789
|
+
|
|
790
|
+
for (const [documentId, heads] of docHeads) {
|
|
791
|
+
if (state?.documents[documentId]) {
|
|
792
|
+
if (!newState) {
|
|
793
|
+
newState = structuredClone(state);
|
|
794
|
+
}
|
|
795
|
+
newState.documents[documentId] = heads;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
if (newState) {
|
|
583
800
|
this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
|
|
584
801
|
collectionsChanged.add(collectionId as CollectionId);
|
|
585
802
|
}
|
|
586
803
|
}
|
|
804
|
+
|
|
587
805
|
for (const collectionId of collectionsChanged) {
|
|
588
806
|
this.collectionStateUpdated.emit({ collectionId });
|
|
589
807
|
}
|
|
@@ -619,35 +837,3 @@ const decodeCollectionState = (state: unknown): CollectionState => {
|
|
|
619
837
|
const encodeCollectionState = (state: CollectionState): unknown => {
|
|
620
838
|
return state;
|
|
621
839
|
};
|
|
622
|
-
|
|
623
|
-
export type CollectionSyncState = {
|
|
624
|
-
peers: PeerSyncState[];
|
|
625
|
-
};
|
|
626
|
-
|
|
627
|
-
export type PeerSyncState = {
|
|
628
|
-
peerId: PeerId;
|
|
629
|
-
/**
|
|
630
|
-
* Documents that are present locally but not on the remote peer.
|
|
631
|
-
*/
|
|
632
|
-
missingOnRemote: number;
|
|
633
|
-
|
|
634
|
-
/**
|
|
635
|
-
* Documents that are present on the remote peer but not locally.
|
|
636
|
-
*/
|
|
637
|
-
missingOnLocal: number;
|
|
638
|
-
|
|
639
|
-
/**
|
|
640
|
-
* Documents that are present on both peers but have different heads.
|
|
641
|
-
*/
|
|
642
|
-
differentDocuments: number;
|
|
643
|
-
|
|
644
|
-
/**
|
|
645
|
-
* Total number of documents locally.
|
|
646
|
-
*/
|
|
647
|
-
localDocumentCount: number;
|
|
648
|
-
|
|
649
|
-
/**
|
|
650
|
-
* Total number of documents on the remote peer.
|
|
651
|
-
*/
|
|
652
|
-
remoteDocumentCount: number;
|
|
653
|
-
};
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
|
+
next as A,
|
|
6
7
|
type Heads,
|
|
7
8
|
change,
|
|
8
9
|
clone,
|
|
@@ -10,7 +11,6 @@ import {
|
|
|
10
11
|
from,
|
|
11
12
|
getBackend,
|
|
12
13
|
getHeads,
|
|
13
|
-
next as A,
|
|
14
14
|
save,
|
|
15
15
|
saveSince,
|
|
16
16
|
} from '@automerge/automerge';
|
|
@@ -19,14 +19,14 @@ import {
|
|
|
19
19
|
type DocHandle,
|
|
20
20
|
type DocumentId,
|
|
21
21
|
type HandleState,
|
|
22
|
-
type StorageAdapterInterface,
|
|
23
22
|
type PeerId,
|
|
24
23
|
Repo,
|
|
25
24
|
type SharePolicy,
|
|
25
|
+
type StorageAdapterInterface,
|
|
26
26
|
generateAutomergeUrl,
|
|
27
27
|
parseAutomergeUrl,
|
|
28
28
|
} from '@automerge/automerge-repo';
|
|
29
|
-
import {
|
|
29
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
30
30
|
|
|
31
31
|
import { asyncTimeout, sleep } from '@dxos/async';
|
|
32
32
|
import { randomBytes } from '@dxos/crypto';
|
|
@@ -36,11 +36,12 @@ import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/
|
|
|
36
36
|
import { openAndClose } from '@dxos/test-utils';
|
|
37
37
|
import { isNonNullable, range } from '@dxos/util';
|
|
38
38
|
|
|
39
|
+
import { TestAdapter, type TestConnectionStateProvider } from '../testing';
|
|
40
|
+
|
|
39
41
|
import { FIND_PARAMS } from './automerge-host';
|
|
40
42
|
import { EchoNetworkAdapter } from './echo-network-adapter';
|
|
41
43
|
import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
|
|
42
44
|
import { MeshEchoReplicator } from './mesh-echo-replicator';
|
|
43
|
-
import { TestAdapter, type TestConnectionStateProvider } from '../testing';
|
|
44
45
|
|
|
45
46
|
const HOST_AND_CLIENT: [string, string] = ['host', 'client'];
|
|
46
47
|
|
|
@@ -420,13 +421,7 @@ describe('AutomergeRepo', () => {
|
|
|
420
421
|
|
|
421
422
|
test('client creates doc and syncs with a Repo', async () => {
|
|
422
423
|
const repo = new Repo({ network: [] });
|
|
423
|
-
const receiveByServer =
|
|
424
|
-
const serverHandle = await repo.find(docId, FIND_PARAMS);
|
|
425
|
-
serverHandle.update((doc) => {
|
|
426
|
-
return A.loadIncremental(doc, blob);
|
|
427
|
-
});
|
|
428
|
-
};
|
|
429
|
-
|
|
424
|
+
const receiveByServer = (blob: Uint8Array, docId: DocumentId) => repo.import<any>(blob, { docId });
|
|
430
425
|
let clientDoc = A.from<{ field?: string }>({});
|
|
431
426
|
const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
|
|
432
427
|
// Sync handshake.
|
|
@@ -434,7 +429,7 @@ describe('AutomergeRepo', () => {
|
|
|
434
429
|
|
|
435
430
|
// Sync protocol.
|
|
436
431
|
const sendDoc = async (doc: A.Doc<any>) => {
|
|
437
|
-
|
|
432
|
+
receiveByServer(saveSince(doc, sentHeads), documentId);
|
|
438
433
|
sentHeads = getHeads(doc);
|
|
439
434
|
};
|
|
440
435
|
|
|
@@ -456,10 +451,9 @@ describe('AutomergeRepo', () => {
|
|
|
456
451
|
|
|
457
452
|
const repo = new Repo({ network: [], storage });
|
|
458
453
|
const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
});
|
|
454
|
+
repo.import<any>(blob, { docId });
|
|
455
|
+
// TODO(mykola): This should not be required. Document is not persisted without it.
|
|
456
|
+
await repo.flush([docId]);
|
|
463
457
|
};
|
|
464
458
|
|
|
465
459
|
let clientDoc = A.from<{ field?: string }>({ field: 'foo' });
|
|
@@ -677,6 +671,63 @@ describe('AutomergeRepo', () => {
|
|
|
677
671
|
await doc.whenReady();
|
|
678
672
|
expect(doc.doc()).to.deep.eq(document.doc());
|
|
679
673
|
});
|
|
674
|
+
|
|
675
|
+
test('document is passively replicated to connected peers', async () => {
|
|
676
|
+
const [spaceKey] = PublicKey.randomSequence();
|
|
677
|
+
|
|
678
|
+
const teleportBuilder = new TeleportBuilder();
|
|
679
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
680
|
+
|
|
681
|
+
const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
682
|
+
const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
683
|
+
|
|
684
|
+
const handle = peer1.repo.create();
|
|
685
|
+
handle.change((doc: any) => (doc.text = 'hello'));
|
|
686
|
+
await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
|
|
687
|
+
|
|
688
|
+
await expect
|
|
689
|
+
.poll(async () => {
|
|
690
|
+
const doc = peer2.repo.handles[handle.documentId];
|
|
691
|
+
await doc.whenReady();
|
|
692
|
+
return doc.doc()!.text;
|
|
693
|
+
})
|
|
694
|
+
.toEqual('hello');
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
test('imported document is passively replicated to connected peers', async () => {
|
|
698
|
+
let blob: Uint8Array;
|
|
699
|
+
let documentId: DocumentId;
|
|
700
|
+
{
|
|
701
|
+
const repo = new Repo();
|
|
702
|
+
const handle = repo.create();
|
|
703
|
+
handle.change((doc: any) => (doc.text = 'hello'));
|
|
704
|
+
blob = A.save(handle.doc()!);
|
|
705
|
+
documentId = handle.documentId;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
const [spaceKey] = PublicKey.randomSequence();
|
|
709
|
+
|
|
710
|
+
const teleportBuilder = new TeleportBuilder();
|
|
711
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
712
|
+
|
|
713
|
+
const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
714
|
+
const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
715
|
+
await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
|
|
716
|
+
|
|
717
|
+
const handle = peer1.repo.import(blob, { docId: documentId });
|
|
718
|
+
await handle.whenReady();
|
|
719
|
+
|
|
720
|
+
await expect
|
|
721
|
+
.poll(
|
|
722
|
+
async () => {
|
|
723
|
+
const doc = peer2.repo.handles[handle.documentId];
|
|
724
|
+
await doc.whenReady();
|
|
725
|
+
return doc.doc()!.text;
|
|
726
|
+
},
|
|
727
|
+
{ timeout: 1_000 },
|
|
728
|
+
)
|
|
729
|
+
.toEqual('hello');
|
|
730
|
+
});
|
|
680
731
|
});
|
|
681
732
|
|
|
682
733
|
const createLevelAdapter = async () => {
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import type { PeerId } from '@automerge/automerge-repo';
|
|
6
|
-
import {
|
|
6
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
7
7
|
|
|
8
8
|
import { sleep } from '@dxos/async';
|
|
9
9
|
|
|
10
|
-
import { CollectionSynchronizer, diffCollectionState
|
|
10
|
+
import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
|
|
11
11
|
|
|
12
12
|
describe('CollectionSynchronizer', () => {
|
|
13
13
|
test('sync two peers', async () => {
|