@dxos/echo-pipeline 0.6.12 → 0.6.13-main.548ca8d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/chunk-PESZVYAN.mjs +2050 -0
- package/dist/lib/browser/chunk-PESZVYAN.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +3463 -17
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +3 -4
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/{chunk-7HHYCGUR.cjs → chunk-6EZVIJNE.cjs} +89 -47
- package/dist/lib/node/chunk-6EZVIJNE.cjs.map +7 -0
- package/dist/lib/node/index.cjs +3440 -35
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +11 -12
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/lib/{browser/chunk-UKXIJW43.mjs → node-esm/chunk-4LW7MDPZ.mjs} +76 -36
- package/dist/lib/node-esm/chunk-4LW7MDPZ.mjs.map +7 -0
- package/dist/lib/{browser/chunk-MPWFDDQK.mjs → node-esm/index.mjs} +1702 -335
- package/dist/lib/node-esm/index.mjs.map +7 -0
- package/dist/lib/node-esm/meta.json +1 -0
- package/dist/lib/node-esm/testing/index.mjs +551 -0
- package/dist/lib/node-esm/testing/index.mjs.map +7 -0
- package/dist/types/src/automerge/automerge-host.d.ts +24 -1
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +2 -0
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts +3 -3
- package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +3 -3
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/space-collection.d.ts +3 -2
- package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
- package/dist/types/src/db-host/automerge-metrics.d.ts +11 -0
- package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -0
- package/dist/types/src/db-host/data-service.d.ts +3 -2
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-root.d.ts +20 -0
- package/dist/types/src/db-host/database-root.d.ts.map +1 -0
- package/dist/types/src/db-host/documents-iterator.d.ts +7 -0
- package/dist/types/src/db-host/documents-iterator.d.ts.map +1 -0
- package/dist/types/src/db-host/echo-host.d.ts +73 -0
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -0
- package/dist/types/src/db-host/index.d.ts +5 -0
- package/dist/types/src/db-host/index.d.ts.map +1 -1
- package/dist/types/src/db-host/migration.d.ts +8 -0
- package/dist/types/src/db-host/migration.d.ts.map +1 -0
- package/dist/types/src/db-host/query-service.d.ts +25 -0
- package/dist/types/src/db-host/query-service.d.ts.map +1 -0
- package/dist/types/src/db-host/query-state.d.ts +41 -0
- package/dist/types/src/db-host/query-state.d.ts.map +1 -0
- package/dist/types/src/db-host/space-state-manager.d.ts +23 -0
- package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -0
- package/dist/types/src/edge/echo-edge-replicator.d.ts +23 -0
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -0
- package/dist/types/src/edge/echo-edge-replicator.test.d.ts +2 -0
- package/dist/types/src/edge/echo-edge-replicator.test.d.ts.map +1 -0
- package/dist/types/src/edge/index.d.ts +2 -0
- package/dist/types/src/edge/index.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +1 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +4 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/test-replicator.d.ts +4 -4
- package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
- package/package.json +40 -50
- package/src/automerge/automerge-host.test.ts +8 -9
- package/src/automerge/automerge-host.ts +46 -7
- package/src/automerge/automerge-repo.test.ts +18 -16
- package/src/automerge/collection-synchronizer.test.ts +10 -5
- package/src/automerge/collection-synchronizer.ts +17 -6
- package/src/automerge/echo-data-monitor.test.ts +1 -3
- package/src/automerge/echo-network-adapter.test.ts +4 -3
- package/src/automerge/echo-network-adapter.ts +5 -4
- package/src/automerge/echo-replicator.ts +3 -3
- package/src/automerge/mesh-echo-replicator-connection.ts +10 -9
- package/src/automerge/mesh-echo-replicator.ts +2 -1
- package/src/automerge/space-collection.ts +3 -2
- package/src/automerge/storage-adapter.test.ts +2 -3
- package/src/db-host/automerge-metrics.ts +38 -0
- package/src/db-host/data-service.ts +29 -14
- package/src/db-host/database-root.ts +86 -0
- package/src/db-host/documents-iterator.ts +73 -0
- package/src/db-host/documents-synchronizer.test.ts +2 -2
- package/src/db-host/echo-host.ts +257 -0
- package/src/db-host/index.ts +6 -1
- package/src/db-host/migration.ts +57 -0
- package/src/db-host/query-service.ts +208 -0
- package/src/db-host/query-state.ts +200 -0
- package/src/db-host/space-state-manager.ts +90 -0
- package/src/edge/echo-edge-replicator.test.ts +96 -0
- package/src/edge/echo-edge-replicator.ts +337 -0
- package/src/edge/index.ts +5 -0
- package/src/index.ts +1 -0
- package/src/metadata/metadata-store.ts +20 -0
- package/src/pipeline/pipeline-stress.test.ts +44 -47
- package/src/pipeline/pipeline.test.ts +3 -4
- package/src/space/control-pipeline.test.ts +2 -3
- package/src/space/control-pipeline.ts +10 -1
- package/src/space/replication.browser.test.ts +2 -8
- package/src/space/space-manager.browser.test.ts +6 -5
- package/src/space/space-protocol.browser.test.ts +29 -34
- package/src/space/space-protocol.test.ts +29 -27
- package/src/space/space.test.ts +28 -11
- package/src/testing/test-agent-builder.ts +2 -2
- package/src/testing/test-replicator.ts +3 -3
- package/dist/lib/browser/chunk-MPWFDDQK.mjs.map +0 -7
- package/dist/lib/browser/chunk-UKXIJW43.mjs.map +0 -7
- package/dist/lib/browser/chunk-XPCF2V5U.mjs +0 -31
- package/dist/lib/browser/chunk-XPCF2V5U.mjs.map +0 -7
- package/dist/lib/browser/light.mjs +0 -32
- package/dist/lib/browser/light.mjs.map +0 -7
- package/dist/lib/node/chunk-5DH4KR2S.cjs +0 -2148
- package/dist/lib/node/chunk-5DH4KR2S.cjs.map +0 -7
- package/dist/lib/node/chunk-7HHYCGUR.cjs.map +0 -7
- package/dist/lib/node/chunk-DZVH7HDD.cjs +0 -43
- package/dist/lib/node/chunk-DZVH7HDD.cjs.map +0 -7
- package/dist/lib/node/light.cjs +0 -52
- package/dist/lib/node/light.cjs.map +0 -7
- package/dist/types/src/light.d.ts +0 -4
- package/dist/types/src/light.d.ts.map +0 -1
- package/src/light.ts +0 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/echo-pipeline",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.13-main.548ca8d",
|
|
4
4
|
"description": "ECHO database.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -10,21 +10,16 @@
|
|
|
10
10
|
".": {
|
|
11
11
|
"browser": "./dist/lib/browser/index.mjs",
|
|
12
12
|
"node": {
|
|
13
|
-
"
|
|
13
|
+
"require": "./dist/lib/node/index.cjs",
|
|
14
|
+
"default": "./dist/lib/node-esm/index.mjs"
|
|
14
15
|
},
|
|
15
16
|
"types": "./dist/types/src/index.d.ts"
|
|
16
17
|
},
|
|
17
|
-
"./light": {
|
|
18
|
-
"browser": "./dist/lib/browser/light.mjs",
|
|
19
|
-
"node": {
|
|
20
|
-
"default": "./dist/lib/node/light.cjs"
|
|
21
|
-
},
|
|
22
|
-
"types": "./dist/types/src/light.d.ts"
|
|
23
|
-
},
|
|
24
18
|
"./testing": {
|
|
25
19
|
"browser": "./dist/lib/browser/testing/index.mjs",
|
|
26
20
|
"node": {
|
|
27
|
-
"
|
|
21
|
+
"require": "./dist/lib/node/testing/index.cjs",
|
|
22
|
+
"default": "./dist/lib/node-esm/testing/index.mjs"
|
|
28
23
|
},
|
|
29
24
|
"types": "./dist/types/src/testing/index.d.ts"
|
|
30
25
|
}
|
|
@@ -32,9 +27,6 @@
|
|
|
32
27
|
"types": "dist/types/src/index.d.ts",
|
|
33
28
|
"typesVersions": {
|
|
34
29
|
"*": {
|
|
35
|
-
"light": [
|
|
36
|
-
"dist/types/src/light.d.ts"
|
|
37
|
-
],
|
|
38
30
|
"testing": [
|
|
39
31
|
"dist/types/src/testing/index.d.ts"
|
|
40
32
|
]
|
|
@@ -47,48 +39,46 @@
|
|
|
47
39
|
"src"
|
|
48
40
|
],
|
|
49
41
|
"dependencies": {
|
|
50
|
-
"abstract-level": "^1.0.2",
|
|
51
42
|
"crc-32": "^1.2.2",
|
|
52
|
-
"level": "^8.0.1",
|
|
53
43
|
"level-transcoder": "^1.0.1",
|
|
54
|
-
"
|
|
55
|
-
"@dxos/
|
|
56
|
-
"@dxos/codec-protobuf": "0.6.
|
|
57
|
-
"@dxos/credentials": "0.6.
|
|
58
|
-
"@dxos/context": "0.6.
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/crypto": "0.6.
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/
|
|
63
|
-
"@dxos/echo-schema": "0.6.
|
|
64
|
-
"@dxos/
|
|
65
|
-
"@dxos/
|
|
66
|
-
"@dxos/
|
|
67
|
-
"@dxos/
|
|
68
|
-
"@dxos/
|
|
69
|
-
"@dxos/
|
|
70
|
-
"@dxos/
|
|
71
|
-
"@dxos/
|
|
72
|
-
"@dxos/
|
|
73
|
-
"@dxos/
|
|
74
|
-
"@dxos/
|
|
75
|
-
"@dxos/
|
|
76
|
-
"@dxos/
|
|
77
|
-
"@dxos/teleport": "0.6.
|
|
78
|
-
"@dxos/
|
|
79
|
-
"@dxos/teleport-extension-
|
|
80
|
-
"@dxos/teleport-extension-automerge-replicator": "0.6.
|
|
81
|
-
"@dxos/teleport-extension-
|
|
82
|
-
"@dxos/
|
|
83
|
-
"@dxos/
|
|
84
|
-
"@dxos/
|
|
85
|
-
"@dxos/
|
|
44
|
+
"lodash.isequal": "^4.5.0",
|
|
45
|
+
"@dxos/async": "0.6.13-main.548ca8d",
|
|
46
|
+
"@dxos/codec-protobuf": "0.6.13-main.548ca8d",
|
|
47
|
+
"@dxos/credentials": "0.6.13-main.548ca8d",
|
|
48
|
+
"@dxos/context": "0.6.13-main.548ca8d",
|
|
49
|
+
"@dxos/automerge": "0.6.13-main.548ca8d",
|
|
50
|
+
"@dxos/crypto": "0.6.13-main.548ca8d",
|
|
51
|
+
"@dxos/debug": "0.6.13-main.548ca8d",
|
|
52
|
+
"@dxos/echo-protocol": "0.6.13-main.548ca8d",
|
|
53
|
+
"@dxos/echo-schema": "0.6.13-main.548ca8d",
|
|
54
|
+
"@dxos/edge-client": "0.6.13-main.548ca8d",
|
|
55
|
+
"@dxos/feed-store": "0.6.13-main.548ca8d",
|
|
56
|
+
"@dxos/hypercore": "0.6.13-main.548ca8d",
|
|
57
|
+
"@dxos/indexing": "0.6.13-main.548ca8d",
|
|
58
|
+
"@dxos/keyring": "0.6.13-main.548ca8d",
|
|
59
|
+
"@dxos/keys": "0.6.13-main.548ca8d",
|
|
60
|
+
"@dxos/invariant": "0.6.13-main.548ca8d",
|
|
61
|
+
"@dxos/kv-store": "0.6.13-main.548ca8d",
|
|
62
|
+
"@dxos/log": "0.6.13-main.548ca8d",
|
|
63
|
+
"@dxos/messaging": "0.6.13-main.548ca8d",
|
|
64
|
+
"@dxos/network-manager": "0.6.13-main.548ca8d",
|
|
65
|
+
"@dxos/protocols": "0.6.13-main.548ca8d",
|
|
66
|
+
"@dxos/random-access-storage": "0.6.13-main.548ca8d",
|
|
67
|
+
"@dxos/teleport": "0.6.13-main.548ca8d",
|
|
68
|
+
"@dxos/node-std": "0.6.13-main.548ca8d",
|
|
69
|
+
"@dxos/teleport-extension-gossip": "0.6.13-main.548ca8d",
|
|
70
|
+
"@dxos/teleport-extension-automerge-replicator": "0.6.13-main.548ca8d",
|
|
71
|
+
"@dxos/teleport-extension-object-sync": "0.6.13-main.548ca8d",
|
|
72
|
+
"@dxos/teleport-extension-replicator": "0.6.13-main.548ca8d",
|
|
73
|
+
"@dxos/timeframe": "0.6.13-main.548ca8d",
|
|
74
|
+
"@dxos/util": "0.6.13-main.548ca8d",
|
|
75
|
+
"@dxos/typings": "0.6.13-main.548ca8d",
|
|
76
|
+
"@dxos/tracing": "0.6.13-main.548ca8d"
|
|
86
77
|
},
|
|
87
78
|
"devDependencies": {
|
|
79
|
+
"@types/lodash.isequal": "^4.5.0",
|
|
88
80
|
"fast-check": "^3.19.0",
|
|
89
|
-
"
|
|
90
|
-
"source-map-support": "^0.5.12",
|
|
91
|
-
"wait-for-expect": "^3.0.2"
|
|
81
|
+
"@dxos/test-utils": "0.6.13-main.548ca8d"
|
|
92
82
|
},
|
|
93
83
|
"publishConfig": {
|
|
94
84
|
"access": "public"
|
|
@@ -2,15 +2,14 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import expect from '
|
|
6
|
-
import waitForExpect from 'wait-for-expect';
|
|
5
|
+
import { onTestFinished, describe, expect, test } from 'vitest';
|
|
7
6
|
|
|
8
7
|
import { getHeads } from '@dxos/automerge/automerge';
|
|
9
8
|
import type { DocumentId, Heads } from '@dxos/automerge/automerge-repo';
|
|
10
9
|
import { IndexMetadataStore } from '@dxos/indexing';
|
|
11
10
|
import type { LevelDB } from '@dxos/kv-store';
|
|
12
11
|
import { createTestLevel } from '@dxos/kv-store/testing';
|
|
13
|
-
import {
|
|
12
|
+
import { openAndClose } from '@dxos/test-utils';
|
|
14
13
|
import { range } from '@dxos/util';
|
|
15
14
|
|
|
16
15
|
import { AutomergeHost } from './automerge-host';
|
|
@@ -114,11 +113,9 @@ describe('AutomergeHost', () => {
|
|
|
114
113
|
await host1.addReplicator(await network.createReplicator());
|
|
115
114
|
await host2.addReplicator(await network.createReplicator());
|
|
116
115
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
});
|
|
116
|
+
for (const documentId of documentIds) {
|
|
117
|
+
await expect.poll(() => host1.getHeads([documentId])).toEqual(await host2.getHeads([documentId]));
|
|
118
|
+
}
|
|
122
119
|
|
|
123
120
|
await host1.close();
|
|
124
121
|
await host2.close();
|
|
@@ -137,6 +134,8 @@ const setupAutomergeHost = async ({ level }: { level: LevelDB }) => {
|
|
|
137
134
|
indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
|
|
138
135
|
});
|
|
139
136
|
await host.open();
|
|
140
|
-
|
|
137
|
+
onTestFinished(async () => {
|
|
138
|
+
await host.close();
|
|
139
|
+
});
|
|
141
140
|
return host;
|
|
142
141
|
};
|
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
type StorageKey,
|
|
26
26
|
} from '@dxos/automerge/automerge-repo';
|
|
27
27
|
import { Context, Resource, cancelWithContext, type Lifecycle } from '@dxos/context';
|
|
28
|
-
import { type SpaceDoc } from '@dxos/echo-protocol';
|
|
28
|
+
import { type CollectionId, type SpaceDoc } from '@dxos/echo-protocol';
|
|
29
29
|
import { type IndexMetadataStore } from '@dxos/indexing';
|
|
30
30
|
import { invariant } from '@dxos/invariant';
|
|
31
31
|
import { PublicKey } from '@dxos/keys';
|
|
@@ -82,6 +82,8 @@ export class AutomergeHost extends Resource {
|
|
|
82
82
|
@trace.info()
|
|
83
83
|
private _peerId!: PeerId;
|
|
84
84
|
|
|
85
|
+
public readonly collectionStateUpdated = new Event<{ collectionId: CollectionId }>();
|
|
86
|
+
|
|
85
87
|
constructor({ db, indexMetadataStore, dataMonitor }: AutomergeHostParams) {
|
|
86
88
|
super();
|
|
87
89
|
this._db = db;
|
|
@@ -128,6 +130,7 @@ export class AutomergeHost extends Resource {
|
|
|
128
130
|
|
|
129
131
|
this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
|
|
130
132
|
this._onRemoteCollectionStateUpdated(collectionId, peerId);
|
|
133
|
+
this.collectionStateUpdated.emit({ collectionId: collectionId as CollectionId });
|
|
131
134
|
});
|
|
132
135
|
|
|
133
136
|
await this._echoNetworkAdapter.open();
|
|
@@ -358,7 +361,10 @@ export class AutomergeHost extends Resource {
|
|
|
358
361
|
async flush({ documentIds }: FlushRequest = {}): Promise<void> {
|
|
359
362
|
// Note: Sync protocol for client and services ensures that all handles should have all changes.
|
|
360
363
|
|
|
361
|
-
|
|
364
|
+
const loadedDocuments = documentIds?.filter(
|
|
365
|
+
(documentId): documentId is DocumentId => !!this._repo.handles[documentId as DocumentId],
|
|
366
|
+
);
|
|
367
|
+
await this._repo.flush(loadedDocuments);
|
|
362
368
|
}
|
|
363
369
|
|
|
364
370
|
async getHeads(documentIds: DocumentId[]): Promise<(Heads | undefined)[]> {
|
|
@@ -416,7 +422,11 @@ export class AutomergeHost extends Resource {
|
|
|
416
422
|
const diff = diffCollectionState(localState, state);
|
|
417
423
|
result.peers.push({
|
|
418
424
|
peerId,
|
|
425
|
+
missingOnRemote: diff.missingOnRemote.length,
|
|
426
|
+
missingOnLocal: diff.missingOnLocal.length,
|
|
419
427
|
differentDocuments: diff.different.length,
|
|
428
|
+
localDocumentCount: Object.keys(localState.documents).length,
|
|
429
|
+
remoteDocumentCount: Object.keys(state.documents).length,
|
|
420
430
|
});
|
|
421
431
|
}
|
|
422
432
|
|
|
@@ -466,31 +476,37 @@ export class AutomergeHost extends Resource {
|
|
|
466
476
|
return;
|
|
467
477
|
}
|
|
468
478
|
|
|
469
|
-
const { different } = diffCollectionState(localState, remoteState);
|
|
479
|
+
const { different, missingOnLocal, missingOnRemote } = diffCollectionState(localState, remoteState);
|
|
480
|
+
const toReplicate = [...missingOnLocal, ...missingOnRemote, ...different];
|
|
470
481
|
|
|
471
|
-
if (
|
|
482
|
+
if (toReplicate.length === 0) {
|
|
472
483
|
return;
|
|
473
484
|
}
|
|
474
485
|
|
|
475
486
|
log.info('replication documents after collection sync', {
|
|
476
|
-
count:
|
|
487
|
+
count: toReplicate.length,
|
|
477
488
|
});
|
|
478
489
|
|
|
479
|
-
// Load the documents
|
|
480
|
-
for (const documentId of
|
|
490
|
+
// Load the documents so they will start syncing.
|
|
491
|
+
for (const documentId of toReplicate) {
|
|
481
492
|
this._repo.find(documentId);
|
|
482
493
|
}
|
|
483
494
|
}
|
|
484
495
|
|
|
485
496
|
private _onHeadsChanged(documentId: DocumentId, heads: Heads) {
|
|
497
|
+
const collectionsChanged = new Set<CollectionId>();
|
|
486
498
|
for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
|
|
487
499
|
const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
488
500
|
if (state?.documents[documentId]) {
|
|
489
501
|
const newState = structuredClone(state);
|
|
490
502
|
newState.documents[documentId] = heads;
|
|
491
503
|
this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
|
|
504
|
+
collectionsChanged.add(collectionId as CollectionId);
|
|
492
505
|
}
|
|
493
506
|
}
|
|
507
|
+
for (const collectionId of collectionsChanged) {
|
|
508
|
+
this.collectionStateUpdated.emit({ collectionId });
|
|
509
|
+
}
|
|
494
510
|
}
|
|
495
511
|
}
|
|
496
512
|
|
|
@@ -540,5 +556,28 @@ export type CollectionSyncState = {
|
|
|
540
556
|
|
|
541
557
|
export type PeerSyncState = {
|
|
542
558
|
peerId: PeerId;
|
|
559
|
+
/**
|
|
560
|
+
* Documents that are present locally but not on the remote peer.
|
|
561
|
+
*/
|
|
562
|
+
missingOnRemote: number;
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Documents that are present on the remote peer but not locally.
|
|
566
|
+
*/
|
|
567
|
+
missingOnLocal: number;
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* Documents that are present on both peers but have different heads.
|
|
571
|
+
*/
|
|
543
572
|
differentDocuments: number;
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Total number of documents locally.
|
|
576
|
+
*/
|
|
577
|
+
localDocumentCount: number;
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Total number of documents on the remote peer.
|
|
581
|
+
*/
|
|
582
|
+
remoteDocumentCount: number;
|
|
544
583
|
};
|
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { expect } from '
|
|
6
|
-
import waitForExpect from 'wait-for-expect';
|
|
5
|
+
import { onTestFinished, describe, expect, test } from 'vitest';
|
|
7
6
|
|
|
8
7
|
import { asyncTimeout, sleep } from '@dxos/async';
|
|
9
8
|
import {
|
|
@@ -34,7 +33,7 @@ import { randomBytes } from '@dxos/crypto';
|
|
|
34
33
|
import { PublicKey } from '@dxos/keys';
|
|
35
34
|
import { createTestLevel } from '@dxos/kv-store/testing';
|
|
36
35
|
import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/teleport/testing';
|
|
37
|
-
import {
|
|
36
|
+
import { openAndClose } from '@dxos/test-utils';
|
|
38
37
|
import { nonNullable, range } from '@dxos/util';
|
|
39
38
|
|
|
40
39
|
import { EchoNetworkAdapter } from './echo-network-adapter';
|
|
@@ -377,10 +376,8 @@ describe('AutomergeRepo', () => {
|
|
|
377
376
|
const hostHandle = peer1.find(url as AutomergeUrl);
|
|
378
377
|
|
|
379
378
|
// Doc should be pushed to peer2
|
|
380
|
-
await
|
|
381
|
-
|
|
382
|
-
expect(peer2.handles[hostHandle.documentId].docSync()).to.deep.eq(hostHandle.docSync());
|
|
383
|
-
});
|
|
379
|
+
await expect.poll(() => hostHandle.docSync().text).not.toBeUndefined();
|
|
380
|
+
await expect.poll(() => peer2.handles[hostHandle.documentId].docSync()).toEqual(hostHandle.docSync());
|
|
384
381
|
});
|
|
385
382
|
|
|
386
383
|
test('client cold-starts and syncs doc from a Repo', async () => {
|
|
@@ -480,7 +477,7 @@ describe('AutomergeRepo', () => {
|
|
|
480
477
|
const [spaceKey] = PublicKey.randomSequence();
|
|
481
478
|
|
|
482
479
|
const teleportBuilder = new TeleportBuilder();
|
|
483
|
-
|
|
480
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
484
481
|
|
|
485
482
|
const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
486
483
|
const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
@@ -493,11 +490,16 @@ describe('AutomergeRepo', () => {
|
|
|
493
490
|
handle.change((doc: any) => {
|
|
494
491
|
doc.text = text;
|
|
495
492
|
});
|
|
496
|
-
await
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
493
|
+
await expect
|
|
494
|
+
.poll(
|
|
495
|
+
async () => {
|
|
496
|
+
const docOnPeer2 = peer2.repo.find(handle.url);
|
|
497
|
+
const doc = await asyncTimeout(docOnPeer2.doc(), 1000);
|
|
498
|
+
return doc.text;
|
|
499
|
+
},
|
|
500
|
+
{ timeout: 1_000 },
|
|
501
|
+
)
|
|
502
|
+
.toEqual(text);
|
|
501
503
|
}
|
|
502
504
|
|
|
503
505
|
const offlineText = 'This has been written while the connection was off';
|
|
@@ -536,7 +538,7 @@ describe('AutomergeRepo', () => {
|
|
|
536
538
|
const [spaceKey] = PublicKey.randomSequence();
|
|
537
539
|
|
|
538
540
|
const teleportBuilder = new TeleportBuilder();
|
|
539
|
-
|
|
541
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
540
542
|
|
|
541
543
|
const peerWithDocs = await createTeleportPeerWithStoredDocs(teleportBuilder, spaceKey, async (repo) => {
|
|
542
544
|
return range(2, (idx) => {
|
|
@@ -569,7 +571,7 @@ describe('AutomergeRepo', () => {
|
|
|
569
571
|
const [spaceKey, anotherSpaceKey] = PublicKey.randomSequence();
|
|
570
572
|
|
|
571
573
|
const teleportBuilder = new TeleportBuilder();
|
|
572
|
-
|
|
574
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
573
575
|
|
|
574
576
|
const peerWithDocs = await createTeleportPeerWithStoredDocs(teleportBuilder, spaceKey, async (repo) => {
|
|
575
577
|
const document = repo.create();
|
|
@@ -600,7 +602,7 @@ describe('AutomergeRepo', () => {
|
|
|
600
602
|
const [spaceKey] = PublicKey.randomSequence();
|
|
601
603
|
|
|
602
604
|
const teleportBuilder = new TeleportBuilder();
|
|
603
|
-
|
|
605
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
604
606
|
|
|
605
607
|
const peerWithDocs = await createTeleportPeerWithStoredDocs(teleportBuilder, spaceKey, async (repo) => {
|
|
606
608
|
const document = repo.create();
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { expect } from '
|
|
5
|
+
import { onTestFinished, describe, expect, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { sleep } from '@dxos/async';
|
|
8
8
|
import type { PeerId } from '@dxos/automerge/automerge-repo';
|
|
9
|
-
import { afterTest, describe, test } from '@dxos/test';
|
|
10
9
|
|
|
11
10
|
import { CollectionSynchronizer, diffCollectionState, type CollectionState } from './collection-synchronizer';
|
|
12
11
|
|
|
@@ -31,7 +30,9 @@ describe('CollectionSynchronizer', () => {
|
|
|
31
30
|
}),
|
|
32
31
|
shouldSyncCollection: () => true,
|
|
33
32
|
}).open();
|
|
34
|
-
|
|
33
|
+
onTestFinished(async () => {
|
|
34
|
+
await peer1.close();
|
|
35
|
+
});
|
|
35
36
|
const peer2 = await new CollectionSynchronizer({
|
|
36
37
|
queryCollectionState: (collectionId, peerId) =>
|
|
37
38
|
queueMicrotask(async () => {
|
|
@@ -45,7 +46,9 @@ describe('CollectionSynchronizer', () => {
|
|
|
45
46
|
}),
|
|
46
47
|
shouldSyncCollection: () => true,
|
|
47
48
|
}).open();
|
|
48
|
-
|
|
49
|
+
onTestFinished(async () => {
|
|
50
|
+
await peer2.close();
|
|
51
|
+
});
|
|
49
52
|
|
|
50
53
|
peer1.onConnectionOpen(peerId2);
|
|
51
54
|
peer2.onConnectionOpen(peerId1);
|
|
@@ -69,7 +72,9 @@ describe('CollectionSynchronizer', () => {
|
|
|
69
72
|
const diff = diffCollectionState(STATE_1, STATE_2);
|
|
70
73
|
|
|
71
74
|
expect(diff).to.deep.equal({
|
|
72
|
-
|
|
75
|
+
missingOnLocal: ['d'],
|
|
76
|
+
missingOnRemote: ['c'],
|
|
77
|
+
different: ['b'],
|
|
73
78
|
});
|
|
74
79
|
});
|
|
75
80
|
});
|
|
@@ -6,6 +6,7 @@ import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from '@dxos/as
|
|
|
6
6
|
import { next as am } from '@dxos/automerge/automerge';
|
|
7
7
|
import type { DocumentId, PeerId } from '@dxos/automerge/automerge-repo';
|
|
8
8
|
import { Resource, type Context } from '@dxos/context';
|
|
9
|
+
import { log } from '@dxos/log';
|
|
9
10
|
import { defaultMap } from '@dxos/util';
|
|
10
11
|
|
|
11
12
|
const MIN_QUERY_INTERVAL = 5_000;
|
|
@@ -64,6 +65,7 @@ export class CollectionSynchronizer extends Resource {
|
|
|
64
65
|
}
|
|
65
66
|
|
|
66
67
|
setLocalCollectionState(collectionId: string, state: CollectionState) {
|
|
68
|
+
log('setLocalCollectionState', { collectionId, state });
|
|
67
69
|
this._getPerCollectionState(collectionId).localState = state;
|
|
68
70
|
|
|
69
71
|
queueMicrotask(async () => {
|
|
@@ -143,6 +145,7 @@ export class CollectionSynchronizer extends Resource {
|
|
|
143
145
|
* Callback when a peer sends the state of a collection.
|
|
144
146
|
*/
|
|
145
147
|
onRemoteStateReceived(collectionId: string, peerId: PeerId, state: CollectionState) {
|
|
148
|
+
log('onRemoteStateReceived', { collectionId, peerId, state });
|
|
146
149
|
validateCollectionState(state);
|
|
147
150
|
const perCollectionState = this._getPerCollectionState(collectionId);
|
|
148
151
|
perCollectionState.remoteStates.set(peerId, state);
|
|
@@ -184,24 +187,32 @@ export type CollectionState = {
|
|
|
184
187
|
};
|
|
185
188
|
|
|
186
189
|
export type CollectionStateDiff = {
|
|
190
|
+
missingOnRemote: DocumentId[];
|
|
191
|
+
missingOnLocal: DocumentId[];
|
|
187
192
|
different: DocumentId[];
|
|
188
193
|
};
|
|
189
194
|
|
|
190
195
|
export const diffCollectionState = (local: CollectionState, remote: CollectionState): CollectionStateDiff => {
|
|
191
196
|
const allDocuments = new Set<DocumentId>([...Object.keys(local.documents), ...Object.keys(remote.documents)] as any);
|
|
192
197
|
|
|
198
|
+
const missingOnRemote: DocumentId[] = [];
|
|
199
|
+
const missingOnLocal: DocumentId[] = [];
|
|
193
200
|
const different: DocumentId[] = [];
|
|
194
201
|
for (const documentId of allDocuments) {
|
|
195
|
-
if (
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
) {
|
|
202
|
+
if (!local.documents[documentId]) {
|
|
203
|
+
missingOnLocal.push(documentId as DocumentId);
|
|
204
|
+
} else if (!remote.documents[documentId]) {
|
|
205
|
+
missingOnRemote.push(documentId as DocumentId);
|
|
206
|
+
} else if (!am.equals(local.documents[documentId], remote.documents[documentId])) {
|
|
200
207
|
different.push(documentId as DocumentId);
|
|
201
208
|
}
|
|
202
209
|
}
|
|
203
210
|
|
|
204
|
-
return {
|
|
211
|
+
return {
|
|
212
|
+
missingOnRemote,
|
|
213
|
+
missingOnLocal,
|
|
214
|
+
different,
|
|
215
|
+
};
|
|
205
216
|
};
|
|
206
217
|
|
|
207
218
|
const validateCollectionState = (state: CollectionState) => {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { expect } from '
|
|
5
|
+
import { onTestFinished, describe, expect, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { sleep, Trigger, waitForCondition } from '@dxos/async';
|
|
8
8
|
import { cbor, type PeerId } from '@dxos/automerge/automerge-repo';
|
|
@@ -14,7 +14,6 @@ import {
|
|
|
14
14
|
type AutomergeReplicatorCallbacks,
|
|
15
15
|
type AutomergeReplicatorFactory,
|
|
16
16
|
} from '@dxos/teleport-extension-automerge-replicator';
|
|
17
|
-
import { afterTest, describe, test } from '@dxos/test';
|
|
18
17
|
|
|
19
18
|
import { EchoNetworkAdapter } from './echo-network-adapter';
|
|
20
19
|
import { MeshEchoReplicator } from './mesh-echo-replicator';
|
|
@@ -111,7 +110,9 @@ describe('EchoNetworkAdapter', () => {
|
|
|
111
110
|
});
|
|
112
111
|
adapter.connect(PEER_ID);
|
|
113
112
|
await adapter.open();
|
|
114
|
-
|
|
113
|
+
onTestFinished(async () => {
|
|
114
|
+
await adapter.close();
|
|
115
|
+
});
|
|
115
116
|
await adapter.addReplicator(replicator);
|
|
116
117
|
return adapter;
|
|
117
118
|
};
|
|
@@ -8,6 +8,7 @@ import { LifecycleState } from '@dxos/context';
|
|
|
8
8
|
import { invariant } from '@dxos/invariant';
|
|
9
9
|
import { type PublicKey } from '@dxos/keys';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
|
+
import type { AutomergeProtocolMessage } from '@dxos/protocols';
|
|
11
12
|
import { nonNullable } from '@dxos/util';
|
|
12
13
|
|
|
13
14
|
import {
|
|
@@ -179,7 +180,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
179
180
|
const writeStart = Date.now();
|
|
180
181
|
// TODO(dmaretskyi): Find a way to enforce backpressure on AM-repo.
|
|
181
182
|
connectionEntry.writer
|
|
182
|
-
.write(message)
|
|
183
|
+
.write(message as AutomergeProtocolMessage)
|
|
183
184
|
.then(() => {
|
|
184
185
|
const durationMs = Date.now() - writeStart;
|
|
185
186
|
this._params.monitor?.recordMessageSent(message, durationMs);
|
|
@@ -220,7 +221,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
220
221
|
break;
|
|
221
222
|
}
|
|
222
223
|
|
|
223
|
-
this._onMessage(value);
|
|
224
|
+
this._onMessage(value as Message);
|
|
224
225
|
}
|
|
225
226
|
} catch (err) {
|
|
226
227
|
if (connectionEntry.isOpen) {
|
|
@@ -282,8 +283,8 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
282
283
|
|
|
283
284
|
type ConnectionEntry = {
|
|
284
285
|
connection: ReplicatorConnection;
|
|
285
|
-
reader: ReadableStreamDefaultReader<
|
|
286
|
-
writer: WritableStreamDefaultWriter<
|
|
286
|
+
reader: ReadableStreamDefaultReader<AutomergeProtocolMessage>;
|
|
287
|
+
writer: WritableStreamDefaultWriter<AutomergeProtocolMessage>;
|
|
287
288
|
isOpen: boolean;
|
|
288
289
|
};
|
|
289
290
|
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { type Message } from '@dxos/automerge/automerge-repo';
|
|
6
5
|
import { type PublicKey, type SpaceId } from '@dxos/keys';
|
|
6
|
+
import type { AutomergeProtocolMessage } from '@dxos/protocols';
|
|
7
7
|
|
|
8
8
|
export interface EchoReplicator {
|
|
9
9
|
/**
|
|
@@ -44,12 +44,12 @@ export interface ReplicatorConnection {
|
|
|
44
44
|
/**
|
|
45
45
|
* Stream to read messages coming from the remote peer.
|
|
46
46
|
*/
|
|
47
|
-
readable: ReadableStream<
|
|
47
|
+
readable: ReadableStream<AutomergeProtocolMessage>;
|
|
48
48
|
|
|
49
49
|
/**
|
|
50
50
|
* Stream to write messages to the remote peer.
|
|
51
51
|
*/
|
|
52
|
-
writable: WritableStream<
|
|
52
|
+
writable: WritableStream<AutomergeProtocolMessage>;
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
55
|
* @returns true if the document should be advertised to this peer.
|
|
@@ -3,11 +3,12 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import * as A from '@dxos/automerge/automerge';
|
|
6
|
-
import { cbor
|
|
6
|
+
import { cbor } from '@dxos/automerge/automerge-repo';
|
|
7
7
|
import { Resource } from '@dxos/context';
|
|
8
8
|
import { invariant } from '@dxos/invariant';
|
|
9
9
|
import { type PublicKey } from '@dxos/keys';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
|
+
import type { AutomergeProtocolMessage } from '@dxos/protocols';
|
|
11
12
|
import { AutomergeReplicator, type AutomergeReplicatorFactory } from '@dxos/teleport-extension-automerge-replicator';
|
|
12
13
|
|
|
13
14
|
import type { ReplicatorConnection, ShouldAdvertiseParams, ShouldSyncCollectionParams } from './echo-replicator';
|
|
@@ -24,8 +25,8 @@ export type MeshReplicatorConnectionParams = {
|
|
|
24
25
|
};
|
|
25
26
|
|
|
26
27
|
export class MeshReplicatorConnection extends Resource implements ReplicatorConnection {
|
|
27
|
-
public readable: ReadableStream<
|
|
28
|
-
public writable: WritableStream<
|
|
28
|
+
public readable: ReadableStream<AutomergeProtocolMessage>;
|
|
29
|
+
public writable: WritableStream<AutomergeProtocolMessage>;
|
|
29
30
|
public remoteDeviceKey: PublicKey | null = null;
|
|
30
31
|
|
|
31
32
|
public readonly replicatorExtension: AutomergeReplicator;
|
|
@@ -36,16 +37,16 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
|
|
|
36
37
|
constructor(private readonly _params: MeshReplicatorConnectionParams) {
|
|
37
38
|
super();
|
|
38
39
|
|
|
39
|
-
let readableStreamController!: ReadableStreamDefaultController<
|
|
40
|
-
this.readable = new ReadableStream<
|
|
40
|
+
let readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
|
|
41
|
+
this.readable = new ReadableStream<AutomergeProtocolMessage>({
|
|
41
42
|
start: (controller) => {
|
|
42
43
|
readableStreamController = controller;
|
|
43
44
|
this._ctx.onDispose(() => controller.close());
|
|
44
45
|
},
|
|
45
46
|
});
|
|
46
47
|
|
|
47
|
-
this.writable = new WritableStream<
|
|
48
|
-
write: async (message:
|
|
48
|
+
this.writable = new WritableStream<AutomergeProtocolMessage>({
|
|
49
|
+
write: async (message: AutomergeProtocolMessage, controller) => {
|
|
49
50
|
invariant(this._isEnabled, 'Writing to a disabled connection');
|
|
50
51
|
try {
|
|
51
52
|
logSendSync(message);
|
|
@@ -89,7 +90,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
|
|
|
89
90
|
if (!this._isEnabled) {
|
|
90
91
|
return;
|
|
91
92
|
}
|
|
92
|
-
const message = cbor.decode(payload) as
|
|
93
|
+
const message = cbor.decode(payload) as AutomergeProtocolMessage;
|
|
93
94
|
// Note: automerge Repo dedup messages.
|
|
94
95
|
readableStreamController.enqueue(message);
|
|
95
96
|
},
|
|
@@ -136,7 +137,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
|
|
|
136
137
|
}
|
|
137
138
|
}
|
|
138
139
|
|
|
139
|
-
const logSendSync = (message:
|
|
140
|
+
const logSendSync = (message: AutomergeProtocolMessage) => {
|
|
140
141
|
log('sendSyncMessage', () => {
|
|
141
142
|
const decodedSyncMessage = message.type === 'sync' && message.data ? A.decodeSyncMessage(message.data) : undefined;
|
|
142
143
|
return {
|