@dxos/echo-pipeline 0.8.3 → 0.8.4-main.28f8d3d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-35I6ERLG.mjs → chunk-2543T5DX.mjs} +224 -145
- package/dist/lib/browser/chunk-2543T5DX.mjs.map +7 -0
- package/dist/lib/browser/{chunk-TQJTKNMS.mjs → chunk-VUXUDIPM.mjs} +2 -2
- package/dist/lib/{node/chunk-HOPOFWAL.cjs.map → browser/chunk-VUXUDIPM.mjs.map} +3 -3
- package/dist/lib/browser/filter/index.mjs +1 -1
- package/dist/lib/browser/index.mjs +419 -242
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +42 -17
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node-esm/{chunk-RVK35BS7.mjs → chunk-PGZYXNYE.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-RVK35BS7.mjs.map → chunk-PGZYXNYE.mjs.map} +2 -2
- package/dist/lib/node-esm/{chunk-5BHLPT24.mjs → chunk-UQI6R3TD.mjs} +224 -145
- package/dist/lib/node-esm/chunk-UQI6R3TD.mjs.map +7 -0
- package/dist/lib/node-esm/filter/index.mjs +1 -1
- package/dist/lib/node-esm/index.mjs +419 -242
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +42 -17
- package/dist/lib/node-esm/testing/index.mjs.map +3 -3
- package/dist/types/src/automerge/automerge-host.d.ts +1 -1
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/index.d.ts +1 -1
- package/dist/types/src/automerge/index.d.ts.map +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +2 -2
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-root.d.ts.map +1 -1
- package/dist/types/src/db-host/documents-synchronizer.d.ts +2 -2
- package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
- package/dist/types/src/db-host/echo-host.d.ts +2 -2
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
- package/dist/types/src/db-host/query-service.d.ts +1 -1
- package/dist/types/src/db-host/query-service.d.ts.map +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
- package/dist/types/src/filter/filter-match.d.ts +1 -1
- package/dist/types/src/filter/filter-match.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/query/errors.d.ts +19 -6
- package/dist/types/src/query/errors.d.ts.map +1 -1
- package/dist/types/src/query/query-executor.d.ts +1 -1
- package/dist/types/src/query/query-executor.d.ts.map +1 -1
- package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +1 -1
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/util.d.ts +1 -1
- package/dist/types/src/util.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +41 -38
- package/src/automerge/automerge-host.test.ts +3 -2
- package/src/automerge/automerge-host.ts +20 -10
- package/src/automerge/automerge-repo.test.ts +67 -16
- package/src/automerge/collection-synchronizer.test.ts +2 -2
- package/src/automerge/collection-synchronizer.ts +2 -2
- package/src/automerge/echo-data-monitor.ts +1 -1
- package/src/automerge/echo-network-adapter.test.ts +3 -3
- package/src/automerge/echo-network-adapter.ts +8 -6
- package/src/automerge/echo-replicator.ts +2 -1
- package/src/automerge/index.ts +1 -1
- package/src/automerge/leveldb-storage-adapter.ts +1 -1
- package/src/automerge/mesh-echo-replicator.ts +2 -1
- package/src/automerge/storage-adapter.test.ts +1 -1
- package/src/common/space-id.ts +1 -1
- package/src/db-host/data-service.ts +8 -7
- package/src/db-host/database-root.ts +2 -2
- package/src/db-host/documents-synchronizer.test.ts +1 -1
- package/src/db-host/documents-synchronizer.ts +39 -26
- package/src/db-host/echo-host.ts +13 -12
- package/src/db-host/query-service.ts +8 -1
- package/src/db-host/space-state-manager.ts +2 -2
- package/src/edge/echo-edge-replicator.test.ts +3 -2
- package/src/edge/echo-edge-replicator.ts +36 -15
- package/src/filter/filter-match.test.ts +2 -2
- package/src/filter/filter-match.ts +1 -1
- package/src/metadata/metadata-store.ts +3 -3
- package/src/pipeline/pipeline-stress.test.ts +4 -2
- package/src/pipeline/pipeline.test.ts +3 -2
- package/src/pipeline/pipeline.ts +8 -5
- package/src/query/query-executor.ts +7 -7
- package/src/query/query-planner.test.ts +2 -1
- package/src/space/admission-discovery-extension.ts +2 -2
- package/src/space/control-pipeline.test.ts +4 -3
- package/src/space/control-pipeline.ts +4 -4
- package/src/space/space-manager.browser.test.ts +1 -1
- package/src/space/space-manager.ts +5 -4
- package/src/space/space-protocol.browser.test.ts +2 -2
- package/src/space/space-protocol.test.ts +3 -2
- package/src/space/space-protocol.ts +6 -3
- package/src/space/space.test.ts +1 -1
- package/src/space/space.ts +3 -2
- package/src/testing/test-agent-builder.ts +4 -3
- package/src/util.ts +1 -1
- package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
- package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
- package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
- package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
- package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
- package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
- package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
- package/dist/lib/node/filter/index.cjs +0 -32
- package/dist/lib/node/filter/index.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -4699
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/testing/index.cjs +0 -753
- package/dist/lib/node/testing/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/echo-pipeline",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.4-main.28f8d3d",
|
|
4
4
|
"description": "ECHO database.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -10,16 +10,19 @@
|
|
|
10
10
|
"type": "module",
|
|
11
11
|
"exports": {
|
|
12
12
|
".": {
|
|
13
|
+
"source": "./src/index.ts",
|
|
13
14
|
"types": "./dist/types/src/index.d.ts",
|
|
14
15
|
"browser": "./dist/lib/browser/index.mjs",
|
|
15
16
|
"node": "./dist/lib/node-esm/index.mjs"
|
|
16
17
|
},
|
|
17
18
|
"./testing": {
|
|
19
|
+
"source": "./src/testing/index.ts",
|
|
18
20
|
"types": "./dist/types/src/testing/index.d.ts",
|
|
19
21
|
"browser": "./dist/lib/browser/testing/index.mjs",
|
|
20
22
|
"node": "./dist/lib/node-esm/testing/index.mjs"
|
|
21
23
|
},
|
|
22
24
|
"./filter": {
|
|
25
|
+
"source": "./src/filter/index.ts",
|
|
23
26
|
"types": "./dist/types/src/filter/index.d.ts",
|
|
24
27
|
"browser": "./dist/lib/browser/filter/index.mjs",
|
|
25
28
|
"node": "./dist/lib/node-esm/filter/index.mjs"
|
|
@@ -40,51 +43,51 @@
|
|
|
40
43
|
"src"
|
|
41
44
|
],
|
|
42
45
|
"dependencies": {
|
|
43
|
-
"@automerge/automerge": "3.
|
|
44
|
-
"@automerge/automerge-repo": "2.0.
|
|
46
|
+
"@automerge/automerge": "3.1.1",
|
|
47
|
+
"@automerge/automerge-repo": "2.3.0-alpha.0",
|
|
45
48
|
"crc-32": "^1.2.2",
|
|
46
|
-
"effect": "3.
|
|
49
|
+
"effect": "3.17.7",
|
|
47
50
|
"level-transcoder": "^1.0.1",
|
|
48
51
|
"lodash.isequal": "^4.5.0",
|
|
49
|
-
"@dxos/
|
|
50
|
-
"@dxos/
|
|
51
|
-
"@dxos/
|
|
52
|
-
"@dxos/
|
|
53
|
-
"@dxos/
|
|
54
|
-
"@dxos/
|
|
55
|
-
"@dxos/
|
|
56
|
-
"@dxos/echo-protocol": "0.8.
|
|
57
|
-
"@dxos/edge-client": "0.8.
|
|
58
|
-
"@dxos/echo-schema": "0.8.
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/
|
|
63
|
-
"@dxos/
|
|
64
|
-
"@dxos/
|
|
65
|
-
"@dxos/
|
|
66
|
-
"@dxos/kv-store": "0.8.
|
|
67
|
-
"@dxos/
|
|
68
|
-
"@dxos/
|
|
69
|
-
"@dxos/network-manager": "0.8.
|
|
70
|
-
"@dxos/node-std": "0.8.
|
|
71
|
-
"@dxos/teleport": "0.8.
|
|
72
|
-
"@dxos/
|
|
73
|
-
"@dxos/
|
|
74
|
-
"@dxos/teleport-extension-
|
|
75
|
-
"@dxos/teleport-extension-
|
|
76
|
-
"@dxos/
|
|
77
|
-
"@dxos/teleport-extension-replicator": "0.8.
|
|
78
|
-
"@dxos/timeframe": "0.8.
|
|
79
|
-
"@dxos/tracing": "0.8.
|
|
80
|
-
"@dxos/
|
|
81
|
-
"@dxos/
|
|
52
|
+
"@dxos/async": "0.8.4-main.28f8d3d",
|
|
53
|
+
"@dxos/context": "0.8.4-main.28f8d3d",
|
|
54
|
+
"@dxos/crypto": "0.8.4-main.28f8d3d",
|
|
55
|
+
"@dxos/codec-protobuf": "0.8.4-main.28f8d3d",
|
|
56
|
+
"@dxos/debug": "0.8.4-main.28f8d3d",
|
|
57
|
+
"@dxos/echo": "0.8.4-main.28f8d3d",
|
|
58
|
+
"@dxos/credentials": "0.8.4-main.28f8d3d",
|
|
59
|
+
"@dxos/echo-protocol": "0.8.4-main.28f8d3d",
|
|
60
|
+
"@dxos/edge-client": "0.8.4-main.28f8d3d",
|
|
61
|
+
"@dxos/echo-schema": "0.8.4-main.28f8d3d",
|
|
62
|
+
"@dxos/errors": "0.8.4-main.28f8d3d",
|
|
63
|
+
"@dxos/feed-store": "0.8.4-main.28f8d3d",
|
|
64
|
+
"@dxos/hypercore": "0.8.4-main.28f8d3d",
|
|
65
|
+
"@dxos/indexing": "0.8.4-main.28f8d3d",
|
|
66
|
+
"@dxos/keyring": "0.8.4-main.28f8d3d",
|
|
67
|
+
"@dxos/keys": "0.8.4-main.28f8d3d",
|
|
68
|
+
"@dxos/log": "0.8.4-main.28f8d3d",
|
|
69
|
+
"@dxos/kv-store": "0.8.4-main.28f8d3d",
|
|
70
|
+
"@dxos/messaging": "0.8.4-main.28f8d3d",
|
|
71
|
+
"@dxos/invariant": "0.8.4-main.28f8d3d",
|
|
72
|
+
"@dxos/network-manager": "0.8.4-main.28f8d3d",
|
|
73
|
+
"@dxos/node-std": "0.8.4-main.28f8d3d",
|
|
74
|
+
"@dxos/teleport": "0.8.4-main.28f8d3d",
|
|
75
|
+
"@dxos/teleport-extension-automerge-replicator": "0.8.4-main.28f8d3d",
|
|
76
|
+
"@dxos/protocols": "0.8.4-main.28f8d3d",
|
|
77
|
+
"@dxos/teleport-extension-gossip": "0.8.4-main.28f8d3d",
|
|
78
|
+
"@dxos/teleport-extension-object-sync": "0.8.4-main.28f8d3d",
|
|
79
|
+
"@dxos/random-access-storage": "0.8.4-main.28f8d3d",
|
|
80
|
+
"@dxos/teleport-extension-replicator": "0.8.4-main.28f8d3d",
|
|
81
|
+
"@dxos/timeframe": "0.8.4-main.28f8d3d",
|
|
82
|
+
"@dxos/tracing": "0.8.4-main.28f8d3d",
|
|
83
|
+
"@dxos/typings": "0.8.4-main.28f8d3d",
|
|
84
|
+
"@dxos/util": "0.8.4-main.28f8d3d"
|
|
82
85
|
},
|
|
83
86
|
"devDependencies": {
|
|
84
87
|
"@types/lodash.isequal": "^4.5.0",
|
|
85
88
|
"fast-check": "^3.19.0",
|
|
86
89
|
"get-port-please": "^3.1.1",
|
|
87
|
-
"@dxos/test-utils": "0.8.
|
|
90
|
+
"@dxos/test-utils": "0.8.4-main.28f8d3d"
|
|
88
91
|
},
|
|
89
92
|
"publishConfig": {
|
|
90
93
|
"access": "public"
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { getHeads } from '@automerge/automerge';
|
|
6
6
|
import type { DocumentId, Heads } from '@automerge/automerge-repo';
|
|
7
|
-
import {
|
|
7
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
8
8
|
|
|
9
9
|
import { IndexMetadataStore } from '@dxos/indexing';
|
|
10
10
|
import type { LevelDB } from '@dxos/kv-store';
|
|
@@ -12,9 +12,10 @@ import { createTestLevel } from '@dxos/kv-store/testing';
|
|
|
12
12
|
import { openAndClose } from '@dxos/test-utils';
|
|
13
13
|
import { range } from '@dxos/util';
|
|
14
14
|
|
|
15
|
-
import { AutomergeHost } from './automerge-host';
|
|
16
15
|
import { TestReplicationNetwork } from '../testing';
|
|
17
16
|
|
|
17
|
+
import { AutomergeHost } from './automerge-host';
|
|
18
|
+
|
|
18
19
|
describe('AutomergeHost', () => {
|
|
19
20
|
test('can create documents', async () => {
|
|
20
21
|
const level = await createLevel();
|
|
@@ -3,32 +3,32 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
|
+
type Doc,
|
|
7
|
+
type Heads,
|
|
6
8
|
getBackend,
|
|
7
9
|
getHeads,
|
|
8
|
-
isAutomerge,
|
|
9
10
|
equals as headsEquals,
|
|
11
|
+
isAutomerge,
|
|
10
12
|
save,
|
|
11
|
-
type Doc,
|
|
12
|
-
type Heads,
|
|
13
13
|
} from '@automerge/automerge';
|
|
14
14
|
import {
|
|
15
|
-
type DocHandleChangePayload,
|
|
16
|
-
Repo,
|
|
17
15
|
type AnyDocumentId,
|
|
18
16
|
type DocHandle,
|
|
17
|
+
type DocHandleChangePayload,
|
|
19
18
|
type DocumentId,
|
|
19
|
+
type HandleState,
|
|
20
20
|
type PeerCandidatePayload,
|
|
21
21
|
type PeerDisconnectedPayload,
|
|
22
22
|
type PeerId,
|
|
23
|
+
Repo,
|
|
23
24
|
type StorageAdapterInterface,
|
|
24
25
|
type StorageKey,
|
|
25
26
|
interpretAsDocumentId,
|
|
26
|
-
type HandleState,
|
|
27
27
|
} from '@automerge/automerge-repo';
|
|
28
28
|
|
|
29
29
|
import { Event, asyncTimeout } from '@dxos/async';
|
|
30
|
-
import { Context, Resource, cancelWithContext
|
|
31
|
-
import {
|
|
30
|
+
import { Context, type Lifecycle, Resource, cancelWithContext } from '@dxos/context';
|
|
31
|
+
import { type CollectionId, DatabaseDirectory } from '@dxos/echo-protocol';
|
|
32
32
|
import { type IndexMetadataStore } from '@dxos/indexing';
|
|
33
33
|
import { invariant } from '@dxos/invariant';
|
|
34
34
|
import { PublicKey } from '@dxos/keys';
|
|
@@ -39,12 +39,12 @@ import { type DocHeadsList, type FlushRequest } from '@dxos/protocols/proto/dxos
|
|
|
39
39
|
import { trace } from '@dxos/tracing';
|
|
40
40
|
import { bufferToArray } from '@dxos/util';
|
|
41
41
|
|
|
42
|
-
import { CollectionSynchronizer, diffCollectionState
|
|
42
|
+
import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
|
|
43
43
|
import { type EchoDataMonitor } from './echo-data-monitor';
|
|
44
44
|
import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
|
|
45
45
|
import { type EchoReplicator, type RemoteDocumentExistenceCheckParams } from './echo-replicator';
|
|
46
46
|
import { HeadsStore } from './heads-store';
|
|
47
|
-
import {
|
|
47
|
+
import { type BeforeSaveParams, LevelDBStorageAdapter } from './leveldb-storage-adapter';
|
|
48
48
|
|
|
49
49
|
export type PeerIdProvider = () => string | undefined;
|
|
50
50
|
|
|
@@ -515,6 +515,16 @@ export class AutomergeHost extends Resource {
|
|
|
515
515
|
heads.map((heads, index) => [documentIds[index], heads ?? []]),
|
|
516
516
|
);
|
|
517
517
|
this._collectionSynchronizer.setLocalCollectionState(collectionId, { documents });
|
|
518
|
+
|
|
519
|
+
// Proactively push our updated local state to peers that are interested in this collection.
|
|
520
|
+
// This reduces reliance on the next periodic query and prevents replication stalls in fast paths
|
|
521
|
+
// where the remote queries before our local state is ready.
|
|
522
|
+
const interestedPeers = this._echoNetworkAdapter.getPeersInterestedInCollection(collectionId);
|
|
523
|
+
if (interestedPeers.length > 0) {
|
|
524
|
+
for (const peerId of interestedPeers) {
|
|
525
|
+
this._sendCollectionState(collectionId, peerId, { documents });
|
|
526
|
+
}
|
|
527
|
+
}
|
|
518
528
|
}
|
|
519
529
|
|
|
520
530
|
async clearLocalCollectionState(collectionId: string): Promise<void> {
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
|
+
next as A,
|
|
6
7
|
type Heads,
|
|
7
8
|
change,
|
|
8
9
|
clone,
|
|
@@ -10,7 +11,6 @@ import {
|
|
|
10
11
|
from,
|
|
11
12
|
getBackend,
|
|
12
13
|
getHeads,
|
|
13
|
-
next as A,
|
|
14
14
|
save,
|
|
15
15
|
saveSince,
|
|
16
16
|
} from '@automerge/automerge';
|
|
@@ -19,14 +19,14 @@ import {
|
|
|
19
19
|
type DocHandle,
|
|
20
20
|
type DocumentId,
|
|
21
21
|
type HandleState,
|
|
22
|
-
type StorageAdapterInterface,
|
|
23
22
|
type PeerId,
|
|
24
23
|
Repo,
|
|
25
24
|
type SharePolicy,
|
|
25
|
+
type StorageAdapterInterface,
|
|
26
26
|
generateAutomergeUrl,
|
|
27
27
|
parseAutomergeUrl,
|
|
28
28
|
} from '@automerge/automerge-repo';
|
|
29
|
-
import {
|
|
29
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
30
30
|
|
|
31
31
|
import { asyncTimeout, sleep } from '@dxos/async';
|
|
32
32
|
import { randomBytes } from '@dxos/crypto';
|
|
@@ -36,11 +36,12 @@ import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/
|
|
|
36
36
|
import { openAndClose } from '@dxos/test-utils';
|
|
37
37
|
import { isNonNullable, range } from '@dxos/util';
|
|
38
38
|
|
|
39
|
+
import { TestAdapter, type TestConnectionStateProvider } from '../testing';
|
|
40
|
+
|
|
39
41
|
import { FIND_PARAMS } from './automerge-host';
|
|
40
42
|
import { EchoNetworkAdapter } from './echo-network-adapter';
|
|
41
43
|
import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
|
|
42
44
|
import { MeshEchoReplicator } from './mesh-echo-replicator';
|
|
43
|
-
import { TestAdapter, type TestConnectionStateProvider } from '../testing';
|
|
44
45
|
|
|
45
46
|
const HOST_AND_CLIENT: [string, string] = ['host', 'client'];
|
|
46
47
|
|
|
@@ -420,13 +421,7 @@ describe('AutomergeRepo', () => {
|
|
|
420
421
|
|
|
421
422
|
test('client creates doc and syncs with a Repo', async () => {
|
|
422
423
|
const repo = new Repo({ network: [] });
|
|
423
|
-
const receiveByServer =
|
|
424
|
-
const serverHandle = await repo.find(docId, FIND_PARAMS);
|
|
425
|
-
serverHandle.update((doc) => {
|
|
426
|
-
return A.loadIncremental(doc, blob);
|
|
427
|
-
});
|
|
428
|
-
};
|
|
429
|
-
|
|
424
|
+
const receiveByServer = (blob: Uint8Array, docId: DocumentId) => repo.import<any>(blob, { docId });
|
|
430
425
|
let clientDoc = A.from<{ field?: string }>({});
|
|
431
426
|
const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
|
|
432
427
|
// Sync handshake.
|
|
@@ -434,7 +429,7 @@ describe('AutomergeRepo', () => {
|
|
|
434
429
|
|
|
435
430
|
// Sync protocol.
|
|
436
431
|
const sendDoc = async (doc: A.Doc<any>) => {
|
|
437
|
-
|
|
432
|
+
receiveByServer(saveSince(doc, sentHeads), documentId);
|
|
438
433
|
sentHeads = getHeads(doc);
|
|
439
434
|
};
|
|
440
435
|
|
|
@@ -456,10 +451,9 @@ describe('AutomergeRepo', () => {
|
|
|
456
451
|
|
|
457
452
|
const repo = new Repo({ network: [], storage });
|
|
458
453
|
const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
});
|
|
454
|
+
repo.import<any>(blob, { docId });
|
|
455
|
+
// TODO(mykola): This should not be required. Document is not persisted without it.
|
|
456
|
+
await repo.flush([docId]);
|
|
463
457
|
};
|
|
464
458
|
|
|
465
459
|
let clientDoc = A.from<{ field?: string }>({ field: 'foo' });
|
|
@@ -677,6 +671,63 @@ describe('AutomergeRepo', () => {
|
|
|
677
671
|
await doc.whenReady();
|
|
678
672
|
expect(doc.doc()).to.deep.eq(document.doc());
|
|
679
673
|
});
|
|
674
|
+
|
|
675
|
+
test('document is passively replicated to connected peers', async () => {
|
|
676
|
+
const [spaceKey] = PublicKey.randomSequence();
|
|
677
|
+
|
|
678
|
+
const teleportBuilder = new TeleportBuilder();
|
|
679
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
680
|
+
|
|
681
|
+
const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
682
|
+
const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
683
|
+
|
|
684
|
+
const handle = peer1.repo.create();
|
|
685
|
+
handle.change((doc: any) => (doc.text = 'hello'));
|
|
686
|
+
await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
|
|
687
|
+
|
|
688
|
+
await expect
|
|
689
|
+
.poll(async () => {
|
|
690
|
+
const doc = peer2.repo.handles[handle.documentId];
|
|
691
|
+
await doc.whenReady();
|
|
692
|
+
return doc.doc()!.text;
|
|
693
|
+
})
|
|
694
|
+
.toEqual('hello');
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
test('imported document is passively replicated to connected peers', async () => {
|
|
698
|
+
let blob: Uint8Array;
|
|
699
|
+
let documentId: DocumentId;
|
|
700
|
+
{
|
|
701
|
+
const repo = new Repo();
|
|
702
|
+
const handle = repo.create();
|
|
703
|
+
handle.change((doc: any) => (doc.text = 'hello'));
|
|
704
|
+
blob = A.save(handle.doc()!);
|
|
705
|
+
documentId = handle.documentId;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
const [spaceKey] = PublicKey.randomSequence();
|
|
709
|
+
|
|
710
|
+
const teleportBuilder = new TeleportBuilder();
|
|
711
|
+
onTestFinished(() => teleportBuilder.destroy());
|
|
712
|
+
|
|
713
|
+
const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
714
|
+
const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
|
|
715
|
+
await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
|
|
716
|
+
|
|
717
|
+
const handle = peer1.repo.import(blob, { docId: documentId });
|
|
718
|
+
await handle.whenReady();
|
|
719
|
+
|
|
720
|
+
await expect
|
|
721
|
+
.poll(
|
|
722
|
+
async () => {
|
|
723
|
+
const doc = peer2.repo.handles[handle.documentId];
|
|
724
|
+
await doc.whenReady();
|
|
725
|
+
return doc.doc()!.text;
|
|
726
|
+
},
|
|
727
|
+
{ timeout: 1_000 },
|
|
728
|
+
)
|
|
729
|
+
.toEqual('hello');
|
|
730
|
+
});
|
|
680
731
|
});
|
|
681
732
|
|
|
682
733
|
const createLevelAdapter = async () => {
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import type { PeerId } from '@automerge/automerge-repo';
|
|
6
|
-
import {
|
|
6
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
7
7
|
|
|
8
8
|
import { sleep } from '@dxos/async';
|
|
9
9
|
|
|
10
|
-
import { CollectionSynchronizer, diffCollectionState
|
|
10
|
+
import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
|
|
11
11
|
|
|
12
12
|
describe('CollectionSynchronizer', () => {
|
|
13
13
|
test('sync two peers', async () => {
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
import { next as am } from '@automerge/automerge';
|
|
6
6
|
import type { DocumentId, PeerId } from '@automerge/automerge-repo';
|
|
7
7
|
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { Event, asyncReturn, scheduleTask, scheduleTaskInterval } from '@dxos/async';
|
|
9
|
+
import { type Context, Resource } from '@dxos/context';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
11
|
import { trace } from '@dxos/tracing';
|
|
12
12
|
import { defaultMap } from '@dxos/util';
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { type Message } from '@automerge/automerge-repo';
|
|
6
6
|
|
|
7
7
|
import { type TimeAware, trace } from '@dxos/tracing';
|
|
8
|
-
import { CircularBuffer,
|
|
8
|
+
import { CircularBuffer, SlidingWindowSummary, type SlidingWindowSummaryConfig, mapValues } from '@dxos/util';
|
|
9
9
|
|
|
10
10
|
import { type NetworkDataMonitor } from './echo-network-adapter';
|
|
11
11
|
import { type StorageAdapterDataMonitor } from './leveldb-storage-adapter';
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { type PeerId, cbor } from '@automerge/automerge-repo';
|
|
6
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
7
7
|
|
|
8
|
-
import {
|
|
8
|
+
import { Trigger, sleep, waitForCondition } from '@dxos/async';
|
|
9
9
|
import { invariant } from '@dxos/invariant';
|
|
10
10
|
import { PublicKey } from '@dxos/keys';
|
|
11
11
|
import { type SyncMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { type Message, NetworkAdapter, type PeerId, type PeerMetadata } from '@automerge/automerge-repo';
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { Trigger, synchronized } from '@dxos/async';
|
|
8
8
|
import { LifecycleState } from '@dxos/context';
|
|
9
9
|
import { invariant } from '@dxos/invariant';
|
|
10
10
|
import { type PublicKey } from '@dxos/keys';
|
|
@@ -12,6 +12,8 @@ import { log } from '@dxos/log';
|
|
|
12
12
|
import type { AutomergeProtocolMessage } from '@dxos/protocols';
|
|
13
13
|
import { isNonNullable } from '@dxos/util';
|
|
14
14
|
|
|
15
|
+
import { createIdFromSpaceKey } from '../common/space-id';
|
|
16
|
+
|
|
15
17
|
import {
|
|
16
18
|
type EchoReplicator,
|
|
17
19
|
type RemoteDocumentExistenceCheckParams,
|
|
@@ -20,12 +22,11 @@ import {
|
|
|
20
22
|
type ShouldSyncCollectionParams,
|
|
21
23
|
} from './echo-replicator';
|
|
22
24
|
import {
|
|
23
|
-
isCollectionQueryMessage,
|
|
24
|
-
isCollectionStateMessage,
|
|
25
25
|
type CollectionQueryMessage,
|
|
26
26
|
type CollectionStateMessage,
|
|
27
|
+
isCollectionQueryMessage,
|
|
28
|
+
isCollectionStateMessage,
|
|
27
29
|
} from './network-protocol';
|
|
28
|
-
import { createIdFromSpaceKey } from '../common/space-id';
|
|
29
30
|
|
|
30
31
|
export interface NetworkDataMonitor {
|
|
31
32
|
recordPeerConnected(peerId: string): void;
|
|
@@ -281,8 +282,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
281
282
|
this.emit('peer-disconnected', { peerId: connection.peerId as PeerId });
|
|
282
283
|
this._params.monitor?.recordPeerDisconnected(connection.peerId);
|
|
283
284
|
|
|
284
|
-
void entry.reader.cancel().catch((err) => log.catch(err));
|
|
285
285
|
void entry.writer.abort().catch((err) => log.catch(err));
|
|
286
|
+
void entry.reader.cancel().catch((err) => log.catch(err));
|
|
287
|
+
|
|
286
288
|
this._connections.delete(connection.peerId as PeerId);
|
|
287
289
|
}
|
|
288
290
|
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { type PublicKey, type SpaceId } from '@dxos/keys';
|
|
6
|
-
import type
|
|
6
|
+
import { type AutomergeProtocolMessage } from '@dxos/protocols';
|
|
7
7
|
|
|
8
|
+
// TODO(burdon): Rename AutomergeReplicator?
|
|
8
9
|
export interface EchoReplicator {
|
|
9
10
|
/**
|
|
10
11
|
* Called on when replicator is added to EchoHost.
|
package/src/automerge/index.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
export * from './automerge-host';
|
|
6
6
|
export * from './leveldb-storage-adapter';
|
|
7
7
|
export * from './mesh-echo-replicator';
|
|
8
|
-
export * from './echo-replicator';
|
|
8
|
+
export type * from './echo-replicator';
|
|
9
9
|
export { diffCollectionState } from './collection-synchronizer';
|
|
10
10
|
export * from './space-collection';
|
|
11
11
|
export * from './echo-data-monitor';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
// s
|
|
4
4
|
|
|
5
|
-
import { type
|
|
5
|
+
import { type Chunk, type StorageAdapterInterface, type StorageKey } from '@automerge/automerge-repo';
|
|
6
6
|
import { type MixedEncoding } from 'level-transcoder';
|
|
7
7
|
|
|
8
8
|
import { LifecycleState, Resource } from '@dxos/context';
|
|
@@ -12,10 +12,11 @@ import {
|
|
|
12
12
|
} from '@dxos/teleport-extension-automerge-replicator';
|
|
13
13
|
import { ComplexSet, defaultMap } from '@dxos/util';
|
|
14
14
|
|
|
15
|
+
import { createIdFromSpaceKey } from '../common/space-id';
|
|
16
|
+
|
|
15
17
|
import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseParams } from './echo-replicator';
|
|
16
18
|
import { MeshReplicatorConnection } from './mesh-echo-replicator-connection';
|
|
17
19
|
import { getSpaceIdFromCollectionId } from './space-collection';
|
|
18
|
-
import { createIdFromSpaceKey } from '../common/space-id';
|
|
19
20
|
|
|
20
21
|
// TODO(dmaretskyi): Move out of @dxos/echo-pipeline.
|
|
21
22
|
|
package/src/common/space-id.ts
CHANGED
|
@@ -18,7 +18,7 @@ export const createIdFromSpaceKey = async (spaceKey: PublicKey): Promise<SpaceId
|
|
|
18
18
|
return cachedValue;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array());
|
|
21
|
+
const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array() as Uint8Array<ArrayBuffer>);
|
|
22
22
|
|
|
23
23
|
const bytes = new Uint8Array(digest).slice(0, SpaceId.byteLength);
|
|
24
24
|
const spaceId = SpaceId.encode(bytes);
|
|
@@ -11,23 +11,24 @@ import { invariant } from '@dxos/invariant';
|
|
|
11
11
|
import { SpaceId } from '@dxos/keys';
|
|
12
12
|
import { log } from '@dxos/log';
|
|
13
13
|
import {
|
|
14
|
+
type BatchedDocumentUpdates,
|
|
14
15
|
type DataService,
|
|
15
16
|
type FlushRequest,
|
|
16
|
-
type SubscribeRequest,
|
|
17
|
-
type BatchedDocumentUpdates,
|
|
18
|
-
type UpdateSubscriptionRequest,
|
|
19
17
|
type GetDocumentHeadsRequest,
|
|
20
18
|
type GetDocumentHeadsResponse,
|
|
21
|
-
type ReIndexHeadsRequest,
|
|
22
|
-
type WaitUntilHeadsReplicatedRequest,
|
|
23
|
-
type UpdateRequest,
|
|
24
19
|
type GetSpaceSyncStateRequest,
|
|
20
|
+
type ReIndexHeadsRequest,
|
|
25
21
|
type SpaceSyncState,
|
|
22
|
+
type SubscribeRequest,
|
|
23
|
+
type UpdateRequest,
|
|
24
|
+
type UpdateSubscriptionRequest,
|
|
25
|
+
type WaitUntilHeadsReplicatedRequest,
|
|
26
26
|
} from '@dxos/protocols/proto/dxos/echo/service';
|
|
27
27
|
|
|
28
|
+
import { type AutomergeHost, deriveCollectionIdFromSpaceId } from '../automerge';
|
|
29
|
+
|
|
28
30
|
import { DocumentsSynchronizer } from './documents-synchronizer';
|
|
29
31
|
import { type SpaceStateManager } from './space-state-manager';
|
|
30
|
-
import { deriveCollectionIdFromSpaceId, type AutomergeHost } from '../automerge';
|
|
31
32
|
|
|
32
33
|
export type DataServiceParams = {
|
|
33
34
|
automergeHost: AutomergeHost;
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import type * as A from '@automerge/automerge';
|
|
6
|
-
import {
|
|
6
|
+
import { type AutomergeUrl, type DocHandle, type DocumentId, interpretAsDocumentId } from '@automerge/automerge-repo';
|
|
7
7
|
|
|
8
8
|
import { DatabaseDirectory, SpaceDocVersion } from '@dxos/echo-protocol';
|
|
9
9
|
import { invariant } from '@dxos/invariant';
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { type DocMetrics, measureDocMetrics } from './automerge-metrics';
|
|
12
12
|
|
|
13
13
|
export class DatabaseRoot {
|
|
14
14
|
static mapLinks(doc: DocHandle<DatabaseDirectory>, mapping: Record<DocumentId, DocumentId>): void {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { next as A } from '@automerge/automerge';
|
|
6
|
-
import { generateAutomergeUrl, parseAutomergeUrl
|
|
6
|
+
import { Repo, generateAutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
|
|
7
7
|
import { describe, expect, test } from 'vitest';
|
|
8
8
|
|
|
9
9
|
import { sleep } from '@dxos/async';
|