@dxos/echo-pipeline 0.6.0 → 0.6.1-main.09a92b0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-HS77A4I4.mjs → chunk-A2LCXJVD.mjs} +16 -1
- package/dist/lib/browser/chunk-A2LCXJVD.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +323 -211
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +1 -1
- package/dist/lib/node/{chunk-Y5U7UXEL.cjs → chunk-GHBIMYZK.cjs} +19 -4
- package/dist/lib/node/chunk-GHBIMYZK.cjs.map +7 -0
- package/dist/lib/node/index.cjs +364 -258
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +11 -11
- package/dist/types/src/automerge/automerge-host.d.ts +13 -9
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +2 -2
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.test.d.ts +2 -0
- package/dist/types/src/automerge/echo-network-adapter.test.d.ts.map +1 -0
- package/dist/types/src/automerge/echo-replicator.d.ts +5 -6
- package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/heads-store.d.ts +13 -0
- package/dist/types/src/automerge/heads-store.d.ts.map +1 -0
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +35 -0
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -0
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts +2 -2
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +4 -2
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/package.json +33 -33
- package/src/automerge/automerge-doc-loader.ts +1 -1
- package/src/automerge/automerge-host.test.ts +34 -17
- package/src/automerge/automerge-host.ts +61 -17
- package/src/automerge/automerge-repo.test.ts +76 -1
- package/src/automerge/echo-network-adapter.test.ts +131 -0
- package/src/automerge/echo-network-adapter.ts +10 -6
- package/src/automerge/echo-replicator.ts +6 -9
- package/src/automerge/heads-store.ts +39 -0
- package/src/automerge/mesh-echo-replicator-connection.ts +130 -0
- package/src/automerge/mesh-echo-replicator.ts +15 -123
- package/src/db-host/data-service.ts +22 -2
- package/dist/lib/browser/chunk-HS77A4I4.mjs.map +0 -7
- package/dist/lib/node/chunk-Y5U7UXEL.cjs.map +0 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/echo-pipeline",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1-main.09a92b0",
|
|
4
4
|
"description": "ECHO database.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -39,38 +39,38 @@
|
|
|
39
39
|
"crc-32": "^1.2.2",
|
|
40
40
|
"level": "^8.0.1",
|
|
41
41
|
"level-transcoder": "^1.0.1",
|
|
42
|
-
"@dxos/async": "0.6.
|
|
43
|
-
"@dxos/automerge": "0.6.
|
|
44
|
-
"@dxos/
|
|
45
|
-
"@dxos/
|
|
46
|
-
"@dxos/
|
|
47
|
-
"@dxos/
|
|
48
|
-
"@dxos/debug": "0.6.
|
|
49
|
-
"@dxos/echo-
|
|
50
|
-
"@dxos/echo-
|
|
51
|
-
"@dxos/
|
|
52
|
-
"@dxos/
|
|
53
|
-
"@dxos/hypercore": "0.6.
|
|
54
|
-
"@dxos/invariant": "0.6.
|
|
55
|
-
"@dxos/keyring": "0.6.
|
|
56
|
-
"@dxos/keys": "0.6.
|
|
57
|
-
"@dxos/kv-store": "0.6.
|
|
58
|
-
"@dxos/log": "0.6.
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/network-manager": "0.6.
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/protocols": "0.6.
|
|
63
|
-
"@dxos/random-access-storage": "0.6.
|
|
64
|
-
"@dxos/
|
|
65
|
-
"@dxos/teleport
|
|
66
|
-
"@dxos/
|
|
67
|
-
"@dxos/teleport-extension-
|
|
68
|
-
"@dxos/teleport-extension-
|
|
69
|
-
"@dxos/teleport-extension-replicator": "0.6.
|
|
70
|
-
"@dxos/
|
|
71
|
-
"@dxos/
|
|
72
|
-
"@dxos/
|
|
73
|
-
"@dxos/
|
|
42
|
+
"@dxos/async": "0.6.1-main.09a92b0",
|
|
43
|
+
"@dxos/automerge": "0.6.1-main.09a92b0",
|
|
44
|
+
"@dxos/codec-protobuf": "0.6.1-main.09a92b0",
|
|
45
|
+
"@dxos/context": "0.6.1-main.09a92b0",
|
|
46
|
+
"@dxos/crypto": "0.6.1-main.09a92b0",
|
|
47
|
+
"@dxos/credentials": "0.6.1-main.09a92b0",
|
|
48
|
+
"@dxos/debug": "0.6.1-main.09a92b0",
|
|
49
|
+
"@dxos/echo-protocol": "0.6.1-main.09a92b0",
|
|
50
|
+
"@dxos/echo-schema": "0.6.1-main.09a92b0",
|
|
51
|
+
"@dxos/feed-store": "0.6.1-main.09a92b0",
|
|
52
|
+
"@dxos/indexing": "0.6.1-main.09a92b0",
|
|
53
|
+
"@dxos/hypercore": "0.6.1-main.09a92b0",
|
|
54
|
+
"@dxos/invariant": "0.6.1-main.09a92b0",
|
|
55
|
+
"@dxos/keyring": "0.6.1-main.09a92b0",
|
|
56
|
+
"@dxos/keys": "0.6.1-main.09a92b0",
|
|
57
|
+
"@dxos/kv-store": "0.6.1-main.09a92b0",
|
|
58
|
+
"@dxos/log": "0.6.1-main.09a92b0",
|
|
59
|
+
"@dxos/messaging": "0.6.1-main.09a92b0",
|
|
60
|
+
"@dxos/network-manager": "0.6.1-main.09a92b0",
|
|
61
|
+
"@dxos/node-std": "0.6.1-main.09a92b0",
|
|
62
|
+
"@dxos/protocols": "0.6.1-main.09a92b0",
|
|
63
|
+
"@dxos/random-access-storage": "0.6.1-main.09a92b0",
|
|
64
|
+
"@dxos/rpc": "0.6.1-main.09a92b0",
|
|
65
|
+
"@dxos/teleport": "0.6.1-main.09a92b0",
|
|
66
|
+
"@dxos/teleport-extension-gossip": "0.6.1-main.09a92b0",
|
|
67
|
+
"@dxos/teleport-extension-automerge-replicator": "0.6.1-main.09a92b0",
|
|
68
|
+
"@dxos/teleport-extension-object-sync": "0.6.1-main.09a92b0",
|
|
69
|
+
"@dxos/teleport-extension-replicator": "0.6.1-main.09a92b0",
|
|
70
|
+
"@dxos/tracing": "0.6.1-main.09a92b0",
|
|
71
|
+
"@dxos/timeframe": "0.6.1-main.09a92b0",
|
|
72
|
+
"@dxos/typings": "0.6.1-main.09a92b0",
|
|
73
|
+
"@dxos/util": "0.6.1-main.09a92b0"
|
|
74
74
|
},
|
|
75
75
|
"devDependencies": {
|
|
76
76
|
"fast-check": "^3.19.0",
|
|
@@ -225,7 +225,7 @@ export class AutomergeDocumentLoaderImpl implements AutomergeDocumentLoader {
|
|
|
225
225
|
|
|
226
226
|
private async _createObjectOnDocumentLoad(handle: DocHandle<SpaceDoc>, objectId: string) {
|
|
227
227
|
try {
|
|
228
|
-
await handle.
|
|
228
|
+
await handle.whenReady();
|
|
229
229
|
const logMeta = { objectId, docUrl: handle.url };
|
|
230
230
|
if (this.onObjectDocumentLoaded.listenerCount() === 0) {
|
|
231
231
|
log.info('document loaded after all listeners were removed', logMeta);
|
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
import expect from 'expect';
|
|
6
6
|
|
|
7
|
+
import { getHeads } from '@dxos/automerge/automerge';
|
|
7
8
|
import { IndexMetadataStore } from '@dxos/indexing';
|
|
9
|
+
import type { LevelDB } from '@dxos/kv-store';
|
|
8
10
|
import { createTestLevel } from '@dxos/kv-store/testing';
|
|
9
11
|
import { afterTest, describe, test } from '@dxos/test';
|
|
10
12
|
|
|
@@ -15,13 +17,8 @@ describe('AutomergeHost', () => {
|
|
|
15
17
|
const level = createTestLevel();
|
|
16
18
|
await level.open();
|
|
17
19
|
afterTest(() => level.close());
|
|
18
|
-
const host = new AutomergeHost({
|
|
19
|
-
db: level.sublevel('automerge'),
|
|
20
|
-
indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
|
|
21
|
-
});
|
|
22
|
-
await host.open();
|
|
23
|
-
afterTest(() => host.close());
|
|
24
20
|
|
|
21
|
+
const host = await setupAutomergeHost({ level });
|
|
25
22
|
const handle = host.repo.create();
|
|
26
23
|
handle.change((doc: any) => {
|
|
27
24
|
doc.text = 'Hello world';
|
|
@@ -35,11 +32,7 @@ describe('AutomergeHost', () => {
|
|
|
35
32
|
await level.open();
|
|
36
33
|
afterTest(() => level.close());
|
|
37
34
|
|
|
38
|
-
const host =
|
|
39
|
-
db: level.sublevel('automerge'),
|
|
40
|
-
indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
|
|
41
|
-
});
|
|
42
|
-
await host.open();
|
|
35
|
+
const host = await setupAutomergeHost({ level });
|
|
43
36
|
const handle = host.repo.create();
|
|
44
37
|
handle.change((doc: any) => {
|
|
45
38
|
doc.text = 'Hello world';
|
|
@@ -49,15 +42,39 @@ describe('AutomergeHost', () => {
|
|
|
49
42
|
await host.repo.flush();
|
|
50
43
|
await host.close();
|
|
51
44
|
|
|
52
|
-
const host2 =
|
|
53
|
-
db: level.sublevel('automerge'),
|
|
54
|
-
indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
|
|
55
|
-
});
|
|
56
|
-
await host2.open();
|
|
57
|
-
afterTest(() => host2.close());
|
|
45
|
+
const host2 = await setupAutomergeHost({ level });
|
|
58
46
|
const handle2 = host2.repo.find(url);
|
|
59
47
|
await handle2.whenReady();
|
|
60
48
|
expect(handle2.docSync().text).toEqual('Hello world');
|
|
61
49
|
await host2.repo.flush();
|
|
62
50
|
});
|
|
51
|
+
|
|
52
|
+
test('query document heads', async () => {
|
|
53
|
+
const level = createTestLevel();
|
|
54
|
+
await level.open();
|
|
55
|
+
afterTest(() => level.close());
|
|
56
|
+
|
|
57
|
+
const host = await setupAutomergeHost({ level });
|
|
58
|
+
const handle = host.createDoc({ text: 'Hello world' });
|
|
59
|
+
const expectedHeads = getHeads(handle.docSync());
|
|
60
|
+
await host.flush();
|
|
61
|
+
|
|
62
|
+
expect(await host.getHeads(handle.documentId)).toEqual(expectedHeads);
|
|
63
|
+
|
|
64
|
+
// Simulate a restart.
|
|
65
|
+
{
|
|
66
|
+
const host = await setupAutomergeHost({ level });
|
|
67
|
+
expect(await host.getHeads(handle.documentId)).toEqual(expectedHeads);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
63
70
|
});
|
|
71
|
+
|
|
72
|
+
const setupAutomergeHost = async ({ level }: { level: LevelDB }) => {
|
|
73
|
+
const host = new AutomergeHost({
|
|
74
|
+
db: level,
|
|
75
|
+
indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
|
|
76
|
+
});
|
|
77
|
+
await host.open();
|
|
78
|
+
afterTest(() => host.close());
|
|
79
|
+
return host;
|
|
80
|
+
};
|
|
@@ -22,11 +22,13 @@ import {
|
|
|
22
22
|
type StorageAdapterInterface,
|
|
23
23
|
} from '@dxos/automerge/automerge-repo';
|
|
24
24
|
import { type Stream } from '@dxos/codec-protobuf';
|
|
25
|
-
import { Context, cancelWithContext, type Lifecycle } from '@dxos/context';
|
|
25
|
+
import { type Context, Resource, cancelWithContext, type Lifecycle } from '@dxos/context';
|
|
26
26
|
import { type SpaceDoc } from '@dxos/echo-protocol';
|
|
27
27
|
import { type IndexMetadataStore } from '@dxos/indexing';
|
|
28
|
+
import { invariant } from '@dxos/invariant';
|
|
28
29
|
import { PublicKey } from '@dxos/keys';
|
|
29
|
-
import { type
|
|
30
|
+
import { type LevelDB } from '@dxos/kv-store';
|
|
31
|
+
import { log } from '@dxos/log';
|
|
30
32
|
import { objectPointerCodec } from '@dxos/protocols';
|
|
31
33
|
import {
|
|
32
34
|
type FlushRequest,
|
|
@@ -39,6 +41,7 @@ import { mapValues } from '@dxos/util';
|
|
|
39
41
|
|
|
40
42
|
import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
|
|
41
43
|
import { type EchoReplicator } from './echo-replicator';
|
|
44
|
+
import { HeadsStore } from './heads-store';
|
|
42
45
|
import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-adapter';
|
|
43
46
|
import { LocalHostNetworkAdapter } from './local-host-network-adapter';
|
|
44
47
|
|
|
@@ -46,7 +49,7 @@ import { LocalHostNetworkAdapter } from './local-host-network-adapter';
|
|
|
46
49
|
export type { DocumentId };
|
|
47
50
|
|
|
48
51
|
export type AutomergeHostParams = {
|
|
49
|
-
db:
|
|
52
|
+
db: LevelDB;
|
|
50
53
|
|
|
51
54
|
indexMetadataStore: IndexMetadataStore;
|
|
52
55
|
};
|
|
@@ -66,9 +69,9 @@ export type CreateDocOptions = {
|
|
|
66
69
|
* Abstracts over the AutomergeRepo.
|
|
67
70
|
*/
|
|
68
71
|
@trace.resource()
|
|
69
|
-
export class AutomergeHost {
|
|
72
|
+
export class AutomergeHost extends Resource {
|
|
73
|
+
private readonly _db: LevelDB;
|
|
70
74
|
private readonly _indexMetadataStore: IndexMetadataStore;
|
|
71
|
-
private readonly _ctx = new Context();
|
|
72
75
|
private readonly _echoNetworkAdapter = new EchoNetworkAdapter({
|
|
73
76
|
getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
|
|
74
77
|
});
|
|
@@ -76,22 +79,26 @@ export class AutomergeHost {
|
|
|
76
79
|
private _repo!: Repo;
|
|
77
80
|
private _clientNetwork!: LocalHostNetworkAdapter;
|
|
78
81
|
private _storage!: StorageAdapterInterface & Lifecycle;
|
|
82
|
+
private readonly _headsStore: HeadsStore;
|
|
79
83
|
|
|
80
84
|
@trace.info()
|
|
81
85
|
private _peerId!: string;
|
|
82
86
|
|
|
83
87
|
constructor({ db, indexMetadataStore }: AutomergeHostParams) {
|
|
88
|
+
super();
|
|
89
|
+
this._db = db;
|
|
84
90
|
this._storage = new LevelDBStorageAdapter({
|
|
85
|
-
db,
|
|
91
|
+
db: db.sublevel('automerge'),
|
|
86
92
|
callbacks: {
|
|
87
93
|
beforeSave: async (params) => this._beforeSave(params),
|
|
88
94
|
afterSave: async () => this._afterSave(),
|
|
89
95
|
},
|
|
90
96
|
});
|
|
97
|
+
this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
|
|
91
98
|
this._indexMetadataStore = indexMetadataStore;
|
|
92
99
|
}
|
|
93
100
|
|
|
94
|
-
async
|
|
101
|
+
protected override async _open() {
|
|
95
102
|
// TODO(burdon): Should this be stable?
|
|
96
103
|
this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
|
|
97
104
|
|
|
@@ -117,7 +124,7 @@ export class AutomergeHost {
|
|
|
117
124
|
await this._echoNetworkAdapter.whenConnected();
|
|
118
125
|
}
|
|
119
126
|
|
|
120
|
-
async
|
|
127
|
+
protected override async _close() {
|
|
121
128
|
await this._storage.close?.();
|
|
122
129
|
await this._clientNetwork.close();
|
|
123
130
|
await this._echoNetworkAdapter.close();
|
|
@@ -131,6 +138,10 @@ export class AutomergeHost {
|
|
|
131
138
|
return this._repo;
|
|
132
139
|
}
|
|
133
140
|
|
|
141
|
+
get loadedDocsCount(): number {
|
|
142
|
+
return Object.keys(this._repo.handles).length;
|
|
143
|
+
}
|
|
144
|
+
|
|
134
145
|
async addReplicator(replicator: EchoReplicator) {
|
|
135
146
|
await this._echoNetworkAdapter.addReplicator(replicator);
|
|
136
147
|
}
|
|
@@ -179,14 +190,32 @@ export class AutomergeHost {
|
|
|
179
190
|
}
|
|
180
191
|
}
|
|
181
192
|
|
|
193
|
+
async reIndexHeads(documentIds: DocumentId[]) {
|
|
194
|
+
for (const documentId of documentIds) {
|
|
195
|
+
log.info('reindexing heads for document', { documentId });
|
|
196
|
+
const handle = this._repo.find(documentId);
|
|
197
|
+
await handle.whenReady(['ready', 'requesting']);
|
|
198
|
+
if (handle.inState(['requesting'])) {
|
|
199
|
+
log.warn('document is not available locally, skipping', { documentId });
|
|
200
|
+
continue; // Handle not available locally.
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const doc = handle.docSync();
|
|
204
|
+
invariant(doc);
|
|
205
|
+
|
|
206
|
+
const heads = getHeads(doc);
|
|
207
|
+
const batch = this._db.batch();
|
|
208
|
+
this._headsStore.setHeads(documentId, heads, batch);
|
|
209
|
+
await batch.write();
|
|
210
|
+
}
|
|
211
|
+
log.info('done reindexing heads');
|
|
212
|
+
}
|
|
213
|
+
|
|
182
214
|
// TODO(dmaretskyi): Share based on HALO permissions and space affinity.
|
|
183
215
|
// Hosts, running in the worker, don't share documents unless requested by other peers.
|
|
184
216
|
// NOTE: If both peers return sharePolicy=false the replication will not happen
|
|
185
217
|
// https://github.com/automerge/automerge-repo/pull/292
|
|
186
|
-
private async _sharePolicy(
|
|
187
|
-
peerId: PeerId /* device key */,
|
|
188
|
-
documentId?: DocumentId /* space key */,
|
|
189
|
-
): Promise<boolean> {
|
|
218
|
+
private async _sharePolicy(peerId: PeerId, documentId?: DocumentId): Promise<boolean> {
|
|
190
219
|
if (peerId.startsWith('client-')) {
|
|
191
220
|
return false; // Only send docs to clients if they are requested.
|
|
192
221
|
}
|
|
@@ -197,7 +226,7 @@ export class AutomergeHost {
|
|
|
197
226
|
|
|
198
227
|
const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
|
|
199
228
|
if (isEchoPeerMetadata(peerMetadata)) {
|
|
200
|
-
return this._echoNetworkAdapter.
|
|
229
|
+
return this._echoNetworkAdapter.shouldAdvertise(peerId, { documentId });
|
|
201
230
|
}
|
|
202
231
|
|
|
203
232
|
return false;
|
|
@@ -215,13 +244,15 @@ export class AutomergeHost {
|
|
|
215
244
|
|
|
216
245
|
const spaceKey = getSpaceKeyFromDoc(doc) ?? undefined;
|
|
217
246
|
|
|
218
|
-
const
|
|
247
|
+
const heads = getHeads(doc);
|
|
248
|
+
|
|
249
|
+
this._headsStore.setHeads(handle.documentId, heads, batch);
|
|
219
250
|
|
|
220
251
|
const objectIds = Object.keys(doc.objects ?? {});
|
|
221
252
|
const encodedIds = objectIds.map((objectId) =>
|
|
222
253
|
objectPointerCodec.encode({ documentId: handle.documentId, objectId, spaceKey }),
|
|
223
254
|
);
|
|
224
|
-
const idToLastHash = new Map(encodedIds.map((id) => [id,
|
|
255
|
+
const idToLastHash = new Map(encodedIds.map((id) => [id, heads]));
|
|
225
256
|
this._indexMetadataStore.markDirty(idToLastHash, batch);
|
|
226
257
|
}
|
|
227
258
|
|
|
@@ -281,7 +312,7 @@ export class AutomergeHost {
|
|
|
281
312
|
* Flush documents to disk.
|
|
282
313
|
*/
|
|
283
314
|
@trace.span({ showInBrowserTimeline: true })
|
|
284
|
-
async flush({ states }: FlushRequest): Promise<void> {
|
|
315
|
+
async flush({ states }: FlushRequest = {}): Promise<void> {
|
|
285
316
|
// Note: Wait for all requested documents to be loaded/synced from thin-client.
|
|
286
317
|
if (states) {
|
|
287
318
|
await Promise.all(
|
|
@@ -289,7 +320,7 @@ export class AutomergeHost {
|
|
|
289
320
|
if (!heads) {
|
|
290
321
|
return;
|
|
291
322
|
}
|
|
292
|
-
const handle = this.
|
|
323
|
+
const handle = this._repo.handles[documentId as DocumentId] ?? this._repo.find(documentId as DocumentId);
|
|
293
324
|
await waitForHeads(handle, heads);
|
|
294
325
|
}) ?? [],
|
|
295
326
|
);
|
|
@@ -298,6 +329,19 @@ export class AutomergeHost {
|
|
|
298
329
|
await this._repo.flush(states?.map(({ documentId }) => documentId as DocumentId));
|
|
299
330
|
}
|
|
300
331
|
|
|
332
|
+
async getHeads(documentId: DocumentId): Promise<Heads | undefined> {
|
|
333
|
+
const handle = this._repo.handles[documentId];
|
|
334
|
+
if (handle) {
|
|
335
|
+
const doc = handle.docSync();
|
|
336
|
+
if (!doc) {
|
|
337
|
+
return undefined;
|
|
338
|
+
}
|
|
339
|
+
return getHeads(doc);
|
|
340
|
+
} else {
|
|
341
|
+
return this._headsStore.getHeads(documentId);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
301
345
|
/**
|
|
302
346
|
* Host <-> Client sync.
|
|
303
347
|
*/
|
|
@@ -7,7 +7,14 @@ import waitForExpect from 'wait-for-expect';
|
|
|
7
7
|
|
|
8
8
|
import { asyncTimeout, sleep } from '@dxos/async';
|
|
9
9
|
import { type Heads, change, clone, equals, from, getBackend, getHeads } from '@dxos/automerge/automerge';
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
type Message,
|
|
12
|
+
Repo,
|
|
13
|
+
type PeerId,
|
|
14
|
+
type DocumentId,
|
|
15
|
+
type HandleState,
|
|
16
|
+
type AutomergeUrl,
|
|
17
|
+
} from '@dxos/automerge/automerge-repo';
|
|
11
18
|
import { randomBytes } from '@dxos/crypto';
|
|
12
19
|
import { PublicKey } from '@dxos/keys';
|
|
13
20
|
import { createTestLevel } from '@dxos/kv-store/testing';
|
|
@@ -92,6 +99,74 @@ describe('AutomergeRepo', () => {
|
|
|
92
99
|
expect(equals(a, c)).to.be.true;
|
|
93
100
|
});
|
|
94
101
|
|
|
102
|
+
test('documents missing from local storage go to requesting state', async () => {
|
|
103
|
+
const hostAdapter: TestAdapter = new TestAdapter({
|
|
104
|
+
send: (message: Message) => {
|
|
105
|
+
console.log('hostAdapter.send', message);
|
|
106
|
+
clientAdapter.receive(message);
|
|
107
|
+
},
|
|
108
|
+
});
|
|
109
|
+
const clientAdapter: TestAdapter = new TestAdapter({
|
|
110
|
+
send: (message: Message) => {
|
|
111
|
+
console.log('clientAdapter.send', message);
|
|
112
|
+
if (message.type !== 'doc-unavailable' && message.type !== 'sync') {
|
|
113
|
+
hostAdapter.receive(message);
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const host = new Repo({
|
|
119
|
+
network: [hostAdapter],
|
|
120
|
+
});
|
|
121
|
+
const _client = new Repo({
|
|
122
|
+
network: [clientAdapter],
|
|
123
|
+
});
|
|
124
|
+
hostAdapter.ready();
|
|
125
|
+
clientAdapter.ready();
|
|
126
|
+
await hostAdapter.onConnect.wait();
|
|
127
|
+
await clientAdapter.onConnect.wait();
|
|
128
|
+
hostAdapter.peerCandidate(clientAdapter.peerId!);
|
|
129
|
+
clientAdapter.peerCandidate(hostAdapter.peerId!);
|
|
130
|
+
|
|
131
|
+
const url = 'automerge:3JN8F3Z4dUWEEKKFN7WE9gEGvVUT';
|
|
132
|
+
const handle = host.find(url as AutomergeUrl);
|
|
133
|
+
await handle.whenReady(['requesting']);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test('documents on disk go to ready state', async () => {
|
|
137
|
+
const level = createTestLevel();
|
|
138
|
+
const storage = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
|
|
139
|
+
await openAndClose(level, storage);
|
|
140
|
+
|
|
141
|
+
let url: AutomergeUrl | undefined;
|
|
142
|
+
{
|
|
143
|
+
const repo = new Repo({
|
|
144
|
+
network: [],
|
|
145
|
+
storage,
|
|
146
|
+
});
|
|
147
|
+
const handle = repo.create<{ field?: string }>();
|
|
148
|
+
url = handle.url;
|
|
149
|
+
await repo.flush();
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
{
|
|
153
|
+
const repo = new Repo({
|
|
154
|
+
network: [],
|
|
155
|
+
storage,
|
|
156
|
+
});
|
|
157
|
+
const handle = repo.find(url as AutomergeUrl);
|
|
158
|
+
|
|
159
|
+
let requestingState = false;
|
|
160
|
+
queueMicrotask(async () => {
|
|
161
|
+
await handle.whenReady(['requesting']);
|
|
162
|
+
requestingState = true;
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
await handle.whenReady(['ready']);
|
|
166
|
+
expect(requestingState).to.be.false;
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
|
|
95
170
|
describe('network', () => {
|
|
96
171
|
test('basic networking', async () => {
|
|
97
172
|
const hostAdapter: TestAdapter = new TestAdapter({
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { expect } from 'chai';
|
|
6
|
+
|
|
7
|
+
import { sleep, Trigger, waitForCondition } from '@dxos/async';
|
|
8
|
+
import { cbor, type PeerId } from '@dxos/automerge/automerge-repo';
|
|
9
|
+
import { invariant } from '@dxos/invariant';
|
|
10
|
+
import { PublicKey } from '@dxos/keys';
|
|
11
|
+
import { type SyncMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';
|
|
12
|
+
import {
|
|
13
|
+
type AutomergeReplicator,
|
|
14
|
+
type AutomergeReplicatorCallbacks,
|
|
15
|
+
type AutomergeReplicatorFactory,
|
|
16
|
+
} from '@dxos/teleport-extension-automerge-replicator';
|
|
17
|
+
import { afterTest, describe, test } from '@dxos/test';
|
|
18
|
+
|
|
19
|
+
import { EchoNetworkAdapter } from './echo-network-adapter';
|
|
20
|
+
import { MeshEchoReplicator } from './mesh-echo-replicator';
|
|
21
|
+
|
|
22
|
+
const PEER_ID = 'peerA' as PeerId;
|
|
23
|
+
const ANOTHER_PEER_ID = 'peerB' as PeerId;
|
|
24
|
+
const PAYLOAD = new Uint8Array([42]);
|
|
25
|
+
|
|
26
|
+
describe('EchoNetworkAdapter', () => {
|
|
27
|
+
test('peer-candidate emitted when replication starts', async () => {
|
|
28
|
+
const controller = createReplicatorController();
|
|
29
|
+
const adapter = await createConnectedAdapter(controller.replicator);
|
|
30
|
+
const peerInfo = new Trigger<any>();
|
|
31
|
+
adapter.on('peer-candidate', (payload) => peerInfo.wake(payload));
|
|
32
|
+
await controller.connectPeer(ANOTHER_PEER_ID);
|
|
33
|
+
const emittedInfo = await peerInfo.wait();
|
|
34
|
+
expect(emittedInfo.peerId).to.eq(ANOTHER_PEER_ID);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('message emitted when sync message is received', async () => {
|
|
38
|
+
const controller = createReplicatorController();
|
|
39
|
+
const adapter = await createConnectedAdapter(controller.replicator);
|
|
40
|
+
const messageReceived = new Trigger<any>();
|
|
41
|
+
adapter.on('message', (message) => messageReceived.wake(message));
|
|
42
|
+
const callbacks = await controller.connectPeer(ANOTHER_PEER_ID);
|
|
43
|
+
await callbacks.onSyncMessage!(encodeSyncPayload(PAYLOAD));
|
|
44
|
+
const receivedMessage = await messageReceived.wait();
|
|
45
|
+
expect(receivedMessage).to.deep.eq(PAYLOAD);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test('peer disconnects when onClose callback is invoked', async () => {
|
|
49
|
+
const controller = createReplicatorController();
|
|
50
|
+
const adapter = await createConnectedAdapter(controller.replicator);
|
|
51
|
+
const callbacks = await controller.connectPeer(ANOTHER_PEER_ID);
|
|
52
|
+
const onDisconnected = new Trigger<any>();
|
|
53
|
+
adapter.on('peer-disconnected', (payload) => onDisconnected.wake(payload));
|
|
54
|
+
await callbacks.onClose!();
|
|
55
|
+
const disconnectedPeer = await onDisconnected.wait();
|
|
56
|
+
expect(disconnectedPeer.peerId).to.eq(ANOTHER_PEER_ID);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('peer disconnects when message sending fails', async () => {
|
|
60
|
+
let errored = false;
|
|
61
|
+
const controller = createReplicatorController(async () => {
|
|
62
|
+
errored = true;
|
|
63
|
+
throw new Error();
|
|
64
|
+
});
|
|
65
|
+
const adapter = await createConnectedAdapter(controller.replicator);
|
|
66
|
+
await controller.connectPeer(ANOTHER_PEER_ID);
|
|
67
|
+
const onDisconnected = new Trigger<any>();
|
|
68
|
+
adapter.on('peer-disconnected', (payload) => onDisconnected.wake(payload));
|
|
69
|
+
adapter.send(newSyncMessage(PEER_ID, ANOTHER_PEER_ID, PAYLOAD));
|
|
70
|
+
const disconnectedPeer = await onDisconnected.wait();
|
|
71
|
+
expect(disconnectedPeer.peerId).to.eq(ANOTHER_PEER_ID);
|
|
72
|
+
expect(errored).to.be.true;
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('message sending is queued', async () => {
|
|
76
|
+
let sentTotal = 0;
|
|
77
|
+
let sendInProgress = false;
|
|
78
|
+
const controller = createReplicatorController(async () => {
|
|
79
|
+
invariant(!sendInProgress);
|
|
80
|
+
sendInProgress = true;
|
|
81
|
+
await sleep(5);
|
|
82
|
+
sendInProgress = false;
|
|
83
|
+
sentTotal++;
|
|
84
|
+
});
|
|
85
|
+
const adapter = await createConnectedAdapter(controller.replicator);
|
|
86
|
+
await controller.connectPeer(ANOTHER_PEER_ID);
|
|
87
|
+
const totalMessages = 5;
|
|
88
|
+
for (let i = 0; i < totalMessages; i++) {
|
|
89
|
+
adapter.send(newSyncMessage(PEER_ID, ANOTHER_PEER_ID, PAYLOAD));
|
|
90
|
+
}
|
|
91
|
+
await waitForCondition({ condition: () => sentTotal === totalMessages });
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const createConnectedAdapter = async (replicator: MeshEchoReplicator) => {
|
|
95
|
+
const adapter = new EchoNetworkAdapter({ getContainingSpaceForDocument: async () => null });
|
|
96
|
+
adapter.connect(PEER_ID);
|
|
97
|
+
await adapter.open();
|
|
98
|
+
afterTest(() => adapter.close());
|
|
99
|
+
await adapter.addReplicator(replicator);
|
|
100
|
+
return adapter;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const createReplicatorController = (sendSyncMessage?: (message: SyncMessage) => Promise<void>) => {
|
|
104
|
+
const replicator = new MeshEchoReplicator();
|
|
105
|
+
return {
|
|
106
|
+
replicator,
|
|
107
|
+
connectPeer: async (peerId: string) => {
|
|
108
|
+
let callbacks: AutomergeReplicatorCallbacks | undefined;
|
|
109
|
+
const extensionFactory: AutomergeReplicatorFactory = (params) => {
|
|
110
|
+
callbacks = params[1];
|
|
111
|
+
return { sendSyncMessage } as AutomergeReplicator;
|
|
112
|
+
};
|
|
113
|
+
replicator.createExtension(extensionFactory);
|
|
114
|
+
invariant(callbacks);
|
|
115
|
+
await callbacks.onStartReplication!({ id: peerId }, PublicKey.random());
|
|
116
|
+
return callbacks;
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const newSyncMessage = (from: PeerId, to: PeerId, payload: Uint8Array) => ({
|
|
122
|
+
type: 'sync',
|
|
123
|
+
senderId: from,
|
|
124
|
+
targetId: to,
|
|
125
|
+
data: payload,
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
const encodeSyncPayload = (payload: Uint8Array): SyncMessage => ({
|
|
129
|
+
payload: cbor.encode(payload),
|
|
130
|
+
});
|
|
131
|
+
});
|
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { synchronized, Trigger } from '@dxos/async';
|
|
6
6
|
import { type Message, NetworkAdapter, type PeerId, type PeerMetadata } from '@dxos/automerge/automerge-repo';
|
|
7
7
|
import { LifecycleState } from '@dxos/context';
|
|
8
8
|
import { invariant } from '@dxos/invariant';
|
|
9
9
|
import { type PublicKey } from '@dxos/keys';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
11
|
|
|
12
|
-
import { type EchoReplicator, type ReplicatorConnection, type
|
|
12
|
+
import { type EchoReplicator, type ReplicatorConnection, type ShouldAdvertiseParams } from './echo-replicator';
|
|
13
13
|
|
|
14
14
|
export type EchoNetworkAdapterParams = {
|
|
15
15
|
getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
|
|
@@ -57,7 +57,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
57
57
|
|
|
58
58
|
@synchronized
|
|
59
59
|
async open() {
|
|
60
|
-
|
|
60
|
+
if (this._lifecycleState === LifecycleState.OPEN) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
61
63
|
this._lifecycleState = LifecycleState.OPEN;
|
|
62
64
|
|
|
63
65
|
log('emit ready');
|
|
@@ -68,7 +70,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
68
70
|
|
|
69
71
|
@synchronized
|
|
70
72
|
async close() {
|
|
71
|
-
|
|
73
|
+
if (this._lifecycleState === LifecycleState.CLOSED) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
72
76
|
|
|
73
77
|
for (const replicator of this._replicators) {
|
|
74
78
|
await replicator.disconnect();
|
|
@@ -106,13 +110,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
106
110
|
this._replicators.delete(replicator);
|
|
107
111
|
}
|
|
108
112
|
|
|
109
|
-
async
|
|
113
|
+
async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseParams): Promise<boolean> {
|
|
110
114
|
const connection = this._connections.get(peerId);
|
|
111
115
|
if (!connection) {
|
|
112
116
|
return false;
|
|
113
117
|
}
|
|
114
118
|
|
|
115
|
-
return connection.connection.
|
|
119
|
+
return connection.connection.shouldAdvertise(params);
|
|
116
120
|
}
|
|
117
121
|
|
|
118
122
|
private _onConnectionOpen(connection: ReplicatorConnection) {
|
|
@@ -23,13 +23,11 @@ export interface EchoReplicatorContext {
|
|
|
23
23
|
*/
|
|
24
24
|
get peerId(): string;
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
|
|
27
27
|
|
|
28
|
+
onConnectionOpen(connection: ReplicatorConnection): void;
|
|
28
29
|
onConnectionClosed(connection: ReplicatorConnection): void;
|
|
29
|
-
|
|
30
30
|
onConnectionAuthScopeChanged(connection: ReplicatorConnection): void;
|
|
31
|
-
|
|
32
|
-
getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
|
|
33
31
|
}
|
|
34
32
|
|
|
35
33
|
export interface ReplicatorConnection {
|
|
@@ -49,13 +47,12 @@ export interface ReplicatorConnection {
|
|
|
49
47
|
writable: WritableStream<Message>;
|
|
50
48
|
|
|
51
49
|
/**
|
|
52
|
-
* @returns true if the document should be
|
|
53
|
-
*
|
|
54
|
-
* The remote peer can still request the document by it's id bypassing this check.
|
|
50
|
+
* @returns true if the document should be advertised to this peer.
|
|
51
|
+
* The remote peer can still request the document by its id bypassing this check.
|
|
55
52
|
*/
|
|
56
|
-
|
|
53
|
+
shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean>;
|
|
57
54
|
}
|
|
58
55
|
|
|
59
|
-
export type
|
|
56
|
+
export type ShouldAdvertiseParams = {
|
|
60
57
|
documentId: string;
|
|
61
58
|
};
|