@dxos/echo-pipeline 0.3.11-main.4eab977 → 0.3.11-main.4faaf44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-W3SSYW3X.mjs → chunk-AUYVKEFF.mjs} +162 -40
- package/dist/lib/browser/chunk-AUYVKEFF.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +30 -8
- package/dist/lib/browser/testing/index.mjs.map +4 -4
- package/dist/lib/node/{chunk-KTFCZMAY.cjs → chunk-ETE7TCKN.cjs} +161 -42
- package/dist/lib/node/chunk-ETE7TCKN.cjs.map +7 -0
- package/dist/lib/node/index.cjs +26 -26
- package/dist/lib/node/index.cjs.map +1 -1
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +43 -22
- package/dist/lib/node/testing/index.cjs.map +4 -4
- package/dist/types/src/automerge/automerge-host.d.ts +35 -2
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/index.d.ts +1 -1
- package/dist/types/src/automerge/index.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +1 -3
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +2 -2
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/testing/change-metadata.d.ts +8 -0
- package/dist/types/src/testing/change-metadata.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +1 -0
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/package.json +33 -33
- package/src/automerge/automerge-host.test.ts +319 -34
- package/src/automerge/automerge-host.ts +115 -20
- package/src/automerge/index.ts +1 -1
- package/src/metadata/metadata-store.ts +12 -2
- package/src/space/space-manager.ts +3 -3
- package/src/testing/change-metadata.ts +27 -0
- package/src/testing/index.ts +1 -0
- package/src/testing/test-agent-builder.ts +1 -1
- package/dist/lib/browser/chunk-W3SSYW3X.mjs.map +0 -7
- package/dist/lib/node/chunk-KTFCZMAY.cjs.map +0 -7
|
@@ -2,16 +2,20 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { randomBytes } from 'crypto';
|
|
5
6
|
import expect from 'expect';
|
|
7
|
+
import waitForExpect from 'wait-for-expect';
|
|
6
8
|
|
|
7
9
|
import { Trigger, asyncTimeout, sleep } from '@dxos/async';
|
|
8
10
|
import { type Message, NetworkAdapter, type PeerId, Repo } from '@dxos/automerge/automerge-repo';
|
|
9
11
|
import { invariant } from '@dxos/invariant';
|
|
10
12
|
import { log } from '@dxos/log';
|
|
11
13
|
import { StorageType, createStorage } from '@dxos/random-access-storage';
|
|
12
|
-
import {
|
|
14
|
+
import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/teleport/testing';
|
|
15
|
+
import { afterTest, describe, test } from '@dxos/test';
|
|
16
|
+
import { arrayToBuffer, bufferToArray } from '@dxos/util';
|
|
13
17
|
|
|
14
|
-
import { AutomergeHost } from './automerge-host';
|
|
18
|
+
import { AutomergeHost, AutomergeStorageAdapter, MeshNetworkAdapter } from './automerge-host';
|
|
15
19
|
|
|
16
20
|
describe('AutomergeHost', () => {
|
|
17
21
|
test('can create documents', () => {
|
|
@@ -44,13 +48,12 @@ describe('AutomergeHost', () => {
|
|
|
44
48
|
});
|
|
45
49
|
|
|
46
50
|
test('basic networking', async () => {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
const clientAdapter = new TestAdapter(context, 'client');
|
|
51
|
+
const hostAdapter: TestAdapter = new TestAdapter({
|
|
52
|
+
send: (message: Message) => clientAdapter.receive(message),
|
|
53
|
+
});
|
|
54
|
+
const clientAdapter: TestAdapter = new TestAdapter({
|
|
55
|
+
send: (message: Message) => hostAdapter.receive(message),
|
|
56
|
+
});
|
|
54
57
|
|
|
55
58
|
const host = new Repo({
|
|
56
59
|
network: [hostAdapter],
|
|
@@ -60,6 +63,10 @@ describe('AutomergeHost', () => {
|
|
|
60
63
|
});
|
|
61
64
|
hostAdapter.ready();
|
|
62
65
|
clientAdapter.ready();
|
|
66
|
+
await hostAdapter.onConnect.wait();
|
|
67
|
+
await clientAdapter.onConnect.wait();
|
|
68
|
+
hostAdapter.peerCandidate(clientAdapter.peerId!);
|
|
69
|
+
clientAdapter.peerCandidate(hostAdapter.peerId!);
|
|
63
70
|
|
|
64
71
|
const handle = host.create();
|
|
65
72
|
const text = 'Hello world';
|
|
@@ -68,17 +75,301 @@ describe('AutomergeHost', () => {
|
|
|
68
75
|
});
|
|
69
76
|
|
|
70
77
|
const docOnClient = client.find(handle.url);
|
|
71
|
-
await asyncTimeout(docOnClient.
|
|
72
|
-
expect(docOnClient.docSync().text).toEqual(text);
|
|
78
|
+
expect((await asyncTimeout(docOnClient.doc(), 1000)).text).toEqual(text);
|
|
73
79
|
});
|
|
74
80
|
|
|
75
|
-
test('
|
|
76
|
-
|
|
81
|
+
test('recovering from a lost connection', async () => {
|
|
82
|
+
let connectionState: 'on' | 'off' = 'on';
|
|
83
|
+
|
|
84
|
+
const hostAdapter: TestAdapter = new TestAdapter({
|
|
85
|
+
send: (message: Message) => connectionState === 'on' && sleep(10).then(() => clientAdapter.receive(message)),
|
|
86
|
+
});
|
|
87
|
+
const clientAdapter: TestAdapter = new TestAdapter({
|
|
88
|
+
send: (message: Message) => connectionState === 'on' && sleep(10).then(() => hostAdapter.receive(message)),
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
const host = new Repo({
|
|
92
|
+
network: [hostAdapter],
|
|
93
|
+
});
|
|
94
|
+
const client = new Repo({
|
|
95
|
+
network: [clientAdapter],
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
// Establish connection.
|
|
99
|
+
hostAdapter.ready();
|
|
100
|
+
clientAdapter.ready();
|
|
101
|
+
await hostAdapter.onConnect.wait();
|
|
102
|
+
await clientAdapter.onConnect.wait();
|
|
103
|
+
hostAdapter.peerCandidate(clientAdapter.peerId!);
|
|
104
|
+
clientAdapter.peerCandidate(hostAdapter.peerId!);
|
|
105
|
+
|
|
106
|
+
const handle = host.create();
|
|
107
|
+
const docOnClient = client.find(handle.url);
|
|
108
|
+
{
|
|
109
|
+
const sanityText = 'Hello world';
|
|
110
|
+
handle.change((doc: any) => {
|
|
111
|
+
doc.sanityText = sanityText;
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
expect((await asyncTimeout(docOnClient.doc(), 1000)).sanityText).toEqual(sanityText);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Disrupt connection.
|
|
118
|
+
const offlineText = 'This has been written while the connection was off';
|
|
119
|
+
{
|
|
120
|
+
connectionState = 'off';
|
|
121
|
+
|
|
122
|
+
handle.change((doc: any) => {
|
|
123
|
+
doc.offlineText = offlineText;
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
await sleep(100);
|
|
127
|
+
expect((await asyncTimeout(docOnClient.doc(), 1000)).offlineText).toBeUndefined();
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Re-establish connection.
|
|
131
|
+
const onlineText = 'This has been written after the connection was re-established';
|
|
132
|
+
{
|
|
133
|
+
connectionState = 'on';
|
|
134
|
+
hostAdapter.peerDisconnected(clientAdapter.peerId!);
|
|
135
|
+
clientAdapter.peerDisconnected(hostAdapter.peerId!);
|
|
136
|
+
hostAdapter.peerCandidate(clientAdapter.peerId!);
|
|
137
|
+
clientAdapter.peerCandidate(hostAdapter.peerId!);
|
|
138
|
+
|
|
139
|
+
handle.change((doc: any) => {
|
|
140
|
+
doc.onlineText = onlineText;
|
|
141
|
+
});
|
|
142
|
+
await sleep(100);
|
|
143
|
+
expect((await asyncTimeout(docOnClient.doc(), 1000)).onlineText).toEqual(onlineText);
|
|
144
|
+
expect((await asyncTimeout(docOnClient.doc(), 1000)).offlineText).toEqual(offlineText);
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test('integration test with teleport', async () => {
|
|
149
|
+
const createAutomergeRepo = () => {
|
|
150
|
+
const meshAdapter = new MeshNetworkAdapter();
|
|
151
|
+
const repo = new Repo({
|
|
152
|
+
network: [meshAdapter],
|
|
153
|
+
});
|
|
154
|
+
meshAdapter.ready();
|
|
155
|
+
return { repo, meshAdapter };
|
|
156
|
+
};
|
|
157
|
+
const peer1 = createAutomergeRepo();
|
|
158
|
+
const peer2 = createAutomergeRepo();
|
|
159
|
+
const handle = peer1.repo.create();
|
|
160
|
+
|
|
161
|
+
const teleportBuilder = new TeleportBuilder();
|
|
162
|
+
afterTest(() => teleportBuilder.destroy());
|
|
163
|
+
|
|
164
|
+
const [teleportPeer1, teleportPeer2] = teleportBuilder.createPeers({ factory: () => new TeleportPeer() });
|
|
165
|
+
{
|
|
166
|
+
// Initiate connection.
|
|
167
|
+
const [connection1, connection2] = await teleportBuilder.connect(teleportPeer1, teleportPeer2);
|
|
168
|
+
connection1.teleport.addExtension('automerge', peer1.meshAdapter.createExtension());
|
|
169
|
+
connection2.teleport.addExtension('automerge', peer2.meshAdapter.createExtension());
|
|
170
|
+
|
|
171
|
+
// Test connection.
|
|
172
|
+
const text = 'Hello world';
|
|
173
|
+
handle.change((doc: any) => {
|
|
174
|
+
doc.text = text;
|
|
175
|
+
});
|
|
176
|
+
const docOnPeer2 = peer2.repo.find(handle.url);
|
|
177
|
+
await waitForExpect(async () => expect((await asyncTimeout(docOnPeer2.doc(), 1000)).text).toEqual(text), 1000);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const offlineText = 'This has been written while the connection was off';
|
|
181
|
+
{
|
|
182
|
+
// Disconnect peers.
|
|
183
|
+
await teleportBuilder.disconnect(teleportPeer1, teleportPeer2);
|
|
184
|
+
|
|
185
|
+
// Make offline changes.
|
|
186
|
+
const offlineText = 'This has been written while the connection was off';
|
|
187
|
+
handle.change((doc: any) => {
|
|
188
|
+
doc.offlineText = offlineText;
|
|
189
|
+
});
|
|
190
|
+
const docOnPeer2 = peer2.repo.find(handle.url);
|
|
191
|
+
await sleep(100);
|
|
192
|
+
expect((await asyncTimeout(docOnPeer2.doc(), 1000)).offlineText).toBeUndefined();
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
{
|
|
196
|
+
// Reconnect peers.
|
|
197
|
+
const [connection1, connection2] = await teleportBuilder.connect(teleportPeer1, teleportPeer2);
|
|
198
|
+
connection1.teleport.addExtension('automerge', peer1.meshAdapter.createExtension());
|
|
199
|
+
connection2.teleport.addExtension('automerge', peer2.meshAdapter.createExtension());
|
|
200
|
+
|
|
201
|
+
// Wait for offline changes to be synced.
|
|
202
|
+
const docOnPeer2 = peer2.repo.find(handle.url);
|
|
203
|
+
await waitForExpect(
|
|
204
|
+
async () => expect((await asyncTimeout(docOnPeer2.doc(), 1000)).offlineText).toEqual(offlineText),
|
|
205
|
+
1000,
|
|
206
|
+
);
|
|
207
|
+
|
|
208
|
+
// Test connection.
|
|
209
|
+
const onlineText = 'This has been written after the connection was re-established';
|
|
210
|
+
handle.change((doc: any) => {
|
|
211
|
+
doc.onlineText = onlineText;
|
|
212
|
+
});
|
|
213
|
+
await waitForExpect(
|
|
214
|
+
async () => expect((await asyncTimeout(docOnPeer2.doc(), 1000)).onlineText).toEqual(onlineText),
|
|
215
|
+
1000,
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
describe('storage', () => {
|
|
221
|
+
test('load range on node', async () => {
|
|
222
|
+
const root = `/tmp/${randomBytes(16).toString('hex')}`;
|
|
223
|
+
{
|
|
224
|
+
const storage = createStorage({ type: StorageType.NODE, root });
|
|
225
|
+
const adapter = new AutomergeStorageAdapter(storage.createDirectory());
|
|
77
226
|
|
|
78
|
-
|
|
227
|
+
await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
|
|
228
|
+
await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
|
|
229
|
+
await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
{
|
|
233
|
+
const storage = createStorage({ type: StorageType.NODE, root });
|
|
234
|
+
const adapter = new AutomergeStorageAdapter(storage.createDirectory());
|
|
235
|
+
|
|
236
|
+
const range = await adapter.loadRange(['test']);
|
|
237
|
+
expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).toEqual(['one', 'two']);
|
|
238
|
+
expect(range.map((chunk) => chunk.key)).toEqual([
|
|
239
|
+
['test', '1'],
|
|
240
|
+
['test', '2'],
|
|
241
|
+
]);
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
test('removeRange on node', async () => {
|
|
246
|
+
const root = `/tmp/${randomBytes(16).toString('hex')}`;
|
|
247
|
+
{
|
|
248
|
+
const storage = createStorage({ type: StorageType.NODE, root });
|
|
249
|
+
const adapter = new AutomergeStorageAdapter(storage.createDirectory());
|
|
250
|
+
await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
|
|
251
|
+
await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
|
|
252
|
+
await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
{
|
|
256
|
+
const storage = createStorage({ type: StorageType.NODE, root });
|
|
257
|
+
const adapter = new AutomergeStorageAdapter(storage.createDirectory());
|
|
258
|
+
await adapter.removeRange(['test']);
|
|
259
|
+
const range = await adapter.loadRange(['test']);
|
|
260
|
+
expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).toEqual([]);
|
|
261
|
+
const range2 = await adapter.loadRange(['bar']);
|
|
262
|
+
expect(range2.map((chunk) => arrayToBuffer(chunk.data!).toString())).toEqual(['bar']);
|
|
263
|
+
expect(range2.map((chunk) => chunk.key)).toEqual([['bar', '1']]);
|
|
264
|
+
}
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
test('replication though a 4 peer chain', async () => {
|
|
269
|
+
const pairAB = TestAdapter.createPair();
|
|
270
|
+
const pairBC = TestAdapter.createPair();
|
|
271
|
+
const pairCD = TestAdapter.createPair();
|
|
272
|
+
|
|
273
|
+
const repoA = new Repo({
|
|
274
|
+
peerId: 'A' as any,
|
|
275
|
+
network: [pairAB[0]],
|
|
276
|
+
sharePolicy: async () => true,
|
|
277
|
+
});
|
|
278
|
+
const _repoB = new Repo({
|
|
279
|
+
peerId: 'B' as any,
|
|
280
|
+
network: [pairAB[1], pairBC[0]],
|
|
281
|
+
sharePolicy: async () => true,
|
|
282
|
+
});
|
|
283
|
+
const _repoC = new Repo({
|
|
284
|
+
peerId: 'C' as any,
|
|
285
|
+
network: [pairBC[1], pairCD[0]],
|
|
286
|
+
sharePolicy: async () => true,
|
|
287
|
+
});
|
|
288
|
+
const repoD = new Repo({
|
|
289
|
+
peerId: 'D' as any,
|
|
290
|
+
network: [pairCD[1]],
|
|
291
|
+
sharePolicy: async () => true,
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
for (const pair of [pairAB, pairBC, pairCD]) {
|
|
295
|
+
pair[0].ready();
|
|
296
|
+
pair[1].ready();
|
|
297
|
+
await pair[0].onConnect.wait();
|
|
298
|
+
await pair[1].onConnect.wait();
|
|
299
|
+
pair[0].peerCandidate(pair[1].peerId!);
|
|
300
|
+
pair[1].peerCandidate(pair[0].peerId!);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const docA = repoA.create();
|
|
304
|
+
// NOTE: Doesn't work if the doc is empty.
|
|
305
|
+
docA.change((doc: any) => {
|
|
306
|
+
doc.text = 'Hello world';
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
// If we wait here for replication to finish naturally, the test will pass.
|
|
310
|
+
|
|
311
|
+
const docD = repoD.find(docA.url);
|
|
312
|
+
|
|
313
|
+
await docD.whenReady();
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test('replication though a 3 peer chain', async () => {
|
|
317
|
+
const pairAB = TestAdapter.createPair();
|
|
318
|
+
const pairBC = TestAdapter.createPair();
|
|
319
|
+
|
|
320
|
+
const repoA = new Repo({
|
|
321
|
+
peerId: 'A' as any,
|
|
322
|
+
network: [pairAB[0]],
|
|
323
|
+
sharePolicy: async () => true,
|
|
324
|
+
});
|
|
325
|
+
const repoB = new Repo({
|
|
326
|
+
peerId: 'B' as any,
|
|
327
|
+
network: [pairAB[1], pairBC[0]],
|
|
328
|
+
sharePolicy: async () => true,
|
|
329
|
+
});
|
|
330
|
+
const repoC = new Repo({
|
|
331
|
+
peerId: 'C' as any,
|
|
332
|
+
network: [pairBC[1]],
|
|
333
|
+
sharePolicy: async () => true,
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
for (const pair of [pairAB, pairBC]) {
|
|
337
|
+
pair[0].ready();
|
|
338
|
+
pair[1].ready();
|
|
339
|
+
await pair[0].onConnect.wait();
|
|
340
|
+
await pair[1].onConnect.wait();
|
|
341
|
+
pair[0].peerCandidate(pair[1].peerId!);
|
|
342
|
+
pair[1].peerCandidate(pair[0].peerId!);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const docA = repoA.create();
|
|
346
|
+
// NOTE: Doesn't work if the doc is empty.
|
|
347
|
+
docA.change((doc: any) => {
|
|
348
|
+
doc.text = 'Hello world';
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
const _docB = repoB.find(docA.url);
|
|
352
|
+
const docC = repoC.find(docA.url);
|
|
353
|
+
|
|
354
|
+
await docC.whenReady();
|
|
355
|
+
});
|
|
356
|
+
});
|
|
79
357
|
|
|
80
358
|
class TestAdapter extends NetworkAdapter {
|
|
81
|
-
|
|
359
|
+
static createPair() {
|
|
360
|
+
const adapter1: TestAdapter = new TestAdapter({
|
|
361
|
+
send: (message: Message) => sleep(10).then(() => adapter2.receive(message)),
|
|
362
|
+
});
|
|
363
|
+
const adapter2: TestAdapter = new TestAdapter({
|
|
364
|
+
send: (message: Message) => sleep(10).then(() => adapter1.receive(message)),
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
return [adapter1, adapter2];
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
public onConnect = new Trigger();
|
|
371
|
+
|
|
372
|
+
constructor(private readonly _params: { send: (message: Message) => void }) {
|
|
82
373
|
super();
|
|
83
374
|
}
|
|
84
375
|
|
|
@@ -90,32 +381,26 @@ class TestAdapter extends NetworkAdapter {
|
|
|
90
381
|
|
|
91
382
|
override connect(peerId: PeerId) {
|
|
92
383
|
this.peerId = peerId;
|
|
93
|
-
this.
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
384
|
+
this.onConnect.wake();
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
peerCandidate(peerId: PeerId) {
|
|
388
|
+
invariant(peerId, 'PeerId is required');
|
|
389
|
+
this.emit('peer-candidate', { peerId, peerMetadata: {} });
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
peerDisconnected(peerId: PeerId) {
|
|
393
|
+
invariant(peerId, 'PeerId is required');
|
|
394
|
+
this.emit('peer-disconnected', { peerId });
|
|
103
395
|
}
|
|
104
396
|
|
|
105
397
|
override send(message: Message) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
.then((adapter) => {
|
|
109
|
-
adapter.receive(message);
|
|
110
|
-
})
|
|
111
|
-
.catch((error) => {
|
|
112
|
-
log.catch(error);
|
|
113
|
-
});
|
|
398
|
+
log('send', { from: message.senderId, to: message.targetId, type: message.type });
|
|
399
|
+
this._params.send(message);
|
|
114
400
|
}
|
|
115
401
|
|
|
116
402
|
override disconnect() {
|
|
117
403
|
this.peerId = undefined;
|
|
118
|
-
this.context[this.role].reset();
|
|
119
404
|
}
|
|
120
405
|
|
|
121
406
|
receive(message: Message) {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { Trigger } from '@dxos/async';
|
|
5
6
|
import {
|
|
6
7
|
Repo,
|
|
7
8
|
NetworkAdapter,
|
|
@@ -12,31 +13,79 @@ import {
|
|
|
12
13
|
type StorageKey,
|
|
13
14
|
cbor,
|
|
14
15
|
} from '@dxos/automerge/automerge-repo';
|
|
16
|
+
import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
|
|
15
17
|
import { Stream } from '@dxos/codec-protobuf';
|
|
16
18
|
import { invariant } from '@dxos/invariant';
|
|
19
|
+
import { PublicKey } from '@dxos/keys';
|
|
17
20
|
import { log } from '@dxos/log';
|
|
18
21
|
import { type HostInfo, type SyncRepoRequest, type SyncRepoResponse } from '@dxos/protocols/proto/dxos/echo/service';
|
|
19
22
|
import { type PeerInfo } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';
|
|
20
|
-
import { type Directory } from '@dxos/random-access-storage';
|
|
23
|
+
import { StorageType, type Directory } from '@dxos/random-access-storage';
|
|
21
24
|
import { AutomergeReplicator } from '@dxos/teleport-extension-automerge-replicator';
|
|
22
|
-
import { arrayToBuffer, bufferToArray } from '@dxos/util';
|
|
25
|
+
import { ComplexMap, ComplexSet, arrayToBuffer, bufferToArray, defaultMap } from '@dxos/util';
|
|
23
26
|
|
|
24
27
|
export class AutomergeHost {
|
|
25
28
|
private readonly _repo: Repo;
|
|
26
29
|
private readonly _meshNetwork: MeshNetworkAdapter;
|
|
27
30
|
private readonly _clientNetwork: LocalHostNetworkAdapter;
|
|
28
|
-
private readonly _storage:
|
|
31
|
+
private readonly _storage: StorageAdapter;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* spaceKey -> deviceKey[]
|
|
35
|
+
*/
|
|
36
|
+
private readonly _authorizedDevices = new ComplexMap<PublicKey, ComplexSet<PublicKey>>(PublicKey.hash);
|
|
29
37
|
|
|
30
38
|
constructor(storageDirectory: Directory) {
|
|
31
39
|
this._meshNetwork = new MeshNetworkAdapter();
|
|
32
40
|
this._clientNetwork = new LocalHostNetworkAdapter();
|
|
33
|
-
|
|
41
|
+
|
|
42
|
+
// TODO(mykola): Delete specific handling of IDB storage.
|
|
43
|
+
this._storage =
|
|
44
|
+
storageDirectory.type === StorageType.IDB
|
|
45
|
+
? new IndexedDBStorageAdapter(storageDirectory.path, 'data')
|
|
46
|
+
: new AutomergeStorageAdapter(storageDirectory);
|
|
34
47
|
this._repo = new Repo({
|
|
48
|
+
peerId: `host-${PublicKey.random().toHex()}` as PeerId,
|
|
35
49
|
network: [this._clientNetwork, this._meshNetwork],
|
|
36
50
|
storage: this._storage,
|
|
37
51
|
|
|
38
52
|
// TODO(dmaretskyi): Share based on HALO permissions and space affinity.
|
|
39
|
-
|
|
53
|
+
// Hosts, running in the worker, don't share documents unless requested by other peers.
|
|
54
|
+
sharePolicy: async (peerId /* device key */, documentId /* space key */) => {
|
|
55
|
+
if (peerId.startsWith('client-')) {
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (!documentId) {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const doc = this._repo.handles[documentId]?.docSync();
|
|
64
|
+
if (!doc) {
|
|
65
|
+
log('doc not found for share policy check', { peerId, documentId });
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
try {
|
|
70
|
+
const spaceKey = PublicKey.from(doc.experimental_spaceKey);
|
|
71
|
+
const authorizedDevices = this._authorizedDevices.get(spaceKey);
|
|
72
|
+
|
|
73
|
+
// TODO(mykola): Hack, stop abusing `peerMetadata` field.
|
|
74
|
+
const deviceKeyHex = (this.repo.peerMetadataByPeerId[peerId] as any)?.dxos_deviceKey;
|
|
75
|
+
if (!deviceKeyHex) {
|
|
76
|
+
log.warn('device key not found for share policy check', { peerId, documentId });
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
const deviceKey = PublicKey.from(deviceKeyHex);
|
|
80
|
+
|
|
81
|
+
const isAuthorized = authorizedDevices?.has(deviceKey) ?? false;
|
|
82
|
+
log.info('share policy check', { peerId, documentId, deviceKey, spaceKey, isAuthorized });
|
|
83
|
+
return isAuthorized;
|
|
84
|
+
} catch (err) {
|
|
85
|
+
log.catch(err);
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
}, // Share everything.
|
|
40
89
|
});
|
|
41
90
|
this._clientNetwork.ready();
|
|
42
91
|
this._meshNetwork.ready();
|
|
@@ -47,6 +96,7 @@ export class AutomergeHost {
|
|
|
47
96
|
}
|
|
48
97
|
|
|
49
98
|
async close() {
|
|
99
|
+
this._storage instanceof AutomergeStorageAdapter && (await this._storage.close());
|
|
50
100
|
await this._clientNetwork.close();
|
|
51
101
|
}
|
|
52
102
|
|
|
@@ -62,7 +112,7 @@ export class AutomergeHost {
|
|
|
62
112
|
return this._clientNetwork.sendSyncMessage(request);
|
|
63
113
|
}
|
|
64
114
|
|
|
65
|
-
getHostInfo(): HostInfo {
|
|
115
|
+
async getHostInfo(): Promise<HostInfo> {
|
|
66
116
|
return this._clientNetwork.getHostInfo();
|
|
67
117
|
}
|
|
68
118
|
|
|
@@ -73,6 +123,10 @@ export class AutomergeHost {
|
|
|
73
123
|
createExtension(): AutomergeReplicator {
|
|
74
124
|
return this._meshNetwork.createExtension();
|
|
75
125
|
}
|
|
126
|
+
|
|
127
|
+
authorizeDevice(spaceKey: PublicKey, deviceKey: PublicKey) {
|
|
128
|
+
defaultMap(this._authorizedDevices, spaceKey, () => new ComplexSet(PublicKey.hash)).add(deviceKey);
|
|
129
|
+
}
|
|
76
130
|
}
|
|
77
131
|
|
|
78
132
|
type ClientSyncState = {
|
|
@@ -98,8 +152,11 @@ class LocalHostNetworkAdapter extends NetworkAdapter {
|
|
|
98
152
|
});
|
|
99
153
|
}
|
|
100
154
|
|
|
155
|
+
private _connected = new Trigger();
|
|
156
|
+
|
|
101
157
|
override connect(peerId: PeerId): void {
|
|
102
158
|
this.peerId = peerId;
|
|
159
|
+
this._connected.wake();
|
|
103
160
|
// No-op. Client always connects first
|
|
104
161
|
}
|
|
105
162
|
|
|
@@ -140,18 +197,26 @@ class LocalHostNetworkAdapter extends NetworkAdapter {
|
|
|
140
197
|
},
|
|
141
198
|
});
|
|
142
199
|
|
|
143
|
-
this.
|
|
144
|
-
|
|
145
|
-
|
|
200
|
+
this._connected
|
|
201
|
+
.wait({ timeout: 1_000 })
|
|
202
|
+
.then(() => {
|
|
203
|
+
this.emit('peer-candidate', {
|
|
204
|
+
peerMetadata: {},
|
|
205
|
+
peerId,
|
|
206
|
+
});
|
|
207
|
+
})
|
|
208
|
+
.catch((err) => log.catch(err));
|
|
146
209
|
});
|
|
147
210
|
}
|
|
148
211
|
|
|
149
212
|
async sendSyncMessage({ id, syncMessage }: SyncRepoRequest): Promise<void> {
|
|
213
|
+
await this._connected.wait({ timeout: 1_000 });
|
|
150
214
|
const message = cbor.decode(syncMessage!) as Message;
|
|
151
215
|
this.emit('message', message);
|
|
152
216
|
}
|
|
153
217
|
|
|
154
|
-
getHostInfo(): HostInfo {
|
|
218
|
+
async getHostInfo(): Promise<HostInfo> {
|
|
219
|
+
await this._connected.wait({ timeout: 1_000 });
|
|
155
220
|
invariant(this.peerId, 'Peer id not set.');
|
|
156
221
|
return {
|
|
157
222
|
peerId: this.peerId,
|
|
@@ -166,7 +231,7 @@ class LocalHostNetworkAdapter extends NetworkAdapter {
|
|
|
166
231
|
/**
|
|
167
232
|
* Used to replicate with other peers over the network.
|
|
168
233
|
*/
|
|
169
|
-
class MeshNetworkAdapter extends NetworkAdapter {
|
|
234
|
+
export class MeshNetworkAdapter extends NetworkAdapter {
|
|
170
235
|
private readonly _extensions: Map<string, AutomergeReplicator> = new Map();
|
|
171
236
|
|
|
172
237
|
/**
|
|
@@ -204,7 +269,7 @@ class MeshNetworkAdapter extends NetworkAdapter {
|
|
|
204
269
|
peerId: this.peerId,
|
|
205
270
|
},
|
|
206
271
|
{
|
|
207
|
-
onStartReplication: async (info) => {
|
|
272
|
+
onStartReplication: async (info, remotePeerId /** Teleport ID */) => {
|
|
208
273
|
// Note: We store only one extension per peer.
|
|
209
274
|
// There can be a case where two connected peers have more than one teleport connection between them
|
|
210
275
|
// and each of them uses different teleport connections to send messages.
|
|
@@ -218,6 +283,10 @@ class MeshNetworkAdapter extends NetworkAdapter {
|
|
|
218
283
|
// TODO(mykola): Fix race condition?
|
|
219
284
|
this._extensions.set(info.id, extension);
|
|
220
285
|
this.emit('peer-candidate', {
|
|
286
|
+
// TODO(mykola): Hack, stop abusing `peerMetadata` field.
|
|
287
|
+
peerMetadata: {
|
|
288
|
+
dxos_deviceKey: remotePeerId.toHex(),
|
|
289
|
+
} as any,
|
|
221
290
|
peerId: info.id as PeerId,
|
|
222
291
|
});
|
|
223
292
|
},
|
|
@@ -227,10 +296,13 @@ class MeshNetworkAdapter extends NetworkAdapter {
|
|
|
227
296
|
this.emit('message', message);
|
|
228
297
|
},
|
|
229
298
|
onClose: async () => {
|
|
230
|
-
peerInfo
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
299
|
+
if (!peerInfo) {
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
this.emit('peer-disconnected', {
|
|
303
|
+
peerId: peerInfo.id as PeerId,
|
|
304
|
+
});
|
|
305
|
+
this._extensions.delete(peerInfo.id);
|
|
234
306
|
},
|
|
235
307
|
},
|
|
236
308
|
);
|
|
@@ -238,12 +310,19 @@ class MeshNetworkAdapter extends NetworkAdapter {
|
|
|
238
310
|
}
|
|
239
311
|
}
|
|
240
312
|
|
|
241
|
-
class AutomergeStorageAdapter extends StorageAdapter {
|
|
313
|
+
export class AutomergeStorageAdapter extends StorageAdapter {
|
|
314
|
+
// TODO(mykola): Hack for restricting automerge Repo to access storage if Host is `closed`.
|
|
315
|
+
// Automerge Repo do not have any lifetime management.
|
|
316
|
+
private _state: 'opened' | 'closed' = 'opened';
|
|
317
|
+
|
|
242
318
|
constructor(private readonly _directory: Directory) {
|
|
243
319
|
super();
|
|
244
320
|
}
|
|
245
321
|
|
|
246
322
|
override async load(key: StorageKey): Promise<Uint8Array | undefined> {
|
|
323
|
+
if (this._state !== 'opened') {
|
|
324
|
+
return undefined;
|
|
325
|
+
}
|
|
247
326
|
const filename = this._getFilename(key);
|
|
248
327
|
const file = this._directory.getOrCreateFile(filename);
|
|
249
328
|
const { size } = await file.stat();
|
|
@@ -255,6 +334,9 @@ class AutomergeStorageAdapter extends StorageAdapter {
|
|
|
255
334
|
}
|
|
256
335
|
|
|
257
336
|
override async save(key: StorageKey, data: Uint8Array): Promise<void> {
|
|
337
|
+
if (this._state !== 'opened') {
|
|
338
|
+
return undefined;
|
|
339
|
+
}
|
|
258
340
|
const filename = this._getFilename(key);
|
|
259
341
|
const file = this._directory.getOrCreateFile(filename);
|
|
260
342
|
await file.write(0, arrayToBuffer(data));
|
|
@@ -264,13 +346,19 @@ class AutomergeStorageAdapter extends StorageAdapter {
|
|
|
264
346
|
}
|
|
265
347
|
|
|
266
348
|
override async remove(key: StorageKey): Promise<void> {
|
|
349
|
+
if (this._state !== 'opened') {
|
|
350
|
+
return undefined;
|
|
351
|
+
}
|
|
267
352
|
// TODO(dmaretskyi): Better deletion.
|
|
268
353
|
const filename = this._getFilename(key);
|
|
269
354
|
const file = this._directory.getOrCreateFile(filename);
|
|
270
|
-
await file.
|
|
355
|
+
await file.destroy();
|
|
271
356
|
}
|
|
272
357
|
|
|
273
358
|
override async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
|
|
359
|
+
if (this._state !== 'opened') {
|
|
360
|
+
return [];
|
|
361
|
+
}
|
|
274
362
|
const filename = this._getFilename(keyPrefix);
|
|
275
363
|
const entries = await this._directory.list();
|
|
276
364
|
return Promise.all(
|
|
@@ -289,18 +377,25 @@ class AutomergeStorageAdapter extends StorageAdapter {
|
|
|
289
377
|
}
|
|
290
378
|
|
|
291
379
|
override async removeRange(keyPrefix: StorageKey): Promise<void> {
|
|
380
|
+
if (this._state !== 'opened') {
|
|
381
|
+
return undefined;
|
|
382
|
+
}
|
|
292
383
|
const filename = this._getFilename(keyPrefix);
|
|
293
384
|
const entries = await this._directory.list();
|
|
294
385
|
await Promise.all(
|
|
295
386
|
entries
|
|
296
387
|
.filter((entry) => entry.startsWith(filename))
|
|
297
388
|
.map(async (entry): Promise<void> => {
|
|
298
|
-
const file = this._directory.getOrCreateFile(
|
|
299
|
-
await file.
|
|
389
|
+
const file = this._directory.getOrCreateFile(entry);
|
|
390
|
+
await file.destroy();
|
|
300
391
|
}),
|
|
301
392
|
);
|
|
302
393
|
}
|
|
303
394
|
|
|
395
|
+
async close(): Promise<void> {
|
|
396
|
+
this._state = 'closed';
|
|
397
|
+
}
|
|
398
|
+
|
|
304
399
|
private _getFilename(key: StorageKey): string {
|
|
305
400
|
return key.map((k) => k.replaceAll('%', '%25').replaceAll('-', '%2D')).join('-');
|
|
306
401
|
}
|