@dxos/echo-pipeline 0.6.0 → 0.6.1-main.09a92b0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/lib/browser/{chunk-HS77A4I4.mjs → chunk-A2LCXJVD.mjs} +16 -1
  2. package/dist/lib/browser/chunk-A2LCXJVD.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +323 -211
  4. package/dist/lib/browser/index.mjs.map +4 -4
  5. package/dist/lib/browser/meta.json +1 -1
  6. package/dist/lib/browser/testing/index.mjs +1 -1
  7. package/dist/lib/node/{chunk-Y5U7UXEL.cjs → chunk-GHBIMYZK.cjs} +19 -4
  8. package/dist/lib/node/chunk-GHBIMYZK.cjs.map +7 -0
  9. package/dist/lib/node/index.cjs +364 -258
  10. package/dist/lib/node/index.cjs.map +4 -4
  11. package/dist/lib/node/meta.json +1 -1
  12. package/dist/lib/node/testing/index.cjs +11 -11
  13. package/dist/types/src/automerge/automerge-host.d.ts +13 -9
  14. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  15. package/dist/types/src/automerge/echo-network-adapter.d.ts +2 -2
  16. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  17. package/dist/types/src/automerge/echo-network-adapter.test.d.ts +2 -0
  18. package/dist/types/src/automerge/echo-network-adapter.test.d.ts.map +1 -0
  19. package/dist/types/src/automerge/echo-replicator.d.ts +5 -6
  20. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  21. package/dist/types/src/automerge/heads-store.d.ts +13 -0
  22. package/dist/types/src/automerge/heads-store.d.ts.map +1 -0
  23. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +35 -0
  24. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -0
  25. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +2 -2
  26. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  27. package/dist/types/src/db-host/data-service.d.ts +4 -2
  28. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  29. package/package.json +33 -33
  30. package/src/automerge/automerge-doc-loader.ts +1 -1
  31. package/src/automerge/automerge-host.test.ts +34 -17
  32. package/src/automerge/automerge-host.ts +61 -17
  33. package/src/automerge/automerge-repo.test.ts +76 -1
  34. package/src/automerge/echo-network-adapter.test.ts +131 -0
  35. package/src/automerge/echo-network-adapter.ts +10 -6
  36. package/src/automerge/echo-replicator.ts +6 -9
  37. package/src/automerge/heads-store.ts +39 -0
  38. package/src/automerge/mesh-echo-replicator-connection.ts +130 -0
  39. package/src/automerge/mesh-echo-replicator.ts +15 -123
  40. package/src/db-host/data-service.ts +22 -2
  41. package/dist/lib/browser/chunk-HS77A4I4.mjs.map +0 -7
  42. package/dist/lib/node/chunk-Y5U7UXEL.cjs.map +0 -7
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/echo-pipeline",
3
- "version": "0.6.0",
3
+ "version": "0.6.1-main.09a92b0",
4
4
  "description": "ECHO database.",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -39,38 +39,38 @@
39
39
  "crc-32": "^1.2.2",
40
40
  "level": "^8.0.1",
41
41
  "level-transcoder": "^1.0.1",
42
- "@dxos/async": "0.6.0",
43
- "@dxos/automerge": "0.6.0",
44
- "@dxos/context": "0.6.0",
45
- "@dxos/codec-protobuf": "0.6.0",
46
- "@dxos/credentials": "0.6.0",
47
- "@dxos/crypto": "0.6.0",
48
- "@dxos/debug": "0.6.0",
49
- "@dxos/echo-schema": "0.6.0",
50
- "@dxos/echo-protocol": "0.6.0",
51
- "@dxos/indexing": "0.6.0",
52
- "@dxos/feed-store": "0.6.0",
53
- "@dxos/hypercore": "0.6.0",
54
- "@dxos/invariant": "0.6.0",
55
- "@dxos/keyring": "0.6.0",
56
- "@dxos/keys": "0.6.0",
57
- "@dxos/kv-store": "0.6.0",
58
- "@dxos/log": "0.6.0",
59
- "@dxos/node-std": "0.6.0",
60
- "@dxos/network-manager": "0.6.0",
61
- "@dxos/messaging": "0.6.0",
62
- "@dxos/protocols": "0.6.0",
63
- "@dxos/random-access-storage": "0.6.0",
64
- "@dxos/teleport": "0.6.0",
65
- "@dxos/teleport-extension-automerge-replicator": "0.6.0",
66
- "@dxos/rpc": "0.6.0",
67
- "@dxos/teleport-extension-object-sync": "0.6.0",
68
- "@dxos/teleport-extension-gossip": "0.6.0",
69
- "@dxos/teleport-extension-replicator": "0.6.0",
70
- "@dxos/timeframe": "0.6.0",
71
- "@dxos/tracing": "0.6.0",
72
- "@dxos/util": "0.6.0",
73
- "@dxos/typings": "0.6.0"
42
+ "@dxos/async": "0.6.1-main.09a92b0",
43
+ "@dxos/automerge": "0.6.1-main.09a92b0",
44
+ "@dxos/codec-protobuf": "0.6.1-main.09a92b0",
45
+ "@dxos/context": "0.6.1-main.09a92b0",
46
+ "@dxos/crypto": "0.6.1-main.09a92b0",
47
+ "@dxos/credentials": "0.6.1-main.09a92b0",
48
+ "@dxos/debug": "0.6.1-main.09a92b0",
49
+ "@dxos/echo-protocol": "0.6.1-main.09a92b0",
50
+ "@dxos/echo-schema": "0.6.1-main.09a92b0",
51
+ "@dxos/feed-store": "0.6.1-main.09a92b0",
52
+ "@dxos/indexing": "0.6.1-main.09a92b0",
53
+ "@dxos/hypercore": "0.6.1-main.09a92b0",
54
+ "@dxos/invariant": "0.6.1-main.09a92b0",
55
+ "@dxos/keyring": "0.6.1-main.09a92b0",
56
+ "@dxos/keys": "0.6.1-main.09a92b0",
57
+ "@dxos/kv-store": "0.6.1-main.09a92b0",
58
+ "@dxos/log": "0.6.1-main.09a92b0",
59
+ "@dxos/messaging": "0.6.1-main.09a92b0",
60
+ "@dxos/network-manager": "0.6.1-main.09a92b0",
61
+ "@dxos/node-std": "0.6.1-main.09a92b0",
62
+ "@dxos/protocols": "0.6.1-main.09a92b0",
63
+ "@dxos/random-access-storage": "0.6.1-main.09a92b0",
64
+ "@dxos/rpc": "0.6.1-main.09a92b0",
65
+ "@dxos/teleport": "0.6.1-main.09a92b0",
66
+ "@dxos/teleport-extension-gossip": "0.6.1-main.09a92b0",
67
+ "@dxos/teleport-extension-automerge-replicator": "0.6.1-main.09a92b0",
68
+ "@dxos/teleport-extension-object-sync": "0.6.1-main.09a92b0",
69
+ "@dxos/teleport-extension-replicator": "0.6.1-main.09a92b0",
70
+ "@dxos/tracing": "0.6.1-main.09a92b0",
71
+ "@dxos/timeframe": "0.6.1-main.09a92b0",
72
+ "@dxos/typings": "0.6.1-main.09a92b0",
73
+ "@dxos/util": "0.6.1-main.09a92b0"
74
74
  },
75
75
  "devDependencies": {
76
76
  "fast-check": "^3.19.0",
@@ -225,7 +225,7 @@ export class AutomergeDocumentLoaderImpl implements AutomergeDocumentLoader {
225
225
 
226
226
  private async _createObjectOnDocumentLoad(handle: DocHandle<SpaceDoc>, objectId: string) {
227
227
  try {
228
- await handle.doc(['ready']);
228
+ await handle.whenReady();
229
229
  const logMeta = { objectId, docUrl: handle.url };
230
230
  if (this.onObjectDocumentLoaded.listenerCount() === 0) {
231
231
  log.info('document loaded after all listeners were removed', logMeta);
@@ -4,7 +4,9 @@
4
4
 
5
5
  import expect from 'expect';
6
6
 
7
+ import { getHeads } from '@dxos/automerge/automerge';
7
8
  import { IndexMetadataStore } from '@dxos/indexing';
9
+ import type { LevelDB } from '@dxos/kv-store';
8
10
  import { createTestLevel } from '@dxos/kv-store/testing';
9
11
  import { afterTest, describe, test } from '@dxos/test';
10
12
 
@@ -15,13 +17,8 @@ describe('AutomergeHost', () => {
15
17
  const level = createTestLevel();
16
18
  await level.open();
17
19
  afterTest(() => level.close());
18
- const host = new AutomergeHost({
19
- db: level.sublevel('automerge'),
20
- indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
21
- });
22
- await host.open();
23
- afterTest(() => host.close());
24
20
 
21
+ const host = await setupAutomergeHost({ level });
25
22
  const handle = host.repo.create();
26
23
  handle.change((doc: any) => {
27
24
  doc.text = 'Hello world';
@@ -35,11 +32,7 @@ describe('AutomergeHost', () => {
35
32
  await level.open();
36
33
  afterTest(() => level.close());
37
34
 
38
- const host = new AutomergeHost({
39
- db: level.sublevel('automerge'),
40
- indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
41
- });
42
- await host.open();
35
+ const host = await setupAutomergeHost({ level });
43
36
  const handle = host.repo.create();
44
37
  handle.change((doc: any) => {
45
38
  doc.text = 'Hello world';
@@ -49,15 +42,39 @@ describe('AutomergeHost', () => {
49
42
  await host.repo.flush();
50
43
  await host.close();
51
44
 
52
- const host2 = new AutomergeHost({
53
- db: level.sublevel('automerge'),
54
- indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
55
- });
56
- await host2.open();
57
- afterTest(() => host2.close());
45
+ const host2 = await setupAutomergeHost({ level });
58
46
  const handle2 = host2.repo.find(url);
59
47
  await handle2.whenReady();
60
48
  expect(handle2.docSync().text).toEqual('Hello world');
61
49
  await host2.repo.flush();
62
50
  });
51
+
52
+ test('query document heads', async () => {
53
+ const level = createTestLevel();
54
+ await level.open();
55
+ afterTest(() => level.close());
56
+
57
+ const host = await setupAutomergeHost({ level });
58
+ const handle = host.createDoc({ text: 'Hello world' });
59
+ const expectedHeads = getHeads(handle.docSync());
60
+ await host.flush();
61
+
62
+ expect(await host.getHeads(handle.documentId)).toEqual(expectedHeads);
63
+
64
+ // Simulate a restart.
65
+ {
66
+ const host = await setupAutomergeHost({ level });
67
+ expect(await host.getHeads(handle.documentId)).toEqual(expectedHeads);
68
+ }
69
+ });
63
70
  });
71
+
72
+ const setupAutomergeHost = async ({ level }: { level: LevelDB }) => {
73
+ const host = new AutomergeHost({
74
+ db: level,
75
+ indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
76
+ });
77
+ await host.open();
78
+ afterTest(() => host.close());
79
+ return host;
80
+ };
@@ -22,11 +22,13 @@ import {
22
22
  type StorageAdapterInterface,
23
23
  } from '@dxos/automerge/automerge-repo';
24
24
  import { type Stream } from '@dxos/codec-protobuf';
25
- import { Context, cancelWithContext, type Lifecycle } from '@dxos/context';
25
+ import { type Context, Resource, cancelWithContext, type Lifecycle } from '@dxos/context';
26
26
  import { type SpaceDoc } from '@dxos/echo-protocol';
27
27
  import { type IndexMetadataStore } from '@dxos/indexing';
28
+ import { invariant } from '@dxos/invariant';
28
29
  import { PublicKey } from '@dxos/keys';
29
- import { type SublevelDB } from '@dxos/kv-store';
30
+ import { type LevelDB } from '@dxos/kv-store';
31
+ import { log } from '@dxos/log';
30
32
  import { objectPointerCodec } from '@dxos/protocols';
31
33
  import {
32
34
  type FlushRequest,
@@ -39,6 +41,7 @@ import { mapValues } from '@dxos/util';
39
41
 
40
42
  import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
41
43
  import { type EchoReplicator } from './echo-replicator';
44
+ import { HeadsStore } from './heads-store';
42
45
  import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-adapter';
43
46
  import { LocalHostNetworkAdapter } from './local-host-network-adapter';
44
47
 
@@ -46,7 +49,7 @@ import { LocalHostNetworkAdapter } from './local-host-network-adapter';
46
49
  export type { DocumentId };
47
50
 
48
51
  export type AutomergeHostParams = {
49
- db: SublevelDB;
52
+ db: LevelDB;
50
53
 
51
54
  indexMetadataStore: IndexMetadataStore;
52
55
  };
@@ -66,9 +69,9 @@ export type CreateDocOptions = {
66
69
  * Abstracts over the AutomergeRepo.
67
70
  */
68
71
  @trace.resource()
69
- export class AutomergeHost {
72
+ export class AutomergeHost extends Resource {
73
+ private readonly _db: LevelDB;
70
74
  private readonly _indexMetadataStore: IndexMetadataStore;
71
- private readonly _ctx = new Context();
72
75
  private readonly _echoNetworkAdapter = new EchoNetworkAdapter({
73
76
  getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
74
77
  });
@@ -76,22 +79,26 @@ export class AutomergeHost {
76
79
  private _repo!: Repo;
77
80
  private _clientNetwork!: LocalHostNetworkAdapter;
78
81
  private _storage!: StorageAdapterInterface & Lifecycle;
82
+ private readonly _headsStore: HeadsStore;
79
83
 
80
84
  @trace.info()
81
85
  private _peerId!: string;
82
86
 
83
87
  constructor({ db, indexMetadataStore }: AutomergeHostParams) {
88
+ super();
89
+ this._db = db;
84
90
  this._storage = new LevelDBStorageAdapter({
85
- db,
91
+ db: db.sublevel('automerge'),
86
92
  callbacks: {
87
93
  beforeSave: async (params) => this._beforeSave(params),
88
94
  afterSave: async () => this._afterSave(),
89
95
  },
90
96
  });
97
+ this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
91
98
  this._indexMetadataStore = indexMetadataStore;
92
99
  }
93
100
 
94
- async open() {
101
+ protected override async _open() {
95
102
  // TODO(burdon): Should this be stable?
96
103
  this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
97
104
 
@@ -117,7 +124,7 @@ export class AutomergeHost {
117
124
  await this._echoNetworkAdapter.whenConnected();
118
125
  }
119
126
 
120
- async close() {
127
+ protected override async _close() {
121
128
  await this._storage.close?.();
122
129
  await this._clientNetwork.close();
123
130
  await this._echoNetworkAdapter.close();
@@ -131,6 +138,10 @@ export class AutomergeHost {
131
138
  return this._repo;
132
139
  }
133
140
 
141
+ get loadedDocsCount(): number {
142
+ return Object.keys(this._repo.handles).length;
143
+ }
144
+
134
145
  async addReplicator(replicator: EchoReplicator) {
135
146
  await this._echoNetworkAdapter.addReplicator(replicator);
136
147
  }
@@ -179,14 +190,32 @@ export class AutomergeHost {
179
190
  }
180
191
  }
181
192
 
193
+ async reIndexHeads(documentIds: DocumentId[]) {
194
+ for (const documentId of documentIds) {
195
+ log.info('reindexing heads for document', { documentId });
196
+ const handle = this._repo.find(documentId);
197
+ await handle.whenReady(['ready', 'requesting']);
198
+ if (handle.inState(['requesting'])) {
199
+ log.warn('document is not available locally, skipping', { documentId });
200
+ continue; // Handle not available locally.
201
+ }
202
+
203
+ const doc = handle.docSync();
204
+ invariant(doc);
205
+
206
+ const heads = getHeads(doc);
207
+ const batch = this._db.batch();
208
+ this._headsStore.setHeads(documentId, heads, batch);
209
+ await batch.write();
210
+ }
211
+ log.info('done reindexing heads');
212
+ }
213
+
182
214
  // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
183
215
  // Hosts, running in the worker, don't share documents unless requested by other peers.
184
216
  // NOTE: If both peers return sharePolicy=false the replication will not happen
185
217
  // https://github.com/automerge/automerge-repo/pull/292
186
- private async _sharePolicy(
187
- peerId: PeerId /* device key */,
188
- documentId?: DocumentId /* space key */,
189
- ): Promise<boolean> {
218
+ private async _sharePolicy(peerId: PeerId, documentId?: DocumentId): Promise<boolean> {
190
219
  if (peerId.startsWith('client-')) {
191
220
  return false; // Only send docs to clients if they are requested.
192
221
  }
@@ -197,7 +226,7 @@ export class AutomergeHost {
197
226
 
198
227
  const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
199
228
  if (isEchoPeerMetadata(peerMetadata)) {
200
- return this._echoNetworkAdapter.shouldAdvertize(peerId, { documentId });
229
+ return this._echoNetworkAdapter.shouldAdvertise(peerId, { documentId });
201
230
  }
202
231
 
203
232
  return false;
@@ -215,13 +244,15 @@ export class AutomergeHost {
215
244
 
216
245
  const spaceKey = getSpaceKeyFromDoc(doc) ?? undefined;
217
246
 
218
- const lastAvailableHash = getHeads(doc);
247
+ const heads = getHeads(doc);
248
+
249
+ this._headsStore.setHeads(handle.documentId, heads, batch);
219
250
 
220
251
  const objectIds = Object.keys(doc.objects ?? {});
221
252
  const encodedIds = objectIds.map((objectId) =>
222
253
  objectPointerCodec.encode({ documentId: handle.documentId, objectId, spaceKey }),
223
254
  );
224
- const idToLastHash = new Map(encodedIds.map((id) => [id, lastAvailableHash]));
255
+ const idToLastHash = new Map(encodedIds.map((id) => [id, heads]));
225
256
  this._indexMetadataStore.markDirty(idToLastHash, batch);
226
257
  }
227
258
 
@@ -281,7 +312,7 @@ export class AutomergeHost {
281
312
  * Flush documents to disk.
282
313
  */
283
314
  @trace.span({ showInBrowserTimeline: true })
284
- async flush({ states }: FlushRequest): Promise<void> {
315
+ async flush({ states }: FlushRequest = {}): Promise<void> {
285
316
  // Note: Wait for all requested documents to be loaded/synced from thin-client.
286
317
  if (states) {
287
318
  await Promise.all(
@@ -289,7 +320,7 @@ export class AutomergeHost {
289
320
  if (!heads) {
290
321
  return;
291
322
  }
292
- const handle = this.repo.handles[documentId as DocumentId] ?? this._repo.find(documentId as DocumentId);
323
+ const handle = this._repo.handles[documentId as DocumentId] ?? this._repo.find(documentId as DocumentId);
293
324
  await waitForHeads(handle, heads);
294
325
  }) ?? [],
295
326
  );
@@ -298,6 +329,19 @@ export class AutomergeHost {
298
329
  await this._repo.flush(states?.map(({ documentId }) => documentId as DocumentId));
299
330
  }
300
331
 
332
+ async getHeads(documentId: DocumentId): Promise<Heads | undefined> {
333
+ const handle = this._repo.handles[documentId];
334
+ if (handle) {
335
+ const doc = handle.docSync();
336
+ if (!doc) {
337
+ return undefined;
338
+ }
339
+ return getHeads(doc);
340
+ } else {
341
+ return this._headsStore.getHeads(documentId);
342
+ }
343
+ }
344
+
301
345
  /**
302
346
  * Host <-> Client sync.
303
347
  */
@@ -7,7 +7,14 @@ import waitForExpect from 'wait-for-expect';
7
7
 
8
8
  import { asyncTimeout, sleep } from '@dxos/async';
9
9
  import { type Heads, change, clone, equals, from, getBackend, getHeads } from '@dxos/automerge/automerge';
10
- import { type Message, Repo, type PeerId, type DocumentId, type HandleState } from '@dxos/automerge/automerge-repo';
10
+ import {
11
+ type Message,
12
+ Repo,
13
+ type PeerId,
14
+ type DocumentId,
15
+ type HandleState,
16
+ type AutomergeUrl,
17
+ } from '@dxos/automerge/automerge-repo';
11
18
  import { randomBytes } from '@dxos/crypto';
12
19
  import { PublicKey } from '@dxos/keys';
13
20
  import { createTestLevel } from '@dxos/kv-store/testing';
@@ -92,6 +99,74 @@ describe('AutomergeRepo', () => {
92
99
  expect(equals(a, c)).to.be.true;
93
100
  });
94
101
 
102
+ test('documents missing from local storage go to requesting state', async () => {
103
+ const hostAdapter: TestAdapter = new TestAdapter({
104
+ send: (message: Message) => {
105
+ console.log('hostAdapter.send', message);
106
+ clientAdapter.receive(message);
107
+ },
108
+ });
109
+ const clientAdapter: TestAdapter = new TestAdapter({
110
+ send: (message: Message) => {
111
+ console.log('clientAdapter.send', message);
112
+ if (message.type !== 'doc-unavailable' && message.type !== 'sync') {
113
+ hostAdapter.receive(message);
114
+ }
115
+ },
116
+ });
117
+
118
+ const host = new Repo({
119
+ network: [hostAdapter],
120
+ });
121
+ const _client = new Repo({
122
+ network: [clientAdapter],
123
+ });
124
+ hostAdapter.ready();
125
+ clientAdapter.ready();
126
+ await hostAdapter.onConnect.wait();
127
+ await clientAdapter.onConnect.wait();
128
+ hostAdapter.peerCandidate(clientAdapter.peerId!);
129
+ clientAdapter.peerCandidate(hostAdapter.peerId!);
130
+
131
+ const url = 'automerge:3JN8F3Z4dUWEEKKFN7WE9gEGvVUT';
132
+ const handle = host.find(url as AutomergeUrl);
133
+ await handle.whenReady(['requesting']);
134
+ });
135
+
136
+ test('documents on disk go to ready state', async () => {
137
+ const level = createTestLevel();
138
+ const storage = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
139
+ await openAndClose(level, storage);
140
+
141
+ let url: AutomergeUrl | undefined;
142
+ {
143
+ const repo = new Repo({
144
+ network: [],
145
+ storage,
146
+ });
147
+ const handle = repo.create<{ field?: string }>();
148
+ url = handle.url;
149
+ await repo.flush();
150
+ }
151
+
152
+ {
153
+ const repo = new Repo({
154
+ network: [],
155
+ storage,
156
+ });
157
+ const handle = repo.find(url as AutomergeUrl);
158
+
159
+ let requestingState = false;
160
+ queueMicrotask(async () => {
161
+ await handle.whenReady(['requesting']);
162
+ requestingState = true;
163
+ });
164
+
165
+ await handle.whenReady(['ready']);
166
+ expect(requestingState).to.be.false;
167
+ }
168
+ });
169
+
95
170
  describe('network', () => {
96
171
  test('basic networking', async () => {
97
172
  const hostAdapter: TestAdapter = new TestAdapter({
@@ -0,0 +1,131 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ //
4
+
5
+ import { expect } from 'chai';
6
+
7
+ import { sleep, Trigger, waitForCondition } from '@dxos/async';
8
+ import { cbor, type PeerId } from '@dxos/automerge/automerge-repo';
9
+ import { invariant } from '@dxos/invariant';
10
+ import { PublicKey } from '@dxos/keys';
11
+ import { type SyncMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';
12
+ import {
13
+ type AutomergeReplicator,
14
+ type AutomergeReplicatorCallbacks,
15
+ type AutomergeReplicatorFactory,
16
+ } from '@dxos/teleport-extension-automerge-replicator';
17
+ import { afterTest, describe, test } from '@dxos/test';
18
+
19
+ import { EchoNetworkAdapter } from './echo-network-adapter';
20
+ import { MeshEchoReplicator } from './mesh-echo-replicator';
21
+
22
+ const PEER_ID = 'peerA' as PeerId;
23
+ const ANOTHER_PEER_ID = 'peerB' as PeerId;
24
+ const PAYLOAD = new Uint8Array([42]);
25
+
26
+ describe('EchoNetworkAdapter', () => {
27
+ test('peer-candidate emitted when replication starts', async () => {
28
+ const controller = createReplicatorController();
29
+ const adapter = await createConnectedAdapter(controller.replicator);
30
+ const peerInfo = new Trigger<any>();
31
+ adapter.on('peer-candidate', (payload) => peerInfo.wake(payload));
32
+ await controller.connectPeer(ANOTHER_PEER_ID);
33
+ const emittedInfo = await peerInfo.wait();
34
+ expect(emittedInfo.peerId).to.eq(ANOTHER_PEER_ID);
35
+ });
36
+
37
+ test('message emitted when sync message is received', async () => {
38
+ const controller = createReplicatorController();
39
+ const adapter = await createConnectedAdapter(controller.replicator);
40
+ const messageReceived = new Trigger<any>();
41
+ adapter.on('message', (message) => messageReceived.wake(message));
42
+ const callbacks = await controller.connectPeer(ANOTHER_PEER_ID);
43
+ await callbacks.onSyncMessage!(encodeSyncPayload(PAYLOAD));
44
+ const receivedMessage = await messageReceived.wait();
45
+ expect(receivedMessage).to.deep.eq(PAYLOAD);
46
+ });
47
+
48
+ test('peer disconnects when onClose callback is invoked', async () => {
49
+ const controller = createReplicatorController();
50
+ const adapter = await createConnectedAdapter(controller.replicator);
51
+ const callbacks = await controller.connectPeer(ANOTHER_PEER_ID);
52
+ const onDisconnected = new Trigger<any>();
53
+ adapter.on('peer-disconnected', (payload) => onDisconnected.wake(payload));
54
+ await callbacks.onClose!();
55
+ const disconnectedPeer = await onDisconnected.wait();
56
+ expect(disconnectedPeer.peerId).to.eq(ANOTHER_PEER_ID);
57
+ });
58
+
59
+ test('peer disconnects when message sending fails', async () => {
60
+ let errored = false;
61
+ const controller = createReplicatorController(async () => {
62
+ errored = true;
63
+ throw new Error();
64
+ });
65
+ const adapter = await createConnectedAdapter(controller.replicator);
66
+ await controller.connectPeer(ANOTHER_PEER_ID);
67
+ const onDisconnected = new Trigger<any>();
68
+ adapter.on('peer-disconnected', (payload) => onDisconnected.wake(payload));
69
+ adapter.send(newSyncMessage(PEER_ID, ANOTHER_PEER_ID, PAYLOAD));
70
+ const disconnectedPeer = await onDisconnected.wait();
71
+ expect(disconnectedPeer.peerId).to.eq(ANOTHER_PEER_ID);
72
+ expect(errored).to.be.true;
73
+ });
74
+
75
+ test('message sending is queued', async () => {
76
+ let sentTotal = 0;
77
+ let sendInProgress = false;
78
+ const controller = createReplicatorController(async () => {
79
+ invariant(!sendInProgress);
80
+ sendInProgress = true;
81
+ await sleep(5);
82
+ sendInProgress = false;
83
+ sentTotal++;
84
+ });
85
+ const adapter = await createConnectedAdapter(controller.replicator);
86
+ await controller.connectPeer(ANOTHER_PEER_ID);
87
+ const totalMessages = 5;
88
+ for (let i = 0; i < totalMessages; i++) {
89
+ adapter.send(newSyncMessage(PEER_ID, ANOTHER_PEER_ID, PAYLOAD));
90
+ }
91
+ await waitForCondition({ condition: () => sentTotal === totalMessages });
92
+ });
93
+
94
+ const createConnectedAdapter = async (replicator: MeshEchoReplicator) => {
95
+ const adapter = new EchoNetworkAdapter({ getContainingSpaceForDocument: async () => null });
96
+ adapter.connect(PEER_ID);
97
+ await adapter.open();
98
+ afterTest(() => adapter.close());
99
+ await adapter.addReplicator(replicator);
100
+ return adapter;
101
+ };
102
+
103
+ const createReplicatorController = (sendSyncMessage?: (message: SyncMessage) => Promise<void>) => {
104
+ const replicator = new MeshEchoReplicator();
105
+ return {
106
+ replicator,
107
+ connectPeer: async (peerId: string) => {
108
+ let callbacks: AutomergeReplicatorCallbacks | undefined;
109
+ const extensionFactory: AutomergeReplicatorFactory = (params) => {
110
+ callbacks = params[1];
111
+ return { sendSyncMessage } as AutomergeReplicator;
112
+ };
113
+ replicator.createExtension(extensionFactory);
114
+ invariant(callbacks);
115
+ await callbacks.onStartReplication!({ id: peerId }, PublicKey.random());
116
+ return callbacks;
117
+ },
118
+ };
119
+ };
120
+
121
+ const newSyncMessage = (from: PeerId, to: PeerId, payload: Uint8Array) => ({
122
+ type: 'sync',
123
+ senderId: from,
124
+ targetId: to,
125
+ data: payload,
126
+ });
127
+
128
+ const encodeSyncPayload = (payload: Uint8Array): SyncMessage => ({
129
+ payload: cbor.encode(payload),
130
+ });
131
+ });
@@ -2,14 +2,14 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import { Trigger, synchronized } from '@dxos/async';
5
+ import { synchronized, Trigger } from '@dxos/async';
6
6
  import { type Message, NetworkAdapter, type PeerId, type PeerMetadata } from '@dxos/automerge/automerge-repo';
7
7
  import { LifecycleState } from '@dxos/context';
8
8
  import { invariant } from '@dxos/invariant';
9
9
  import { type PublicKey } from '@dxos/keys';
10
10
  import { log } from '@dxos/log';
11
11
 
12
- import { type EchoReplicator, type ReplicatorConnection, type ShouldAdvertizeParams } from './echo-replicator';
12
+ import { type EchoReplicator, type ReplicatorConnection, type ShouldAdvertiseParams } from './echo-replicator';
13
13
 
14
14
  export type EchoNetworkAdapterParams = {
15
15
  getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
@@ -57,7 +57,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
57
57
 
58
58
  @synchronized
59
59
  async open() {
60
- invariant(this._lifecycleState === LifecycleState.CLOSED);
60
+ if (this._lifecycleState === LifecycleState.OPEN) {
61
+ return;
62
+ }
61
63
  this._lifecycleState = LifecycleState.OPEN;
62
64
 
63
65
  log('emit ready');
@@ -68,7 +70,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
68
70
 
69
71
  @synchronized
70
72
  async close() {
71
- invariant(this._lifecycleState === LifecycleState.OPEN);
73
+ if (this._lifecycleState === LifecycleState.CLOSED) {
74
+ return;
75
+ }
72
76
 
73
77
  for (const replicator of this._replicators) {
74
78
  await replicator.disconnect();
@@ -106,13 +110,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
106
110
  this._replicators.delete(replicator);
107
111
  }
108
112
 
109
- async shouldAdvertize(peerId: PeerId, params: ShouldAdvertizeParams): Promise<boolean> {
113
+ async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseParams): Promise<boolean> {
110
114
  const connection = this._connections.get(peerId);
111
115
  if (!connection) {
112
116
  return false;
113
117
  }
114
118
 
115
- return connection.connection.shouldAdvertize(params);
119
+ return connection.connection.shouldAdvertise(params);
116
120
  }
117
121
 
118
122
  private _onConnectionOpen(connection: ReplicatorConnection) {
@@ -23,13 +23,11 @@ export interface EchoReplicatorContext {
23
23
  */
24
24
  get peerId(): string;
25
25
 
26
- onConnectionOpen(connection: ReplicatorConnection): void;
26
+ getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
27
27
 
28
+ onConnectionOpen(connection: ReplicatorConnection): void;
28
29
  onConnectionClosed(connection: ReplicatorConnection): void;
29
-
30
30
  onConnectionAuthScopeChanged(connection: ReplicatorConnection): void;
31
-
32
- getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
33
31
  }
34
32
 
35
33
  export interface ReplicatorConnection {
@@ -49,13 +47,12 @@ export interface ReplicatorConnection {
49
47
  writable: WritableStream<Message>;
50
48
 
51
49
  /**
52
- * @returns true if the document should be advertized to this peer.
53
- *
54
- * The remote peer can still request the document by it's id bypassing this check.
50
+ * @returns true if the document should be advertised to this peer.
51
+ * The remote peer can still request the document by its id bypassing this check.
55
52
  */
56
- shouldAdvertize(params: ShouldAdvertizeParams): Promise<boolean>;
53
+ shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean>;
57
54
  }
58
55
 
59
- export type ShouldAdvertizeParams = {
56
+ export type ShouldAdvertiseParams = {
60
57
  documentId: string;
61
58
  };