@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1068cf700f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/dist/lib/neutral/chunk-FJPXA75J.mjs +235 -0
  2. package/dist/lib/neutral/chunk-FJPXA75J.mjs.map +7 -0
  3. package/dist/lib/{browser/chunk-35I6ERLG.mjs → neutral/chunk-WIQUYO7H.mjs} +300 -213
  4. package/dist/lib/neutral/chunk-WIQUYO7H.mjs.map +7 -0
  5. package/dist/lib/{browser → neutral}/filter/index.mjs +3 -2
  6. package/dist/lib/{browser → neutral}/index.mjs +2832 -946
  7. package/dist/lib/neutral/index.mjs.map +7 -0
  8. package/dist/lib/neutral/meta.json +1 -0
  9. package/dist/lib/{browser → neutral}/testing/index.mjs +73 -224
  10. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  11. package/dist/types/src/automerge/automerge-host.d.ts +53 -42
  12. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  13. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  14. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  15. package/dist/types/src/automerge/collection-synchronizer.d.ts +6 -4
  16. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  17. package/dist/types/src/automerge/echo-network-adapter.d.ts +22 -10
  18. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  19. package/dist/types/src/automerge/echo-replicator.d.ts +37 -15
  20. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  21. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  22. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  23. package/dist/types/src/automerge/index.d.ts +1 -1
  24. package/dist/types/src/automerge/index.d.ts.map +1 -1
  25. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +5 -5
  26. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  27. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +9 -8
  28. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  29. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +5 -5
  30. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  31. package/dist/types/src/common/codec.d.ts +1 -1
  32. package/dist/types/src/common/codec.d.ts.map +1 -1
  33. package/dist/types/src/db-host/automerge-data-source.d.ts +34 -0
  34. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  35. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  36. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  37. package/dist/types/src/db-host/data-service.d.ts +5 -4
  38. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  39. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  40. package/dist/types/src/db-host/documents-synchronizer.d.ts +6 -5
  41. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  42. package/dist/types/src/db-host/echo-host.d.ts +44 -23
  43. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  44. package/dist/types/src/db-host/index.d.ts +1 -0
  45. package/dist/types/src/db-host/index.d.ts.map +1 -1
  46. package/dist/types/src/db-host/local-queue-service.d.ts +16 -0
  47. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  48. package/dist/types/src/db-host/query-service.d.ts +14 -9
  49. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  50. package/dist/types/src/db-host/queue-data-source.d.ts +28 -0
  51. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  52. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  53. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  54. package/dist/types/src/db-host/space-state-manager.d.ts +2 -1
  55. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  56. package/dist/types/src/db-host/stub.d.ts +10 -0
  57. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  58. package/dist/types/src/edge/echo-edge-replicator.d.ts +8 -6
  59. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  60. package/dist/types/src/filter/filter-match.d.ts +4 -1
  61. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  62. package/dist/types/src/metadata/metadata-store.d.ts +1 -1
  63. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  64. package/dist/types/src/pipeline/pipeline.d.ts +3 -3
  65. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  66. package/dist/types/src/query/errors.d.ts +21 -9
  67. package/dist/types/src/query/errors.d.ts.map +1 -1
  68. package/dist/types/src/query/plan.d.ts +57 -5
  69. package/dist/types/src/query/plan.d.ts.map +1 -1
  70. package/dist/types/src/query/query-executor.d.ts +49 -10
  71. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  72. package/dist/types/src/query/query-planner.d.ts +9 -0
  73. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  74. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  75. package/dist/types/src/space/auth.d.ts +3 -3
  76. package/dist/types/src/space/auth.d.ts.map +1 -1
  77. package/dist/types/src/space/control-pipeline.d.ts +3 -3
  78. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  79. package/dist/types/src/space/space-manager.d.ts +7 -7
  80. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  81. package/dist/types/src/space/space-protocol.d.ts +6 -6
  82. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  83. package/dist/types/src/space/space.d.ts +4 -4
  84. package/dist/types/src/space/space.d.ts.map +1 -1
  85. package/dist/types/src/testing/index.d.ts +0 -2
  86. package/dist/types/src/testing/index.d.ts.map +1 -1
  87. package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
  88. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  89. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  90. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  91. package/dist/types/src/testing/test-replicator.d.ts +14 -13
  92. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  93. package/dist/types/src/util.d.ts +1 -1
  94. package/dist/types/src/util.d.ts.map +1 -1
  95. package/dist/types/tsconfig.tsbuildinfo +1 -1
  96. package/package.json +53 -45
  97. package/src/automerge/automerge-host.test.ts +55 -24
  98. package/src/automerge/automerge-host.ts +399 -126
  99. package/src/automerge/automerge-repo.test.ts +214 -46
  100. package/src/automerge/collection-synchronizer.test.ts +22 -15
  101. package/src/automerge/collection-synchronizer.ts +57 -22
  102. package/src/automerge/echo-data-monitor.ts +1 -1
  103. package/src/automerge/echo-network-adapter.test.ts +3 -3
  104. package/src/automerge/echo-network-adapter.ts +73 -28
  105. package/src/automerge/echo-replicator.ts +40 -15
  106. package/src/automerge/heads-store.ts +42 -3
  107. package/src/automerge/index.ts +1 -1
  108. package/src/automerge/leveldb-storage-adapter.ts +11 -11
  109. package/src/automerge/mesh-echo-replicator-connection.ts +12 -8
  110. package/src/automerge/mesh-echo-replicator.ts +15 -11
  111. package/src/automerge/storage-adapter.test.ts +1 -1
  112. package/src/common/space-id.ts +1 -1
  113. package/src/db-host/automerge-data-source.test.ts +292 -0
  114. package/src/db-host/automerge-data-source.ts +144 -0
  115. package/src/db-host/data-service.ts +19 -20
  116. package/src/db-host/database-root.ts +2 -2
  117. package/src/db-host/documents-synchronizer.test.ts +73 -13
  118. package/src/db-host/documents-synchronizer.ts +58 -44
  119. package/src/db-host/echo-host.ts +194 -84
  120. package/src/db-host/index.ts +1 -0
  121. package/src/db-host/local-queue-service.ts +147 -0
  122. package/src/db-host/query-service.ts +28 -95
  123. package/src/db-host/queue-data-source.ts +132 -0
  124. package/src/db-host/queue-service.test.ts +145 -0
  125. package/src/db-host/space-state-manager.ts +6 -2
  126. package/src/db-host/stub.ts +28 -0
  127. package/src/edge/echo-edge-replicator.test.ts +8 -6
  128. package/src/edge/echo-edge-replicator.ts +124 -42
  129. package/src/filter/filter-match.test.ts +58 -32
  130. package/src/filter/filter-match.ts +163 -15
  131. package/src/metadata/metadata-store.ts +8 -5
  132. package/src/pipeline/pipeline-stress.test.ts +4 -2
  133. package/src/pipeline/pipeline.test.ts +3 -2
  134. package/src/pipeline/pipeline.ts +10 -7
  135. package/src/query/errors.ts +3 -1
  136. package/src/query/plan.ts +75 -5
  137. package/src/query/query-executor.ts +812 -117
  138. package/src/query/query-planner.test.ts +652 -34
  139. package/src/query/query-planner.ts +276 -13
  140. package/src/space/admission-discovery-extension.ts +3 -3
  141. package/src/space/auth.ts +6 -6
  142. package/src/space/control-pipeline.test.ts +4 -3
  143. package/src/space/control-pipeline.ts +11 -8
  144. package/src/space/space-manager.browser.test.ts +1 -1
  145. package/src/space/space-manager.ts +11 -10
  146. package/src/space/space-protocol.browser.test.ts +2 -2
  147. package/src/space/space-protocol.test.ts +3 -2
  148. package/src/space/space-protocol.ts +17 -14
  149. package/src/space/space.test.ts +1 -1
  150. package/src/space/space.ts +6 -5
  151. package/src/testing/index.ts +0 -2
  152. package/src/testing/test-agent-builder.ts +4 -3
  153. package/src/testing/test-network-adapter.ts +16 -3
  154. package/src/testing/test-replicator.ts +23 -18
  155. package/src/util.ts +7 -3
  156. package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
  157. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  158. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  159. package/dist/lib/browser/chunk-TQJTKNMS.mjs +0 -126
  160. package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
  161. package/dist/lib/browser/index.mjs.map +0 -7
  162. package/dist/lib/browser/meta.json +0 -1
  163. package/dist/lib/browser/testing/index.mjs.map +0 -7
  164. package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
  165. package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
  166. package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
  167. package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
  168. package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
  169. package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
  170. package/dist/lib/node/filter/index.cjs +0 -32
  171. package/dist/lib/node/filter/index.cjs.map +0 -7
  172. package/dist/lib/node/index.cjs +0 -4699
  173. package/dist/lib/node/index.cjs.map +0 -7
  174. package/dist/lib/node/meta.json +0 -1
  175. package/dist/lib/node/testing/index.cjs +0 -753
  176. package/dist/lib/node/testing/index.cjs.map +0 -7
  177. package/dist/lib/node-esm/chunk-5BHLPT24.mjs +0 -2067
  178. package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
  179. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  180. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  181. package/dist/lib/node-esm/chunk-RVK35BS7.mjs +0 -126
  182. package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +0 -7
  183. package/dist/lib/node-esm/filter/index.mjs +0 -11
  184. package/dist/lib/node-esm/filter/index.mjs.map +0 -7
  185. package/dist/lib/node-esm/index.mjs +0 -4743
  186. package/dist/lib/node-esm/index.mjs.map +0 -7
  187. package/dist/lib/node-esm/meta.json +0 -1
  188. package/dist/lib/node-esm/testing/index.mjs +0 -742
  189. package/dist/lib/node-esm/testing/index.mjs.map +0 -7
  190. package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
  191. package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
  192. package/dist/types/src/testing/test-data.d.ts +0 -18
  193. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  194. package/dist/types/src/testing/test-schema.d.ts +0 -37
  195. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  196. package/src/db-host/documents-iterator.ts +0 -74
  197. package/src/testing/test-data.ts +0 -127
  198. package/src/testing/test-schema.ts +0 -53
  199. /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
@@ -2,9 +2,16 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import { NetworkAdapter, type Message, type PeerId, type PeerMetadata } from '@automerge/automerge-repo';
6
-
7
- import { synchronized, Trigger } from '@dxos/async';
5
+ import {
6
+ type DocumentId,
7
+ type Heads,
8
+ type Message,
9
+ NetworkAdapter,
10
+ type PeerId,
11
+ type PeerMetadata,
12
+ } from '@automerge/automerge-repo';
13
+
14
+ import { Event, Trigger, synchronized } from '@dxos/async';
8
15
  import { LifecycleState } from '@dxos/context';
9
16
  import { invariant } from '@dxos/invariant';
10
17
  import { type PublicKey } from '@dxos/keys';
@@ -12,20 +19,21 @@ import { log } from '@dxos/log';
12
19
  import type { AutomergeProtocolMessage } from '@dxos/protocols';
13
20
  import { isNonNullable } from '@dxos/util';
14
21
 
22
+ import { createIdFromSpaceKey } from '../common/space-id';
23
+
15
24
  import {
16
- type EchoReplicator,
17
- type RemoteDocumentExistenceCheckParams,
18
- type ReplicatorConnection,
19
- type ShouldAdvertiseParams,
20
- type ShouldSyncCollectionParams,
25
+ type AutomergeReplicator,
26
+ type AutomergeReplicatorConnection,
27
+ type RemoteDocumentExistenceCheckProps,
28
+ type ShouldAdvertiseProps,
29
+ type ShouldSyncCollectionProps,
21
30
  } from './echo-replicator';
22
31
  import {
23
- isCollectionQueryMessage,
24
- isCollectionStateMessage,
25
32
  type CollectionQueryMessage,
26
33
  type CollectionStateMessage,
34
+ isCollectionQueryMessage,
35
+ isCollectionStateMessage,
27
36
  } from './network-protocol';
28
- import { createIdFromSpaceKey } from '../common/space-id';
29
37
 
30
38
  export interface NetworkDataMonitor {
31
39
  recordPeerConnected(peerId: string): void;
@@ -35,9 +43,9 @@ export interface NetworkDataMonitor {
35
43
  recordMessageSendingFailed(message: Message): void;
36
44
  }
37
45
 
38
- export type EchoNetworkAdapterParams = {
46
+ export type EchoNetworkAdapterProps = {
39
47
  getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
40
- isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckParams) => Promise<boolean>;
48
+ isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckProps) => Promise<boolean>;
41
49
  onCollectionStateQueried: (collectionId: string, peerId: PeerId) => void;
42
50
  onCollectionStateReceived: (collectionId: string, peerId: PeerId, state: unknown) => void;
43
51
  monitor?: NetworkDataMonitor;
@@ -45,16 +53,17 @@ export type EchoNetworkAdapterParams = {
45
53
 
46
54
  type ConnectionEntry = {
47
55
  isOpen: boolean;
48
- connection: ReplicatorConnection;
56
+ connection: AutomergeReplicatorConnection;
49
57
  reader: ReadableStreamDefaultReader<AutomergeProtocolMessage>;
50
58
  writer: WritableStreamDefaultWriter<AutomergeProtocolMessage>;
59
+ requestedDocuments: Set<DocumentId>;
51
60
  };
52
61
 
53
62
  /**
54
- * Manages a set of {@link EchoReplicator} instances.
63
+ * Manages a set of {@link AutomergeReplicator} instances.
55
64
  */
56
65
  export class EchoNetworkAdapter extends NetworkAdapter {
57
- private readonly _replicators = new Set<EchoReplicator>();
66
+ private readonly _replicators = new Set<AutomergeReplicator>();
58
67
  /**
59
68
  * Remote peer id -> connection.
60
69
  */
@@ -63,7 +72,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
63
72
  private readonly _connected = new Trigger();
64
73
  private readonly _ready = new Trigger();
65
74
 
66
- constructor(private readonly _params: EchoNetworkAdapterParams) {
75
+ public readonly documentRequested = new Event<{ documentId: DocumentId; peerId: PeerId }>();
76
+
77
+ constructor(private readonly _params: EchoNetworkAdapterProps) {
67
78
  super();
68
79
  }
69
80
 
@@ -125,7 +136,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
125
136
  }
126
137
 
127
138
  @synchronized
128
- async addReplicator(replicator: EchoReplicator): Promise<void> {
139
+ async addReplicator(replicator: AutomergeReplicator): Promise<void> {
129
140
  invariant(this._lifecycleState === LifecycleState.OPEN);
130
141
  invariant(this.peerId);
131
142
  invariant(!this._replicators.has(replicator));
@@ -146,14 +157,14 @@ export class EchoNetworkAdapter extends NetworkAdapter {
146
157
  }
147
158
 
148
159
  @synchronized
149
- async removeReplicator(replicator: EchoReplicator): Promise<void> {
160
+ async removeReplicator(replicator: AutomergeReplicator): Promise<void> {
150
161
  invariant(this._lifecycleState === LifecycleState.OPEN);
151
162
  invariant(this._replicators.has(replicator));
152
163
  await replicator.disconnect();
153
164
  this._replicators.delete(replicator);
154
165
  }
155
166
 
156
- async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseParams): Promise<boolean> {
167
+ async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseProps): Promise<boolean> {
157
168
  const connection = this._connections.get(peerId);
158
169
  if (!connection) {
159
170
  return false;
@@ -162,7 +173,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
162
173
  return connection.connection.shouldAdvertise(params);
163
174
  }
164
175
 
165
- shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionParams): boolean {
176
+ shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionProps): boolean {
166
177
  const connection = this._connections.get(peerId);
167
178
  if (!connection) {
168
179
  return false;
@@ -203,6 +214,30 @@ export class EchoNetworkAdapter extends NetworkAdapter {
203
214
  .filter(isNonNullable);
204
215
  }
205
216
 
217
+ bundleSyncEnabledForPeer(peerId: PeerId): boolean {
218
+ const connection = this._connections.get(peerId);
219
+ if (!connection) {
220
+ return false;
221
+ }
222
+ return connection.connection.bundleSyncEnabled;
223
+ }
224
+
225
+ async pushBundle(peerId: PeerId, bundle: { documentId: DocumentId; data: Uint8Array; heads: Heads }[]) {
226
+ const connection = this._connections.get(peerId);
227
+ if (!connection) {
228
+ throw new Error('Connection not found.');
229
+ }
230
+ return connection.connection.pushBundle!(bundle);
231
+ }
232
+
233
+ async pullBundle(peerId: PeerId, docHeads: Record<DocumentId, Heads>) {
234
+ const connection = this._connections.get(peerId);
235
+ if (!connection) {
236
+ throw new Error('Connection not found.');
237
+ }
238
+ return connection.connection.pullBundle!(docHeads);
239
+ }
240
+
206
241
  private _send(message: Message): void {
207
242
  const connectionEntry = this._connections.get(message.targetId);
208
243
  if (!connectionEntry) {
@@ -225,7 +260,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
225
260
  });
226
261
  }
227
262
 
228
- private _onConnectionOpen(connection: ReplicatorConnection): void {
263
+ private _onConnectionOpen(connection: AutomergeReplicatorConnection): void {
229
264
  log('connection opened', { peerId: connection.peerId });
230
265
  invariant(!this._connections.has(connection.peerId as PeerId));
231
266
  const connectionEntry: ConnectionEntry = {
@@ -233,6 +268,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
233
268
  connection,
234
269
  reader: connection.readable.getReader(),
235
270
  writer: connection.writable.getWriter(),
271
+ requestedDocuments: new Set(),
236
272
  };
237
273
 
238
274
  this._connections.set(connection.peerId as PeerId, connectionEntry);
@@ -247,7 +283,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
247
283
  break;
248
284
  }
249
285
 
250
- this._onMessage(value as Message);
286
+ this._onMessage(connectionEntry, value as Message);
251
287
  }
252
288
  } catch (err) {
253
289
  if (connectionEntry.isOpen) {
@@ -261,7 +297,15 @@ export class EchoNetworkAdapter extends NetworkAdapter {
261
297
  this._params.monitor?.recordPeerConnected(connection.peerId);
262
298
  }
263
299
 
264
- private _onMessage(message: Message): void {
300
+ private _onMessage(connectionEntry: ConnectionEntry, message: Message): void {
301
+ const amMessage = message as AutomergeProtocolMessage;
302
+ if (amMessage.type === 'request') {
303
+ this.documentRequested.emit({
304
+ documentId: amMessage.documentId as DocumentId,
305
+ peerId: connectionEntry.connection.peerId as PeerId,
306
+ });
307
+ }
308
+
265
309
  if (isCollectionQueryMessage(message)) {
266
310
  this._params.onCollectionStateQueried(message.collectionId, message.senderId);
267
311
  } else if (isCollectionStateMessage(message)) {
@@ -272,7 +316,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
272
316
  this._params.monitor?.recordMessageReceived(message);
273
317
  }
274
318
 
275
- private _onConnectionClosed(connection: ReplicatorConnection): void {
319
+ private _onConnectionClosed(connection: AutomergeReplicatorConnection): void {
276
320
  log('connection closed', { peerId: connection.peerId });
277
321
  const entry = this._connections.get(connection.peerId as PeerId);
278
322
  invariant(entry);
@@ -281,8 +325,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
281
325
  this.emit('peer-disconnected', { peerId: connection.peerId as PeerId });
282
326
  this._params.monitor?.recordPeerDisconnected(connection.peerId);
283
327
 
284
- void entry.reader.cancel().catch((err) => log.catch(err));
285
328
  void entry.writer.abort().catch((err) => log.catch(err));
329
+ void entry.reader.cancel().catch((err) => log.catch(err));
330
+
286
331
  this._connections.delete(connection.peerId as PeerId);
287
332
  }
288
333
 
@@ -290,7 +335,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
290
335
  * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
291
336
  * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
292
337
  */
293
- private _onConnectionAuthScopeChanged(connection: ReplicatorConnection): void {
338
+ private _onConnectionAuthScopeChanged(connection: AutomergeReplicatorConnection): void {
294
339
  log('Connection auth scope changed', { peerId: connection.peerId });
295
340
  const entry = this._connections.get(connection.peerId as PeerId);
296
341
  invariant(entry);
@@ -298,7 +343,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
298
343
  this._emitPeerCandidate(connection);
299
344
  }
300
345
 
301
- private _emitPeerCandidate(connection: ReplicatorConnection): void {
346
+ private _emitPeerCandidate(connection: AutomergeReplicatorConnection): void {
302
347
  this.emit('peer-candidate', {
303
348
  peerId: connection.peerId as PeerId,
304
349
  peerMetadata: createEchoPeerMetadata(),
@@ -2,14 +2,18 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ import { type Heads } from '@automerge/automerge';
6
+ import { type DocumentId } from '@automerge/automerge-repo';
7
+
5
8
  import { type PublicKey, type SpaceId } from '@dxos/keys';
6
- import type { AutomergeProtocolMessage } from '@dxos/protocols';
9
+ import { type AutomergeProtocolMessage } from '@dxos/protocols';
7
10
 
8
- export interface EchoReplicator {
11
+ // TODO(burdon): Rename AutomergeReplicator?
12
+ export interface AutomergeReplicator {
9
13
  /**
10
14
  * Called on when replicator is added to EchoHost.
11
15
  */
12
- connect(context: EchoReplicatorContext): Promise<void>;
16
+ connect(context: AutomergeReplicatorContext): Promise<void>;
13
17
 
14
18
  /**
15
19
  * Called on when replicator is removed from EchoHost.
@@ -17,7 +21,7 @@ export interface EchoReplicator {
17
21
  disconnect(): Promise<void>;
18
22
  }
19
23
 
20
- export interface EchoReplicatorContext {
24
+ export interface AutomergeReplicatorContext {
21
25
  /**
22
26
  * Our own peer id.
23
27
  */
@@ -28,16 +32,20 @@ export interface EchoReplicatorContext {
28
32
  */
29
33
  getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
30
34
  getContainingSpaceIdForDocument(documentId: string): Promise<SpaceId | null>;
31
- isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckParams): Promise<boolean>;
32
35
 
33
- onConnectionOpen(connection: ReplicatorConnection): void;
34
- onConnectionClosed(connection: ReplicatorConnection): void;
35
- onConnectionAuthScopeChanged(connection: ReplicatorConnection): void;
36
+ /**
37
+ * Returns false if collection sync hasn't happened yet.
38
+ */
39
+ isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckProps): Promise<boolean>;
40
+
41
+ onConnectionOpen(connection: AutomergeReplicatorConnection): void;
42
+ onConnectionClosed(connection: AutomergeReplicatorConnection): void;
43
+ onConnectionAuthScopeChanged(connection: AutomergeReplicatorConnection): void;
36
44
  }
37
45
 
38
- export interface ReplicatorConnection {
46
+ export interface AutomergeReplicatorConnection {
39
47
  /**
40
- * Remove peer id.
48
+ * Remote peer id.
41
49
  */
42
50
  get peerId(): string;
43
51
 
@@ -55,23 +63,40 @@ export interface ReplicatorConnection {
55
63
  * @returns true if the document should be advertised to this peer.
56
64
  * The remote peer can still request the document by its id bypassing this check.
57
65
  */
58
- shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean>;
66
+ shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean>;
59
67
 
60
68
  /**
61
69
  * @returns true if the collection should be synced to this peer.
62
70
  */
63
- shouldSyncCollection(params: ShouldSyncCollectionParams): boolean;
71
+ shouldSyncCollection(params: ShouldSyncCollectionProps): boolean;
72
+
73
+ /**
74
+ * Batch syncing considered enabled if AutomergeReplicatorConnection implements `pushBatch` and `pullBatch` methods.
75
+ * @returns true if the batch syncing is enabled.
76
+ */
77
+ get bundleSyncEnabled(): boolean;
78
+
79
+ /**
80
+ * Pushes the batch of documents to the remote peer.
81
+ */
82
+ pushBundle?(bundle: { documentId: DocumentId; data: Uint8Array; heads: Heads }[]): Promise<void>;
83
+
84
+ /**
85
+ * Pulls the batch of documents from the remote peer.
86
+ */
87
+ // TODO(mykola): Use automerge-repo-bundles Bundle type here.
88
+ pullBundle?(docHeads: Record<DocumentId, Heads>): Promise<Record<DocumentId, Uint8Array>>;
64
89
  }
65
90
 
66
- export type ShouldAdvertiseParams = {
91
+ export type ShouldAdvertiseProps = {
67
92
  documentId: string;
68
93
  };
69
94
 
70
- export type ShouldSyncCollectionParams = {
95
+ export type ShouldSyncCollectionProps = {
71
96
  collectionId: string;
72
97
  };
73
98
 
74
- export type RemoteDocumentExistenceCheckParams = {
99
+ export type RemoteDocumentExistenceCheckProps = {
75
100
  peerId: string;
76
101
  documentId: string;
77
102
  };
@@ -4,18 +4,45 @@
4
4
 
5
5
  import type { Heads } from '@automerge/automerge';
6
6
  import type { DocumentId } from '@automerge/automerge-repo';
7
+ import { type MixedEncoding } from 'level-transcoder';
7
8
 
8
- import { headsEncoding } from '@dxos/indexing';
9
+ import type { ProtoCodec } from '@dxos/codec-protobuf';
9
10
  import type { BatchLevel, SublevelDB } from '@dxos/kv-store';
11
+ import { log } from '@dxos/log';
12
+ import { schema } from '@dxos/protocols/proto';
13
+ import type { Heads as HeadsProto } from '@dxos/protocols/proto/dxos/echo/query';
10
14
 
11
- export type HeadsStoreParams = {
15
+ // NOTE: Lazy so that code that doesn't use indexing doesn't need to load the codec (breaks in workerd).
16
+ let headsCodec: ProtoCodec<HeadsProto>;
17
+ const getHeadsCodec = () => (headsCodec ??= schema.getCodecForType('dxos.echo.query.Heads'));
18
+
19
+ const headsEncoding: MixedEncoding<Heads, Uint8Array, Heads> = {
20
+ encode: (value: Heads): Uint8Array => getHeadsCodec().encode({ hashes: value }),
21
+ decode: (encodedValue: Uint8Array): Heads => {
22
+ try {
23
+ return getHeadsCodec().decode(encodedValue).hashes!;
24
+ } catch {
25
+ // Legacy encoding migration path.
26
+ log.warn('Detected legacy encoding of heads in storage.');
27
+ const concatenatedHeads = Buffer.from(encodedValue).toString('utf8').replace(/"/g, '');
28
+ const heads = [];
29
+ for (let i = 0; i < concatenatedHeads.length; i += 64) {
30
+ heads.push(concatenatedHeads.slice(i, i + 64));
31
+ }
32
+ return heads;
33
+ }
34
+ },
35
+ format: 'buffer',
36
+ };
37
+
38
+ export type HeadsStoreProps = {
12
39
  db: SublevelDB;
13
40
  };
14
41
 
15
42
  export class HeadsStore {
16
43
  private readonly _db: SublevelDB;
17
44
 
18
- constructor({ db }: HeadsStoreParams) {
45
+ constructor({ db }: HeadsStoreProps) {
19
46
  this._db = db;
20
47
  }
21
48
 
@@ -34,4 +61,16 @@ export class HeadsStore {
34
61
  valueEncoding: headsEncoding,
35
62
  });
36
63
  }
64
+
65
+ /**
66
+ * Iterate over all document IDs and their heads.
67
+ */
68
+ async *iterateAll(): AsyncGenerator<{ documentId: DocumentId; heads: Heads }> {
69
+ for await (const [documentId, heads] of this._db.iterator<DocumentId, Heads>({
70
+ keyEncoding: 'utf8',
71
+ valueEncoding: headsEncoding,
72
+ })) {
73
+ yield { documentId, heads };
74
+ }
75
+ }
37
76
  }
@@ -5,7 +5,7 @@
5
5
  export * from './automerge-host';
6
6
  export * from './leveldb-storage-adapter';
7
7
  export * from './mesh-echo-replicator';
8
- export * from './echo-replicator';
8
+ export type * from './echo-replicator';
9
9
  export { diffCollectionState } from './collection-synchronizer';
10
10
  export * from './space-collection';
11
11
  export * from './echo-data-monitor';
@@ -2,10 +2,10 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  // s
4
4
 
5
- import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@automerge/automerge-repo';
5
+ import { type Chunk, type StorageAdapterInterface, type StorageKey } from '@automerge/automerge-repo';
6
6
  import { type MixedEncoding } from 'level-transcoder';
7
7
 
8
- import { LifecycleState, Resource } from '@dxos/context';
8
+ import { Resource } from '@dxos/context';
9
9
  import { type BatchLevel, type SublevelDB } from '@dxos/kv-store';
10
10
  import { type MaybePromise } from '@dxos/util';
11
11
 
@@ -16,27 +16,27 @@ export interface StorageAdapterDataMonitor {
16
16
  recordStoreDuration(durationMs: number): void;
17
17
  }
18
18
 
19
- export type LevelDBStorageAdapterParams = {
19
+ export type LevelDBStorageAdapterProps = {
20
20
  db: SublevelDB;
21
21
  callbacks?: StorageCallbacks;
22
22
  monitor?: StorageAdapterDataMonitor;
23
23
  };
24
24
 
25
- export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
25
+ export type BeforeSaveProps = { path: StorageKey; batch: BatchLevel };
26
26
 
27
27
  export interface StorageCallbacks {
28
- beforeSave(params: BeforeSaveParams): MaybePromise<void>;
28
+ beforeSave(params: BeforeSaveProps): MaybePromise<void>;
29
29
  afterSave(path: StorageKey): MaybePromise<void>;
30
30
  }
31
31
 
32
32
  export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
33
- constructor(private readonly _params: LevelDBStorageAdapterParams) {
33
+ constructor(private readonly _params: LevelDBStorageAdapterProps) {
34
34
  super();
35
35
  }
36
36
 
37
37
  async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
38
38
  try {
39
- if (this._lifecycleState !== LifecycleState.OPEN) {
39
+ if (!this.isOpen) {
40
40
  // TODO(mykola): this should be an error.
41
41
  return undefined;
42
42
  }
@@ -54,7 +54,7 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
54
54
  }
55
55
 
56
56
  async save(keyArray: StorageKey, binary: Uint8Array): Promise<void> {
57
- if (this._lifecycleState !== LifecycleState.OPEN) {
57
+ if (!this.isOpen) {
58
58
  return undefined;
59
59
  }
60
60
  const startMs = Date.now();
@@ -72,14 +72,14 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
72
72
  }
73
73
 
74
74
  async remove(keyArray: StorageKey): Promise<void> {
75
- if (this._lifecycleState !== LifecycleState.OPEN) {
75
+ if (!this.isOpen) {
76
76
  return undefined;
77
77
  }
78
78
  await this._params.db.del<StorageKey>(keyArray, { ...encodingOptions });
79
79
  }
80
80
 
81
81
  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
82
- if (this._lifecycleState !== LifecycleState.OPEN) {
82
+ if (!this.isOpen) {
83
83
  return [];
84
84
  }
85
85
  const startMs = Date.now();
@@ -100,7 +100,7 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
100
100
  }
101
101
 
102
102
  async removeRange(keyPrefix: StorageKey): Promise<void> {
103
- if (this._lifecycleState !== LifecycleState.OPEN) {
103
+ if (!this.isOpen) {
104
104
  return undefined;
105
105
  }
106
106
  const batch = this._params.db.batch();
@@ -12,20 +12,20 @@ import { log } from '@dxos/log';
12
12
  import type { AutomergeProtocolMessage } from '@dxos/protocols';
13
13
  import { AutomergeReplicator, type AutomergeReplicatorFactory } from '@dxos/teleport-extension-automerge-replicator';
14
14
 
15
- import type { ReplicatorConnection, ShouldAdvertiseParams, ShouldSyncCollectionParams } from './echo-replicator';
15
+ import type { AutomergeReplicatorConnection, ShouldAdvertiseProps, ShouldSyncCollectionProps } from './echo-replicator';
16
16
 
17
17
  const DEFAULT_FACTORY: AutomergeReplicatorFactory = (params) => new AutomergeReplicator(...params);
18
18
 
19
- export type MeshReplicatorConnectionParams = {
19
+ export type MeshReplicatorConnectionProps = {
20
20
  ownPeerId: string;
21
21
  onRemoteConnected: () => void;
22
22
  onRemoteDisconnected: () => void;
23
- shouldAdvertise: (params: ShouldAdvertiseParams) => Promise<boolean>;
24
- shouldSyncCollection: (params: ShouldSyncCollectionParams) => boolean;
23
+ shouldAdvertise: (params: ShouldAdvertiseProps) => Promise<boolean>;
24
+ shouldSyncCollection: (params: ShouldSyncCollectionProps) => boolean;
25
25
  replicatorFactory?: AutomergeReplicatorFactory;
26
26
  };
27
27
 
28
- export class MeshReplicatorConnection extends Resource implements ReplicatorConnection {
28
+ export class MeshReplicatorConnection extends Resource implements AutomergeReplicatorConnection {
29
29
  public readable: ReadableStream<AutomergeProtocolMessage>;
30
30
  public writable: WritableStream<AutomergeProtocolMessage>;
31
31
  public remoteDeviceKey: PublicKey | null = null;
@@ -35,7 +35,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
35
35
  private _remotePeerId: string | null = null;
36
36
  private _isEnabled = false;
37
37
 
38
- constructor(private readonly _params: MeshReplicatorConnectionParams) {
38
+ constructor(private readonly _params: MeshReplicatorConnectionProps) {
39
39
  super();
40
40
 
41
41
  let readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
@@ -111,11 +111,15 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
111
111
  return this._isEnabled;
112
112
  }
113
113
 
114
- async shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean> {
114
+ get bundleSyncEnabled(): boolean {
115
+ return false;
116
+ }
117
+
118
+ async shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean> {
115
119
  return this._params.shouldAdvertise(params);
116
120
  }
117
121
 
118
- shouldSyncCollection(params: ShouldSyncCollectionParams): boolean {
122
+ shouldSyncCollection(params: ShouldSyncCollectionProps): boolean {
119
123
  return this._params.shouldSyncCollection(params);
120
124
  }
121
125
 
@@ -6,23 +6,25 @@ import type { CollectionId } from '@dxos/echo-protocol';
6
6
  import { invariant } from '@dxos/invariant';
7
7
  import { PublicKey, type SpaceId } from '@dxos/keys';
8
8
  import { log } from '@dxos/log';
9
- import {
10
- type AutomergeReplicator,
11
- type AutomergeReplicatorFactory,
12
- } from '@dxos/teleport-extension-automerge-replicator';
9
+ import type * as TeleportAutomergeReplicator from '@dxos/teleport-extension-automerge-replicator';
13
10
  import { ComplexSet, defaultMap } from '@dxos/util';
14
11
 
15
- import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseParams } from './echo-replicator';
12
+ import { createIdFromSpaceKey } from '../common/space-id';
13
+
14
+ import {
15
+ type AutomergeReplicator,
16
+ type AutomergeReplicatorContext,
17
+ type ShouldAdvertiseProps,
18
+ } from './echo-replicator';
16
19
  import { MeshReplicatorConnection } from './mesh-echo-replicator-connection';
17
20
  import { getSpaceIdFromCollectionId } from './space-collection';
18
- import { createIdFromSpaceKey } from '../common/space-id';
19
21
 
20
22
  // TODO(dmaretskyi): Move out of @dxos/echo-pipeline.
21
23
 
22
24
  /**
23
25
  * Used to replicate with other peers over the network.
24
26
  */
25
- export class MeshEchoReplicator implements EchoReplicator {
27
+ export class MeshEchoReplicator implements AutomergeReplicator {
26
28
  /**
27
29
  * We might have multiple connections open with a peer (one per space), but there'll be only one enabled
28
30
  * connection at any given moment, because there's a single repo for all the spaces.
@@ -40,9 +42,9 @@ export class MeshEchoReplicator implements EchoReplicator {
40
42
  */
41
43
  private readonly _authorizedDevices = new Map<SpaceId, ComplexSet<PublicKey>>();
42
44
 
43
- private _context: EchoReplicatorContext | null = null;
45
+ private _context: AutomergeReplicatorContext | null = null;
44
46
 
45
- async connect(context: EchoReplicatorContext): Promise<void> {
47
+ async connect(context: AutomergeReplicatorContext): Promise<void> {
46
48
  this._context = context;
47
49
  }
48
50
 
@@ -63,7 +65,9 @@ export class MeshEchoReplicator implements EchoReplicator {
63
65
  this._context = null;
64
66
  }
65
67
 
66
- createExtension(extensionFactory?: AutomergeReplicatorFactory): AutomergeReplicator {
68
+ createExtension(
69
+ extensionFactory?: TeleportAutomergeReplicator.AutomergeReplicatorFactory,
70
+ ): TeleportAutomergeReplicator.AutomergeReplicator {
67
71
  invariant(this._context);
68
72
 
69
73
  const connection: MeshReplicatorConnection = new MeshReplicatorConnection({
@@ -110,7 +114,7 @@ export class MeshEchoReplicator implements EchoReplicator {
110
114
  }
111
115
  }
112
116
  },
113
- shouldAdvertise: async (params: ShouldAdvertiseParams) => {
117
+ shouldAdvertise: async (params: ShouldAdvertiseProps) => {
114
118
  log('shouldAdvertise', { peerId: connection.peerId, documentId: params.documentId });
115
119
  invariant(this._context);
116
120
  try {
@@ -2,7 +2,7 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import { onTestFinished, describe, expect, test } from 'vitest';
5
+ import { describe, expect, onTestFinished, test } from 'vitest';
6
6
 
7
7
  import { randomBytes } from '@dxos/crypto';
8
8
  import { PublicKey } from '@dxos/keys';
@@ -18,7 +18,7 @@ export const createIdFromSpaceKey = async (spaceKey: PublicKey): Promise<SpaceId
18
18
  return cachedValue;
19
19
  }
20
20
 
21
- const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array());
21
+ const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array() as Uint8Array<ArrayBuffer>);
22
22
 
23
23
  const bytes = new Uint8Array(digest).slice(0, SpaceId.byteLength);
24
24
  const spaceId = SpaceId.encode(bytes);