@dxos/echo-pipeline 0.8.3 → 0.8.4-main.05e74ebcff

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/LICENSE +102 -5
  2. package/README.md +1 -1
  3. package/dist/lib/neutral/chunk-46DCRDA7.mjs +271 -0
  4. package/dist/lib/neutral/chunk-46DCRDA7.mjs.map +7 -0
  5. package/dist/lib/{browser/chunk-35I6ERLG.mjs → neutral/chunk-LR7UFVQX.mjs} +313 -709
  6. package/dist/lib/neutral/chunk-LR7UFVQX.mjs.map +7 -0
  7. package/dist/lib/{browser → neutral}/filter/index.mjs +3 -2
  8. package/dist/lib/neutral/index.mjs +7332 -0
  9. package/dist/lib/neutral/index.mjs.map +7 -0
  10. package/dist/lib/neutral/meta.json +1 -0
  11. package/dist/lib/{browser → neutral}/testing/index.mjs +90 -311
  12. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  13. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts +2 -0
  14. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts.map +1 -0
  15. package/dist/types/src/automerge/automerge-host.d.ts +119 -54
  16. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  17. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts +2 -0
  18. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts.map +1 -0
  19. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  20. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  21. package/dist/types/src/automerge/automerge-subduction.test.d.ts +2 -0
  22. package/dist/types/src/automerge/automerge-subduction.test.d.ts.map +1 -0
  23. package/dist/types/src/automerge/collection-synchronizer.d.ts +25 -4
  24. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  25. package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
  26. package/dist/types/src/automerge/echo-network-adapter.d.ts +23 -10
  27. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  28. package/dist/types/src/automerge/echo-replicator.d.ts +46 -15
  29. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  30. package/dist/types/src/automerge/errors.d.ts +35 -0
  31. package/dist/types/src/automerge/errors.d.ts.map +1 -0
  32. package/dist/types/src/automerge/handle-state.d.ts +52 -0
  33. package/dist/types/src/automerge/handle-state.d.ts.map +1 -0
  34. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  35. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  36. package/dist/types/src/automerge/index.d.ts +3 -1
  37. package/dist/types/src/automerge/index.d.ts.map +1 -1
  38. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +11 -5
  39. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  40. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +9 -8
  41. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  42. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +6 -5
  43. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  44. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  45. package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
  46. package/dist/types/src/common/codec.d.ts +1 -1
  47. package/dist/types/src/common/codec.d.ts.map +1 -1
  48. package/dist/types/src/common/feeds.d.ts.map +1 -1
  49. package/dist/types/src/common/space-id.d.ts.map +1 -1
  50. package/dist/types/src/db-host/automerge-data-source.d.ts +35 -0
  51. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  52. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  53. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  54. package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
  55. package/dist/types/src/db-host/data-service.d.ts +13 -7
  56. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  57. package/dist/types/src/db-host/database-root.d.ts +9 -3
  58. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  59. package/dist/types/src/db-host/documents-synchronizer.d.ts +30 -8
  60. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  61. package/dist/types/src/db-host/echo-host.d.ts +74 -28
  62. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  63. package/dist/types/src/db-host/index.d.ts +2 -0
  64. package/dist/types/src/db-host/index.d.ts.map +1 -1
  65. package/dist/types/src/db-host/invalidation-hint.d.ts +19 -0
  66. package/dist/types/src/db-host/invalidation-hint.d.ts.map +1 -0
  67. package/dist/types/src/db-host/local-queue-service.d.ts +19 -0
  68. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  69. package/dist/types/src/db-host/query-invalidation.test.d.ts +2 -0
  70. package/dist/types/src/db-host/query-invalidation.test.d.ts.map +1 -0
  71. package/dist/types/src/db-host/query-service.d.ts +19 -11
  72. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  73. package/dist/types/src/db-host/queue-data-source.d.ts +38 -0
  74. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  75. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  76. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  77. package/dist/types/src/db-host/space-state-manager.d.ts +4 -3
  78. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  79. package/dist/types/src/db-host/stub.d.ts +11 -0
  80. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  81. package/dist/types/src/edge/echo-edge-replicator.d.ts +10 -7
  82. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  83. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts +39 -0
  84. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts.map +1 -0
  85. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts +2 -0
  86. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts.map +1 -0
  87. package/dist/types/src/edge/index.d.ts +1 -0
  88. package/dist/types/src/edge/index.d.ts.map +1 -1
  89. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
  90. package/dist/types/src/filter/filter-match.d.ts +4 -1
  91. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  92. package/dist/types/src/metadata/metadata-store.d.ts +1 -1
  93. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  94. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  95. package/dist/types/src/pipeline/pipeline.d.ts +3 -3
  96. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  97. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
  98. package/dist/types/src/query/errors.d.ts +22 -10
  99. package/dist/types/src/query/errors.d.ts.map +1 -1
  100. package/dist/types/src/query/plan.d.ts +66 -8
  101. package/dist/types/src/query/plan.d.ts.map +1 -1
  102. package/dist/types/src/query/query-executor.d.ts +66 -11
  103. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  104. package/dist/types/src/query/query-planner.d.ts +12 -1
  105. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  106. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  107. package/dist/types/src/space/auth.d.ts +3 -3
  108. package/dist/types/src/space/auth.d.ts.map +1 -1
  109. package/dist/types/src/space/control-pipeline.d.ts +5 -4
  110. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  111. package/dist/types/src/space/space-manager.d.ts +8 -8
  112. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  113. package/dist/types/src/space/space-protocol.d.ts +10 -9
  114. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  115. package/dist/types/src/space/space.d.ts +6 -6
  116. package/dist/types/src/space/space.d.ts.map +1 -1
  117. package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
  118. package/dist/types/src/testing/index.d.ts +0 -2
  119. package/dist/types/src/testing/index.d.ts.map +1 -1
  120. package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
  121. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  122. package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -1
  123. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  124. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  125. package/dist/types/src/testing/test-replicator.d.ts +14 -13
  126. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  127. package/dist/types/src/util.d.ts +1 -1
  128. package/dist/types/src/util.d.ts.map +1 -1
  129. package/dist/types/tsconfig.tsbuildinfo +1 -1
  130. package/package.json +59 -55
  131. package/src/automerge/automerge-host-subduction.test.ts +263 -0
  132. package/src/automerge/automerge-host.test.ts +162 -39
  133. package/src/automerge/automerge-host.ts +679 -201
  134. package/src/automerge/automerge-repo-subduction.test.ts +1054 -0
  135. package/src/automerge/automerge-repo.test.ts +349 -91
  136. package/src/automerge/automerge-subduction.test.ts +330 -0
  137. package/src/automerge/collection-synchronizer.test.ts +98 -17
  138. package/src/automerge/collection-synchronizer.ts +141 -37
  139. package/src/automerge/echo-data-monitor.ts +1 -1
  140. package/src/automerge/echo-network-adapter.test.ts +5 -4
  141. package/src/automerge/echo-network-adapter.ts +76 -31
  142. package/src/automerge/echo-replicator.ts +50 -15
  143. package/src/automerge/errors.ts +10 -0
  144. package/src/automerge/handle-state.ts +62 -0
  145. package/src/automerge/heads-store.ts +42 -3
  146. package/src/automerge/index.ts +3 -1
  147. package/src/automerge/leveldb-storage-adapter.ts +77 -11
  148. package/src/automerge/mesh-echo-replicator-connection.ts +12 -8
  149. package/src/automerge/mesh-echo-replicator.ts +15 -11
  150. package/src/automerge/storage-adapter.test.ts +1 -1
  151. package/src/common/space-id.ts +1 -1
  152. package/src/db-host/automerge-data-source.test.ts +299 -0
  153. package/src/db-host/automerge-data-source.ts +187 -0
  154. package/src/db-host/data-service.ts +35 -26
  155. package/src/db-host/database-root.ts +20 -8
  156. package/src/db-host/documents-synchronizer.test.ts +145 -14
  157. package/src/db-host/documents-synchronizer.ts +163 -51
  158. package/src/db-host/echo-host.ts +384 -99
  159. package/src/db-host/index.ts +2 -0
  160. package/src/db-host/invalidation-hint.ts +50 -0
  161. package/src/db-host/local-queue-service.ts +127 -0
  162. package/src/db-host/query-invalidation.test.ts +270 -0
  163. package/src/db-host/query-service.ts +111 -106
  164. package/src/db-host/queue-data-source.ts +160 -0
  165. package/src/db-host/queue-service.test.ts +145 -0
  166. package/src/db-host/space-state-manager.ts +12 -10
  167. package/src/db-host/stub.ts +33 -0
  168. package/src/edge/echo-edge-replicator.test.ts +12 -10
  169. package/src/edge/echo-edge-replicator.ts +144 -58
  170. package/src/edge/echo-edge-subduction-replicator.test.ts +158 -0
  171. package/src/edge/echo-edge-subduction-replicator.ts +377 -0
  172. package/src/edge/index.ts +1 -0
  173. package/src/filter/filter-match.test.ts +68 -32
  174. package/src/filter/filter-match.ts +230 -21
  175. package/src/metadata/metadata-store.ts +11 -8
  176. package/src/pipeline/pipeline-stress.test.ts +1 -1
  177. package/src/pipeline/pipeline.test.ts +2 -2
  178. package/src/pipeline/pipeline.ts +9 -7
  179. package/src/query/errors.ts +3 -1
  180. package/src/query/plan.ts +84 -9
  181. package/src/query/query-executor.ts +1197 -165
  182. package/src/query/query-planner.test.ts +993 -81
  183. package/src/query/query-planner.ts +503 -27
  184. package/src/space/admission-discovery-extension.ts +3 -3
  185. package/src/space/auth.ts +6 -6
  186. package/src/space/control-pipeline.test.ts +5 -4
  187. package/src/space/control-pipeline.ts +16 -13
  188. package/src/space/space-manager.browser.test.ts +1 -1
  189. package/src/space/space-manager.ts +16 -20
  190. package/src/space/space-protocol.browser.test.ts +15 -14
  191. package/src/space/space-protocol.test.ts +15 -14
  192. package/src/space/space-protocol.ts +22 -18
  193. package/src/space/space.test.ts +3 -3
  194. package/src/space/space.ts +11 -11
  195. package/src/testing/index.ts +0 -2
  196. package/src/testing/test-agent-builder.ts +3 -3
  197. package/src/testing/test-network-adapter.ts +24 -3
  198. package/src/testing/test-replicator.ts +19 -20
  199. package/src/util.ts +7 -3
  200. package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
  201. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  202. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  203. package/dist/lib/browser/chunk-TQJTKNMS.mjs +0 -126
  204. package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
  205. package/dist/lib/browser/index.mjs +0 -4743
  206. package/dist/lib/browser/index.mjs.map +0 -7
  207. package/dist/lib/browser/meta.json +0 -1
  208. package/dist/lib/browser/testing/index.mjs.map +0 -7
  209. package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
  210. package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
  211. package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
  212. package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
  213. package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
  214. package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
  215. package/dist/lib/node/filter/index.cjs +0 -32
  216. package/dist/lib/node/filter/index.cjs.map +0 -7
  217. package/dist/lib/node/index.cjs +0 -4699
  218. package/dist/lib/node/index.cjs.map +0 -7
  219. package/dist/lib/node/meta.json +0 -1
  220. package/dist/lib/node/testing/index.cjs +0 -753
  221. package/dist/lib/node/testing/index.cjs.map +0 -7
  222. package/dist/lib/node-esm/chunk-5BHLPT24.mjs +0 -2067
  223. package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
  224. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  225. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  226. package/dist/lib/node-esm/chunk-RVK35BS7.mjs +0 -126
  227. package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +0 -7
  228. package/dist/lib/node-esm/filter/index.mjs +0 -11
  229. package/dist/lib/node-esm/filter/index.mjs.map +0 -7
  230. package/dist/lib/node-esm/index.mjs +0 -4743
  231. package/dist/lib/node-esm/index.mjs.map +0 -7
  232. package/dist/lib/node-esm/meta.json +0 -1
  233. package/dist/lib/node-esm/testing/index.mjs +0 -742
  234. package/dist/lib/node-esm/testing/index.mjs.map +0 -7
  235. package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
  236. package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
  237. package/dist/types/src/testing/test-data.d.ts +0 -18
  238. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  239. package/dist/types/src/testing/test-schema.d.ts +0 -37
  240. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  241. package/src/db-host/documents-iterator.ts +0 -74
  242. package/src/testing/test-data.ts +0 -127
  243. package/src/testing/test-schema.ts +0 -53
  244. /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
@@ -2,20 +2,24 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import { next as am } from '@automerge/automerge';
5
+ import { next as A, type Heads } from '@automerge/automerge';
6
6
  import type { DocumentId, PeerId } from '@automerge/automerge-repo';
7
+ import * as Array from 'effect/Array';
8
+ import * as Record from 'effect/Record';
7
9
 
8
- import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from '@dxos/async';
9
- import { Resource, type Context } from '@dxos/context';
10
+ import { Event, asyncReturn, scheduleTask, scheduleTaskInterval } from '@dxos/async';
11
+ import { type Context, Resource } from '@dxos/context';
10
12
  import { log } from '@dxos/log';
11
13
  import { trace } from '@dxos/tracing';
12
14
  import { defaultMap } from '@dxos/util';
13
15
 
16
+ import { PeerNotFoundError } from './errors';
17
+
14
18
  const MIN_QUERY_INTERVAL = 5_000;
15
19
 
16
- const POLL_INTERVAL = 30_000;
20
+ const POLL_INTERVAL = 10_000;
17
21
 
18
- export type CollectionSynchronizerParams = {
22
+ export type CollectionSynchronizerProps = {
19
23
  sendCollectionState: (collectionId: string, peerId: PeerId, state: CollectionState) => void;
20
24
  queryCollectionState: (collectionId: string, peerId: PeerId) => void;
21
25
  shouldSyncCollection: (collectionId: string, peerId: PeerId) => boolean;
@@ -24,11 +28,11 @@ export type CollectionSynchronizerParams = {
24
28
  /**
25
29
  * Implements collection sync protocol.
26
30
  */
27
- @trace.resource()
31
+ @trace.resource({ lifecycle: true })
28
32
  export class CollectionSynchronizer extends Resource {
29
- private readonly _sendCollectionState: CollectionSynchronizerParams['sendCollectionState'];
30
- private readonly _queryCollectionState: CollectionSynchronizerParams['queryCollectionState'];
31
- private readonly _shouldSyncCollection: CollectionSynchronizerParams['shouldSyncCollection'];
33
+ private readonly _sendCollectionState: CollectionSynchronizerProps['sendCollectionState'];
34
+ private readonly _queryCollectionState: CollectionSynchronizerProps['queryCollectionState'];
35
+ private readonly _shouldSyncCollection: CollectionSynchronizerProps['shouldSyncCollection'];
32
36
 
33
37
  /**
34
38
  * CollectionId -> State.
@@ -40,7 +44,7 @@ export class CollectionSynchronizer extends Resource {
40
44
 
41
45
  public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId; newDocsAppeared: boolean }>();
42
46
 
43
- constructor(params: CollectionSynchronizerParams) {
47
+ constructor(params: CollectionSynchronizerProps) {
44
48
  super();
45
49
  this._sendCollectionState = params.sendCollectionState;
46
50
  this._queryCollectionState = params.queryCollectionState;
@@ -74,13 +78,29 @@ export class CollectionSynchronizer extends Resource {
74
78
  this._activeCollections.add(collectionId);
75
79
 
76
80
  log('setLocalCollectionState', { collectionId, state });
77
- this._getOrCreatePerCollectionState(collectionId).localState = state;
81
+ const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
82
+ perCollectionState.localState = state;
78
83
 
79
- queueMicrotask(async () => {
80
- if (!this._ctx.disposed && this._activeCollections.has(collectionId)) {
81
- this._refreshInterestedPeers(collectionId);
82
- this.refreshCollection(collectionId);
83
- }
84
+ for (const peerId of this._connectedPeers) {
85
+ this._diffCollectionState(collectionId, peerId);
86
+ }
87
+
88
+ this._scheduleBroadcast(collectionId);
89
+ }
90
+
91
+ /**
92
+ * Coalesce bursts of `setLocalCollectionState` (e.g. from `_onHeadsChanged` during an
93
+ * import) into one broadcast microtask per collection. If another call arrives while
94
+ * the microtask is pending, we just flag the collection dirty — the in-flight run
95
+ * already reads the latest `localState`. If it arrives during the run itself, we
96
+ * schedule exactly one follow-up.
97
+ */
98
+ private _scheduleBroadcast(collectionId: string): void {
99
+ const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
100
+ queueMicrotask(() => {
101
+ this._refreshInterestedPeers(collectionId);
102
+ this._broadcastLocalState(collectionId);
103
+ this.refreshCollection(collectionId);
84
104
  });
85
105
  }
86
106
 
@@ -117,10 +137,14 @@ export class CollectionSynchronizer extends Resource {
117
137
  * Callback when a connection to a peer is established.
118
138
  */
119
139
  onConnectionOpen(peerId: PeerId): void {
120
- const spanId = getSpanName(peerId);
121
- trace.spanStart({
140
+ log('onConnectionOpen', { peerId });
141
+ const spanId = getSpanId(peerId);
142
+ // Browser-timeline-only span; the derived ctx is intentionally discarded because
143
+ // the downstream `_queryCollectionState` hop is user-supplied callback land with
144
+ // no ctx plumbing.
145
+ void trace.spanStart({
122
146
  id: spanId,
123
- methodName: spanId,
147
+ methodName: SYNC_SPAN_METHOD,
124
148
  instance: this,
125
149
  parentCtx: this._ctx,
126
150
  showInBrowserTimeline: true,
@@ -146,21 +170,37 @@ export class CollectionSynchronizer extends Resource {
146
170
  * Callback when a connection to a peer is closed.
147
171
  */
148
172
  onConnectionClosed(peerId: PeerId): void {
173
+ log('onConnectionClosed', { peerId });
174
+
149
175
  this._connectedPeers.delete(peerId);
150
176
 
151
177
  for (const perCollectionState of this._perCollectionStates.values()) {
152
178
  perCollectionState.remoteStates.delete(peerId);
179
+ perCollectionState.interestedPeers.delete(peerId);
180
+ perCollectionState.lastQueried.delete(peerId);
181
+ perCollectionState.lastBroadcast.delete(peerId);
153
182
  }
154
183
  }
155
184
 
156
185
  /**
157
186
  * Callback when a peer queries the state of a collection.
187
+ *
188
+ * If we have no local state yet we silently drop the query; the peer will receive our
189
+ * state via {@link _broadcastLocalState} once `setLocalCollectionState` is called.
158
190
  */
159
191
  onCollectionStateQueried(collectionId: string, peerId: PeerId): void {
160
192
  const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
161
193
 
162
194
  if (perCollectionState.localState) {
163
- this._sendCollectionState(collectionId, peerId, perCollectionState.localState);
195
+ try {
196
+ this._sendCollectionState(collectionId, peerId, perCollectionState.localState);
197
+ } catch (error) {
198
+ if (PeerNotFoundError.is(error)) {
199
+ log('peer not found when sending collection state callback', { error });
200
+ return;
201
+ }
202
+ throw error;
203
+ }
164
204
  }
165
205
  }
166
206
 
@@ -171,24 +211,48 @@ export class CollectionSynchronizer extends Resource {
171
211
  log('onRemoteStateReceived', { collectionId, peerId, state });
172
212
  validateCollectionState(state);
173
213
  const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
174
- const existingState = perCollectionState.remoteStates.get(peerId) ?? { documents: {} };
175
- const diff = diffCollectionState(existingState, state);
176
- const spanId = getSpanName(peerId);
214
+ perCollectionState.remoteStates.set(peerId, state);
215
+ this._diffCollectionState(collectionId, peerId);
216
+ }
217
+
218
+ private _diffCollectionState(collectionId: string, peerId: PeerId) {
219
+ const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
220
+ const remoteState = perCollectionState.remoteStates.get(peerId);
221
+ if (!remoteState) {
222
+ return;
223
+ }
224
+
225
+ log('diffCollectionState', { collectionId, peerId });
226
+ const localState = perCollectionState.localState ?? { documents: {} };
227
+ const diff = diffCollectionState(localState, remoteState);
228
+ const spanId = getSpanId(peerId);
177
229
  if (diff.different.length === 0) {
178
230
  trace.spanEnd(spanId);
179
231
  } else {
180
- trace.spanStart({
232
+ // Browser-timeline-only span; see note in onConnectionOpen.
233
+ void trace.spanStart({
181
234
  id: spanId,
182
- methodName: spanId,
235
+ methodName: SYNC_SPAN_METHOD,
183
236
  instance: this,
184
237
  parentCtx: this._ctx,
185
238
  showInBrowserTimeline: true,
186
239
  attributes: { peerId },
187
240
  });
188
241
  }
189
- if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
190
- perCollectionState.remoteStates.set(peerId, state);
191
- this.remoteStateUpdated.emit({ peerId, collectionId, newDocsAppeared: diff.missingOnLocal.length > 0 });
242
+ log('diff', {
243
+ localState: localState.documents,
244
+ remoteState: remoteState.documents,
245
+ missingOnLocal: diff.missingOnLocal,
246
+ missingOnRemote: diff.missingOnRemote,
247
+ different: diff.different,
248
+ });
249
+ if (diff.missingOnLocal.length > 0 || diff.different.length > 0 || diff.missingOnRemote.length > 0) {
250
+ log('emit remote state update');
251
+ this.remoteStateUpdated.emit({
252
+ peerId,
253
+ collectionId,
254
+ newDocsAppeared: diff.missingOnLocal.length > 0,
255
+ });
192
256
  }
193
257
  }
194
258
 
@@ -198,9 +262,43 @@ export class CollectionSynchronizer extends Resource {
198
262
  remoteStates: new Map(),
199
263
  interestedPeers: new Set(),
200
264
  lastQueried: new Map(),
265
+ lastBroadcast: new Map(),
201
266
  }));
202
267
  }
203
268
 
269
+ /**
270
+ * Push local state to interested peers whose last-known remote state differs from local
271
+ * (or is unknown), then pull from peers via {@link refreshCollection}.
272
+ *
273
+ * Diff-gating avoids spamming peers that are already in sync; this matters because
274
+ * {@link setLocalCollectionState} is called on every local heads change.
275
+ */
276
+ private _broadcastLocalState(collectionId: string): void {
277
+ if (this._ctx.disposed || !this._activeCollections.has(collectionId)) {
278
+ return;
279
+ }
280
+ const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
281
+ const localState = perCollectionState.localState;
282
+ if (!localState) {
283
+ return;
284
+ }
285
+ for (const peerId of perCollectionState.interestedPeers) {
286
+ const lastBroadcast = perCollectionState.lastBroadcast.get(peerId) ?? 0;
287
+ if (Date.now() - lastBroadcast > MIN_QUERY_INTERVAL) {
288
+ perCollectionState.lastBroadcast.set(peerId, Date.now());
289
+ try {
290
+ this._sendCollectionState(collectionId, peerId, localState);
291
+ } catch (error) {
292
+ if (PeerNotFoundError.is(error)) {
293
+ log('peer not found when broadcasting collection state', { error });
294
+ return;
295
+ }
296
+ throw error;
297
+ }
298
+ }
299
+ }
300
+ }
301
+
204
302
  private _refreshInterestedPeers(collectionId: string): void {
205
303
  for (const peerId of this._connectedPeers) {
206
304
  if (this._shouldSyncCollection(collectionId, peerId)) {
@@ -217,13 +315,14 @@ type PerCollectionState = {
217
315
  remoteStates: Map<PeerId, CollectionState>;
218
316
  interestedPeers: Set<PeerId>;
219
317
  lastQueried: Map<PeerId, number>;
318
+ lastBroadcast: Map<PeerId, number>;
220
319
  };
221
320
 
222
321
  export type CollectionState = {
223
322
  /**
224
323
  * DocumentId -> Heads.
225
324
  */
226
- documents: Record<string, string[]>;
325
+ documents: Record<DocumentId, Heads>;
227
326
  };
228
327
 
229
328
  export type CollectionStateDiff = {
@@ -233,18 +332,21 @@ export type CollectionStateDiff = {
233
332
  };
234
333
 
235
334
  export const diffCollectionState = (local: CollectionState, remote: CollectionState): CollectionStateDiff => {
236
- const allDocuments = new Set<DocumentId>([...Object.keys(local.documents), ...Object.keys(remote.documents)] as any);
335
+ const localDocuments = Record.filter(local.documents, (heads) => heads.length > 0);
336
+ const remoteDocuments = Record.filter(remote.documents, (heads) => heads.length > 0);
337
+ // NOTE: Using `Array.union` is slow.
338
+ const allDocuments = [...new Set([...Record.keys(localDocuments), ...Record.keys(remoteDocuments)])] as DocumentId[];
237
339
 
238
340
  const missingOnRemote: DocumentId[] = [];
239
341
  const missingOnLocal: DocumentId[] = [];
240
342
  const different: DocumentId[] = [];
241
343
  for (const documentId of allDocuments) {
242
- if (!local.documents[documentId]) {
243
- missingOnLocal.push(documentId as DocumentId);
244
- } else if (!remote.documents[documentId]) {
245
- missingOnRemote.push(documentId as DocumentId);
246
- } else if (!am.equals(local.documents[documentId], remote.documents[documentId])) {
247
- different.push(documentId as DocumentId);
344
+ if (!localDocuments[documentId]) {
345
+ missingOnLocal.push(documentId);
346
+ } else if (!remoteDocuments[documentId]) {
347
+ missingOnRemote.push(documentId);
348
+ } else if (!A.equals(local.documents[documentId], remote.documents[documentId])) {
349
+ different.push(documentId);
248
350
  }
249
351
  }
250
352
 
@@ -270,6 +372,8 @@ const isValidDocumentId = (documentId: DocumentId) => {
270
372
  return typeof documentId === 'string' && !documentId.includes(':');
271
373
  };
272
374
 
273
- const getSpanName = (peerId: PeerId) => {
375
+ const SYNC_SPAN_METHOD = 'syncPeer';
376
+
377
+ const getSpanId = (peerId: PeerId) => {
274
378
  return `collection-sync-${peerId}`;
275
379
  };
@@ -5,7 +5,7 @@
5
5
  import { type Message } from '@automerge/automerge-repo';
6
6
 
7
7
  import { type TimeAware, trace } from '@dxos/tracing';
8
- import { CircularBuffer, mapValues, SlidingWindowSummary, type SlidingWindowSummaryConfig } from '@dxos/util';
8
+ import { CircularBuffer, SlidingWindowSummary, type SlidingWindowSummaryConfig, mapValues } from '@dxos/util';
9
9
 
10
10
  import { type NetworkDataMonitor } from './echo-network-adapter';
11
11
  import { type StorageAdapterDataMonitor } from './leveldb-storage-adapter';
@@ -2,10 +2,11 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import { cbor, type PeerId } from '@automerge/automerge-repo';
6
- import { onTestFinished, describe, expect, test } from 'vitest';
5
+ import { type PeerId, cbor } from '@automerge/automerge-repo';
6
+ import { describe, expect, onTestFinished, test } from 'vitest';
7
7
 
8
- import { sleep, Trigger, waitForCondition } from '@dxos/async';
8
+ import { Trigger, sleep, waitForCondition } from '@dxos/async';
9
+ import { Context } from '@dxos/context';
9
10
  import { invariant } from '@dxos/invariant';
10
11
  import { PublicKey } from '@dxos/keys';
11
12
  import { type SyncMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';
@@ -113,7 +114,7 @@ describe('EchoNetworkAdapter', () => {
113
114
  onTestFinished(async () => {
114
115
  await adapter.close();
115
116
  });
116
- await adapter.addReplicator(replicator);
117
+ await adapter.addReplicator(Context.default(), replicator);
117
118
  return adapter;
118
119
  };
119
120
 
@@ -2,30 +2,38 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import { NetworkAdapter, type Message, type PeerId, type PeerMetadata } from '@automerge/automerge-repo';
6
-
7
- import { synchronized, Trigger } from '@dxos/async';
8
- import { LifecycleState } from '@dxos/context';
5
+ import {
6
+ type DocumentId,
7
+ type Heads,
8
+ type Message,
9
+ NetworkAdapter,
10
+ type PeerId,
11
+ type PeerMetadata,
12
+ } from '@automerge/automerge-repo';
13
+
14
+ import { Event, Trigger, synchronized } from '@dxos/async';
15
+ import { type Context, LifecycleState } from '@dxos/context';
9
16
  import { invariant } from '@dxos/invariant';
10
17
  import { type PublicKey } from '@dxos/keys';
11
18
  import { log } from '@dxos/log';
12
19
  import type { AutomergeProtocolMessage } from '@dxos/protocols';
13
20
  import { isNonNullable } from '@dxos/util';
14
21
 
22
+ import { createIdFromSpaceKey } from '../common/space-id';
15
23
  import {
16
- type EchoReplicator,
17
- type RemoteDocumentExistenceCheckParams,
18
- type ReplicatorConnection,
19
- type ShouldAdvertiseParams,
20
- type ShouldSyncCollectionParams,
24
+ type AutomergeReplicator,
25
+ type AutomergeReplicatorConnection,
26
+ type RemoteDocumentExistenceCheckProps,
27
+ type ShouldAdvertiseProps,
28
+ type ShouldSyncCollectionProps,
21
29
  } from './echo-replicator';
30
+ import { PeerNotFoundError } from './errors';
22
31
  import {
23
- isCollectionQueryMessage,
24
- isCollectionStateMessage,
25
32
  type CollectionQueryMessage,
26
33
  type CollectionStateMessage,
34
+ isCollectionQueryMessage,
35
+ isCollectionStateMessage,
27
36
  } from './network-protocol';
28
- import { createIdFromSpaceKey } from '../common/space-id';
29
37
 
30
38
  export interface NetworkDataMonitor {
31
39
  recordPeerConnected(peerId: string): void;
@@ -35,9 +43,9 @@ export interface NetworkDataMonitor {
35
43
  recordMessageSendingFailed(message: Message): void;
36
44
  }
37
45
 
38
- export type EchoNetworkAdapterParams = {
46
+ export type EchoNetworkAdapterProps = {
39
47
  getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
40
- isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckParams) => Promise<boolean>;
48
+ isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckProps) => Promise<boolean>;
41
49
  onCollectionStateQueried: (collectionId: string, peerId: PeerId) => void;
42
50
  onCollectionStateReceived: (collectionId: string, peerId: PeerId, state: unknown) => void;
43
51
  monitor?: NetworkDataMonitor;
@@ -45,16 +53,17 @@ export type EchoNetworkAdapterParams = {
45
53
 
46
54
  type ConnectionEntry = {
47
55
  isOpen: boolean;
48
- connection: ReplicatorConnection;
56
+ connection: AutomergeReplicatorConnection;
49
57
  reader: ReadableStreamDefaultReader<AutomergeProtocolMessage>;
50
58
  writer: WritableStreamDefaultWriter<AutomergeProtocolMessage>;
59
+ requestedDocuments: Set<DocumentId>;
51
60
  };
52
61
 
53
62
  /**
54
- * Manages a set of {@link EchoReplicator} instances.
63
+ * Manages a set of {@link AutomergeReplicator} instances.
55
64
  */
56
65
  export class EchoNetworkAdapter extends NetworkAdapter {
57
- private readonly _replicators = new Set<EchoReplicator>();
66
+ private readonly _replicators = new Set<AutomergeReplicator>();
58
67
  /**
59
68
  * Remote peer id -> connection.
60
69
  */
@@ -63,7 +72,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
63
72
  private readonly _connected = new Trigger();
64
73
  private readonly _ready = new Trigger();
65
74
 
66
- constructor(private readonly _params: EchoNetworkAdapterParams) {
75
+ public readonly documentRequested = new Event<{ documentId: DocumentId; peerId: PeerId }>();
76
+
77
+ constructor(private readonly _params: EchoNetworkAdapterProps) {
67
78
  super();
68
79
  }
69
80
 
@@ -125,13 +136,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
125
136
  }
126
137
 
127
138
  @synchronized
128
- async addReplicator(replicator: EchoReplicator): Promise<void> {
139
+ async addReplicator(ctx: Context, replicator: AutomergeReplicator): Promise<void> {
129
140
  invariant(this._lifecycleState === LifecycleState.OPEN);
130
141
  invariant(this.peerId);
131
142
  invariant(!this._replicators.has(replicator));
132
143
 
133
144
  this._replicators.add(replicator);
134
- await replicator.connect({
145
+ await replicator.connect(ctx, {
135
146
  peerId: this.peerId,
136
147
  onConnectionOpen: this._onConnectionOpen.bind(this),
137
148
  onConnectionClosed: this._onConnectionClosed.bind(this),
@@ -146,14 +157,14 @@ export class EchoNetworkAdapter extends NetworkAdapter {
146
157
  }
147
158
 
148
159
  @synchronized
149
- async removeReplicator(replicator: EchoReplicator): Promise<void> {
160
+ async removeReplicator(replicator: AutomergeReplicator): Promise<void> {
150
161
  invariant(this._lifecycleState === LifecycleState.OPEN);
151
162
  invariant(this._replicators.has(replicator));
152
163
  await replicator.disconnect();
153
164
  this._replicators.delete(replicator);
154
165
  }
155
166
 
156
- async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseParams): Promise<boolean> {
167
+ async shouldAdvertise(peerId: PeerId, params: ShouldAdvertiseProps): Promise<boolean> {
157
168
  const connection = this._connections.get(peerId);
158
169
  if (!connection) {
159
170
  return false;
@@ -162,7 +173,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
162
173
  return connection.connection.shouldAdvertise(params);
163
174
  }
164
175
 
165
- shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionParams): boolean {
176
+ shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionProps): boolean {
166
177
  const connection = this._connections.get(peerId);
167
178
  if (!connection) {
168
179
  return false;
@@ -203,10 +214,34 @@ export class EchoNetworkAdapter extends NetworkAdapter {
203
214
  .filter(isNonNullable);
204
215
  }
205
216
 
217
+ bundleSyncEnabledForPeer(peerId: PeerId): boolean {
218
+ const connection = this._connections.get(peerId);
219
+ if (!connection) {
220
+ return false;
221
+ }
222
+ return connection.connection.bundleSyncEnabled;
223
+ }
224
+
225
+ async pushBundle(ctx: Context, peerId: PeerId, bundle: { documentId: DocumentId; data: Uint8Array; heads: Heads }[]) {
226
+ const connection = this._connections.get(peerId);
227
+ if (!connection) {
228
+ throw new PeerNotFoundError({ context: { peerId } });
229
+ }
230
+ return connection.connection.pushBundle!(ctx, bundle);
231
+ }
232
+
233
+ async pullBundle(ctx: Context, peerId: PeerId, docHeads: Record<DocumentId, Heads>) {
234
+ const connection = this._connections.get(peerId);
235
+ if (!connection) {
236
+ throw new PeerNotFoundError({ context: { peerId } });
237
+ }
238
+ return connection.connection.pullBundle!(ctx, docHeads);
239
+ }
240
+
206
241
  private _send(message: Message): void {
207
242
  const connectionEntry = this._connections.get(message.targetId);
208
243
  if (!connectionEntry) {
209
- throw new Error('Connection not found.');
244
+ throw new PeerNotFoundError({ context: { peerId: message.targetId } });
210
245
  }
211
246
 
212
247
  // TODO(dmaretskyi): Find a way to enforce backpressure on AM-repo.
@@ -225,7 +260,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
225
260
  });
226
261
  }
227
262
 
228
- private _onConnectionOpen(connection: ReplicatorConnection): void {
263
+ private _onConnectionOpen(connection: AutomergeReplicatorConnection): void {
229
264
  log('connection opened', { peerId: connection.peerId });
230
265
  invariant(!this._connections.has(connection.peerId as PeerId));
231
266
  const connectionEntry: ConnectionEntry = {
@@ -233,6 +268,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
233
268
  connection,
234
269
  reader: connection.readable.getReader(),
235
270
  writer: connection.writable.getWriter(),
271
+ requestedDocuments: new Set(),
236
272
  };
237
273
 
238
274
  this._connections.set(connection.peerId as PeerId, connectionEntry);
@@ -247,7 +283,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
247
283
  break;
248
284
  }
249
285
 
250
- this._onMessage(value as Message);
286
+ this._onMessage(connectionEntry, value as Message);
251
287
  }
252
288
  } catch (err) {
253
289
  if (connectionEntry.isOpen) {
@@ -261,7 +297,15 @@ export class EchoNetworkAdapter extends NetworkAdapter {
261
297
  this._params.monitor?.recordPeerConnected(connection.peerId);
262
298
  }
263
299
 
264
- private _onMessage(message: Message): void {
300
+ private _onMessage(connectionEntry: ConnectionEntry, message: Message): void {
301
+ const amMessage = message as AutomergeProtocolMessage;
302
+ if (amMessage.type === 'request') {
303
+ this.documentRequested.emit({
304
+ documentId: amMessage.documentId as DocumentId,
305
+ peerId: connectionEntry.connection.peerId as PeerId,
306
+ });
307
+ }
308
+
265
309
  if (isCollectionQueryMessage(message)) {
266
310
  this._params.onCollectionStateQueried(message.collectionId, message.senderId);
267
311
  } else if (isCollectionStateMessage(message)) {
@@ -272,7 +316,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
272
316
  this._params.monitor?.recordMessageReceived(message);
273
317
  }
274
318
 
275
- private _onConnectionClosed(connection: ReplicatorConnection): void {
319
+ private _onConnectionClosed(connection: AutomergeReplicatorConnection): void {
276
320
  log('connection closed', { peerId: connection.peerId });
277
321
  const entry = this._connections.get(connection.peerId as PeerId);
278
322
  invariant(entry);
@@ -281,8 +325,9 @@ export class EchoNetworkAdapter extends NetworkAdapter {
281
325
  this.emit('peer-disconnected', { peerId: connection.peerId as PeerId });
282
326
  this._params.monitor?.recordPeerDisconnected(connection.peerId);
283
327
 
284
- void entry.reader.cancel().catch((err) => log.catch(err));
285
328
  void entry.writer.abort().catch((err) => log.catch(err));
329
+ void entry.reader.cancel().catch((err) => log.catch(err));
330
+
286
331
  this._connections.delete(connection.peerId as PeerId);
287
332
  }
288
333
 
@@ -290,7 +335,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
290
335
  * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
291
336
  * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
292
337
  */
293
- private _onConnectionAuthScopeChanged(connection: ReplicatorConnection): void {
338
+ private _onConnectionAuthScopeChanged(connection: AutomergeReplicatorConnection): void {
294
339
  log('Connection auth scope changed', { peerId: connection.peerId });
295
340
  const entry = this._connections.get(connection.peerId as PeerId);
296
341
  invariant(entry);
@@ -298,7 +343,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
298
343
  this._emitPeerCandidate(connection);
299
344
  }
300
345
 
301
- private _emitPeerCandidate(connection: ReplicatorConnection): void {
346
+ private _emitPeerCandidate(connection: AutomergeReplicatorConnection): void {
302
347
  this.emit('peer-candidate', {
303
348
  peerId: connection.peerId as PeerId,
304
349
  peerMetadata: createEchoPeerMetadata(),