@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/LICENSE +102 -5
  2. package/README.md +1 -1
  3. package/dist/lib/{browser/chunk-WJJXJTNS.mjs → neutral/chunk-46DCRDA7.mjs} +44 -13
  4. package/dist/lib/neutral/chunk-46DCRDA7.mjs.map +7 -0
  5. package/dist/lib/{browser/chunk-FR6GJ4S4.mjs → neutral/chunk-LR7UFVQX.mjs} +145 -618
  6. package/dist/lib/neutral/chunk-LR7UFVQX.mjs.map +7 -0
  7. package/dist/lib/{browser → neutral}/filter/index.mjs +1 -2
  8. package/dist/lib/neutral/index.mjs +7332 -0
  9. package/dist/lib/neutral/index.mjs.map +7 -0
  10. package/dist/lib/neutral/meta.json +1 -0
  11. package/dist/lib/{browser → neutral}/testing/index.mjs +49 -292
  12. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  13. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts +2 -0
  14. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts.map +1 -0
  15. package/dist/types/src/automerge/automerge-host.d.ts +105 -27
  16. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  17. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts +2 -0
  18. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts.map +1 -0
  19. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  20. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  21. package/dist/types/src/automerge/automerge-subduction.test.d.ts +2 -0
  22. package/dist/types/src/automerge/automerge-subduction.test.d.ts.map +1 -0
  23. package/dist/types/src/automerge/collection-synchronizer.d.ts +22 -2
  24. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  25. package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
  26. package/dist/types/src/automerge/echo-network-adapter.d.ts +17 -11
  27. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  28. package/dist/types/src/automerge/echo-replicator.d.ts +28 -16
  29. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  30. package/dist/types/src/automerge/errors.d.ts +35 -0
  31. package/dist/types/src/automerge/errors.d.ts.map +1 -0
  32. package/dist/types/src/automerge/handle-state.d.ts +52 -0
  33. package/dist/types/src/automerge/handle-state.d.ts.map +1 -0
  34. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  35. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  36. package/dist/types/src/automerge/index.d.ts +2 -0
  37. package/dist/types/src/automerge/index.d.ts.map +1 -1
  38. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +10 -4
  39. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  40. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +8 -8
  41. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  42. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +6 -5
  43. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  44. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  45. package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
  46. package/dist/types/src/common/feeds.d.ts.map +1 -1
  47. package/dist/types/src/common/space-id.d.ts.map +1 -1
  48. package/dist/types/src/db-host/automerge-data-source.d.ts +35 -0
  49. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  50. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  51. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  52. package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
  53. package/dist/types/src/db-host/data-service.d.ts +12 -6
  54. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  55. package/dist/types/src/db-host/database-root.d.ts +9 -3
  56. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  57. package/dist/types/src/db-host/documents-synchronizer.d.ts +30 -8
  58. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  59. package/dist/types/src/db-host/echo-host.d.ts +74 -28
  60. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  61. package/dist/types/src/db-host/index.d.ts +2 -0
  62. package/dist/types/src/db-host/index.d.ts.map +1 -1
  63. package/dist/types/src/db-host/invalidation-hint.d.ts +19 -0
  64. package/dist/types/src/db-host/invalidation-hint.d.ts.map +1 -0
  65. package/dist/types/src/db-host/local-queue-service.d.ts +19 -0
  66. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  67. package/dist/types/src/db-host/query-invalidation.test.d.ts +2 -0
  68. package/dist/types/src/db-host/query-invalidation.test.d.ts.map +1 -0
  69. package/dist/types/src/db-host/query-service.d.ts +18 -10
  70. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  71. package/dist/types/src/db-host/queue-data-source.d.ts +38 -0
  72. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  73. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  74. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  75. package/dist/types/src/db-host/space-state-manager.d.ts +3 -2
  76. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  77. package/dist/types/src/db-host/stub.d.ts +11 -0
  78. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  79. package/dist/types/src/edge/echo-edge-replicator.d.ts +7 -6
  80. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  81. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts +39 -0
  82. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts.map +1 -0
  83. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts +2 -0
  84. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts.map +1 -0
  85. package/dist/types/src/edge/index.d.ts +1 -0
  86. package/dist/types/src/edge/index.d.ts.map +1 -1
  87. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
  88. package/dist/types/src/filter/filter-match.d.ts +1 -1
  89. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  90. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  91. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  92. package/dist/types/src/pipeline/pipeline.d.ts +2 -2
  93. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  94. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
  95. package/dist/types/src/query/errors.d.ts +16 -20
  96. package/dist/types/src/query/errors.d.ts.map +1 -1
  97. package/dist/types/src/query/plan.d.ts +59 -8
  98. package/dist/types/src/query/plan.d.ts.map +1 -1
  99. package/dist/types/src/query/query-executor.d.ts +55 -12
  100. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  101. package/dist/types/src/query/query-planner.d.ts +10 -1
  102. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  103. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  104. package/dist/types/src/space/auth.d.ts +3 -3
  105. package/dist/types/src/space/auth.d.ts.map +1 -1
  106. package/dist/types/src/space/control-pipeline.d.ts +4 -3
  107. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  108. package/dist/types/src/space/space-manager.d.ts +7 -7
  109. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  110. package/dist/types/src/space/space-protocol.d.ts +9 -8
  111. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  112. package/dist/types/src/space/space.d.ts +5 -5
  113. package/dist/types/src/space/space.d.ts.map +1 -1
  114. package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
  115. package/dist/types/src/testing/index.d.ts +0 -2
  116. package/dist/types/src/testing/index.d.ts.map +1 -1
  117. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  118. package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -1
  119. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  120. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  121. package/dist/types/src/testing/test-replicator.d.ts +13 -13
  122. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  123. package/dist/types/src/util.d.ts.map +1 -1
  124. package/dist/types/tsconfig.tsbuildinfo +1 -1
  125. package/package.json +56 -54
  126. package/src/automerge/automerge-host-subduction.test.ts +263 -0
  127. package/src/automerge/automerge-host.test.ts +145 -32
  128. package/src/automerge/automerge-host.ts +442 -163
  129. package/src/automerge/automerge-repo-subduction.test.ts +1054 -0
  130. package/src/automerge/automerge-repo.test.ts +284 -77
  131. package/src/automerge/automerge-subduction.test.ts +330 -0
  132. package/src/automerge/collection-synchronizer.test.ts +96 -16
  133. package/src/automerge/collection-synchronizer.ts +125 -26
  134. package/src/automerge/echo-network-adapter.test.ts +2 -1
  135. package/src/automerge/echo-network-adapter.ts +44 -32
  136. package/src/automerge/echo-replicator.ts +30 -16
  137. package/src/automerge/errors.ts +10 -0
  138. package/src/automerge/handle-state.ts +62 -0
  139. package/src/automerge/heads-store.ts +42 -3
  140. package/src/automerge/index.ts +2 -0
  141. package/src/automerge/leveldb-storage-adapter.ts +70 -4
  142. package/src/automerge/mesh-echo-replicator-connection.ts +8 -8
  143. package/src/automerge/mesh-echo-replicator.ts +14 -11
  144. package/src/db-host/automerge-data-source.test.ts +299 -0
  145. package/src/db-host/automerge-data-source.ts +187 -0
  146. package/src/db-host/data-service.ts +27 -10
  147. package/src/db-host/database-root.ts +19 -7
  148. package/src/db-host/documents-synchronizer.test.ts +145 -14
  149. package/src/db-host/documents-synchronizer.ts +158 -59
  150. package/src/db-host/echo-host.ts +376 -90
  151. package/src/db-host/index.ts +2 -0
  152. package/src/db-host/invalidation-hint.ts +50 -0
  153. package/src/db-host/local-queue-service.ts +127 -0
  154. package/src/db-host/query-invalidation.test.ts +270 -0
  155. package/src/db-host/query-service.ts +108 -111
  156. package/src/db-host/queue-data-source.ts +160 -0
  157. package/src/db-host/queue-service.test.ts +145 -0
  158. package/src/db-host/space-state-manager.ts +11 -9
  159. package/src/db-host/stub.ts +33 -0
  160. package/src/edge/echo-edge-replicator.test.ts +9 -9
  161. package/src/edge/echo-edge-replicator.ts +50 -42
  162. package/src/edge/echo-edge-subduction-replicator.test.ts +158 -0
  163. package/src/edge/echo-edge-subduction-replicator.ts +377 -0
  164. package/src/edge/index.ts +1 -0
  165. package/src/filter/filter-match.test.ts +41 -25
  166. package/src/filter/filter-match.ts +75 -20
  167. package/src/metadata/metadata-store.ts +8 -5
  168. package/src/pipeline/pipeline-stress.test.ts +1 -3
  169. package/src/pipeline/pipeline.test.ts +0 -1
  170. package/src/pipeline/pipeline.ts +2 -3
  171. package/src/query/errors.ts +2 -2
  172. package/src/query/plan.ts +73 -9
  173. package/src/query/query-executor.ts +1064 -172
  174. package/src/query/query-planner.test.ts +854 -86
  175. package/src/query/query-planner.ts +444 -38
  176. package/src/space/admission-discovery-extension.ts +1 -1
  177. package/src/space/auth.ts +6 -6
  178. package/src/space/control-pipeline.test.ts +2 -2
  179. package/src/space/control-pipeline.ts +7 -7
  180. package/src/space/space-manager.ts +12 -17
  181. package/src/space/space-protocol.browser.test.ts +13 -12
  182. package/src/space/space-protocol.test.ts +13 -13
  183. package/src/space/space-protocol.ts +16 -15
  184. package/src/space/space.test.ts +2 -2
  185. package/src/space/space.ts +9 -10
  186. package/src/testing/index.ts +0 -2
  187. package/src/testing/test-agent-builder.ts +0 -1
  188. package/src/testing/test-network-adapter.ts +24 -3
  189. package/src/testing/test-replicator.ts +15 -20
  190. package/src/util.ts +7 -3
  191. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  192. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  193. package/dist/lib/browser/chunk-FR6GJ4S4.mjs.map +0 -7
  194. package/dist/lib/browser/chunk-WJJXJTNS.mjs.map +0 -7
  195. package/dist/lib/browser/index.mjs +0 -5492
  196. package/dist/lib/browser/index.mjs.map +0 -7
  197. package/dist/lib/browser/meta.json +0 -1
  198. package/dist/lib/browser/testing/index.mjs.map +0 -7
  199. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  200. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  201. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs +0 -2144
  202. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs.map +0 -7
  203. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs +0 -240
  204. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs.map +0 -7
  205. package/dist/lib/node-esm/filter/index.mjs +0 -13
  206. package/dist/lib/node-esm/filter/index.mjs.map +0 -7
  207. package/dist/lib/node-esm/index.mjs +0 -5492
  208. package/dist/lib/node-esm/index.mjs.map +0 -7
  209. package/dist/lib/node-esm/meta.json +0 -1
  210. package/dist/lib/node-esm/testing/index.mjs +0 -764
  211. package/dist/lib/node-esm/testing/index.mjs.map +0 -7
  212. package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
  213. package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
  214. package/dist/types/src/testing/test-data.d.ts +0 -18
  215. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  216. package/dist/types/src/testing/test-schema.d.ts +0 -37
  217. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  218. package/src/db-host/documents-iterator.ts +0 -74
  219. package/src/testing/test-data.ts +0 -127
  220. package/src/testing/test-schema.ts +0 -53
  221. /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
@@ -16,44 +16,47 @@ import {
16
16
  type DocHandle,
17
17
  type DocHandleChangePayload,
18
18
  type DocumentId,
19
- type HandleState,
19
+ type DocumentQuery,
20
20
  type PeerCandidatePayload,
21
21
  type PeerDisconnectedPayload,
22
22
  type PeerId,
23
23
  Repo,
24
24
  type StorageAdapterInterface,
25
25
  type StorageKey,
26
+ type SubductionPolicy,
26
27
  interpretAsDocumentId,
28
+ initSubduction,
27
29
  } from '@automerge/automerge-repo';
30
+ import { type MemorySigner } from '@automerge/automerge-subduction';
28
31
 
29
32
  import { DeferredTask, Event, asyncTimeout } from '@dxos/async';
30
33
  import { Context, type Lifecycle, Resource, cancelWithContext } from '@dxos/context';
31
34
  import { type CollectionId, DatabaseDirectory } from '@dxos/echo-protocol';
32
- import { type IndexMetadataStore } from '@dxos/indexing';
33
35
  import { invariant } from '@dxos/invariant';
34
36
  import { PublicKey } from '@dxos/keys';
35
37
  import { type LevelDB } from '@dxos/kv-store';
36
38
  import { log } from '@dxos/log';
37
- import { objectPointerCodec } from '@dxos/protocols';
38
39
  import { type SpaceSyncState } from '@dxos/protocols/proto/dxos/echo/service';
39
40
  import { type DocHeadsList, type FlushRequest } from '@dxos/protocols/proto/dxos/echo/service';
40
41
  import { trace } from '@dxos/tracing';
41
- import { ComplexSet, bufferToArray, isNonNullable, range } from '@dxos/util';
42
+ import { ComplexSet, bufferToArray, defaultMap, isNonNullable, range } from '@dxos/util';
42
43
 
43
44
  import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
44
45
  import { type EchoDataMonitor } from './echo-data-monitor';
45
46
  import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
46
- import { type EchoReplicator, type RemoteDocumentExistenceCheckParams } from './echo-replicator';
47
+ import { type AutomergeReplicator, type RemoteDocumentExistenceCheckProps } from './echo-replicator';
48
+ import { getHandleState } from './handle-state';
47
49
  import { HeadsStore } from './heads-store';
48
- import { type BeforeSaveParams, LevelDBStorageAdapter } from './leveldb-storage-adapter';
50
+ import { type BeforeSaveProps, LevelDBStorageAdapter } from './leveldb-storage-adapter';
49
51
 
50
52
  export type PeerIdProvider = () => string | undefined;
51
53
 
52
54
  export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | undefined;
53
55
 
54
- export type AutomergeHostParams = {
56
+ const SUBDUCTION_SERVICE_NAME = 'dxos-subduction';
57
+
58
+ export type AutomergeHostProps = {
55
59
  db: LevelDB;
56
- indexMetadataStore: IndexMetadataStore;
57
60
  dataMonitor?: EchoDataMonitor;
58
61
 
59
62
  /**
@@ -61,10 +64,34 @@ export type AutomergeHostParams = {
61
64
  */
62
65
  peerIdProvider?: PeerIdProvider;
63
66
  getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
67
+
68
+ /**
69
+ * Enable Subduction sedimentree transport.
70
+ *
71
+ * When `false` (default), the host wires {@link EchoNetworkAdapter} as a classical
72
+ * automerge-repo network adapter and skips all Subduction-specific initialization
73
+ * (WASM init, signer generation, subduction policy / adapters). When `true`, the
74
+ * host runs Subduction as the document byte transport.
75
+ */
76
+ useSubduction?: boolean;
64
77
  };
65
78
 
66
79
  export type LoadDocOptions = {
67
80
  timeout?: number;
81
+
82
+ /**
83
+ * Controls whether `loadDoc` is allowed to wait on the network.
84
+ *
85
+ * - `true` / unset (default): announce that we want the doc and wait
86
+ * until any source (storage OR network) delivers it.
87
+ * - `false`: probe local storage only. If chunks for the doc exist
88
+ * on disk, wait for storage to populate the handle; otherwise
89
+ * throw `'unavailable'` immediately without ever consulting the
90
+ * network. Note that this does not guarantee that the document
91
+ * will not be fetched from the network — an inbound peer announce
92
+ * can still deliver bytes — but the host will never request it.
93
+ */
94
+ fetchFromNetwork?: boolean;
68
95
  };
69
96
 
70
97
  export type CreateDocOptions = {
@@ -72,32 +99,42 @@ export type CreateDocOptions = {
72
99
  * Import the document together with its history.
73
100
  */
74
101
  preserveHistory?: boolean;
75
- };
76
102
 
77
- export const FIND_PARAMS = {
78
- allowableStates: ['ready', 'requesting'] satisfies HandleState[],
103
+ documentId?: DocumentId;
79
104
  };
80
105
 
81
106
  /**
82
107
  * Maximum amount of documents to sync in a single bundle.
83
108
  */
84
109
  const BUNDLE_SIZE = 100;
110
+
85
111
  /**
86
112
  * Maximum amount of concurrent tasks to run when pushing or pulling bundles.
87
113
  */
88
114
  const BUNDLE_SYNC_CONCURRENCY = 2;
115
+
89
116
  /**
90
117
  * If the number of documents to sync is greater than this threshold, we will use bundles.
91
118
  */
92
119
  const BUNDLE_SYNC_THRESHOLD = 50;
93
120
 
121
+ /**
122
+ * Only announce documents that are known to require sync.
123
+ */
124
+ const OPTIMIZED_SHARE_POLICY = true;
125
+
94
126
  /**
95
127
  * Abstracts over the AutomergeRepo.
128
+ *
129
+ * Runs Subduction as the document byte transport ({@link Repo.subductionAdapters}), while
130
+ * the DXOS-specific {@link CollectionSynchronizer} rides on the same {@link EchoNetworkAdapter}
131
+ * via `sync-request` / `sync-state` control messages that are intercepted at the adapter
132
+ * level and never reach the Subduction sedimentree layer. Bundle sync remains available
133
+ * for replicators that opt in.
96
134
  */
97
- @trace.resource()
135
+ @trace.resource({ lifecycle: true })
98
136
  export class AutomergeHost extends Resource {
99
137
  private readonly _db: LevelDB;
100
- private readonly _indexMetadataStore: IndexMetadataStore;
101
138
  private readonly _echoNetworkAdapter: EchoNetworkAdapter;
102
139
 
103
140
  private readonly _collectionSynchronizer = new CollectionSynchronizer({
@@ -132,17 +169,43 @@ export class AutomergeHost extends Resource {
132
169
  public readonly documentsSaved = new Event();
133
170
 
134
171
  private readonly _headsUpdates = new Map<DocumentId, Heads>();
135
- private _onHeadsChangedTask?: DeferredTask | undefined;
172
+ private _onHeadsChangedTask?: DeferredTask;
173
+
174
+ /**
175
+ * Documents created in this session.
176
+ */
177
+ private _createdDocuments = new Set<DocumentId>();
178
+
179
+ /**
180
+ * Documents that need to be synced based on the result of collection-sync.
181
+ */
182
+ private _documentsToSync = new Set<DocumentId>();
183
+
184
+ /**
185
+ * Documents that are not available locally that should be requested.
186
+ */
187
+ private _documentsToRequest = new Set<DocumentId>();
188
+
189
+ /**
190
+ * Documents requested by remote peers.
191
+ */
192
+ private _documentsRequested = new Map<PeerId, Set<DocumentId>>();
193
+
194
+ private _sharePolicyChangedTask?: DeferredTask;
195
+
196
+ private _signer: MemorySigner | undefined = undefined;
197
+ private readonly _useSubduction: boolean;
136
198
 
137
199
  constructor({
138
200
  db,
139
- indexMetadataStore,
140
201
  dataMonitor,
141
202
  peerIdProvider,
142
203
  getSpaceKeyByRootDocumentId,
143
- }: AutomergeHostParams) {
204
+ useSubduction = false,
205
+ }: AutomergeHostProps) {
144
206
  super();
145
207
  this._db = db;
208
+ this._useSubduction = useSubduction;
146
209
  this._storage = new LevelDBStorageAdapter({
147
210
  db: db.sublevel('automerge'),
148
211
  callbacks: {
@@ -158,13 +221,18 @@ export class AutomergeHost extends Resource {
158
221
  onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
159
222
  monitor: dataMonitor,
160
223
  });
224
+ this._echoNetworkAdapter.documentRequested.on(({ peerId, documentId }) => {
225
+ defaultMap(this._documentsRequested, peerId, () => new Set()).add(documentId);
226
+ if (!this._useSubduction) {
227
+ this._sharePolicyChangedTask!.schedule();
228
+ }
229
+ });
161
230
  this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
162
- this._indexMetadataStore = indexMetadataStore;
163
231
  this._peerIdProvider = peerIdProvider;
164
232
  this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
165
233
  }
166
234
 
167
- protected override async _open(): Promise<void> {
235
+ protected override async _open(ctx: Context): Promise<void> {
168
236
  this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}` as PeerId;
169
237
 
170
238
  this._onHeadsChangedTask = new DeferredTask(this._ctx, async () => {
@@ -175,16 +243,49 @@ export class AutomergeHost extends Resource {
175
243
 
176
244
  await this._storage.open?.();
177
245
 
178
- // Construct the automerge repo.
179
- this._repo = new Repo({
180
- peerId: this._peerId as PeerId,
181
- sharePolicy: this._sharePolicy.bind(this),
182
- storage: this._storage,
183
- network: [
184
- // Upstream swarm.
185
- this._echoNetworkAdapter,
186
- ],
187
- });
246
+ // `Repo` unconditionally constructs a Subduction `SubductionSource` (with a fresh
247
+ // `MemorySigner` when none is injected) regardless of whether we register any
248
+ // `subductionAdapters`. That source's signer needs the Subduction WASM module
249
+ // initialized first — `Repo` imports from `@automerge/automerge-subduction/slim`,
250
+ // which does not auto-init. So WASM init runs in both modes.
251
+ await initSubduction();
252
+
253
+ if (this._useSubduction) {
254
+ const { MemorySigner } = await import('@automerge/automerge-subduction');
255
+ this._signer ??= MemorySigner.generate();
256
+
257
+ this._repo = new Repo({
258
+ peerId: this._peerId as PeerId,
259
+ shareConfig: this._shareConfig,
260
+ subductionPolicy: this._subductionPolicy,
261
+ storage: this._storage,
262
+ network: [],
263
+ signer: this._signer,
264
+ subductionAdapters: [
265
+ {
266
+ adapter: this._echoNetworkAdapter,
267
+ serviceName: SUBDUCTION_SERVICE_NAME,
268
+ // DXOS hosts are always clients — the edge DO is the single `accept` peer in
269
+ // the DXOS-client <-> edge topology. `connect` uses Subduction's discovery mode,
270
+ // so peer-to-peer connections (e.g., mesh replicator, test networks) also work
271
+ // with `connect` on both sides.
272
+ role: 'connect',
273
+ },
274
+ ],
275
+ });
276
+ } else {
277
+ // Classical automerge-repo wiring: the EchoNetworkAdapter is registered as a
278
+ // network adapter and document bytes flow through the standard sync protocol.
279
+ // `Repo` will internally construct an unused `MemorySigner` for its always-on
280
+ // `SubductionSource`; with no `subductionAdapters` passed in that source has no
281
+ // peers to talk to, so it's effectively dormant.
282
+ this._repo = new Repo({
283
+ peerId: this._peerId as PeerId,
284
+ shareConfig: this._shareConfig,
285
+ storage: this._storage,
286
+ network: [this._echoNetworkAdapter],
287
+ });
288
+ }
188
289
 
189
290
  let updatingAuthScope = false;
190
291
  Event.wrap(this._echoNetworkAdapter, 'peer-candidate').on(
@@ -199,8 +300,7 @@ export class AutomergeHost extends Resource {
199
300
  this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
200
301
  this._onRemoteCollectionStateUpdated(collectionId, peerId);
201
302
  this.collectionStateUpdated.emit({ collectionId: collectionId as CollectionId });
202
- // We use collection lookups during share policy check, so we might need to update share policy for the new doc
203
- if (newDocsAppeared) {
303
+ if (!this._useSubduction && newDocsAppeared) {
204
304
  updatingAuthScope = true;
205
305
  try {
206
306
  this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
@@ -212,13 +312,14 @@ export class AutomergeHost extends Resource {
212
312
 
213
313
  this._syncTask = new DeferredTask(this._ctx, async () => {
214
314
  const collectionToSync = Array.from(this._collectionsToSync.values());
315
+ this._collectionsToSync.clear();
215
316
  if (collectionToSync.length === 0) {
216
317
  return;
217
318
  }
218
319
  await Promise.all(
219
320
  collectionToSync.map(async ({ collectionId, peerId }) => {
220
321
  try {
221
- await this._handleCollectionSync(collectionId, peerId);
322
+ await this._handleCollectionSync(this._ctx, collectionId, peerId);
222
323
  } catch (err) {
223
324
  log.error('failed to sync collection', { collectionId, peerId, err });
224
325
  }
@@ -226,25 +327,33 @@ export class AutomergeHost extends Resource {
226
327
  );
227
328
  });
228
329
 
330
+ this._sharePolicyChangedTask = new DeferredTask(this._ctx, async () => {
331
+ log('share policy changed');
332
+ this._repo.shareConfigChanged();
333
+ });
334
+
229
335
  await this._echoNetworkAdapter.open();
230
- await this._collectionSynchronizer.open();
231
- await this._echoNetworkAdapter.open();
336
+ await this._collectionSynchronizer.open(ctx);
232
337
  await this._echoNetworkAdapter.whenConnected();
233
338
  }
234
339
 
235
- protected override async _close(): Promise<void> {
236
- await this._collectionSynchronizer.close();
340
+ protected override async _close(ctx: Context): Promise<void> {
341
+ // Drain any in-flight `_onHeadsChangedTask` before the `Resource` base
342
+ // disposes `this._ctx`. Without this, a concurrent `flush()` call (e.g.
343
+ // from a client `db.flush()` RPC racing with shutdown) hits its
344
+ // `runBlocking` after the ctx has been disposed and throws
345
+ // `ContextDisposedError`, which escapes as an unhandled rejection at the
346
+ // fire-and-forget originating caller.
347
+ await this._onHeadsChangedTask?.join();
348
+
349
+ await this._collectionSynchronizer.close(ctx);
350
+ await this._repo.flush().catch((err) => log.warn('failed to flush repo before shutdown', { err }));
351
+ await this._repo.shutdown().catch((err) => log.warn('failed to shutdown repo', { err }));
237
352
  await this._storage.close?.();
238
353
  await this._echoNetworkAdapter.close();
239
354
  this._syncTask = undefined;
240
355
  this._onHeadsChangedTask = undefined;
241
- }
242
-
243
- /**
244
- * @deprecated To be abstracted away.
245
- */
246
- get repo(): Repo {
247
- return this._repo;
356
+ this._sharePolicyChangedTask = undefined;
248
357
  }
249
358
 
250
359
  get peerId(): PeerId {
@@ -255,43 +364,85 @@ export class AutomergeHost extends Resource {
255
364
  return Object.keys(this._repo.handles).length;
256
365
  }
257
366
 
258
- async addReplicator(replicator: EchoReplicator): Promise<void> {
367
+ get handles(): Readonly<Record<DocumentId, DocHandle<any>>> {
368
+ return this._repo.handles;
369
+ }
370
+
371
+ async addReplicator(ctx: Context, replicator: AutomergeReplicator): Promise<void> {
259
372
  invariant(this.isOpen, 'AutomergeHost is not open');
260
- await this._echoNetworkAdapter.addReplicator(replicator);
373
+ await this._echoNetworkAdapter.addReplicator(ctx, replicator);
261
374
  }
262
375
 
263
- async removeReplicator(replicator: EchoReplicator): Promise<void> {
376
+ async removeReplicator(replicator: AutomergeReplicator): Promise<void> {
264
377
  invariant(this.isOpen, 'AutomergeHost is not open');
265
378
  await this._echoNetworkAdapter.removeReplicator(replicator);
266
379
  }
267
380
 
268
- /**
269
- * Loads the document handle from the repo and waits for it to be ready.
270
- */
271
- async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T>> {
381
+ async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T> | null> {
272
382
  invariant(this.isOpen, 'AutomergeHost is not open');
273
- let handle: DocHandle<T> | undefined;
274
- if (typeof documentId === 'string') {
275
- // NOTE: documentId might also be a URL, in which case this lookup will fail.
276
- handle = this._repo.handles[documentId as DocumentId];
383
+ // Readiness lives on the `DocumentQuery`, not the `DocHandle` — see
384
+ // {@link getHandleState}.
385
+ const progress = this._repo.findWithProgress<T>(documentId as DocumentId);
386
+ const initial = progress.peek();
387
+ if (initial.state === 'ready') {
388
+ return initial.handle;
277
389
  }
278
- if (!handle) {
279
- handle = await this._repo.find(documentId as DocumentId, FIND_PARAMS);
390
+
391
+ // Default: when `fetchFromNetwork` is unset, behave as if it were
392
+ // `true` — wait on any source. Only an explicit `false` activates
393
+ // the storage-only branch.
394
+ if (opts?.fetchFromNetwork !== false) {
395
+ // Network branch: announce that we want the doc, then fall through
396
+ // to `whenReady()` and wait for any source (storage or network) to
397
+ // deliver it.
398
+ if (!this._useSubduction) {
399
+ this._documentsToRequest.add(progress.documentId);
400
+ this._sharePolicyChangedTask!.schedule();
401
+ }
402
+ } else {
403
+ // Note: This is a Hack.
404
+ // Storage-only branch. The subduction fork's `DocumentQuery` merged
405
+ // storage/network into a single `'loading'` state, so we can't tell
406
+ // them apart via `progress.peek()`. Workaround: probe storage
407
+ // directly and throw `'unavailable'` if nothing is on disk, without
408
+ // ever scheduling a network announce. See `fetchFromNetwork` JSDoc
409
+ // for the residual inbound-announce race.
410
+ // TODO(mykola): replace with per-source state inspection once the
411
+ // patched fork exposes it on `DocumentProgress`.
412
+ const chunks = await this._storage.loadRange([progress.documentId]);
413
+ const onDisk = chunks.some((chunk) => chunk.data && chunk.data.length > 0);
414
+ if (!onDisk) {
415
+ return null;
416
+ }
280
417
  }
281
418
 
282
- // `whenReady` creates a timeout so we guard it with an if to skip it if the handle is already ready.
283
- if (!handle.isReady()) {
284
- if (!opts?.timeout) {
285
- await cancelWithContext(ctx, handle.whenReady());
286
- } else {
287
- await cancelWithContext(ctx, asyncTimeout(handle.whenReady(), opts.timeout));
419
+ const readyPromise = progress.whenReady();
420
+ try {
421
+ return opts?.timeout
422
+ ? await cancelWithContext(ctx, asyncTimeout(readyPromise, opts.timeout))
423
+ : await cancelWithContext(ctx, readyPromise);
424
+ } catch (error: any) {
425
+ if (isDocumentUnavailableError(error) && opts?.fetchFromNetwork === false) {
426
+ return null;
288
427
  }
428
+ throw error;
289
429
  }
430
+ }
290
431
 
291
- return handle;
432
+ /**
433
+ * Returns the live {@link DocumentQuery} for a document — exposes the
434
+ * always-attached `DocHandle` and the actual readiness state
435
+ * (`'loading' | 'ready' | 'unavailable' | 'failed'`) via `peek()` /
436
+ * `subscribe()` / `whenReady()`. Use this to observe sync state without
437
+ * forcing a wait. See {@link getHandleState} for why callers must read
438
+ * liveness off the query rather than the `DocHandle` itself.
439
+ */
440
+ findWithProgress<T>(documentId: AnyDocumentId): DocumentQuery<T> {
441
+ invariant(this.isOpen, 'AutomergeHost is not open');
442
+ return this._repo.findWithProgress<T>(documentId as DocumentId) as DocumentQuery<T>;
292
443
  }
293
444
 
294
- async exportDoc(ctx: Context, id: AnyDocumentId): Promise<Uint8Array> {
445
+ async exportDoc(id: AnyDocumentId): Promise<Uint8Array> {
295
446
  invariant(this.isOpen, 'AutomergeHost is not open');
296
447
  const documentId = interpretAsDocumentId(id);
297
448
 
@@ -299,14 +450,37 @@ export class AutomergeHost extends Resource {
299
450
  return bufferToArray(Buffer.concat(chunks.map((c) => c.data!)));
300
451
  }
301
452
 
453
+ /**
454
+ * Probe local storage to determine whether the document has any persisted
455
+ * chunks. Does not request the document from the network and does not
456
+ * touch the in-memory `Repo`. Returns `true` iff at least one non-empty
457
+ * chunk exists on disk for the document.
458
+ *
459
+ * Intended for query-driven (disk-only) loads that need to know quickly
460
+ * whether a document is locally available without waiting on network
461
+ * latency. See `DocumentsSynchronizer.addDocuments` and the `requesting`
462
+ * transition in `DocHandleProxy`.
463
+ */
464
+ async hasDocOnDisk(id: AnyDocumentId): Promise<boolean> {
465
+ invariant(this.isOpen, 'AutomergeHost is not open');
466
+ const documentId = interpretAsDocumentId(id);
467
+ const chunks = await this._storage.loadRange([documentId]);
468
+ return chunks.some((chunk) => chunk.data != null && chunk.data.length > 0);
469
+ }
470
+
302
471
  /**
303
472
  * Create new persisted document.
304
473
  */
305
- createDoc<T>(initialValue?: T | Doc<T> | Uint8Array, opts?: CreateDocOptions): DocHandle<T> {
474
+ async createDoc<T>(initialValue?: T | Doc<T> | Uint8Array, opts?: CreateDocOptions): Promise<DocHandle<T>> {
306
475
  invariant(this.isOpen, 'AutomergeHost is not open');
307
476
  if (opts?.preserveHistory) {
308
477
  if (initialValue instanceof Uint8Array) {
309
- return this._repo.import(initialValue);
478
+ const handle = this._repo.import<T>(initialValue, { docId: opts?.documentId });
479
+ this._createdDocuments.add(handle.documentId);
480
+ if (!this._useSubduction) {
481
+ this._sharePolicyChangedTask!.schedule();
482
+ }
483
+ return handle;
310
484
  }
311
485
 
312
486
  if (!isAutomerge(initialValue)) {
@@ -314,17 +488,30 @@ export class AutomergeHost extends Resource {
314
488
  }
315
489
 
316
490
  // TODO(dmaretskyi): There's a more efficient way.
317
- return this._repo.import(save(initialValue as Doc<T>));
491
+ const handle = this._repo.import<T>(save(initialValue as Doc<T>), { docId: opts?.documentId });
492
+ this._createdDocuments.add(handle.documentId);
493
+ if (!this._useSubduction) {
494
+ this._sharePolicyChangedTask!.schedule();
495
+ }
496
+ return handle;
318
497
  } else {
319
498
  if (initialValue instanceof Uint8Array) {
320
499
  throw new Error('Cannot create document from Uint8Array without preserving history');
321
500
  }
322
501
 
323
- return this._repo.create(initialValue);
502
+ if (opts?.documentId) {
503
+ throw new Error('Cannot prefil document id when not importing an existing doc');
504
+ }
505
+ const handle = await this._repo.create2<T>(initialValue);
506
+ this._createdDocuments.add(handle.documentId);
507
+ if (!this._useSubduction) {
508
+ this._sharePolicyChangedTask!.schedule();
509
+ }
510
+ return handle;
324
511
  }
325
512
  }
326
513
 
327
- async waitUntilHeadsReplicated(heads: DocHeadsList): Promise<void> {
514
+ async waitUntilHeadsReplicated(ctx: Context, heads: DocHeadsList): Promise<void> {
328
515
  invariant(this.isOpen, 'AutomergeHost is not open');
329
516
  const entries = heads.entries;
330
517
  if (!entries?.length) {
@@ -338,34 +525,40 @@ export class AutomergeHost extends Resource {
338
525
  return false;
339
526
  }
340
527
  const currentHeads = documentHeads[index];
341
- return !(currentHeads !== null && headsEquals(currentHeads, targetHeads));
528
+ return !(currentHeads !== null && currentHeads !== undefined && headsEquals(currentHeads, targetHeads));
342
529
  });
343
530
  if (headsToWait.length > 0) {
344
531
  await Promise.all(
345
532
  headsToWait.map(async (entry) => {
346
- const handle = await this.loadDoc<DatabaseDirectory>(Context.default(), entry.documentId as DocumentId);
533
+ const handle = await this.loadDoc<DatabaseDirectory>(ctx, entry.documentId as DocumentId);
534
+ invariant(handle, 'Document handle must be available when waiting for heads replication.');
347
535
  await waitForHeads(handle, entry.heads!);
348
536
  }),
349
537
  );
350
538
  }
351
539
 
352
540
  // Flush to disk handles loaded to memory also so that the indexer can pick up the changes.
353
- await this._repo.flush(
354
- documentIds.filter((documentId) => this._repo.handles[documentId] && this._repo.handles[documentId].isReady()),
355
- );
541
+ await this._repo.flush(documentIds.filter((documentId) => getHandleState(this._repo, documentId) === 'ready'));
356
542
  }
357
543
 
358
544
  async reIndexHeads(documentIds: DocumentId[]): Promise<void> {
359
545
  invariant(this.isOpen, 'AutomergeHost is not open');
360
546
  for (const documentId of documentIds) {
361
547
  log('re-indexing heads for document', { documentId });
362
- const handle = await this._repo.find(documentId, FIND_PARAMS);
363
- if (!handle.isReady()) {
364
- log.warn('document is not available locally, skipping', { documentId });
365
- continue; // Handle not available locally.
548
+ // `Repo.find()` resolves on `'ready'` and rejects on `'unavailable'`,
549
+ // so the handle is guaranteed to hold data here.
550
+ let handle: DocHandle<any>;
551
+ try {
552
+ handle = await this._repo.find(documentId);
553
+ } catch (err) {
554
+ log.error('failed to find document', { documentId, err });
555
+ continue;
366
556
  }
367
557
 
368
558
  const heads = handle.heads();
559
+ if (!heads) {
560
+ continue;
561
+ }
369
562
  const batch = this._db.batch();
370
563
  this._headsStore.setHeads(documentId, heads, batch);
371
564
  await batch.write();
@@ -373,26 +566,62 @@ export class AutomergeHost extends Resource {
373
566
  log('done re-indexing heads');
374
567
  }
375
568
 
376
- // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
377
- // Hosts, running in the worker, don't share documents unless requested by other peers.
378
- // NOTE: If both peers return sharePolicy=false the replication will not happen
379
- // https://github.com/automerge/automerge-repo/pull/292
380
- private async _sharePolicy(peerId: PeerId, documentId?: DocumentId): Promise<boolean> {
381
- if (!documentId) {
382
- return false;
383
- }
569
+ /**
570
+ * Share policy for the Repo's classical automerge-repo sync path (CollectionSynchronizer /
571
+ * DocSynchronizer — see `src/synchronizer/DocSynchronizer.ts#resolveSharePolicy`).
572
+ *
573
+ * NOTE: Subduction replication does NOT consult `shareConfig`.
574
+ * {@link AutomergeReplicator}s via {@link EchoNetworkAdapter}.
575
+ */
576
+ private readonly _shareConfig = {
577
+ access: async (_peerId: PeerId, _documentId?: DocumentId): Promise<boolean> => {
578
+ // Access-on-request is always allowed; per-doc authorization happens in the replicator.
579
+ return true;
580
+ },
581
+
582
+ // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
583
+ // Hosts, running in the worker, don't share documents unless requested by other peers.
584
+ // NOTE: If both peers return sharePolicy=false the replication will not happen
585
+ // https://github.com/automerge/automerge-repo/pull/292
586
+ // Called for all loaded documents so they could be advertised to the sync server.
587
+ announce: async (peerId: PeerId, documentId?: DocumentId): Promise<boolean> => {
588
+ if (!documentId) {
589
+ return false;
590
+ }
384
591
 
385
- const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
386
- if (isEchoPeerMetadata(peerMetadata)) {
387
- return this._echoNetworkAdapter.shouldAdvertise(peerId, { documentId });
388
- }
592
+ if (OPTIMIZED_SHARE_POLICY) {
593
+ if (
594
+ !this._createdDocuments.has(documentId) &&
595
+ !this._documentsToSync.has(documentId) &&
596
+ !this._documentsToRequest.has(documentId)
597
+ ) {
598
+ // Skip advertising documents that don't need to be synced.
599
+ return false;
600
+ }
601
+ }
389
602
 
390
- return false;
391
- }
603
+ const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
604
+ if (isEchoPeerMetadata(peerMetadata)) {
605
+ return this._echoNetworkAdapter.shouldAdvertise(peerId, { documentId });
606
+ }
607
+
608
+ return false;
609
+ },
610
+ };
392
611
 
393
- private async _beforeSave({ path, batch }: BeforeSaveParams): Promise<void> {
612
+ /**
613
+ * Authorization policy consulted by the Subduction sedimentree protocol.
614
+ */
615
+ private readonly _subductionPolicy: SubductionPolicy = {
616
+ authorizeConnect: async (_peerId) => {},
617
+ authorizeFetch: async (_peerId, _sedimentreeId) => {},
618
+ authorizePut: async (_requestor, _author, _sedimentreeId) => {},
619
+ filterAuthorizedFetch: async (_peerId, sedimentreeIds) => sedimentreeIds,
620
+ };
621
+
622
+ private async _beforeSave({ path, batch }: BeforeSaveProps): Promise<void> {
394
623
  const handle = this._repo.handles[path[0] as DocumentId];
395
- if (!handle || !handle.isReady()) {
624
+ if (!handle || getHandleState(this._repo, handle.documentId) !== 'ready') {
396
625
  return;
397
626
  }
398
627
  const doc = handle.doc();
@@ -402,23 +631,13 @@ export class AutomergeHost extends Resource {
402
631
 
403
632
  const heads = getHeads(doc);
404
633
  this._headsStore.setHeads(handle.documentId, heads, batch);
405
-
406
- const spaceKey = DatabaseDirectory.getSpaceKey(doc) ?? undefined;
407
- const objectIds = Object.keys(doc.objects ?? {});
408
- const encodedIds = objectIds.map((objectId) =>
409
- objectPointerCodec.encode({ documentId: handle.documentId, objectId, spaceKey }),
410
- );
411
- const idToLastHash = new Map(encodedIds.map((id) => [id, heads]));
412
- this._indexMetadataStore.markDirty(idToLastHash, batch);
413
634
  }
414
635
 
415
636
  private _shouldSyncCollection(collectionId: string, peerId: PeerId): boolean {
416
- const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
417
- if (isEchoPeerMetadata(peerMetadata)) {
418
- return this._echoNetworkAdapter.shouldSyncCollection(peerId, { collectionId });
419
- }
420
-
421
- return false;
637
+ // Under Subduction the Repo's `peerMetadataByPeerId` is not populated for peers that only
638
+ // speak Subduction (no classical peer message). Query the adapter directly — it maps
639
+ // peerId -> connection and the per-connection `shouldSyncCollection` gates the answer.
640
+ return this._echoNetworkAdapter.shouldSyncCollection(peerId, { collectionId });
422
641
  }
423
642
 
424
643
  /**
@@ -429,10 +648,9 @@ export class AutomergeHost extends Resource {
429
648
  return undefined;
430
649
  }
431
650
 
432
- this._indexMetadataStore.notifyMarkedDirty();
433
651
  const documentId = path[0] as DocumentId;
434
652
  const handle = this._repo.handles[documentId];
435
- if (!handle || !handle.isReady()) {
653
+ if (!handle) {
436
654
  return;
437
655
  }
438
656
  const document = handle.doc();
@@ -447,12 +665,7 @@ export class AutomergeHost extends Resource {
447
665
  this.documentsSaved.emit();
448
666
  }
449
667
 
450
- @trace.info({ depth: null })
451
- private _automergePeers(): PeerId[] {
452
- return this._repo.peers;
453
- }
454
-
455
- private async _isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckParams): Promise<boolean> {
668
+ private async _isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckProps): Promise<boolean> {
456
669
  for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
457
670
  const remoteCollections = this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
458
671
  const remotePeerDocs = remoteCollections.get(params.peerId as PeerId)?.documents;
@@ -464,21 +677,28 @@ export class AutomergeHost extends Resource {
464
677
  }
465
678
 
466
679
  private async _getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null> {
680
+ // This runs inside share-policy resolution (see `MeshEchoReplicator.shouldAdvertise` ->
681
+ // `_shareConfig.announce` -> `DocSynchronizer.#resolveSharePolicy`). It must NOT block
682
+ // on the document becoming ready: under classical sync the network source's availability
683
+ // is itself gated on share policy returning, so awaiting `progress.whenReady()` here
684
+ // deadlocks the load (the document never transitions out of `'loading'`).
685
+ //
686
+ // Read the spaceKey iff the document is already loaded; otherwise let the share policy
687
+ // fall through to the `_getSpaceKeyByRootDocumentId` lookup or the
688
+ // `isDocumentInRemoteCollection` check on the caller.
467
689
  const handle = this._repo.handles[documentId as any];
468
- if (handle.state === 'loading') {
469
- await handle.whenReady();
470
- }
471
- if (handle && handle.isReady() && handle.doc()) {
472
- const spaceKeyHex = DatabaseDirectory.getSpaceKey(handle.doc());
473
- if (spaceKeyHex) {
474
- return PublicKey.from(spaceKeyHex);
690
+ if (handle && getHandleState(this._repo, documentId as DocumentId) === 'ready') {
691
+ const doc = handle.doc();
692
+ if (doc) {
693
+ const spaceKeyHex = DatabaseDirectory.getSpaceKey(doc);
694
+ if (spaceKeyHex) {
695
+ return PublicKey.from(spaceKeyHex);
696
+ }
475
697
  }
476
698
  }
477
- /**
478
- * Edge case on the initial space setup.
479
- * A peer is maybe trying to share space root document with us after a successful invitation.
480
- * We don't have a document to check access block locally, so we need to rely on external sources (space metada).
481
- */
699
+
700
+ // Edge case on initial space setup: a peer may be sharing the space root document
701
+ // with us after a successful invitation, before our local handle has any data.
482
702
  const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
483
703
  if (rootDocSpaceKey) {
484
704
  return rootDocSpaceKey;
@@ -490,30 +710,43 @@ export class AutomergeHost extends Resource {
490
710
  /**
491
711
  * Flush documents to disk.
492
712
  */
493
- @trace.span({ showInBrowserTimeline: true })
494
- async flush({ documentIds }: FlushRequest = {}): Promise<void> {
495
- // Note: Sync protocol for client and services ensures that all handles should have all changes.
496
-
713
+ @trace.span({ showInBrowserTimeline: true, showInRemoteTracing: false })
714
+ async flush(ctx: Context, { documentIds }: FlushRequest = {}): Promise<void> {
715
+ // Concurrent `db.flush()` RPCs can arrive while the host is in or past
716
+ // teardown. `runBlocking` below would throw `ContextDisposedError` on a
717
+ // disposed ctx; treat a closed host as a no-op flush instead (nothing to
718
+ // propagate, the resource is gone). The host can also transition from
719
+ // open to closed *during* the `_repo.flush()` await — re-check before
720
+ // reaching `runBlocking`.
721
+ if (!this.isOpen) {
722
+ return;
723
+ }
497
724
  const loadedDocuments = (documentIds ?? Object.keys(this._repo.handles)).filter(
498
- (documentId): documentId is DocumentId => {
499
- const handle = this._repo.handles[documentId as DocumentId];
500
- return handle && handle.isReady();
501
- },
725
+ (documentId): documentId is DocumentId => getHandleState(this._repo, documentId as DocumentId) === 'ready',
502
726
  );
503
727
  await this._repo.flush(loadedDocuments);
728
+ if (!this.isOpen) {
729
+ return;
730
+ }
504
731
 
505
- // Ensure that document verions have propagated accross the system.
506
- // This is important for the case where we are doing flush and then waiting for sync to happen.
732
+ // Ensure that document versions have propagated across the system.
507
733
  await this._onHeadsChangedTask?.runBlocking();
508
734
  }
509
735
 
736
+ /**
737
+ * Returns current heads of each requested document.
738
+ *
739
+ * Loaded handles are read directly; unloaded documents fall back to the {@link HeadsStore},
740
+ * then to reconstruction from the automerge storage chunks (for docs persisted before
741
+ * the HeadsStore was populated).
742
+ */
510
743
  async getHeads(documentIds: DocumentId[]): Promise<(Heads | undefined)[]> {
511
744
  const result: (Heads | undefined)[] = [];
512
745
  const storeRequestIds: DocumentId[] = [];
513
746
  const storeResultIndices: number[] = [];
514
747
  for (const documentId of documentIds) {
515
748
  const handle = this._repo.handles[documentId];
516
- if (handle && handle.isReady() && handle.doc()) {
749
+ if (handle && getHandleState(this._repo, documentId) === 'ready' && handle.doc()) {
517
750
  result.push(getHeads(handle.doc()!));
518
751
  } else {
519
752
  storeRequestIds.push(documentId);
@@ -530,6 +763,13 @@ export class AutomergeHost extends Resource {
530
763
  return result;
531
764
  }
532
765
 
766
+ /**
767
+ * Iterate over all document heads stored on disk.
768
+ */
769
+ listDocumentHeads(): AsyncGenerator<{ documentId: DocumentId; heads: Heads }> {
770
+ return this._headsStore.iterateAll();
771
+ }
772
+
533
773
  //
534
774
  // Collection sync.
535
775
  //
@@ -567,7 +807,6 @@ export class AutomergeHost extends Resource {
567
807
  differentDocuments: diff.different.length,
568
808
  localDocumentCount: Object.entries(localState.documents).filter(([_, heads]) => heads.length > 0).length,
569
809
  remoteDocumentCount: Object.entries(state.documents).filter(([_, heads]) => heads.length > 0).length,
570
-
571
810
  totalDocumentCount: new Set([...Object.keys(localState.documents), ...Object.keys(state.documents)]).size,
572
811
  unsyncedDocumentCount: new Set([...diff.missingOnLocal, ...diff.missingOnRemote, ...diff.different]).size,
573
812
  });
@@ -587,8 +826,8 @@ export class AutomergeHost extends Resource {
587
826
  this._collectionSynchronizer.setLocalCollectionState(collectionId, { documents });
588
827
 
589
828
  // Proactively push our updated local state to peers that are interested in this collection.
590
- // This reduces reliance on the next periodic query and prevents replication stalls in fast paths
591
- // where the remote queries before our local state is ready.
829
+ // This reduces reliance on the next periodic query and prevents replication stalls in fast
830
+ // paths where the remote queries before our local state is ready.
592
831
  const interestedPeers = this._echoNetworkAdapter.getPeersInterestedInCollection(collectionId);
593
832
  if (interestedPeers.length > 0) {
594
833
  for (const peerId of interestedPeers) {
@@ -630,7 +869,7 @@ export class AutomergeHost extends Resource {
630
869
  this._syncTask?.schedule();
631
870
  }
632
871
 
633
- private async _handleCollectionSync(collectionId: string, peerId: PeerId) {
872
+ private async _handleCollectionSync(ctx: Context, collectionId: string, peerId: PeerId) {
634
873
  const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
635
874
  const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
636
875
 
@@ -648,7 +887,7 @@ export class AutomergeHost extends Resource {
648
887
  const bundleSyncEnabled = this._echoNetworkAdapter.bundleSyncEnabledForPeer(peerId);
649
888
  if (bundleSyncEnabled && missingOnRemote.length >= BUNDLE_SYNC_THRESHOLD) {
650
889
  log('pushing bundle', { amount: missingOnRemote.length });
651
- const { syncInteractively } = await this._pushInBundles(peerId, missingOnRemote);
890
+ const { syncInteractively } = await this._pushInBundles(ctx, peerId, missingOnRemote);
652
891
  toReplicateWithoutBatching.push(...syncInteractively);
653
892
  } else {
654
893
  log.verbose('failed to push bundle, replicating interactively', {
@@ -660,7 +899,7 @@ export class AutomergeHost extends Resource {
660
899
  }
661
900
  if (bundleSyncEnabled && missingOnLocal.length >= BUNDLE_SYNC_THRESHOLD) {
662
901
  log('pulling bundle', { amount: missingOnLocal.length });
663
- const { syncInteractively } = await this._pullInBundles(peerId, missingOnLocal);
902
+ const { syncInteractively } = await this._pullInBundles(ctx, peerId, missingOnLocal);
664
903
  toReplicateWithoutBatching.push(...syncInteractively);
665
904
  } else {
666
905
  log.verbose('failed to pull bundle, replicating interactively', {
@@ -682,15 +921,23 @@ export class AutomergeHost extends Resource {
682
921
  count: toReplicateWithoutBatching.length,
683
922
  });
684
923
 
685
- // Load the documents so they will start syncing.
924
+ // Trigger replication of the missing documents. `findWithProgress` is the
925
+ // fire-and-forget trigger — it creates a DocHandle and attaches sources. Under classical
926
+ // sync this triggers automerge-repo's doc-synchronizer; under Subduction it registers a
927
+ // query for the sedimentreeId. Either way, once bytes arrive `_afterSave` populates
928
+ // `HeadsStore` so collection sync sees the updated heads on the next diff.
686
929
  for (const documentId of toReplicateWithoutBatching) {
687
- this._repo.findWithProgress(documentId);
930
+ this._documentsToSync.add(documentId);
931
+ this._repo.findWithProgress(documentId as DocumentId);
932
+ }
933
+ if (!this._useSubduction) {
934
+ this._sharePolicyChangedTask!.schedule();
688
935
  }
689
936
  }
690
937
 
691
938
  // TODO(mykola): Add retries of batches https://gist.github.com/mykola-vrmchk/fde270259e9209fcbf1331e5abbf12cf
692
- // TODO(mykola): Use effect to retry batches.
693
939
  private async _pushInBundles(
940
+ ctx: Context,
694
941
  peerId: PeerId,
695
942
  documentIds: DocumentId[],
696
943
  ): Promise<{ syncInteractively: DocumentId[] }> {
@@ -705,7 +952,7 @@ export class AutomergeHost extends Resource {
705
952
  if (bundle.length === 0) {
706
953
  return;
707
954
  }
708
- await this._pushBundle(peerId, bundle).catch((err) => {
955
+ await this._pushBundle(ctx, peerId, bundle).catch((err) => {
709
956
  log.warn('failed to push bundle, replicating interactively', { peerId, bundle, err });
710
957
  syncInteractively.push(...bundle);
711
958
  });
@@ -716,17 +963,17 @@ export class AutomergeHost extends Resource {
716
963
  return { syncInteractively };
717
964
  }
718
965
 
719
- private async _pushBundle(peerId: PeerId, documentIds: DocumentId[]): Promise<void> {
966
+ private async _pushBundle(ctx: Context, peerId: PeerId, documentIds: DocumentId[]): Promise<void> {
720
967
  if (this._ctx.disposed) {
721
968
  return;
722
969
  }
723
970
 
724
971
  const docs = documentIds.map((documentId) => {
725
- const handle = this._repo.handles[documentId];
726
- if (!handle || !handle.isReady()) {
972
+ if (getHandleState(this._repo, documentId) !== 'ready') {
727
973
  log.warn('document not ready, skipping', { documentId });
728
974
  return;
729
975
  }
976
+ const handle = this._repo.handles[documentId];
730
977
  const doc = handle.doc();
731
978
  if (!doc) {
732
979
  log.warn('document not available, skipping', { documentId });
@@ -739,10 +986,11 @@ export class AutomergeHost extends Resource {
739
986
  };
740
987
  });
741
988
 
742
- await this._echoNetworkAdapter.pushBundle(peerId, docs.filter(isNonNullable));
989
+ await this._echoNetworkAdapter.pushBundle(ctx, peerId, docs.filter(isNonNullable));
743
990
  }
744
991
 
745
992
  private async _pullInBundles(
993
+ ctx: Context,
746
994
  peerId: PeerId,
747
995
  documentIds: DocumentId[],
748
996
  ): Promise<{ syncInteractively: DocumentId[] }> {
@@ -758,7 +1006,7 @@ export class AutomergeHost extends Resource {
758
1006
  if (bundle.length === 0) {
759
1007
  return;
760
1008
  }
761
- const result = await this._pullBundle(peerId, bundle).catch((err) => {
1009
+ const result = await this._pullBundle(ctx, peerId, bundle).catch((err) => {
762
1010
  log.warn('failed to pull bundle, replicating interactively', { peerId, bundle, err });
763
1011
  syncInteractively.push(...bundle);
764
1012
  });
@@ -778,6 +1026,7 @@ export class AutomergeHost extends Resource {
778
1026
  }
779
1027
 
780
1028
  private async _pullBundle(
1029
+ ctx: Context,
781
1030
  peerId: PeerId,
782
1031
  documentIds: DocumentId[],
783
1032
  ): Promise<{ docsToImport: Record<DocumentId, Uint8Array> } | undefined> {
@@ -786,7 +1035,7 @@ export class AutomergeHost extends Resource {
786
1035
  }
787
1036
  // NOTE: We are expecting that documents that are being pulled are not present locally, so we are pulling all changes.
788
1037
  const docHeads = Object.fromEntries(documentIds.map((documentId) => [documentId, []]));
789
- const bundle = await this._echoNetworkAdapter.pullBundle(peerId, docHeads);
1038
+ const bundle = await this._echoNetworkAdapter.pullBundle(ctx, peerId, docHeads);
790
1039
  return { docsToImport: bundle };
791
1040
  }
792
1041
 
@@ -801,12 +1050,24 @@ export class AutomergeHost extends Resource {
801
1050
  let newState: CollectionState | undefined;
802
1051
 
803
1052
  for (const [documentId, heads] of docHeads) {
804
- if (documentId in state.documents) {
805
- if (!newState) {
806
- newState = structuredClone(state);
807
- }
808
- newState.documents[documentId] = heads;
1053
+ const current = state.documents[documentId];
1054
+ // Collection membership is owned by `updateLocalCollectionState` (driven by
1055
+ // `SpaceStateManager.spaceDocumentListUpdated`). `_afterSave` fires for every
1056
+ // chunk written — including the space root, system docs, and transiently-fetched
1057
+ // docs that haven't been admitted to any collection — so we only refresh heads
1058
+ // for documents the membership path has already registered. Adding new keys
1059
+ // here would leak non-collection docs into the broadcast state and race with
1060
+ // the authoritative rebuild in `updateLocalCollectionState`.
1061
+ if (current === undefined) {
1062
+ continue;
809
1063
  }
1064
+ if (headsEquals(current, heads)) {
1065
+ continue;
1066
+ }
1067
+ if (!newState) {
1068
+ newState = structuredClone(state);
1069
+ }
1070
+ newState.documents[documentId] = heads;
810
1071
  }
811
1072
 
812
1073
  if (newState) {
@@ -824,15 +1085,29 @@ export class AutomergeHost extends Resource {
824
1085
  const waitForHeads = async (handle: DocHandle<DatabaseDirectory>, heads: Heads) => {
825
1086
  const unavailableHeads = new Set(heads);
826
1087
 
827
- await handle.whenReady();
828
- await Event.wrap<DocHandleChangePayload<DatabaseDirectory>>(handle, 'change').waitForCondition(() => {
829
- // Check if unavailable heads became available.
1088
+ // Check the current doc first, then subscribe to `change` to catch later
1089
+ // updates. (We can't use `handle.whenReady()` to gate the subscription —
1090
+ // see {@link getHandleState} for why `DocHandle.*` state is unusable in
1091
+ // this fork.)
1092
+ const checkPresentHeads = () => {
1093
+ const doc = handle.doc();
1094
+ if (!doc) {
1095
+ return;
1096
+ }
830
1097
  for (const changeHash of unavailableHeads.values()) {
831
- if (changeIsPresentInDoc(handle.doc()!, changeHash)) {
1098
+ if (changeIsPresentInDoc(doc, changeHash)) {
832
1099
  unavailableHeads.delete(changeHash);
833
1100
  }
834
1101
  }
1102
+ };
1103
+
1104
+ checkPresentHeads();
1105
+ if (unavailableHeads.size === 0) {
1106
+ return;
1107
+ }
835
1108
 
1109
+ await Event.wrap<DocHandleChangePayload<DatabaseDirectory>>(handle, 'change').waitForCondition(() => {
1110
+ checkPresentHeads();
836
1111
  return unavailableHeads.size === 0;
837
1112
  });
838
1113
  };
@@ -850,3 +1125,7 @@ const decodeCollectionState = (state: unknown): CollectionState => {
850
1125
  const encodeCollectionState = (state: CollectionState): unknown => {
851
1126
  return state;
852
1127
  };
1128
+
1129
+ const isDocumentUnavailableError = (error: Error): boolean => {
1130
+ return error.message.includes('unavailable');
1131
+ };