@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/LICENSE +102 -5
  2. package/README.md +1 -1
  3. package/dist/lib/{browser/chunk-WJJXJTNS.mjs → neutral/chunk-46DCRDA7.mjs} +44 -13
  4. package/dist/lib/neutral/chunk-46DCRDA7.mjs.map +7 -0
  5. package/dist/lib/{browser/chunk-FR6GJ4S4.mjs → neutral/chunk-LR7UFVQX.mjs} +145 -618
  6. package/dist/lib/neutral/chunk-LR7UFVQX.mjs.map +7 -0
  7. package/dist/lib/{browser → neutral}/filter/index.mjs +1 -2
  8. package/dist/lib/neutral/index.mjs +7332 -0
  9. package/dist/lib/neutral/index.mjs.map +7 -0
  10. package/dist/lib/neutral/meta.json +1 -0
  11. package/dist/lib/{browser → neutral}/testing/index.mjs +49 -292
  12. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  13. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts +2 -0
  14. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts.map +1 -0
  15. package/dist/types/src/automerge/automerge-host.d.ts +105 -27
  16. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  17. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts +2 -0
  18. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts.map +1 -0
  19. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  20. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  21. package/dist/types/src/automerge/automerge-subduction.test.d.ts +2 -0
  22. package/dist/types/src/automerge/automerge-subduction.test.d.ts.map +1 -0
  23. package/dist/types/src/automerge/collection-synchronizer.d.ts +22 -2
  24. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  25. package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
  26. package/dist/types/src/automerge/echo-network-adapter.d.ts +17 -11
  27. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  28. package/dist/types/src/automerge/echo-replicator.d.ts +28 -16
  29. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  30. package/dist/types/src/automerge/errors.d.ts +35 -0
  31. package/dist/types/src/automerge/errors.d.ts.map +1 -0
  32. package/dist/types/src/automerge/handle-state.d.ts +52 -0
  33. package/dist/types/src/automerge/handle-state.d.ts.map +1 -0
  34. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  35. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  36. package/dist/types/src/automerge/index.d.ts +2 -0
  37. package/dist/types/src/automerge/index.d.ts.map +1 -1
  38. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +10 -4
  39. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  40. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +8 -8
  41. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  42. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +6 -5
  43. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  44. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  45. package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
  46. package/dist/types/src/common/feeds.d.ts.map +1 -1
  47. package/dist/types/src/common/space-id.d.ts.map +1 -1
  48. package/dist/types/src/db-host/automerge-data-source.d.ts +35 -0
  49. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  50. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  51. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  52. package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
  53. package/dist/types/src/db-host/data-service.d.ts +12 -6
  54. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  55. package/dist/types/src/db-host/database-root.d.ts +9 -3
  56. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  57. package/dist/types/src/db-host/documents-synchronizer.d.ts +30 -8
  58. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  59. package/dist/types/src/db-host/echo-host.d.ts +74 -28
  60. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  61. package/dist/types/src/db-host/index.d.ts +2 -0
  62. package/dist/types/src/db-host/index.d.ts.map +1 -1
  63. package/dist/types/src/db-host/invalidation-hint.d.ts +19 -0
  64. package/dist/types/src/db-host/invalidation-hint.d.ts.map +1 -0
  65. package/dist/types/src/db-host/local-queue-service.d.ts +19 -0
  66. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  67. package/dist/types/src/db-host/query-invalidation.test.d.ts +2 -0
  68. package/dist/types/src/db-host/query-invalidation.test.d.ts.map +1 -0
  69. package/dist/types/src/db-host/query-service.d.ts +18 -10
  70. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  71. package/dist/types/src/db-host/queue-data-source.d.ts +38 -0
  72. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  73. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  74. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  75. package/dist/types/src/db-host/space-state-manager.d.ts +3 -2
  76. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  77. package/dist/types/src/db-host/stub.d.ts +11 -0
  78. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  79. package/dist/types/src/edge/echo-edge-replicator.d.ts +7 -6
  80. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  81. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts +39 -0
  82. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts.map +1 -0
  83. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts +2 -0
  84. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts.map +1 -0
  85. package/dist/types/src/edge/index.d.ts +1 -0
  86. package/dist/types/src/edge/index.d.ts.map +1 -1
  87. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
  88. package/dist/types/src/filter/filter-match.d.ts +1 -1
  89. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  90. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  91. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  92. package/dist/types/src/pipeline/pipeline.d.ts +2 -2
  93. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  94. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
  95. package/dist/types/src/query/errors.d.ts +16 -20
  96. package/dist/types/src/query/errors.d.ts.map +1 -1
  97. package/dist/types/src/query/plan.d.ts +59 -8
  98. package/dist/types/src/query/plan.d.ts.map +1 -1
  99. package/dist/types/src/query/query-executor.d.ts +55 -12
  100. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  101. package/dist/types/src/query/query-planner.d.ts +10 -1
  102. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  103. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  104. package/dist/types/src/space/auth.d.ts +3 -3
  105. package/dist/types/src/space/auth.d.ts.map +1 -1
  106. package/dist/types/src/space/control-pipeline.d.ts +4 -3
  107. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  108. package/dist/types/src/space/space-manager.d.ts +7 -7
  109. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  110. package/dist/types/src/space/space-protocol.d.ts +9 -8
  111. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  112. package/dist/types/src/space/space.d.ts +5 -5
  113. package/dist/types/src/space/space.d.ts.map +1 -1
  114. package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
  115. package/dist/types/src/testing/index.d.ts +0 -2
  116. package/dist/types/src/testing/index.d.ts.map +1 -1
  117. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  118. package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -1
  119. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  120. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  121. package/dist/types/src/testing/test-replicator.d.ts +13 -13
  122. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  123. package/dist/types/src/util.d.ts.map +1 -1
  124. package/dist/types/tsconfig.tsbuildinfo +1 -1
  125. package/package.json +56 -54
  126. package/src/automerge/automerge-host-subduction.test.ts +263 -0
  127. package/src/automerge/automerge-host.test.ts +145 -32
  128. package/src/automerge/automerge-host.ts +442 -163
  129. package/src/automerge/automerge-repo-subduction.test.ts +1054 -0
  130. package/src/automerge/automerge-repo.test.ts +284 -77
  131. package/src/automerge/automerge-subduction.test.ts +330 -0
  132. package/src/automerge/collection-synchronizer.test.ts +96 -16
  133. package/src/automerge/collection-synchronizer.ts +125 -26
  134. package/src/automerge/echo-network-adapter.test.ts +2 -1
  135. package/src/automerge/echo-network-adapter.ts +44 -32
  136. package/src/automerge/echo-replicator.ts +30 -16
  137. package/src/automerge/errors.ts +10 -0
  138. package/src/automerge/handle-state.ts +62 -0
  139. package/src/automerge/heads-store.ts +42 -3
  140. package/src/automerge/index.ts +2 -0
  141. package/src/automerge/leveldb-storage-adapter.ts +70 -4
  142. package/src/automerge/mesh-echo-replicator-connection.ts +8 -8
  143. package/src/automerge/mesh-echo-replicator.ts +14 -11
  144. package/src/db-host/automerge-data-source.test.ts +299 -0
  145. package/src/db-host/automerge-data-source.ts +187 -0
  146. package/src/db-host/data-service.ts +27 -10
  147. package/src/db-host/database-root.ts +19 -7
  148. package/src/db-host/documents-synchronizer.test.ts +145 -14
  149. package/src/db-host/documents-synchronizer.ts +158 -59
  150. package/src/db-host/echo-host.ts +376 -90
  151. package/src/db-host/index.ts +2 -0
  152. package/src/db-host/invalidation-hint.ts +50 -0
  153. package/src/db-host/local-queue-service.ts +127 -0
  154. package/src/db-host/query-invalidation.test.ts +270 -0
  155. package/src/db-host/query-service.ts +108 -111
  156. package/src/db-host/queue-data-source.ts +160 -0
  157. package/src/db-host/queue-service.test.ts +145 -0
  158. package/src/db-host/space-state-manager.ts +11 -9
  159. package/src/db-host/stub.ts +33 -0
  160. package/src/edge/echo-edge-replicator.test.ts +9 -9
  161. package/src/edge/echo-edge-replicator.ts +50 -42
  162. package/src/edge/echo-edge-subduction-replicator.test.ts +158 -0
  163. package/src/edge/echo-edge-subduction-replicator.ts +377 -0
  164. package/src/edge/index.ts +1 -0
  165. package/src/filter/filter-match.test.ts +41 -25
  166. package/src/filter/filter-match.ts +75 -20
  167. package/src/metadata/metadata-store.ts +8 -5
  168. package/src/pipeline/pipeline-stress.test.ts +1 -3
  169. package/src/pipeline/pipeline.test.ts +0 -1
  170. package/src/pipeline/pipeline.ts +2 -3
  171. package/src/query/errors.ts +2 -2
  172. package/src/query/plan.ts +73 -9
  173. package/src/query/query-executor.ts +1064 -172
  174. package/src/query/query-planner.test.ts +854 -86
  175. package/src/query/query-planner.ts +444 -38
  176. package/src/space/admission-discovery-extension.ts +1 -1
  177. package/src/space/auth.ts +6 -6
  178. package/src/space/control-pipeline.test.ts +2 -2
  179. package/src/space/control-pipeline.ts +7 -7
  180. package/src/space/space-manager.ts +12 -17
  181. package/src/space/space-protocol.browser.test.ts +13 -12
  182. package/src/space/space-protocol.test.ts +13 -13
  183. package/src/space/space-protocol.ts +16 -15
  184. package/src/space/space.test.ts +2 -2
  185. package/src/space/space.ts +9 -10
  186. package/src/testing/index.ts +0 -2
  187. package/src/testing/test-agent-builder.ts +0 -1
  188. package/src/testing/test-network-adapter.ts +24 -3
  189. package/src/testing/test-replicator.ts +15 -20
  190. package/src/util.ts +7 -3
  191. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  192. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  193. package/dist/lib/browser/chunk-FR6GJ4S4.mjs.map +0 -7
  194. package/dist/lib/browser/chunk-WJJXJTNS.mjs.map +0 -7
  195. package/dist/lib/browser/index.mjs +0 -5492
  196. package/dist/lib/browser/index.mjs.map +0 -7
  197. package/dist/lib/browser/meta.json +0 -1
  198. package/dist/lib/browser/testing/index.mjs.map +0 -7
  199. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  200. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  201. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs +0 -2144
  202. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs.map +0 -7
  203. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs +0 -240
  204. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs.map +0 -7
  205. package/dist/lib/node-esm/filter/index.mjs +0 -13
  206. package/dist/lib/node-esm/filter/index.mjs.map +0 -7
  207. package/dist/lib/node-esm/index.mjs +0 -5492
  208. package/dist/lib/node-esm/index.mjs.map +0 -7
  209. package/dist/lib/node-esm/meta.json +0 -1
  210. package/dist/lib/node-esm/testing/index.mjs +0 -764
  211. package/dist/lib/node-esm/testing/index.mjs.map +0 -7
  212. package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
  213. package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
  214. package/dist/types/src/testing/test-data.d.ts +0 -18
  215. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  216. package/dist/types/src/testing/test-schema.d.ts +0 -37
  217. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  218. package/src/db-host/documents-iterator.ts +0 -74
  219. package/src/testing/test-data.ts +0 -127
  220. package/src/testing/test-schema.ts +0 -53
  221. /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
@@ -4,18 +4,45 @@
4
4
 
5
5
  import type { Heads } from '@automerge/automerge';
6
6
  import type { DocumentId } from '@automerge/automerge-repo';
7
+ import { type MixedEncoding } from 'level-transcoder';
7
8
 
8
- import { headsEncoding } from '@dxos/indexing';
9
+ import type { ProtoCodec } from '@dxos/codec-protobuf';
9
10
  import type { BatchLevel, SublevelDB } from '@dxos/kv-store';
11
+ import { log } from '@dxos/log';
12
+ import { schema } from '@dxos/protocols/proto';
13
+ import type { Heads as HeadsProto } from '@dxos/protocols/proto/dxos/echo/query';
10
14
 
11
- export type HeadsStoreParams = {
15
+ // NOTE: Lazy so that code that doesn't use indexing doesn't need to load the codec (breaks in workerd).
16
+ let headsCodec: ProtoCodec<HeadsProto>;
17
+ const getHeadsCodec = () => (headsCodec ??= schema.getCodecForType('dxos.echo.query.Heads'));
18
+
19
+ const headsEncoding: MixedEncoding<Heads, Uint8Array, Heads> = {
20
+ encode: (value: Heads): Uint8Array => getHeadsCodec().encode({ hashes: value }),
21
+ decode: (encodedValue: Uint8Array): Heads => {
22
+ try {
23
+ return getHeadsCodec().decode(encodedValue).hashes!;
24
+ } catch {
25
+ // Legacy encoding migration path.
26
+ log.warn('Detected legacy encoding of heads in storage.');
27
+ const concatenatedHeads = Buffer.from(encodedValue).toString('utf8').replace(/"/g, '');
28
+ const heads = [];
29
+ for (let i = 0; i < concatenatedHeads.length; i += 64) {
30
+ heads.push(concatenatedHeads.slice(i, i + 64));
31
+ }
32
+ return heads;
33
+ }
34
+ },
35
+ format: 'buffer',
36
+ };
37
+
38
+ export type HeadsStoreProps = {
12
39
  db: SublevelDB;
13
40
  };
14
41
 
15
42
  export class HeadsStore {
16
43
  private readonly _db: SublevelDB;
17
44
 
18
- constructor({ db }: HeadsStoreParams) {
45
+ constructor({ db }: HeadsStoreProps) {
19
46
  this._db = db;
20
47
  }
21
48
 
@@ -34,4 +61,16 @@ export class HeadsStore {
34
61
  valueEncoding: headsEncoding,
35
62
  });
36
63
  }
64
+
65
+ /**
66
+ * Iterate over all document IDs and their heads.
67
+ */
68
+ async *iterateAll(): AsyncGenerator<{ documentId: DocumentId; heads: Heads }> {
69
+ for await (const [documentId, heads] of this._db.iterator<DocumentId, Heads>({
70
+ keyEncoding: 'utf8',
71
+ valueEncoding: headsEncoding,
72
+ })) {
73
+ yield { documentId, heads };
74
+ }
75
+ }
37
76
  }
@@ -3,9 +3,11 @@
3
3
  //
4
4
 
5
5
  export * from './automerge-host';
6
+ export * from './handle-state';
6
7
  export * from './leveldb-storage-adapter';
7
8
  export * from './mesh-echo-replicator';
8
9
  export type * from './echo-replicator';
9
10
  export { diffCollectionState } from './collection-synchronizer';
10
11
  export * from './space-collection';
11
12
  export * from './echo-data-monitor';
13
+ export * from './errors';
@@ -16,24 +16,53 @@ export interface StorageAdapterDataMonitor {
16
16
  recordStoreDuration(durationMs: number): void;
17
17
  }
18
18
 
19
- export type LevelDBStorageAdapterParams = {
19
+ export type LevelDBStorageAdapterProps = {
20
20
  db: SublevelDB;
21
21
  callbacks?: StorageCallbacks;
22
22
  monitor?: StorageAdapterDataMonitor;
23
23
  };
24
24
 
25
- export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
25
+ export type BeforeSaveProps = { path: StorageKey; batch: BatchLevel };
26
26
 
27
27
  export interface StorageCallbacks {
28
- beforeSave(params: BeforeSaveParams): MaybePromise<void>;
28
+ beforeSave(params: BeforeSaveProps): MaybePromise<void>;
29
29
  afterSave(path: StorageKey): MaybePromise<void>;
30
30
  }
31
31
 
32
32
  export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
33
- constructor(private readonly _params: LevelDBStorageAdapterParams) {
33
+ /**
34
+ * In-flight `loadRange` / `removeRange` iterations. Awaited in `_close` so
35
+ * the adapter doesn't return from close while a `for await` on the sublevel
36
+ * is still pending — otherwise the kv owner upstream would close the
37
+ * sublevel and the iterator's next `.next()` would reject with
38
+ * `LEVEL_ITERATOR_NOT_OPEN`.
39
+ *
40
+ * Promises stored here resolve regardless of outcome (the wrapper
41
+ * `.catch`es) so `Promise.all` won't reject.
42
+ */
43
+ readonly #inFlightIterations = new Set<Promise<unknown>>();
44
+
45
+ constructor(private readonly _params: LevelDBStorageAdapterProps) {
34
46
  super();
35
47
  }
36
48
 
49
+ protected override async _close(): Promise<void> {
50
+ // New `loadRange`/`removeRange` calls already short-circuit on `!isOpen`
51
+ // (the Resource base flips state before invoking `_close`). Wait for any
52
+ // iterations that started while we were still open to finish before
53
+ // returning, so the upstream `kv` owner can safely close the sublevel.
54
+ if (this.#inFlightIterations.size > 0) {
55
+ await Promise.all(this.#inFlightIterations);
56
+ }
57
+ }
58
+
59
+ #trackIteration<T>(work: Promise<T>): Promise<T> {
60
+ const settled = work.catch(() => undefined);
61
+ this.#inFlightIterations.add(settled);
62
+ void settled.finally(() => this.#inFlightIterations.delete(settled));
63
+ return work;
64
+ }
65
+
37
66
  async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
38
67
  try {
39
68
  if (!this.isOpen) {
@@ -71,6 +100,35 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
71
100
  this._params.monitor?.recordStoreDuration(Date.now() - startMs);
72
101
  }
73
102
 
103
+ /**
104
+ * Atomically persist multiple key/value entries in a single LevelDB batch write.
105
+ */
106
+ async saveBatch(entries: Array<[StorageKey, Uint8Array]>): Promise<void> {
107
+ if (!this.isOpen || entries.length === 0) {
108
+ return undefined;
109
+ }
110
+ const startMs = Date.now();
111
+ const batch = this._params.db.batch();
112
+
113
+ for (const [keyArray] of entries) {
114
+ await this._params.callbacks?.beforeSave?.({ path: keyArray, batch });
115
+ }
116
+ for (const [keyArray, binary] of entries) {
117
+ batch.put<StorageKey, Uint8Array>(keyArray, Buffer.from(binary), {
118
+ ...encodingOptions,
119
+ });
120
+ }
121
+ await batch.write();
122
+
123
+ let bytesStored = 0;
124
+ for (const [keyArray, binary] of entries) {
125
+ bytesStored += binary.byteLength;
126
+ await this._params.callbacks?.afterSave?.(keyArray);
127
+ }
128
+ this._params.monitor?.recordBytesStored(bytesStored);
129
+ this._params.monitor?.recordStoreDuration(Date.now() - startMs);
130
+ }
131
+
74
132
  async remove(keyArray: StorageKey): Promise<void> {
75
133
  if (!this.isOpen) {
76
134
  return undefined;
@@ -82,6 +140,10 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
82
140
  if (!this.isOpen) {
83
141
  return [];
84
142
  }
143
+ return this.#trackIteration(this.#doLoadRange(keyPrefix));
144
+ }
145
+
146
+ async #doLoadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
85
147
  const startMs = Date.now();
86
148
  const result: Chunk[] = [];
87
149
  for await (const [key, value] of this._params.db.iterator<StorageKey, Uint8Array>({
@@ -103,6 +165,10 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
103
165
  if (!this.isOpen) {
104
166
  return undefined;
105
167
  }
168
+ await this.#trackIteration(this.#doRemoveRange(keyPrefix));
169
+ }
170
+
171
+ async #doRemoveRange(keyPrefix: StorageKey): Promise<void> {
106
172
  const batch = this._params.db.batch();
107
173
 
108
174
  for await (const [key] of this._params.db.iterator<StorageKey, Uint8Array>({
@@ -12,20 +12,20 @@ import { log } from '@dxos/log';
12
12
  import type { AutomergeProtocolMessage } from '@dxos/protocols';
13
13
  import { AutomergeReplicator, type AutomergeReplicatorFactory } from '@dxos/teleport-extension-automerge-replicator';
14
14
 
15
- import type { ReplicatorConnection, ShouldAdvertiseParams, ShouldSyncCollectionParams } from './echo-replicator';
15
+ import type { AutomergeReplicatorConnection, ShouldAdvertiseProps, ShouldSyncCollectionProps } from './echo-replicator';
16
16
 
17
17
  const DEFAULT_FACTORY: AutomergeReplicatorFactory = (params) => new AutomergeReplicator(...params);
18
18
 
19
- export type MeshReplicatorConnectionParams = {
19
+ export type MeshReplicatorConnectionProps = {
20
20
  ownPeerId: string;
21
21
  onRemoteConnected: () => void;
22
22
  onRemoteDisconnected: () => void;
23
- shouldAdvertise: (params: ShouldAdvertiseParams) => Promise<boolean>;
24
- shouldSyncCollection: (params: ShouldSyncCollectionParams) => boolean;
23
+ shouldAdvertise: (params: ShouldAdvertiseProps) => Promise<boolean>;
24
+ shouldSyncCollection: (params: ShouldSyncCollectionProps) => boolean;
25
25
  replicatorFactory?: AutomergeReplicatorFactory;
26
26
  };
27
27
 
28
- export class MeshReplicatorConnection extends Resource implements ReplicatorConnection {
28
+ export class MeshReplicatorConnection extends Resource implements AutomergeReplicatorConnection {
29
29
  public readable: ReadableStream<AutomergeProtocolMessage>;
30
30
  public writable: WritableStream<AutomergeProtocolMessage>;
31
31
  public remoteDeviceKey: PublicKey | null = null;
@@ -35,7 +35,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
35
35
  private _remotePeerId: string | null = null;
36
36
  private _isEnabled = false;
37
37
 
38
- constructor(private readonly _params: MeshReplicatorConnectionParams) {
38
+ constructor(private readonly _params: MeshReplicatorConnectionProps) {
39
39
  super();
40
40
 
41
41
  let readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
@@ -115,11 +115,11 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
115
115
  return false;
116
116
  }
117
117
 
118
- async shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean> {
118
+ async shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean> {
119
119
  return this._params.shouldAdvertise(params);
120
120
  }
121
121
 
122
- shouldSyncCollection(params: ShouldSyncCollectionParams): boolean {
122
+ shouldSyncCollection(params: ShouldSyncCollectionProps): boolean {
123
123
  return this._params.shouldSyncCollection(params);
124
124
  }
125
125
 
@@ -2,19 +2,20 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ import { type Context } from '@dxos/context';
5
6
  import type { CollectionId } from '@dxos/echo-protocol';
6
7
  import { invariant } from '@dxos/invariant';
7
8
  import { PublicKey, type SpaceId } from '@dxos/keys';
8
9
  import { log } from '@dxos/log';
9
- import {
10
- type AutomergeReplicator,
11
- type AutomergeReplicatorFactory,
12
- } from '@dxos/teleport-extension-automerge-replicator';
10
+ import type * as TeleportAutomergeReplicator from '@dxos/teleport-extension-automerge-replicator';
13
11
  import { ComplexSet, defaultMap } from '@dxos/util';
14
12
 
15
13
  import { createIdFromSpaceKey } from '../common/space-id';
16
-
17
- import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseParams } from './echo-replicator';
14
+ import {
15
+ type AutomergeReplicator,
16
+ type AutomergeReplicatorContext,
17
+ type ShouldAdvertiseProps,
18
+ } from './echo-replicator';
18
19
  import { MeshReplicatorConnection } from './mesh-echo-replicator-connection';
19
20
  import { getSpaceIdFromCollectionId } from './space-collection';
20
21
 
@@ -23,7 +24,7 @@ import { getSpaceIdFromCollectionId } from './space-collection';
23
24
  /**
24
25
  * Used to replicate with other peers over the network.
25
26
  */
26
- export class MeshEchoReplicator implements EchoReplicator {
27
+ export class MeshEchoReplicator implements AutomergeReplicator {
27
28
  /**
28
29
  * We might have multiple connections open with a peer (one per space), but there'll be only one enabled
29
30
  * connection at any given moment, because there's a single repo for all the spaces.
@@ -41,9 +42,9 @@ export class MeshEchoReplicator implements EchoReplicator {
41
42
  */
42
43
  private readonly _authorizedDevices = new Map<SpaceId, ComplexSet<PublicKey>>();
43
44
 
44
- private _context: EchoReplicatorContext | null = null;
45
+ private _context: AutomergeReplicatorContext | null = null;
45
46
 
46
- async connect(context: EchoReplicatorContext): Promise<void> {
47
+ async connect(_ctx: Context, context: AutomergeReplicatorContext): Promise<void> {
47
48
  this._context = context;
48
49
  }
49
50
 
@@ -64,7 +65,9 @@ export class MeshEchoReplicator implements EchoReplicator {
64
65
  this._context = null;
65
66
  }
66
67
 
67
- createExtension(extensionFactory?: AutomergeReplicatorFactory): AutomergeReplicator {
68
+ createExtension(
69
+ extensionFactory?: TeleportAutomergeReplicator.AutomergeReplicatorFactory,
70
+ ): TeleportAutomergeReplicator.AutomergeReplicator {
68
71
  invariant(this._context);
69
72
 
70
73
  const connection: MeshReplicatorConnection = new MeshReplicatorConnection({
@@ -111,7 +114,7 @@ export class MeshEchoReplicator implements EchoReplicator {
111
114
  }
112
115
  }
113
116
  },
114
- shouldAdvertise: async (params: ShouldAdvertiseParams) => {
117
+ shouldAdvertise: async (params: ShouldAdvertiseProps) => {
115
118
  log('shouldAdvertise', { peerId: connection.peerId, documentId: params.documentId });
116
119
  invariant(this._context);
117
120
  try {
@@ -0,0 +1,299 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import { getHeads } from '@automerge/automerge';
6
+ import { describe, expect, onTestFinished, test } from 'vitest';
7
+
8
+ import { Context } from '@dxos/context';
9
+ import { type DatabaseDirectory, ObjectStructure, SpaceDocVersion } from '@dxos/echo-protocol';
10
+ import { runAndForwardErrors } from '@dxos/effect';
11
+ import { type IndexCursor } from '@dxos/index-core';
12
+ import { DXN, SpaceId } from '@dxos/keys';
13
+ import { type LevelDB } from '@dxos/kv-store';
14
+ import { createTestLevel } from '@dxos/kv-store/testing';
15
+ import { openAndClose } from '@dxos/test-utils';
16
+
17
+ import { AutomergeHost } from '../automerge';
18
+ import { AutomergeDataSource, headsCodec } from './automerge-data-source';
19
+
20
+ const TEST_TYPE = DXN.parse('dxn:type:com.example.type.test:0.1.0').toString();
21
+ const OTHER_TYPE = DXN.parse('dxn:type:com.example.type.other:0.1.0').toString();
22
+ const PERSON_TYPE = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
23
+
24
+ /**
25
+ * Set up a real AutomergeHost with LevelDB storage.
26
+ */
27
+ const setupAutomergeHost = async (level: LevelDB): Promise<AutomergeHost> => {
28
+ const host = new AutomergeHost({
29
+ db: level,
30
+ });
31
+ await host.open();
32
+ onTestFinished(async () => {
33
+ await host.close();
34
+ });
35
+ return host;
36
+ };
37
+
38
+ /**
39
+ * Create a DatabaseDirectory document with the given objects.
40
+ */
41
+ const createDatabaseDirectory = async (
42
+ host: AutomergeHost,
43
+ spaceKey: string,
44
+ objects: Record<string, ObjectStructure>,
45
+ ): Promise<Awaited<ReturnType<typeof host.createDoc<DatabaseDirectory>>>> => {
46
+ const handle = await host.createDoc<DatabaseDirectory>({
47
+ version: SpaceDocVersion.CURRENT,
48
+ access: { spaceKey },
49
+ objects: {},
50
+ links: {},
51
+ });
52
+
53
+ handle.change((doc) => {
54
+ doc.objects = objects;
55
+ });
56
+
57
+ return handle;
58
+ };
59
+
60
+ describe('AutomergeDataSource', () => {
61
+ test('headsCodec encodes and decodes heads correctly', () => {
62
+ const heads = ['abc123', 'def456'];
63
+ const encoded = headsCodec.encode(heads);
64
+
65
+ expect(encoded).toBe('abc123|def456');
66
+ expect(headsCodec.decode(encoded)).toEqual(['abc123', 'def456']);
67
+ });
68
+
69
+ test('headsCodec sorts heads for consistent encoding', () => {
70
+ const heads1 = ['def456', 'abc123'];
71
+ const heads2 = ['abc123', 'def456'];
72
+
73
+ expect(headsCodec.encode(heads1)).toBe(headsCodec.encode(heads2));
74
+ });
75
+
76
+ test('returns empty when no documents exist', async () => {
77
+ const level = createTestLevel();
78
+ await openAndClose(level);
79
+ const host = await setupAutomergeHost(level);
80
+
81
+ const dataSource = new AutomergeDataSource(host);
82
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
83
+
84
+ expect(result.objects).toHaveLength(0);
85
+ expect(result.cursors).toHaveLength(0);
86
+ });
87
+
88
+ test('returns new documents that have no cursor', async () => {
89
+ const level = createTestLevel();
90
+ await openAndClose(level);
91
+ const host = await setupAutomergeHost(level);
92
+ const spaceKey = SpaceId.random();
93
+
94
+ const handle = await createDatabaseDirectory(host, spaceKey, {
95
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Test Document' } }),
96
+ });
97
+ await host.flush(Context.default());
98
+
99
+ const dataSource = new AutomergeDataSource(host);
100
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
101
+
102
+ expect(result.objects).toHaveLength(1);
103
+ expect(result.objects[0].documentId).toBe(handle.documentId);
104
+ expect(result.objects[0].data.title).toBe('Test Document');
105
+
106
+ expect(result.cursors).toHaveLength(1);
107
+ expect(result.cursors[0].resourceId).toBe(handle.documentId);
108
+ expect(result.cursors[0].cursor).toBe(headsCodec.encode(getHeads(handle.doc()!)));
109
+ });
110
+
111
+ test('returns documents with changed heads', async () => {
112
+ const level = createTestLevel();
113
+ await openAndClose(level);
114
+ const host = await setupAutomergeHost(level);
115
+ const spaceKey = SpaceId.random();
116
+
117
+ const handle1 = await createDatabaseDirectory(host, spaceKey, {
118
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Doc 1' } }),
119
+ });
120
+ const handle2 = await createDatabaseDirectory(host, spaceKey, {
121
+ 'obj-2': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Doc 2' } }),
122
+ });
123
+ await host.flush(Context.default());
124
+
125
+ // Capture heads before mutation.
126
+ const doc1HeadsBefore = headsCodec.encode(getHeads(handle1.doc()!));
127
+ const doc2Heads = headsCodec.encode(getHeads(handle2.doc()!));
128
+
129
+ // Modify doc1 to have new heads.
130
+ handle1.change((doc) => {
131
+ doc.objects!['obj-1'].data.title = 'Doc 1 Updated';
132
+ });
133
+ await host.flush(Context.default());
134
+
135
+ const dataSource = new AutomergeDataSource(host);
136
+ const cursors: IndexCursor[] = [
137
+ {
138
+ indexName: 'fts',
139
+ spaceId: null,
140
+ sourceName: 'automerge',
141
+ resourceId: handle1.documentId,
142
+ cursor: doc1HeadsBefore,
143
+ },
144
+ { indexName: 'fts', spaceId: null, sourceName: 'automerge', resourceId: handle2.documentId, cursor: doc2Heads },
145
+ ];
146
+
147
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), cursors));
148
+
149
+ // Only doc1 changed.
150
+ expect(result.objects).toHaveLength(1);
151
+ expect(result.objects[0].documentId).toBe(handle1.documentId);
152
+ });
153
+
154
+ test('skips documents with unchanged heads', async () => {
155
+ const level = createTestLevel();
156
+ await openAndClose(level);
157
+ const host = await setupAutomergeHost(level);
158
+ const spaceKey = SpaceId.random();
159
+
160
+ const handle = await createDatabaseDirectory(host, spaceKey, {
161
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Doc 1' } }),
162
+ });
163
+ await host.flush(Context.default());
164
+
165
+ const currentHeads = headsCodec.encode(getHeads(handle.doc()!));
166
+
167
+ const dataSource = new AutomergeDataSource(host);
168
+ const cursors: IndexCursor[] = [
169
+ {
170
+ indexName: 'fts',
171
+ spaceId: null,
172
+ sourceName: 'automerge',
173
+ resourceId: handle.documentId,
174
+ cursor: currentHeads,
175
+ },
176
+ ];
177
+
178
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), cursors));
179
+
180
+ expect(result.objects).toHaveLength(0);
181
+ expect(result.cursors).toHaveLength(0);
182
+ });
183
+
184
+ test('respects limit option', async () => {
185
+ const level = createTestLevel();
186
+ await openAndClose(level);
187
+ const host = await setupAutomergeHost(level);
188
+ const spaceKey = SpaceId.random();
189
+
190
+ // Create 3 documents.
191
+ for (let i = 1; i <= 3; i++) {
192
+ await createDatabaseDirectory(host, spaceKey, {
193
+ [`obj-${i}`]: ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: `Doc ${i}` } }),
194
+ });
195
+ }
196
+ await host.flush(Context.default());
197
+
198
+ const dataSource = new AutomergeDataSource(host);
199
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), [], { limit: 2 }));
200
+
201
+ expect(result.objects).toHaveLength(2);
202
+ expect(result.cursors).toHaveLength(2);
203
+ });
204
+
205
+ test('extracts multiple objects from a document', async () => {
206
+ const level = createTestLevel();
207
+ await openAndClose(level);
208
+ const host = await setupAutomergeHost(level);
209
+ const spaceKey = SpaceId.random();
210
+
211
+ await createDatabaseDirectory(host, spaceKey, {
212
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Object 1' } }),
213
+ 'obj-2': ObjectStructure.makeObject({ type: OTHER_TYPE as DXN.String, data: { title: 'Object 2' } }),
214
+ });
215
+ await host.flush(Context.default());
216
+
217
+ const dataSource = new AutomergeDataSource(host);
218
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
219
+
220
+ expect(result.objects).toHaveLength(2);
221
+ expect(result.objects.map((o) => o.data.id)).toContain('obj-1');
222
+ expect(result.objects.map((o) => o.data.id)).toContain('obj-2');
223
+ });
224
+
225
+ test('skips outdated documents', async () => {
226
+ const level = createTestLevel();
227
+ await openAndClose(level);
228
+ const host = await setupAutomergeHost(level);
229
+ const spaceKey = SpaceId.random();
230
+
231
+ // Create a document with outdated version.
232
+ const handle = await host.createDoc<DatabaseDirectory>({
233
+ version: 0 as SpaceDocVersion, // Outdated version.
234
+ access: { spaceKey },
235
+ objects: {},
236
+ links: {},
237
+ });
238
+ handle.change((doc) => {
239
+ doc.objects = {
240
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Test' } }),
241
+ };
242
+ });
243
+ await host.flush(Context.default());
244
+
245
+ const dataSource = new AutomergeDataSource(host);
246
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
247
+
248
+ expect(result.objects).toHaveLength(0);
249
+ });
250
+
251
+ test('extracts object attributes correctly', async () => {
252
+ const level = createTestLevel();
253
+ await openAndClose(level);
254
+ const host = await setupAutomergeHost(level);
255
+ const spaceKey = SpaceId.random();
256
+
257
+ await createDatabaseDirectory(host, spaceKey, {
258
+ 'person-1': ObjectStructure.makeObject({
259
+ type: PERSON_TYPE as DXN.String,
260
+ data: { name: 'Alice', age: 30 },
261
+ }),
262
+ });
263
+ await host.flush(Context.default());
264
+
265
+ const dataSource = new AutomergeDataSource(host);
266
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
267
+
268
+ expect(result.objects).toHaveLength(1);
269
+ const obj = result.objects[0];
270
+ expect(obj.data.id).toBe('person-1');
271
+ expect(obj.data['@type']).toBe(PERSON_TYPE);
272
+ expect(obj.data.name).toBe('Alice');
273
+ expect(obj.data.age).toBe(30);
274
+ });
275
+
276
+ test('skips documents without spaceKey', async () => {
277
+ const level = createTestLevel();
278
+ await openAndClose(level);
279
+ const host = await setupAutomergeHost(level);
280
+
281
+ // Create a document without access.spaceKey.
282
+ const handle = await host.createDoc<DatabaseDirectory>({
283
+ version: SpaceDocVersion.CURRENT,
284
+ objects: {},
285
+ links: {},
286
+ });
287
+ handle.change((doc) => {
288
+ doc.objects = {
289
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'No Space' } }),
290
+ };
291
+ });
292
+ await host.flush(Context.default());
293
+
294
+ const dataSource = new AutomergeDataSource(host);
295
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
296
+
297
+ expect(result.objects).toHaveLength(0);
298
+ });
299
+ });