@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1c7ec43d41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/dist/lib/{browser/chunk-35I6ERLG.mjs → neutral/chunk-VIHYDCCC.mjs} +313 -709
  2. package/dist/lib/neutral/chunk-VIHYDCCC.mjs.map +7 -0
  3. package/dist/lib/neutral/chunk-YBC6ONBZ.mjs +247 -0
  4. package/dist/lib/neutral/chunk-YBC6ONBZ.mjs.map +7 -0
  5. package/dist/lib/{browser → neutral}/filter/index.mjs +3 -2
  6. package/dist/lib/neutral/index.mjs +7065 -0
  7. package/dist/lib/neutral/index.mjs.map +7 -0
  8. package/dist/lib/neutral/meta.json +1 -0
  9. package/dist/lib/{browser → neutral}/testing/index.mjs +90 -311
  10. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  11. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts +2 -0
  12. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts.map +1 -0
  13. package/dist/types/src/automerge/automerge-host.d.ts +99 -54
  14. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  15. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts +2 -0
  16. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts.map +1 -0
  17. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  18. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  19. package/dist/types/src/automerge/automerge-subduction.test.d.ts +2 -0
  20. package/dist/types/src/automerge/automerge-subduction.test.d.ts.map +1 -0
  21. package/dist/types/src/automerge/collection-synchronizer.d.ts +25 -4
  22. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  23. package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
  24. package/dist/types/src/automerge/echo-network-adapter.d.ts +23 -10
  25. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  26. package/dist/types/src/automerge/echo-replicator.d.ts +38 -15
  27. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  28. package/dist/types/src/automerge/handle-state.d.ts +52 -0
  29. package/dist/types/src/automerge/handle-state.d.ts.map +1 -0
  30. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  31. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  32. package/dist/types/src/automerge/index.d.ts +2 -1
  33. package/dist/types/src/automerge/index.d.ts.map +1 -1
  34. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +9 -5
  35. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  36. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +9 -8
  37. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  38. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +6 -5
  39. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  40. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  41. package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
  42. package/dist/types/src/common/codec.d.ts +1 -1
  43. package/dist/types/src/common/codec.d.ts.map +1 -1
  44. package/dist/types/src/common/feeds.d.ts.map +1 -1
  45. package/dist/types/src/common/space-id.d.ts.map +1 -1
  46. package/dist/types/src/db-host/automerge-data-source.d.ts +35 -0
  47. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  48. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  49. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  50. package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
  51. package/dist/types/src/db-host/data-service.d.ts +8 -7
  52. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  53. package/dist/types/src/db-host/database-root.d.ts +9 -3
  54. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  55. package/dist/types/src/db-host/documents-synchronizer.d.ts +9 -8
  56. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  57. package/dist/types/src/db-host/echo-host.d.ts +62 -27
  58. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  59. package/dist/types/src/db-host/index.d.ts +2 -0
  60. package/dist/types/src/db-host/index.d.ts.map +1 -1
  61. package/dist/types/src/db-host/invalidation-hint.d.ts +19 -0
  62. package/dist/types/src/db-host/invalidation-hint.d.ts.map +1 -0
  63. package/dist/types/src/db-host/local-queue-service.d.ts +19 -0
  64. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  65. package/dist/types/src/db-host/query-invalidation.test.d.ts +2 -0
  66. package/dist/types/src/db-host/query-invalidation.test.d.ts.map +1 -0
  67. package/dist/types/src/db-host/query-service.d.ts +19 -11
  68. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  69. package/dist/types/src/db-host/queue-data-source.d.ts +38 -0
  70. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  71. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  72. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  73. package/dist/types/src/db-host/space-state-manager.d.ts +4 -3
  74. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  75. package/dist/types/src/db-host/stub.d.ts +11 -0
  76. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  77. package/dist/types/src/edge/echo-edge-replicator.d.ts +10 -7
  78. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  79. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts +39 -0
  80. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts.map +1 -0
  81. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts +2 -0
  82. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts.map +1 -0
  83. package/dist/types/src/edge/index.d.ts +1 -0
  84. package/dist/types/src/edge/index.d.ts.map +1 -1
  85. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
  86. package/dist/types/src/filter/filter-match.d.ts +4 -1
  87. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  88. package/dist/types/src/metadata/metadata-store.d.ts +1 -1
  89. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  90. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  91. package/dist/types/src/pipeline/pipeline.d.ts +3 -3
  92. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  93. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
  94. package/dist/types/src/query/errors.d.ts +22 -10
  95. package/dist/types/src/query/errors.d.ts.map +1 -1
  96. package/dist/types/src/query/plan.d.ts +66 -8
  97. package/dist/types/src/query/plan.d.ts.map +1 -1
  98. package/dist/types/src/query/query-executor.d.ts +66 -11
  99. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  100. package/dist/types/src/query/query-planner.d.ts +12 -1
  101. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  102. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  103. package/dist/types/src/space/auth.d.ts +3 -3
  104. package/dist/types/src/space/auth.d.ts.map +1 -1
  105. package/dist/types/src/space/control-pipeline.d.ts +5 -4
  106. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  107. package/dist/types/src/space/space-manager.d.ts +8 -8
  108. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  109. package/dist/types/src/space/space-protocol.d.ts +10 -9
  110. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  111. package/dist/types/src/space/space.d.ts +6 -6
  112. package/dist/types/src/space/space.d.ts.map +1 -1
  113. package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
  114. package/dist/types/src/testing/index.d.ts +0 -2
  115. package/dist/types/src/testing/index.d.ts.map +1 -1
  116. package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
  117. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  118. package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -1
  119. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  120. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  121. package/dist/types/src/testing/test-replicator.d.ts +14 -13
  122. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  123. package/dist/types/src/util.d.ts +1 -1
  124. package/dist/types/src/util.d.ts.map +1 -1
  125. package/dist/types/tsconfig.tsbuildinfo +1 -1
  126. package/package.json +56 -54
  127. package/src/automerge/automerge-host-subduction.test.ts +247 -0
  128. package/src/automerge/automerge-host.test.ts +65 -39
  129. package/src/automerge/automerge-host.ts +589 -206
  130. package/src/automerge/automerge-repo-subduction.test.ts +1049 -0
  131. package/src/automerge/automerge-repo.test.ts +349 -91
  132. package/src/automerge/automerge-subduction.test.ts +330 -0
  133. package/src/automerge/collection-synchronizer.test.ts +53 -17
  134. package/src/automerge/collection-synchronizer.ts +119 -36
  135. package/src/automerge/echo-data-monitor.ts +1 -1
  136. package/src/automerge/echo-network-adapter.test.ts +5 -4
  137. package/src/automerge/echo-network-adapter.ts +74 -30
  138. package/src/automerge/echo-replicator.ts +41 -15
  139. package/src/automerge/handle-state.ts +62 -0
  140. package/src/automerge/heads-store.ts +42 -3
  141. package/src/automerge/index.ts +2 -1
  142. package/src/automerge/leveldb-storage-adapter.ts +40 -11
  143. package/src/automerge/mesh-echo-replicator-connection.ts +12 -8
  144. package/src/automerge/mesh-echo-replicator.ts +15 -11
  145. package/src/automerge/storage-adapter.test.ts +1 -1
  146. package/src/common/space-id.ts +1 -1
  147. package/src/db-host/automerge-data-source.test.ts +299 -0
  148. package/src/db-host/automerge-data-source.ts +187 -0
  149. package/src/db-host/data-service.ts +25 -26
  150. package/src/db-host/database-root.ts +20 -8
  151. package/src/db-host/documents-synchronizer.test.ts +143 -14
  152. package/src/db-host/documents-synchronizer.ts +72 -55
  153. package/src/db-host/echo-host.ts +348 -96
  154. package/src/db-host/index.ts +2 -0
  155. package/src/db-host/invalidation-hint.ts +50 -0
  156. package/src/db-host/local-queue-service.ts +127 -0
  157. package/src/db-host/query-invalidation.test.ts +270 -0
  158. package/src/db-host/query-service.ts +111 -106
  159. package/src/db-host/queue-data-source.ts +160 -0
  160. package/src/db-host/queue-service.test.ts +145 -0
  161. package/src/db-host/space-state-manager.ts +12 -10
  162. package/src/db-host/stub.ts +33 -0
  163. package/src/edge/echo-edge-replicator.test.ts +12 -10
  164. package/src/edge/echo-edge-replicator.ts +144 -58
  165. package/src/edge/echo-edge-subduction-replicator.test.ts +158 -0
  166. package/src/edge/echo-edge-subduction-replicator.ts +377 -0
  167. package/src/edge/index.ts +1 -0
  168. package/src/filter/filter-match.test.ts +68 -32
  169. package/src/filter/filter-match.ts +184 -20
  170. package/src/metadata/metadata-store.ts +11 -8
  171. package/src/pipeline/pipeline-stress.test.ts +1 -1
  172. package/src/pipeline/pipeline.test.ts +2 -2
  173. package/src/pipeline/pipeline.ts +9 -7
  174. package/src/query/errors.ts +3 -1
  175. package/src/query/plan.ts +82 -8
  176. package/src/query/query-executor.ts +1199 -164
  177. package/src/query/query-planner.test.ts +993 -81
  178. package/src/query/query-planner.ts +467 -27
  179. package/src/space/admission-discovery-extension.ts +3 -3
  180. package/src/space/auth.ts +6 -6
  181. package/src/space/control-pipeline.test.ts +5 -4
  182. package/src/space/control-pipeline.ts +16 -13
  183. package/src/space/space-manager.browser.test.ts +1 -1
  184. package/src/space/space-manager.ts +16 -20
  185. package/src/space/space-protocol.browser.test.ts +15 -14
  186. package/src/space/space-protocol.test.ts +15 -14
  187. package/src/space/space-protocol.ts +22 -18
  188. package/src/space/space.test.ts +3 -3
  189. package/src/space/space.ts +11 -11
  190. package/src/testing/index.ts +0 -2
  191. package/src/testing/test-agent-builder.ts +3 -3
  192. package/src/testing/test-network-adapter.ts +24 -3
  193. package/src/testing/test-replicator.ts +19 -20
  194. package/src/util.ts +7 -3
  195. package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
  196. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  197. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  198. package/dist/lib/browser/chunk-TQJTKNMS.mjs +0 -126
  199. package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
  200. package/dist/lib/browser/index.mjs +0 -4743
  201. package/dist/lib/browser/index.mjs.map +0 -7
  202. package/dist/lib/browser/meta.json +0 -1
  203. package/dist/lib/browser/testing/index.mjs.map +0 -7
  204. package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
  205. package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
  206. package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
  207. package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
  208. package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
  209. package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
  210. package/dist/lib/node/filter/index.cjs +0 -32
  211. package/dist/lib/node/filter/index.cjs.map +0 -7
  212. package/dist/lib/node/index.cjs +0 -4699
  213. package/dist/lib/node/index.cjs.map +0 -7
  214. package/dist/lib/node/meta.json +0 -1
  215. package/dist/lib/node/testing/index.cjs +0 -753
  216. package/dist/lib/node/testing/index.cjs.map +0 -7
  217. package/dist/lib/node-esm/chunk-5BHLPT24.mjs +0 -2067
  218. package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
  219. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  220. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  221. package/dist/lib/node-esm/chunk-RVK35BS7.mjs +0 -126
  222. package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +0 -7
  223. package/dist/lib/node-esm/filter/index.mjs +0 -11
  224. package/dist/lib/node-esm/filter/index.mjs.map +0 -7
  225. package/dist/lib/node-esm/index.mjs +0 -4743
  226. package/dist/lib/node-esm/index.mjs.map +0 -7
  227. package/dist/lib/node-esm/meta.json +0 -1
  228. package/dist/lib/node-esm/testing/index.mjs +0 -742
  229. package/dist/lib/node-esm/testing/index.mjs.map +0 -7
  230. package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
  231. package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
  232. package/dist/types/src/testing/test-data.d.ts +0 -18
  233. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  234. package/dist/types/src/testing/test-schema.d.ts +0 -37
  235. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  236. package/src/db-host/documents-iterator.ts +0 -74
  237. package/src/testing/test-data.ts +0 -127
  238. package/src/testing/test-schema.ts +0 -53
  239. /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
@@ -0,0 +1,299 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import { getHeads } from '@automerge/automerge';
6
+ import { describe, expect, onTestFinished, test } from 'vitest';
7
+
8
+ import { Context } from '@dxos/context';
9
+ import { type DatabaseDirectory, ObjectStructure, SpaceDocVersion } from '@dxos/echo-protocol';
10
+ import { runAndForwardErrors } from '@dxos/effect';
11
+ import { type IndexCursor } from '@dxos/index-core';
12
+ import { DXN, SpaceId } from '@dxos/keys';
13
+ import { type LevelDB } from '@dxos/kv-store';
14
+ import { createTestLevel } from '@dxos/kv-store/testing';
15
+ import { openAndClose } from '@dxos/test-utils';
16
+
17
+ import { AutomergeHost } from '../automerge';
18
+ import { AutomergeDataSource, headsCodec } from './automerge-data-source';
19
+
20
+ const TEST_TYPE = DXN.parse('dxn:type:com.example.type.test:0.1.0').toString();
21
+ const OTHER_TYPE = DXN.parse('dxn:type:com.example.type.other:0.1.0').toString();
22
+ const PERSON_TYPE = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
23
+
24
+ /**
25
+ * Set up a real AutomergeHost with LevelDB storage.
26
+ */
27
+ const setupAutomergeHost = async (level: LevelDB): Promise<AutomergeHost> => {
28
+ const host = new AutomergeHost({
29
+ db: level,
30
+ });
31
+ await host.open();
32
+ onTestFinished(async () => {
33
+ await host.close();
34
+ });
35
+ return host;
36
+ };
37
+
38
+ /**
39
+ * Create a DatabaseDirectory document with the given objects.
40
+ */
41
+ const createDatabaseDirectory = async (
42
+ host: AutomergeHost,
43
+ spaceKey: string,
44
+ objects: Record<string, ObjectStructure>,
45
+ ): Promise<Awaited<ReturnType<typeof host.createDoc<DatabaseDirectory>>>> => {
46
+ const handle = await host.createDoc<DatabaseDirectory>({
47
+ version: SpaceDocVersion.CURRENT,
48
+ access: { spaceKey },
49
+ objects: {},
50
+ links: {},
51
+ });
52
+
53
+ handle.change((doc) => {
54
+ doc.objects = objects;
55
+ });
56
+
57
+ return handle;
58
+ };
59
+
60
+ describe('AutomergeDataSource', () => {
61
+ test('headsCodec encodes and decodes heads correctly', () => {
62
+ const heads = ['abc123', 'def456'];
63
+ const encoded = headsCodec.encode(heads);
64
+
65
+ expect(encoded).toBe('abc123|def456');
66
+ expect(headsCodec.decode(encoded)).toEqual(['abc123', 'def456']);
67
+ });
68
+
69
+ test('headsCodec sorts heads for consistent encoding', () => {
70
+ const heads1 = ['def456', 'abc123'];
71
+ const heads2 = ['abc123', 'def456'];
72
+
73
+ expect(headsCodec.encode(heads1)).toBe(headsCodec.encode(heads2));
74
+ });
75
+
76
+ test('returns empty when no documents exist', async () => {
77
+ const level = createTestLevel();
78
+ await openAndClose(level);
79
+ const host = await setupAutomergeHost(level);
80
+
81
+ const dataSource = new AutomergeDataSource(host);
82
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
83
+
84
+ expect(result.objects).toHaveLength(0);
85
+ expect(result.cursors).toHaveLength(0);
86
+ });
87
+
88
+ test('returns new documents that have no cursor', async () => {
89
+ const level = createTestLevel();
90
+ await openAndClose(level);
91
+ const host = await setupAutomergeHost(level);
92
+ const spaceKey = SpaceId.random();
93
+
94
+ const handle = await createDatabaseDirectory(host, spaceKey, {
95
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Test Document' } }),
96
+ });
97
+ await host.flush(Context.default());
98
+
99
+ const dataSource = new AutomergeDataSource(host);
100
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
101
+
102
+ expect(result.objects).toHaveLength(1);
103
+ expect(result.objects[0].documentId).toBe(handle.documentId);
104
+ expect(result.objects[0].data.title).toBe('Test Document');
105
+
106
+ expect(result.cursors).toHaveLength(1);
107
+ expect(result.cursors[0].resourceId).toBe(handle.documentId);
108
+ expect(result.cursors[0].cursor).toBe(headsCodec.encode(getHeads(handle.doc()!)));
109
+ });
110
+
111
+ test('returns documents with changed heads', async () => {
112
+ const level = createTestLevel();
113
+ await openAndClose(level);
114
+ const host = await setupAutomergeHost(level);
115
+ const spaceKey = SpaceId.random();
116
+
117
+ const handle1 = await createDatabaseDirectory(host, spaceKey, {
118
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Doc 1' } }),
119
+ });
120
+ const handle2 = await createDatabaseDirectory(host, spaceKey, {
121
+ 'obj-2': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Doc 2' } }),
122
+ });
123
+ await host.flush(Context.default());
124
+
125
+ // Capture heads before mutation.
126
+ const doc1HeadsBefore = headsCodec.encode(getHeads(handle1.doc()!));
127
+ const doc2Heads = headsCodec.encode(getHeads(handle2.doc()!));
128
+
129
+ // Modify doc1 to have new heads.
130
+ handle1.change((doc) => {
131
+ doc.objects!['obj-1'].data.title = 'Doc 1 Updated';
132
+ });
133
+ await host.flush(Context.default());
134
+
135
+ const dataSource = new AutomergeDataSource(host);
136
+ const cursors: IndexCursor[] = [
137
+ {
138
+ indexName: 'fts',
139
+ spaceId: null,
140
+ sourceName: 'automerge',
141
+ resourceId: handle1.documentId,
142
+ cursor: doc1HeadsBefore,
143
+ },
144
+ { indexName: 'fts', spaceId: null, sourceName: 'automerge', resourceId: handle2.documentId, cursor: doc2Heads },
145
+ ];
146
+
147
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), cursors));
148
+
149
+ // Only doc1 changed.
150
+ expect(result.objects).toHaveLength(1);
151
+ expect(result.objects[0].documentId).toBe(handle1.documentId);
152
+ });
153
+
154
+ test('skips documents with unchanged heads', async () => {
155
+ const level = createTestLevel();
156
+ await openAndClose(level);
157
+ const host = await setupAutomergeHost(level);
158
+ const spaceKey = SpaceId.random();
159
+
160
+ const handle = await createDatabaseDirectory(host, spaceKey, {
161
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Doc 1' } }),
162
+ });
163
+ await host.flush(Context.default());
164
+
165
+ const currentHeads = headsCodec.encode(getHeads(handle.doc()!));
166
+
167
+ const dataSource = new AutomergeDataSource(host);
168
+ const cursors: IndexCursor[] = [
169
+ {
170
+ indexName: 'fts',
171
+ spaceId: null,
172
+ sourceName: 'automerge',
173
+ resourceId: handle.documentId,
174
+ cursor: currentHeads,
175
+ },
176
+ ];
177
+
178
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), cursors));
179
+
180
+ expect(result.objects).toHaveLength(0);
181
+ expect(result.cursors).toHaveLength(0);
182
+ });
183
+
184
+ test('respects limit option', async () => {
185
+ const level = createTestLevel();
186
+ await openAndClose(level);
187
+ const host = await setupAutomergeHost(level);
188
+ const spaceKey = SpaceId.random();
189
+
190
+ // Create 3 documents.
191
+ for (let i = 1; i <= 3; i++) {
192
+ await createDatabaseDirectory(host, spaceKey, {
193
+ [`obj-${i}`]: ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: `Doc ${i}` } }),
194
+ });
195
+ }
196
+ await host.flush(Context.default());
197
+
198
+ const dataSource = new AutomergeDataSource(host);
199
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), [], { limit: 2 }));
200
+
201
+ expect(result.objects).toHaveLength(2);
202
+ expect(result.cursors).toHaveLength(2);
203
+ });
204
+
205
+ test('extracts multiple objects from a document', async () => {
206
+ const level = createTestLevel();
207
+ await openAndClose(level);
208
+ const host = await setupAutomergeHost(level);
209
+ const spaceKey = SpaceId.random();
210
+
211
+ await createDatabaseDirectory(host, spaceKey, {
212
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Object 1' } }),
213
+ 'obj-2': ObjectStructure.makeObject({ type: OTHER_TYPE as DXN.String, data: { title: 'Object 2' } }),
214
+ });
215
+ await host.flush(Context.default());
216
+
217
+ const dataSource = new AutomergeDataSource(host);
218
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
219
+
220
+ expect(result.objects).toHaveLength(2);
221
+ expect(result.objects.map((o) => o.data.id)).toContain('obj-1');
222
+ expect(result.objects.map((o) => o.data.id)).toContain('obj-2');
223
+ });
224
+
225
+ test('skips outdated documents', async () => {
226
+ const level = createTestLevel();
227
+ await openAndClose(level);
228
+ const host = await setupAutomergeHost(level);
229
+ const spaceKey = SpaceId.random();
230
+
231
+ // Create a document with outdated version.
232
+ const handle = await host.createDoc<DatabaseDirectory>({
233
+ version: 0 as SpaceDocVersion, // Outdated version.
234
+ access: { spaceKey },
235
+ objects: {},
236
+ links: {},
237
+ });
238
+ handle.change((doc) => {
239
+ doc.objects = {
240
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'Test' } }),
241
+ };
242
+ });
243
+ await host.flush(Context.default());
244
+
245
+ const dataSource = new AutomergeDataSource(host);
246
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
247
+
248
+ expect(result.objects).toHaveLength(0);
249
+ });
250
+
251
+ test('extracts object attributes correctly', async () => {
252
+ const level = createTestLevel();
253
+ await openAndClose(level);
254
+ const host = await setupAutomergeHost(level);
255
+ const spaceKey = SpaceId.random();
256
+
257
+ await createDatabaseDirectory(host, spaceKey, {
258
+ 'person-1': ObjectStructure.makeObject({
259
+ type: PERSON_TYPE as DXN.String,
260
+ data: { name: 'Alice', age: 30 },
261
+ }),
262
+ });
263
+ await host.flush(Context.default());
264
+
265
+ const dataSource = new AutomergeDataSource(host);
266
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
267
+
268
+ expect(result.objects).toHaveLength(1);
269
+ const obj = result.objects[0];
270
+ expect(obj.data.id).toBe('person-1');
271
+ expect(obj.data['@type']).toBe(PERSON_TYPE);
272
+ expect(obj.data.name).toBe('Alice');
273
+ expect(obj.data.age).toBe(30);
274
+ });
275
+
276
+ test('skips documents without spaceKey', async () => {
277
+ const level = createTestLevel();
278
+ await openAndClose(level);
279
+ const host = await setupAutomergeHost(level);
280
+
281
+ // Create a document without access.spaceKey.
282
+ const handle = await host.createDoc<DatabaseDirectory>({
283
+ version: SpaceDocVersion.CURRENT,
284
+ objects: {},
285
+ links: {},
286
+ });
287
+ handle.change((doc) => {
288
+ doc.objects = {
289
+ 'obj-1': ObjectStructure.makeObject({ type: TEST_TYPE as DXN.String, data: { title: 'No Space' } }),
290
+ };
291
+ });
292
+ await host.flush(Context.default());
293
+
294
+ const dataSource = new AutomergeDataSource(host);
295
+ const result = await runAndForwardErrors(dataSource.getChangedObjects(Context.default(), []));
296
+
297
+ expect(result.objects).toHaveLength(0);
298
+ });
299
+ });
@@ -0,0 +1,187 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as A from '@automerge/automerge';
6
+ import { type DocumentId } from '@automerge/automerge-repo';
7
+ import * as Effect from 'effect/Effect';
8
+
9
+ import { type Context } from '@dxos/context';
10
+ import { DatabaseDirectory, SpaceDocVersion } from '@dxos/echo-protocol';
11
+ import { objectStructureToJson } from '@dxos/echo/internal';
12
+ import { type DataSourceCursor, type IndexDataSource, type IndexerObject } from '@dxos/index-core';
13
+ import { PublicKey } from '@dxos/keys';
14
+ import { log } from '@dxos/log';
15
+
16
+ import { type AutomergeHost } from '../automerge';
17
+ import { createIdFromSpaceKey } from '../common';
18
+
19
+ const HEADS_DELIMITER = '|';
20
+
21
+ /**
22
+ * Codec for serializing/deserializing Automerge heads to cursor strings.
23
+ */
24
+ export const headsCodec = {
25
+ /**
26
+ * Serialize automerge heads to a cursor string.
27
+ * Heads are sorted to ensure consistent comparison.
28
+ */
29
+ encode: (heads: A.Heads): string => [...heads].sort().join(HEADS_DELIMITER),
30
+
31
+ /**
32
+ * Deserialize a cursor string back to heads array.
33
+ */
34
+ decode: (cursor: string): string[] => (cursor ? cursor.split(HEADS_DELIMITER) : []),
35
+ };
36
+
37
+ /**
38
+ * Check if document has changed by comparing cursor with current heads.
39
+ */
40
+ const hasChanged = (cursor: string | undefined, currentHeads: A.Heads): boolean => {
41
+ if (!cursor) {
42
+ return true; // New document.
43
+ }
44
+ return cursor !== headsCodec.encode(currentHeads);
45
+ };
46
+
47
+ /**
48
+ * Data source that fetches objects from AutomergeHost.
49
+ * Iterates all documents from HeadsStore and tracks document heads as cursors to detect changes.
50
+ */
51
+ export class AutomergeDataSource implements IndexDataSource {
52
+ readonly sourceName = 'automerge';
53
+
54
+ readonly #automergeHost: AutomergeHost;
55
+
56
+ constructor(automergeHost: AutomergeHost) {
57
+ this.#automergeHost = automergeHost;
58
+ }
59
+
60
+ getChangedObjects(
61
+ ctx: Context,
62
+ cursors: DataSourceCursor[],
63
+ opts?: { limit?: number },
64
+ ): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }> {
65
+ return Effect.gen(this, function* () {
66
+ // Build a map of documentId -> cursor for quick lookup.
67
+ const cursorMap = new Map<string, string>();
68
+ for (const cursor of cursors) {
69
+ if (cursor.resourceId) {
70
+ cursorMap.set(cursor.resourceId, String(cursor.cursor));
71
+ }
72
+ }
73
+
74
+ // Find changed documents by iterating all documents from HeadsStore.
75
+ const changedDocuments = yield* Effect.promise(async () => {
76
+ const result: { documentId: DocumentId; heads: A.Heads }[] = [];
77
+ const limit = opts?.limit ?? Infinity;
78
+
79
+ for await (const { documentId, heads } of this.#automergeHost.listDocumentHeads()) {
80
+ const existingCursor = cursorMap.get(documentId);
81
+ if (hasChanged(existingCursor, heads)) {
82
+ result.push({ documentId, heads });
83
+ if (result.length >= limit) {
84
+ break;
85
+ }
86
+ }
87
+ }
88
+ return result;
89
+ });
90
+
91
+ // Load changed documents and extract objects.
92
+ const objects: IndexerObject[] = [];
93
+ const updatedCursors: DataSourceCursor[] = [];
94
+
95
+ for (const { documentId, heads: docHeads } of changedDocuments) {
96
+ try {
97
+ const handle = yield* Effect.promise(() => this.#automergeHost.loadDoc<DatabaseDirectory>(ctx, documentId));
98
+ const doc = handle.doc();
99
+ if (!doc) {
100
+ continue;
101
+ }
102
+
103
+ // Skip outdated docs.
104
+ if (doc.version !== SpaceDocVersion.CURRENT) {
105
+ continue;
106
+ }
107
+
108
+ // Extract spaceId from document.
109
+ const spaceKeyHex = DatabaseDirectory.getSpaceKey(doc);
110
+ if (!spaceKeyHex) {
111
+ // Skip documents without a space key.
112
+ continue;
113
+ }
114
+ const spaceKey = PublicKey.fromHex(spaceKeyHex);
115
+ const spaceId = yield* Effect.promise(() => createIdFromSpaceKey(spaceKey));
116
+
117
+ const existingCursor = cursorMap.get(documentId);
118
+ const { changedObjectIds, updatedAt } = inspectDocChanges(doc, existingCursor);
119
+
120
+ const docObjects = doc.objects ?? {};
121
+ for (const [objectId, structure] of Object.entries(docObjects)) {
122
+ if (changedObjectIds && !changedObjectIds.has(objectId)) {
123
+ continue;
124
+ }
125
+ objects.push({
126
+ spaceId,
127
+ documentId,
128
+ queueId: null,
129
+ queueNamespace: null,
130
+ recordId: null,
131
+ data: objectStructureToJson(objectId, structure),
132
+ updatedAt,
133
+ });
134
+ }
135
+
136
+ // Update cursor for this document.
137
+ updatedCursors.push({
138
+ spaceId,
139
+ resourceId: documentId,
140
+ cursor: headsCodec.encode(docHeads),
141
+ });
142
+ } catch (error) {
143
+ log.error('Error loading document for indexing', { documentId, error });
144
+ }
145
+ }
146
+
147
+ return { objects, cursors: updatedCursors };
148
+ });
149
+ }
150
+ }
151
+
152
+ /**
153
+ * Determines which ECHO objects changed and the document-level max change timestamp.
154
+ *
155
+ * Uses `A.diff` to extract changed objectIds from patch paths (`["objects", objectId, ...]`),
156
+ * and `A.getChangesMetaSince` for the max timestamp (second-level precision).
157
+ * Returns `changedObjectIds: null` when all objects should be indexed (new document).
158
+ */
159
+ const inspectDocChanges = (
160
+ doc: DatabaseDirectory,
161
+ existingCursor: string | undefined,
162
+ ): { changedObjectIds: Set<string> | null; updatedAt: number } => {
163
+ if (!existingCursor) {
164
+ return { changedObjectIds: null, updatedAt: Date.now() };
165
+ }
166
+
167
+ const oldHeads = headsCodec.decode(existingCursor);
168
+
169
+ const patches = A.diff(doc, oldHeads, A.getHeads(doc));
170
+ const changedObjectIds = new Set<string>();
171
+ for (const patch of patches) {
172
+ if (patch.path.length >= 2 && patch.path[0] === 'objects') {
173
+ changedObjectIds.add(String(patch.path[1]));
174
+ }
175
+ }
176
+
177
+ const changes = A.getChangesMetaSince(doc, oldHeads);
178
+ let maxTime = 0;
179
+ for (const change of changes) {
180
+ if (change.time > maxTime) {
181
+ maxTime = change.time;
182
+ }
183
+ }
184
+ const updatedAt = maxTime > 0 ? maxTime * 1000 : Date.now();
185
+
186
+ return { changedObjectIds, updatedAt };
187
+ };
@@ -7,29 +7,32 @@ import { type DocumentId } from '@automerge/automerge-repo';
7
7
  import { UpdateScheduler } from '@dxos/async';
8
8
  import { type RequestOptions } from '@dxos/codec-protobuf';
9
9
  import { Stream } from '@dxos/codec-protobuf/stream';
10
+ import { Context } from '@dxos/context';
10
11
  import { invariant } from '@dxos/invariant';
11
12
  import { SpaceId } from '@dxos/keys';
12
13
  import { log } from '@dxos/log';
13
14
  import {
15
+ type BatchedDocumentUpdates,
16
+ type CreateDocumentRequest,
17
+ type CreateDocumentResponse,
14
18
  type DataService,
15
19
  type FlushRequest,
16
- type SubscribeRequest,
17
- type BatchedDocumentUpdates,
18
- type UpdateSubscriptionRequest,
19
20
  type GetDocumentHeadsRequest,
20
21
  type GetDocumentHeadsResponse,
21
- type ReIndexHeadsRequest,
22
- type WaitUntilHeadsReplicatedRequest,
23
- type UpdateRequest,
24
22
  type GetSpaceSyncStateRequest,
23
+ type ReIndexHeadsRequest,
25
24
  type SpaceSyncState,
25
+ type SubscribeRequest,
26
+ type UpdateRequest,
27
+ type UpdateSubscriptionRequest,
28
+ type WaitUntilHeadsReplicatedRequest,
26
29
  } from '@dxos/protocols/proto/dxos/echo/service';
27
30
 
31
+ import { type AutomergeHost, deriveCollectionIdFromSpaceId } from '../automerge';
28
32
  import { DocumentsSynchronizer } from './documents-synchronizer';
29
33
  import { type SpaceStateManager } from './space-state-manager';
30
- import { deriveCollectionIdFromSpaceId, type AutomergeHost } from '../automerge';
31
34
 
32
- export type DataServiceParams = {
35
+ export type DataServiceProps = {
33
36
  automergeHost: AutomergeHost;
34
37
  spaceStateManager: SpaceStateManager;
35
38
  updateIndexes: () => Promise<void>;
@@ -50,7 +53,7 @@ export class DataServiceImpl implements DataService {
50
53
  private readonly _spaceStateManager: SpaceStateManager;
51
54
  private readonly _updateIndexes: () => Promise<void>;
52
55
 
53
- constructor(params: DataServiceParams) {
56
+ constructor(params: DataServiceProps) {
54
57
  this._automergeHost = params.automergeHost;
55
58
  this._spaceStateManager = params.spaceStateManager;
56
59
  this._updateIndexes = params.updateIndexes;
@@ -59,7 +62,7 @@ export class DataServiceImpl implements DataService {
59
62
  subscribe(request: SubscribeRequest): Stream<BatchedDocumentUpdates> {
60
63
  return new Stream<BatchedDocumentUpdates>(({ next, ready }) => {
61
64
  const synchronizer = new DocumentsSynchronizer({
62
- repo: this._automergeHost.repo,
65
+ automergeHost: this._automergeHost,
63
66
  sendUpdates: (updates) => next(updates),
64
67
  });
65
68
  synchronizer
@@ -85,18 +88,23 @@ export class DataServiceImpl implements DataService {
85
88
  }
86
89
  }
87
90
 
88
- async update(request: UpdateRequest): Promise<void> {
91
+ async createDocument(request: CreateDocumentRequest): Promise<CreateDocumentResponse> {
92
+ const handle = await this._automergeHost.createDoc(request.initialValue);
93
+ return { documentId: handle.documentId };
94
+ }
95
+
96
+ async update(request: UpdateRequest, options?: RequestOptions): Promise<void> {
89
97
  if (!request.updates) {
90
98
  return;
91
99
  }
92
100
  const synchronizer = this._subscriptions.get(request.subscriptionId);
93
101
  invariant(synchronizer, 'Subscription not found');
94
102
 
95
- await synchronizer.update(request.updates);
103
+ await synchronizer.update(options?.ctx ?? Context.default(), request.updates);
96
104
  }
97
105
 
98
- async flush(request: FlushRequest): Promise<void> {
99
- await this._automergeHost.flush(request);
106
+ async flush(request: FlushRequest, options?: RequestOptions): Promise<void> {
107
+ await this._automergeHost.flush(options?.ctx ?? Context.default(), request);
100
108
  }
101
109
 
102
110
  async getDocumentHeads(request: GetDocumentHeadsRequest): Promise<GetDocumentHeadsResponse> {
@@ -116,10 +124,10 @@ export class DataServiceImpl implements DataService {
116
124
  request: WaitUntilHeadsReplicatedRequest,
117
125
  options?: RequestOptions | undefined,
118
126
  ): Promise<void> {
119
- await this._automergeHost.waitUntilHeadsReplicated(request.heads);
127
+ await this._automergeHost.waitUntilHeadsReplicated(options?.ctx ?? Context.default(), request.heads);
120
128
  }
121
129
 
122
- async reIndexHeads(request: ReIndexHeadsRequest, options?: RequestOptions): Promise<void> {
130
+ async reIndexHeads(request: ReIndexHeadsRequest, _options?: RequestOptions): Promise<void> {
123
131
  await this._automergeHost.reIndexHeads((request.documentIds ?? []) as DocumentId[]);
124
132
  }
125
133
 
@@ -145,16 +153,7 @@ export class DataServiceImpl implements DataService {
145
153
  const scheduler = new UpdateScheduler(ctx, async () => {
146
154
  const state = collectionId ? await this._automergeHost.getCollectionSyncState(collectionId) : { peers: [] };
147
155
 
148
- next({
149
- peers: state.peers.map((peer) => ({
150
- peerId: peer.peerId,
151
- missingOnRemote: peer.missingOnRemote,
152
- missingOnLocal: peer.missingOnLocal,
153
- differentDocuments: peer.differentDocuments,
154
- localDocumentCount: peer.localDocumentCount,
155
- remoteDocumentCount: peer.remoteDocumentCount,
156
- })),
157
- });
156
+ next({ peers: state.peers });
158
157
  });
159
158
 
160
159
  this._automergeHost.collectionStateUpdated.on(ctx, (e) => {
@@ -3,12 +3,18 @@
3
3
  //
4
4
 
5
5
  import type * as A from '@automerge/automerge';
6
- import { interpretAsDocumentId, type AutomergeUrl, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
6
+ import {
7
+ type AutomergeUrl,
8
+ type DocHandle,
9
+ type DocumentId,
10
+ type DocumentQuery,
11
+ interpretAsDocumentId,
12
+ } from '@automerge/automerge-repo';
7
13
 
8
14
  import { DatabaseDirectory, SpaceDocVersion } from '@dxos/echo-protocol';
9
15
  import { invariant } from '@dxos/invariant';
10
16
 
11
- import { measureDocMetrics, type DocMetrics } from './automerge-metrics';
17
+ import { type DocMetrics, measureDocMetrics } from './automerge-metrics';
12
18
 
13
19
  export class DatabaseRoot {
14
20
  static mapLinks(doc: DocHandle<DatabaseDirectory>, mapping: Record<DocumentId, DocumentId>): void {
@@ -25,26 +31,32 @@ export class DatabaseRoot {
25
31
  });
26
32
  }
27
33
 
28
- constructor(private readonly _rootHandle: DocHandle<DatabaseDirectory>) {}
34
+ /**
35
+ * @param _query - Live `DocumentQuery` for the root doc. Carries both the
36
+ * `DocHandle` (synchronously created with the query) and the actual
37
+ * readiness state. See `getHandleState` in `@dxos/echo-pipeline` for why
38
+ * we read liveness off the query rather than `DocHandle.*` predicates.
39
+ */
40
+ constructor(private readonly _query: DocumentQuery<DatabaseDirectory>) {}
29
41
 
30
42
  get documentId(): DocumentId {
31
- return this._rootHandle.documentId;
43
+ return this._query.documentId;
32
44
  }
33
45
 
34
46
  get url() {
35
- return this._rootHandle.url;
47
+ return this._query.handle.url;
36
48
  }
37
49
 
38
50
  get isLoaded(): boolean {
39
- return this._rootHandle.isReady();
51
+ return this._query.peek().state === 'ready';
40
52
  }
41
53
 
42
54
  get handle(): DocHandle<DatabaseDirectory> {
43
- return this._rootHandle;
55
+ return this._query.handle;
44
56
  }
45
57
 
46
58
  doc(): A.Doc<DatabaseDirectory> | null {
47
- return this._rootHandle.isReady() ? this._rootHandle.doc() : null;
59
+ return this.isLoaded ? this._query.handle.doc() : null;
48
60
  }
49
61
 
50
62
  getVersion(): SpaceDocVersion | null {