@dxos/echo-pipeline 0.8.2-staging.7ac8446 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/dist/lib/browser/{chunk-32WDI3LB.mjs → chunk-3XSXS5EX.mjs} +15 -21
  2. package/dist/lib/browser/chunk-3XSXS5EX.mjs.map +7 -0
  3. package/dist/lib/browser/chunk-CGS2ULMK.mjs +11 -0
  4. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +7 -0
  5. package/dist/lib/browser/chunk-TQJTKNMS.mjs +126 -0
  6. package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +7 -0
  7. package/dist/lib/browser/filter/index.mjs +11 -0
  8. package/dist/lib/browser/filter/index.mjs.map +7 -0
  9. package/dist/lib/browser/index.mjs +1380 -516
  10. package/dist/lib/browser/index.mjs.map +4 -4
  11. package/dist/lib/browser/meta.json +1 -1
  12. package/dist/lib/browser/testing/index.mjs +202 -22
  13. package/dist/lib/browser/testing/index.mjs.map +4 -4
  14. package/dist/lib/node/chunk-HOPOFWAL.cjs +147 -0
  15. package/dist/lib/node/chunk-HOPOFWAL.cjs.map +7 -0
  16. package/dist/lib/node/chunk-Q7SFCCGT.cjs +33 -0
  17. package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +7 -0
  18. package/dist/lib/node/{chunk-TC2PRBEU.cjs → chunk-SG2PL5RH.cjs} +18 -24
  19. package/dist/lib/node/chunk-SG2PL5RH.cjs.map +7 -0
  20. package/dist/lib/node/filter/index.cjs +32 -0
  21. package/dist/lib/node/filter/index.cjs.map +7 -0
  22. package/dist/lib/node/index.cjs +1381 -525
  23. package/dist/lib/node/index.cjs.map +4 -4
  24. package/dist/lib/node/meta.json +1 -1
  25. package/dist/lib/node/testing/index.cjs +207 -31
  26. package/dist/lib/node/testing/index.cjs.map +4 -4
  27. package/dist/lib/node-esm/{chunk-UKOLB3LW.mjs → chunk-3BZP75TJ.mjs} +15 -21
  28. package/dist/lib/node-esm/chunk-3BZP75TJ.mjs.map +7 -0
  29. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +11 -0
  30. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +7 -0
  31. package/dist/lib/node-esm/chunk-RVK35BS7.mjs +126 -0
  32. package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +7 -0
  33. package/dist/lib/node-esm/filter/index.mjs +11 -0
  34. package/dist/lib/node-esm/filter/index.mjs.map +7 -0
  35. package/dist/lib/node-esm/index.mjs +1380 -516
  36. package/dist/lib/node-esm/index.mjs.map +4 -4
  37. package/dist/lib/node-esm/meta.json +1 -1
  38. package/dist/lib/node-esm/testing/index.mjs +202 -22
  39. package/dist/lib/node-esm/testing/index.mjs.map +4 -4
  40. package/dist/types/src/automerge/automerge-host.d.ts +6 -4
  41. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  42. package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -1
  43. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  44. package/dist/types/src/automerge/echo-data-monitor.d.ts +6 -6
  45. package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
  46. package/dist/types/src/automerge/echo-network-adapter.d.ts +4 -1
  47. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  48. package/dist/types/src/automerge/heads-store.d.ts +2 -2
  49. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  50. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
  51. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  52. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  53. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  54. package/dist/types/src/automerge/network-protocol.d.ts +1 -1
  55. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  56. package/dist/types/src/automerge/space-collection.d.ts +1 -1
  57. package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
  58. package/dist/types/src/common/feeds.d.ts.map +1 -1
  59. package/dist/types/src/common/space-id.d.ts.map +1 -1
  60. package/dist/types/src/db-host/automerge-metrics.d.ts +1 -1
  61. package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
  62. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  63. package/dist/types/src/db-host/database-root.d.ts +7 -7
  64. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  65. package/dist/types/src/db-host/documents-iterator.d.ts +1 -1
  66. package/dist/types/src/db-host/documents-iterator.d.ts.map +1 -1
  67. package/dist/types/src/db-host/documents-synchronizer.d.ts +4 -4
  68. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  69. package/dist/types/src/db-host/echo-host.d.ts +13 -2
  70. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  71. package/dist/types/src/db-host/index.d.ts +0 -1
  72. package/dist/types/src/db-host/index.d.ts.map +1 -1
  73. package/dist/types/src/db-host/query-service.d.ts +2 -0
  74. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  75. package/dist/types/src/db-host/space-state-manager.d.ts +4 -3
  76. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  77. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  78. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
  79. package/dist/types/src/filter/filter-match.d.ts +13 -0
  80. package/dist/types/src/filter/filter-match.d.ts.map +1 -0
  81. package/dist/types/src/filter/filter-match.test.d.ts +2 -0
  82. package/dist/types/src/filter/filter-match.test.d.ts.map +1 -0
  83. package/dist/types/src/filter/index.d.ts +2 -0
  84. package/dist/types/src/filter/index.d.ts.map +1 -0
  85. package/dist/types/src/index.d.ts +1 -0
  86. package/dist/types/src/index.d.ts.map +1 -1
  87. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  88. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  89. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  90. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
  91. package/dist/types/src/query/errors.d.ts +23 -0
  92. package/dist/types/src/query/errors.d.ts.map +1 -0
  93. package/dist/types/src/query/index.d.ts +5 -0
  94. package/dist/types/src/query/index.d.ts.map +1 -0
  95. package/dist/types/src/query/plan.d.ts +132 -0
  96. package/dist/types/src/query/plan.d.ts.map +1 -0
  97. package/dist/types/src/query/query-executor.d.ts +83 -0
  98. package/dist/types/src/query/query-executor.d.ts.map +1 -0
  99. package/dist/types/src/query/query-planner.d.ts +33 -0
  100. package/dist/types/src/query/query-planner.d.ts.map +1 -0
  101. package/dist/types/src/query/query-planner.test.d.ts +2 -0
  102. package/dist/types/src/query/query-planner.test.d.ts.map +1 -0
  103. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  104. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  105. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  106. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  107. package/dist/types/src/space/space.d.ts.map +1 -1
  108. package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
  109. package/dist/types/src/testing/index.d.ts +2 -0
  110. package/dist/types/src/testing/index.d.ts.map +1 -1
  111. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  112. package/dist/types/src/testing/test-data.d.ts +18 -0
  113. package/dist/types/src/testing/test-data.d.ts.map +1 -0
  114. package/dist/types/src/testing/test-network-adapter.d.ts +3 -2
  115. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  116. package/dist/types/src/testing/test-schema.d.ts +39 -0
  117. package/dist/types/src/testing/test-schema.d.ts.map +1 -0
  118. package/dist/types/src/util.d.ts +2 -2
  119. package/dist/types/src/util.d.ts.map +1 -1
  120. package/dist/types/tsconfig.tsbuildinfo +1 -1
  121. package/package.json +43 -34
  122. package/src/automerge/automerge-host.test.ts +7 -7
  123. package/src/automerge/automerge-host.ts +58 -60
  124. package/src/automerge/automerge-repo.test.ts +65 -65
  125. package/src/automerge/collection-synchronizer.test.ts +1 -1
  126. package/src/automerge/collection-synchronizer.ts +11 -10
  127. package/src/automerge/echo-data-monitor.ts +21 -20
  128. package/src/automerge/echo-network-adapter.test.ts +1 -1
  129. package/src/automerge/echo-network-adapter.ts +25 -18
  130. package/src/automerge/heads-store.ts +4 -3
  131. package/src/automerge/leveldb-storage-adapter.ts +1 -1
  132. package/src/automerge/mesh-echo-replicator-connection.ts +6 -5
  133. package/src/automerge/mesh-echo-replicator.ts +2 -2
  134. package/src/automerge/network-protocol.ts +2 -1
  135. package/src/automerge/space-collection.ts +2 -1
  136. package/src/db-host/automerge-metrics.ts +2 -1
  137. package/src/db-host/data-service.ts +4 -3
  138. package/src/db-host/database-root.ts +17 -22
  139. package/src/db-host/documents-iterator.ts +9 -8
  140. package/src/db-host/documents-synchronizer.test.ts +2 -2
  141. package/src/db-host/documents-synchronizer.ts +20 -18
  142. package/src/db-host/echo-host.ts +44 -15
  143. package/src/db-host/index.ts +0 -1
  144. package/src/db-host/query-service.ts +43 -37
  145. package/src/db-host/space-state-manager.ts +14 -4
  146. package/src/edge/echo-edge-replicator.test.ts +3 -3
  147. package/src/edge/echo-edge-replicator.ts +9 -8
  148. package/src/edge/inflight-request-limiter.ts +4 -4
  149. package/src/filter/filter-match.test.ts +101 -0
  150. package/src/filter/filter-match.ts +174 -0
  151. package/src/filter/index.ts +5 -0
  152. package/src/index.ts +1 -0
  153. package/src/metadata/metadata-store.ts +13 -13
  154. package/src/pipeline/pipeline-stress.test.ts +9 -9
  155. package/src/pipeline/pipeline.ts +13 -13
  156. package/src/pipeline/timeframe-clock.ts +5 -5
  157. package/src/query/errors.ts +7 -0
  158. package/src/query/index.ts +8 -0
  159. package/src/query/plan.ts +179 -0
  160. package/src/query/query-executor.ts +648 -0
  161. package/src/query/query-planner.test.ts +613 -0
  162. package/src/query/query-planner.ts +470 -0
  163. package/src/space/admission-discovery-extension.ts +2 -2
  164. package/src/space/control-pipeline.ts +8 -8
  165. package/src/space/space-manager.ts +5 -4
  166. package/src/space/space-protocol.browser.test.ts +1 -0
  167. package/src/space/space-protocol.test.ts +1 -0
  168. package/src/space/space-protocol.ts +4 -4
  169. package/src/space/space.ts +5 -5
  170. package/src/testing/index.ts +2 -0
  171. package/src/testing/test-agent-builder.ts +6 -6
  172. package/src/testing/test-data.ts +127 -0
  173. package/src/testing/test-network-adapter.ts +15 -12
  174. package/src/testing/test-replicator.ts +2 -2
  175. package/src/testing/test-schema.ts +53 -0
  176. package/src/util.ts +7 -3
  177. package/dist/lib/browser/chunk-32WDI3LB.mjs.map +0 -7
  178. package/dist/lib/node/chunk-TC2PRBEU.cjs.map +0 -7
  179. package/dist/lib/node-esm/chunk-UKOLB3LW.mjs.map +0 -7
  180. package/dist/types/src/db-host/query-state.d.ts +0 -41
  181. package/dist/types/src/db-host/query-state.d.ts.map +0 -1
  182. package/src/db-host/query-state.ts +0 -217
@@ -8,15 +8,16 @@ import {
8
8
  type DocHandle,
9
9
  type DocumentId,
10
10
  type Repo,
11
- } from '@dxos/automerge/automerge-repo';
11
+ } from '@automerge/automerge-repo';
12
+
12
13
  import { LifecycleState, Resource, type Context } from '@dxos/context';
13
14
  import { todo } from '@dxos/debug';
14
- import { createIdFromSpaceKey, SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
15
+ import { createIdFromSpaceKey, SpaceDocVersion, type DatabaseDirectory } from '@dxos/echo-protocol';
15
16
  import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
16
17
  import { invariant } from '@dxos/invariant';
17
18
  import { type PublicKey, type SpaceId } from '@dxos/keys';
18
19
  import { type LevelDB } from '@dxos/kv-store';
19
- import { IndexKind, type IndexConfig } from '@dxos/protocols/proto/dxos/echo/indexing';
20
+ import { IndexKind } from '@dxos/protocols/proto/dxos/echo/indexing';
20
21
  import { trace } from '@dxos/tracing';
21
22
 
22
23
  import { DataServiceImpl } from './data-service';
@@ -26,6 +27,7 @@ import { QueryServiceImpl } from './query-service';
26
27
  import { SpaceStateManager } from './space-state-manager';
27
28
  import {
28
29
  AutomergeHost,
30
+ FIND_PARAMS,
29
31
  EchoDataMonitor,
30
32
  deriveCollectionIdFromSpaceId,
31
33
  type LoadDocOptions,
@@ -36,15 +38,30 @@ import {
36
38
  type RootDocumentSpaceKeyProvider,
37
39
  } from '../automerge';
38
40
 
39
- const INDEXER_CONFIG: IndexConfig = {
40
- enabled: true,
41
- indexes: [{ kind: IndexKind.Kind.SCHEMA_MATCH }],
41
+ export interface EchoHostIndexingConfig {
42
+ /**
43
+ * @default true
44
+ */
45
+ fullText: boolean;
46
+
47
+ /**
48
+ * @default false
49
+ */
50
+ vector: boolean;
51
+ }
52
+
53
+ const DEFAULT_INDEXING_CONFIG: EchoHostIndexingConfig = {
54
+ // TODO(dmaretskyi): Disabled by default since embedding generation is expensive.
55
+ fullText: false,
56
+ vector: false,
42
57
  };
43
58
 
44
59
  export type EchoHostParams = {
45
60
  kv: LevelDB;
46
61
  peerIdProvider?: PeerIdProvider;
47
62
  getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
63
+
64
+ indexing?: Partial<EchoHostIndexingConfig>;
48
65
  };
49
66
 
50
67
  /**
@@ -62,13 +79,13 @@ export class EchoHost extends Resource {
62
79
  private readonly _spaceStateManager = new SpaceStateManager();
63
80
  private readonly _echoDataMonitor: EchoDataMonitor;
64
81
 
65
- constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
82
+ constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
66
83
  super();
67
84
 
68
- this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
85
+ const indexingConfig = { ...DEFAULT_INDEXING_CONFIG, ...indexing };
69
86
 
87
+ this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
70
88
  this._echoDataMonitor = new EchoDataMonitor();
71
-
72
89
  this._automergeHost = new AutomergeHost({
73
90
  db: kv,
74
91
  dataMonitor: this._echoDataMonitor,
@@ -84,11 +101,22 @@ export class EchoHost extends Resource {
84
101
  loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
85
102
  indexCooldownTime: process.env.NODE_ENV === 'test' ? 0 : undefined,
86
103
  });
87
- this._indexer.setConfig(INDEXER_CONFIG);
104
+ void this._indexer.setConfig({
105
+ enabled: true,
106
+ indexes: [
107
+ //
108
+ { kind: IndexKind.Kind.SCHEMA_MATCH },
109
+ { kind: IndexKind.Kind.GRAPH },
110
+
111
+ ...(indexingConfig.fullText ? [{ kind: IndexKind.Kind.FULL_TEXT }] : []),
112
+ ...(indexingConfig.vector ? [{ kind: IndexKind.Kind.VECTOR }] : []),
113
+ ],
114
+ });
88
115
 
89
116
  this._queryService = new QueryServiceImpl({
90
117
  automergeHost: this._automergeHost,
91
118
  indexer: this._indexer,
119
+ spaceStateManager: this._spaceStateManager,
92
120
  });
93
121
 
94
122
  this._dataService = new DataServiceImpl({
@@ -179,8 +207,8 @@ export class EchoHost extends Resource {
179
207
  }
180
208
 
181
209
  protected override async _close(ctx: Context): Promise<void> {
182
- await this._spaceStateManager.close();
183
210
  await this._queryService.close(ctx);
211
+ await this._spaceStateManager.close(ctx);
184
212
  await this._indexer.close(ctx);
185
213
  await this._automergeHost.close();
186
214
  }
@@ -188,14 +216,14 @@ export class EchoHost extends Resource {
188
216
  /**
189
217
  * Flush all pending writes to the underlying storage.
190
218
  */
191
- async flush() {
219
+ async flush(): Promise<void> {
192
220
  await this._automergeHost.repo.flush();
193
221
  }
194
222
 
195
223
  /**
196
224
  * Perform any pending index updates.
197
225
  */
198
- async updateIndexes() {
226
+ async updateIndexes(): Promise<void> {
199
227
  await this._indexer.updateIndexes();
200
228
  }
201
229
 
@@ -224,7 +252,7 @@ export class EchoHost extends Resource {
224
252
  invariant(this._lifecycleState === LifecycleState.OPEN);
225
253
  const spaceId = await createIdFromSpaceKey(spaceKey);
226
254
 
227
- const automergeRoot = this._automergeHost.createDoc<SpaceDoc>({
255
+ const automergeRoot = this._automergeHost.createDoc<DatabaseDirectory>({
228
256
  version: SpaceDocVersion.CURRENT,
229
257
  access: { spaceKey: spaceKey.toHex() },
230
258
 
@@ -241,7 +269,8 @@ export class EchoHost extends Resource {
241
269
  // TODO(dmaretskyi): Change to document id.
242
270
  async openSpaceRoot(spaceId: SpaceId, automergeUrl: AutomergeUrl): Promise<DatabaseRoot> {
243
271
  invariant(this._lifecycleState === LifecycleState.OPEN);
244
- const handle = this._automergeHost.repo.find<SpaceDoc>(automergeUrl);
272
+ const handle = await this._automergeHost.repo.find<DatabaseDirectory>(automergeUrl, FIND_PARAMS);
273
+ await handle.whenReady();
245
274
 
246
275
  return this._spaceStateManager.assignRootToSpace(spaceId, handle);
247
276
  }
@@ -6,6 +6,5 @@ export * from './data-service';
6
6
  export * from './documents-synchronizer';
7
7
  export * from './echo-host';
8
8
  export * from './database-root';
9
- export * from './query-state';
10
9
  export * from './query-service';
11
10
  export * from './space-state-manager';
@@ -2,12 +2,15 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ import { getHeads } from '@automerge/automerge';
6
+ import { type DocHandle, type DocumentId } from '@automerge/automerge-repo';
7
+ import { Schema } from 'effect';
8
+
5
9
  import { DeferredTask } from '@dxos/async';
6
- import { getHeads } from '@dxos/automerge/automerge';
7
- import { type DocHandle, type DocumentId } from '@dxos/automerge/automerge-repo';
8
10
  import { Stream } from '@dxos/codec-protobuf/stream';
9
11
  import { Context, Resource } from '@dxos/context';
10
- import { type SpaceDoc } from '@dxos/echo-protocol';
12
+ import { raise } from '@dxos/debug';
13
+ import { DatabaseDirectory, QueryAST } from '@dxos/echo-protocol';
11
14
  import { type IdToHeads, type Indexer, type ObjectSnapshot } from '@dxos/indexing';
12
15
  import { log } from '@dxos/log';
13
16
  import { objectPointerCodec } from '@dxos/protocols';
@@ -20,19 +23,21 @@ import {
20
23
  } from '@dxos/protocols/proto/dxos/echo/query';
21
24
  import { trace } from '@dxos/tracing';
22
25
 
23
- import { QueryState } from './query-state';
24
- import { getSpaceKeyFromDoc, type AutomergeHost } from '../automerge';
26
+ import type { SpaceStateManager } from './space-state-manager';
27
+ import { type AutomergeHost } from '../automerge';
28
+ import { QueryExecutor } from '../query';
25
29
 
26
30
  export type QueryServiceParams = {
27
31
  indexer: Indexer;
28
32
  automergeHost: AutomergeHost;
33
+ spaceStateManager: SpaceStateManager;
29
34
  };
30
35
 
31
36
  /**
32
37
  * Represents an active query (stream and query state connected to that stream).
33
38
  */
34
39
  type ActiveQuery = {
35
- state: QueryState;
40
+ executor: QueryExecutor;
36
41
  sendResults: (results: QueryResult[]) => void;
37
42
  close: () => Promise<void>;
38
43
  };
@@ -44,9 +49,9 @@ export class QueryServiceImpl extends Resource implements QueryService {
44
49
  await Promise.all(
45
50
  Array.from(this._queries).map(async (query) => {
46
51
  try {
47
- const { changed } = await query.state.execQuery();
52
+ const { changed } = await query.executor.execQuery();
48
53
  if (changed) {
49
- query.sendResults(query.state.getResults());
54
+ query.sendResults(query.executor.getResults());
50
55
  }
51
56
  } catch (err) {
52
57
  log.catch(err);
@@ -65,37 +70,39 @@ export class QueryServiceImpl extends Resource implements QueryService {
65
70
  fetch: () => {
66
71
  return Array.from(this._queries).map((query) => {
67
72
  return {
68
- filter: JSON.stringify(query.state.filter),
69
- metrics: query.state.metrics,
73
+ query: JSON.stringify(query.executor.query),
74
+ plan: JSON.stringify(query.executor.plan),
75
+ trace: JSON.stringify(query.executor.trace),
70
76
  };
71
77
  });
72
78
  },
73
79
  });
74
80
  }
75
81
 
76
- override async _open() {
82
+ override async _open(): Promise<void> {
77
83
  this._params.indexer.updated.on(this._ctx, () => this._updateQueries.schedule());
78
84
  }
79
85
 
80
- override async _close() {
86
+ override async _close(): Promise<void> {
81
87
  await Promise.all(Array.from(this._queries).map((query) => query.close()));
82
88
  }
83
89
 
84
90
  async setConfig(config: IndexConfig): Promise<void> {
85
- if (this._params.indexer.initialized) {
86
- log.warn('Indexer already initialized.');
87
- return;
88
- }
89
- this._params.indexer.setConfig(config);
91
+ await this._params.indexer.setConfig(config);
90
92
  }
91
93
 
92
94
  execQuery(request: QueryRequest): Stream<QueryResponse> {
93
95
  return new Stream<QueryResponse>(({ next, close, ctx }) => {
94
- const query: ActiveQuery = {
95
- state: new QueryState({
96
+ const parsedQuery = QueryAST.Query.pipe(Schema.decodeUnknownSync)(JSON.parse(request.query));
97
+
98
+ const queryEntry: ActiveQuery = {
99
+ executor: new QueryExecutor({
96
100
  indexer: this._params.indexer,
97
101
  automergeHost: this._params.automergeHost,
98
- request,
102
+ queryId: request.queryId ?? raise(new Error('query id required')),
103
+ query: parsedQuery,
104
+ reactivity: request.reactivity,
105
+ spaceStateManager: this._params.spaceStateManager,
99
106
  }),
100
107
  sendResults: (results) => {
101
108
  if (ctx.disposed) {
@@ -105,33 +112,32 @@ export class QueryServiceImpl extends Resource implements QueryService {
105
112
  },
106
113
  close: async () => {
107
114
  close();
108
- await query.state.close();
109
- this._queries.delete(query);
115
+ await queryEntry.executor.close();
116
+ this._queries.delete(queryEntry);
110
117
  },
111
118
  };
112
- this._queries.add(query);
119
+ this._queries.add(queryEntry);
113
120
 
114
121
  queueMicrotask(async () => {
115
- await query.state.open();
122
+ await queryEntry.executor.open();
116
123
 
117
124
  try {
118
- const { changed } = await query.state.execQuery();
119
- if (changed) {
120
- query.sendResults(query.state.getResults());
121
- }
122
- } catch (error) {
123
- log.catch(error);
125
+ await queryEntry.executor.execQuery();
126
+ // Always send first result set.
127
+ queryEntry.sendResults(queryEntry.executor.getResults());
128
+ } catch (error: any) {
129
+ close(error);
124
130
  }
125
131
  });
126
132
 
127
- return query.close;
133
+ return queryEntry.close;
128
134
  });
129
135
  }
130
136
 
131
137
  /**
132
138
  * Re-index all loaded documents.
133
139
  */
134
- async reindex() {
140
+ async reindex(): Promise<void> {
135
141
  log.info('Reindexing all documents...');
136
142
  const iterator = createDocumentsIterator(this._params.automergeHost);
137
143
  const ids: IdToHeads = new Map();
@@ -162,13 +168,13 @@ const createDocumentsIterator = (automergeHost: AutomergeHost) =>
162
168
  /** visited automerge handles */
163
169
  const visited = new Set<string>();
164
170
 
165
- async function* getObjectsFromHandle(handle: DocHandle<SpaceDoc>): AsyncGenerator<ObjectSnapshot[]> {
166
- if (visited.has(handle.documentId)) {
171
+ async function* getObjectsFromHandle(handle: DocHandle<DatabaseDirectory>): AsyncGenerator<ObjectSnapshot[]> {
172
+ if (visited.has(handle.documentId) || !handle.isReady()) {
167
173
  return;
168
174
  }
169
- const doc = handle.docSync()!;
175
+ const doc = handle.doc()!;
170
176
 
171
- const spaceKey = getSpaceKeyFromDoc(doc) ?? undefined;
177
+ const spaceKey = DatabaseDirectory.getSpaceKey(doc) ?? undefined;
172
178
 
173
179
  if (doc.objects) {
174
180
  yield Object.entries(doc.objects as { [key: string]: any }).map(([objectId, object]) => {
@@ -186,7 +192,7 @@ const createDocumentsIterator = (automergeHost: AutomergeHost) =>
186
192
  if (visited.has(urlString)) {
187
193
  continue;
188
194
  }
189
- const linkHandle = await automergeHost.loadDoc<SpaceDoc>(Context.default(), urlString as DocumentId);
195
+ const linkHandle = await automergeHost.loadDoc<DatabaseDirectory>(Context.default(), urlString as DocumentId);
190
196
  for await (const result of getObjectsFromHandle(linkHandle)) {
191
197
  yield result;
192
198
  }
@@ -2,12 +2,13 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ import { interpretAsDocumentId, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
5
6
  import isEqual from 'lodash.isequal';
6
7
 
7
8
  import { Event, UpdateScheduler } from '@dxos/async';
8
- import { interpretAsDocumentId, type DocHandle, type DocumentId } from '@dxos/automerge/automerge-repo';
9
- import { Resource, Context } from '@dxos/context';
10
- import { type SpaceDoc } from '@dxos/echo-protocol';
9
+ import { Resource, Context, LifecycleState } from '@dxos/context';
10
+ import { type DatabaseDirectory } from '@dxos/echo-protocol';
11
+ import { invariant } from '@dxos/invariant';
11
12
  import { type SpaceId } from '@dxos/keys';
12
13
 
13
14
  import { DatabaseRoot } from './database-root';
@@ -39,7 +40,16 @@ export class SpaceStateManager extends Resource {
39
40
  return this._rootBySpace.get(spaceId);
40
41
  }
41
42
 
42
- async assignRootToSpace(spaceId: SpaceId, handle: DocHandle<SpaceDoc>): Promise<DatabaseRoot> {
43
+ getRootBySpaceId(spaceId: SpaceId): DatabaseRoot | undefined {
44
+ invariant(this._lifecycleState === LifecycleState.OPEN);
45
+ const documentId = this._rootBySpace.get(spaceId);
46
+ if (!documentId) {
47
+ return undefined;
48
+ }
49
+ return this._roots.get(documentId);
50
+ }
51
+
52
+ async assignRootToSpace(spaceId: SpaceId, handle: DocHandle<DatabaseDirectory>): Promise<DatabaseRoot> {
43
53
  let root: DatabaseRoot;
44
54
  if (this._roots.has(handle.documentId)) {
45
55
  root = this._roots.get(handle.documentId)!;
@@ -2,11 +2,11 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ import { cbor } from '@automerge/automerge-repo';
5
6
  import { getRandomPort } from 'get-port-please';
6
7
  import { describe, expect, onTestFinished, test } from 'vitest';
7
8
 
8
9
  import { Event } from '@dxos/async';
9
- import { cbor } from '@dxos/automerge/automerge-repo';
10
10
  import { createEphemeralEdgeIdentity, EdgeClient, MessageSchema } from '@dxos/edge-client';
11
11
  import { createTestEdgeWsServer } from '@dxos/edge-client/testing';
12
12
  import { PublicKey, SpaceId } from '@dxos/keys';
@@ -33,12 +33,12 @@ describe('EchoEdgeReplicator', () => {
33
33
  await connectionOpen.waitForCount(1);
34
34
 
35
35
  const forbidden = createForbiddenMessage({ identityKey: client.identityKey, peerKey: client.peerKey }, spaceId);
36
- server.sendMessage(forbidden);
36
+ await server.sendMessage(forbidden);
37
37
  await connectionOpen.waitForCount(1);
38
38
 
39
39
  // Double restart to check for race conditions.
40
40
  client.setIdentity(await createEphemeralEdgeIdentity());
41
- server.sendMessage(forbidden);
41
+ await server.sendMessage(forbidden);
42
42
  await connectionOpen.waitForCount(1);
43
43
 
44
44
  await replicator.disconnect();
@@ -2,8 +2,9 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ import { cbor } from '@automerge/automerge-repo';
6
+
5
7
  import { Mutex, scheduleTask, scheduleMicroTask } from '@dxos/async';
6
- import { cbor } from '@dxos/automerge/automerge-repo';
7
8
  import { Context, Resource } from '@dxos/context';
8
9
  import { randomUUID } from '@dxos/crypto';
9
10
  import type { CollectionId } from '@dxos/echo-protocol';
@@ -68,7 +69,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
68
69
  );
69
70
  }
70
71
 
71
- private async _handleReconnect() {
72
+ private async _handleReconnect(): Promise<void> {
72
73
  using _guard = await this._mutex.acquire();
73
74
 
74
75
  const spaces = [...this._connectedSpaces];
@@ -94,7 +95,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
94
95
  this._connections.clear();
95
96
  }
96
97
 
97
- async connectToSpace(spaceId: SpaceId) {
98
+ async connectToSpace(spaceId: SpaceId): Promise<void> {
98
99
  using _guard = await this._mutex.acquire();
99
100
 
100
101
  if (this._connectedSpaces.has(spaceId)) {
@@ -108,7 +109,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
108
109
  }
109
110
  }
110
111
 
111
- async disconnectFromSpace(spaceId: SpaceId) {
112
+ async disconnectFromSpace(spaceId: SpaceId): Promise<void> {
112
113
  using _guard = await this._mutex.acquire();
113
114
 
114
115
  this._connectedSpaces.delete(spaceId);
@@ -120,7 +121,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
120
121
  }
121
122
  }
122
123
 
123
- private async _openConnection(spaceId: SpaceId, reconnects: number = 0) {
124
+ private async _openConnection(spaceId: SpaceId, reconnects: number = 0): Promise<void> {
124
125
  invariant(this._context);
125
126
  invariant(!this._connections.has(spaceId));
126
127
 
@@ -310,7 +311,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
310
311
  return spaceId === this._spaceId && params.collectionId.split(':').length === 3;
311
312
  }
312
313
 
313
- private _onMessage(message: RouterMessage) {
314
+ private _onMessage(message: RouterMessage): void {
314
315
  if (message.serviceId !== this._targetServiceId) {
315
316
  return;
316
317
  }
@@ -327,7 +328,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
327
328
  this._processMessage(payload);
328
329
  }
329
330
 
330
- private _processMessage(message: AutomergeProtocolMessage) {
331
+ private _processMessage(message: AutomergeProtocolMessage): void {
331
332
  // There's a race between the credentials being replicated that are needed for access control and the data replication.
332
333
  // AutomergeReplicator might return a Forbidden error if the credentials are not yet replicated.
333
334
  // We restart the connection with some delay to account for that.
@@ -341,7 +342,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
341
342
  this._readableStreamController.enqueue(message);
342
343
  }
343
344
 
344
- private async _sendMessage(message: AutomergeProtocolMessage) {
345
+ private async _sendMessage(message: AutomergeProtocolMessage): Promise<void> {
345
346
  // Fix the peer id.
346
347
  (message as any).targetId = this._targetServiceId as PeerId;
347
348
 
@@ -26,19 +26,19 @@ export class InflightRequestLimiter extends Resource {
26
26
  super();
27
27
  }
28
28
 
29
- protected override async _open() {
29
+ protected override async _open(): Promise<void> {
30
30
  this._inflightRequestBalance = 0;
31
31
  this._requestBarrier.reset();
32
32
  this._requestBarrier.wake();
33
33
  }
34
34
 
35
- protected override async _close() {
35
+ protected override async _close(): Promise<void> {
36
36
  this._inflightRequestBalance = 0;
37
37
  this._requestBarrier.throw(new Error('Rate limiter closed.'));
38
38
  clearTimeout(this._resetBalanceTimeout);
39
39
  }
40
40
 
41
- public async rateLimit(message: AutomergeProtocolMessage) {
41
+ public async rateLimit(message: AutomergeProtocolMessage): Promise<void> {
42
42
  if (message.type !== 'sync') {
43
43
  return;
44
44
  }
@@ -56,7 +56,7 @@ export class InflightRequestLimiter extends Resource {
56
56
  }
57
57
  }
58
58
 
59
- public handleResponse(message: AutomergeProtocolMessage) {
59
+ public handleResponse(message: AutomergeProtocolMessage): void {
60
60
  if (message.type !== 'sync') {
61
61
  return;
62
62
  }
@@ -0,0 +1,101 @@
1
+ //
2
+ // Copyright 2025 DXOS.org
3
+ //
4
+
5
+ import { describe, expect, test } from 'vitest';
6
+
7
+ import { ObjectStructure } from '@dxos/echo-protocol';
8
+ import { Expando, EXPANDO_TYPENAME, Filter, Ref } from '@dxos/echo-schema';
9
+ import { DXN, ObjectId, SpaceId } from '@dxos/keys';
10
+
11
+ import { filterMatchObject, type MatchedObject } from './filter-match';
12
+
13
+ describe('filterMatch', () => {
14
+ test('properties', () => {
15
+ expect(filterMatchObject(Filter.type(Expando, { title: 'test' }).ast, OBJECT_1)).to.be.true;
16
+ expect(filterMatchObject(Filter.type(Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
17
+ expect(filterMatchObject(Filter.type(Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
18
+ expect(filterMatchObject(Filter.type(Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
19
+ });
20
+
21
+ test('and', () => {
22
+ const filter1 = Filter.type(Expando, { title: 'test' });
23
+ const filter2 = Filter.type(Expando, { value: 100 });
24
+ const filter3 = Filter.type(Expando, { complete: true });
25
+
26
+ expect(filterMatchObject(Filter.and(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
27
+ });
28
+
29
+ test('or', () => {
30
+ const filter1 = Filter.type(Expando, { value: 200 });
31
+ const filter2 = Filter.type(Expando, { title: 'test' });
32
+ const filter3 = Filter.type(Expando, { complete: false });
33
+
34
+ expect(filterMatchObject(Filter.or(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
35
+ });
36
+
37
+ test('not', () => {
38
+ const filter1 = Filter.type(Expando, { title: 'test' });
39
+ const filter2 = Filter.not(filter1);
40
+
41
+ expect(filterMatchObject(filter1.ast, OBJECT_1)).to.be.true;
42
+ expect(filterMatchObject(filter2.ast, OBJECT_1)).to.be.false;
43
+ });
44
+
45
+ test('complex', () => {
46
+ const filter1 = Filter.type(Expando, { title: 'bad' });
47
+ const filter2 = Filter.type(Expando, { value: 0 });
48
+ expect(filterMatchObject(Filter.not(Filter.or(filter1, filter2)).ast, OBJECT_1)).to.be.true;
49
+ });
50
+
51
+ test('ids', () => {
52
+ const filter = Filter.ids(OBJECT_1.id);
53
+ expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.true;
54
+ expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
55
+ });
56
+
57
+ test('everything', () => {
58
+ expect(filterMatchObject(Filter.everything().ast, OBJECT_1)).to.be.true;
59
+ expect(filterMatchObject(Filter.everything().ast, OBJECT_2)).to.be.true;
60
+ });
61
+
62
+ test('nothing', () => {
63
+ expect(filterMatchObject(Filter.nothing().ast, OBJECT_1)).to.be.false;
64
+ expect(filterMatchObject(Filter.nothing().ast, OBJECT_2)).to.be.false;
65
+ });
66
+
67
+ test('refs', () => {
68
+ const filter = Filter.type(Expando, { parent: Ref.fromDXN(DXN.fromLocalObjectId(OBJECT_1.id)) });
69
+ expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.false;
70
+ expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
71
+ expect(filterMatchObject(filter.ast, OBJECT_3)).to.be.true;
72
+ });
73
+ });
74
+
75
+ const OBJECT_1: MatchedObject = {
76
+ id: ObjectId.make('01JVS9YYT5VMVJW0GGTM1YHCCH'),
77
+ spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
78
+ doc: ObjectStructure.makeObject({
79
+ type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
80
+ data: { title: 'test', value: 100, complete: true },
81
+ }),
82
+ };
83
+
84
+ const OBJECT_2: MatchedObject = {
85
+ id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
86
+ spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
87
+ doc: ObjectStructure.makeObject({
88
+ type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
89
+ data: { title: 'test', value: 100, complete: true },
90
+ deleted: true,
91
+ }),
92
+ };
93
+
94
+ const OBJECT_3: MatchedObject = {
95
+ id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
96
+ spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
97
+ doc: ObjectStructure.makeObject({
98
+ type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
99
+ data: { title: 'test', value: 100, complete: true, parent: { '/': DXN.fromLocalObjectId(OBJECT_1.id).toString() } },
100
+ }),
101
+ };