@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/LICENSE +102 -5
  2. package/README.md +1 -1
  3. package/dist/lib/{browser/chunk-WJJXJTNS.mjs → neutral/chunk-46DCRDA7.mjs} +44 -13
  4. package/dist/lib/neutral/chunk-46DCRDA7.mjs.map +7 -0
  5. package/dist/lib/{browser/chunk-FR6GJ4S4.mjs → neutral/chunk-LR7UFVQX.mjs} +145 -618
  6. package/dist/lib/neutral/chunk-LR7UFVQX.mjs.map +7 -0
  7. package/dist/lib/{browser → neutral}/filter/index.mjs +1 -2
  8. package/dist/lib/neutral/index.mjs +7332 -0
  9. package/dist/lib/neutral/index.mjs.map +7 -0
  10. package/dist/lib/neutral/meta.json +1 -0
  11. package/dist/lib/{browser → neutral}/testing/index.mjs +49 -292
  12. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  13. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts +2 -0
  14. package/dist/types/src/automerge/automerge-host-subduction.test.d.ts.map +1 -0
  15. package/dist/types/src/automerge/automerge-host.d.ts +105 -27
  16. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  17. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts +2 -0
  18. package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts.map +1 -0
  19. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  20. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  21. package/dist/types/src/automerge/automerge-subduction.test.d.ts +2 -0
  22. package/dist/types/src/automerge/automerge-subduction.test.d.ts.map +1 -0
  23. package/dist/types/src/automerge/collection-synchronizer.d.ts +22 -2
  24. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  25. package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
  26. package/dist/types/src/automerge/echo-network-adapter.d.ts +17 -11
  27. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  28. package/dist/types/src/automerge/echo-replicator.d.ts +28 -16
  29. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  30. package/dist/types/src/automerge/errors.d.ts +35 -0
  31. package/dist/types/src/automerge/errors.d.ts.map +1 -0
  32. package/dist/types/src/automerge/handle-state.d.ts +52 -0
  33. package/dist/types/src/automerge/handle-state.d.ts.map +1 -0
  34. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  35. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  36. package/dist/types/src/automerge/index.d.ts +2 -0
  37. package/dist/types/src/automerge/index.d.ts.map +1 -1
  38. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +10 -4
  39. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  40. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +8 -8
  41. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  42. package/dist/types/src/automerge/mesh-echo-replicator.d.ts +6 -5
  43. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  44. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  45. package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
  46. package/dist/types/src/common/feeds.d.ts.map +1 -1
  47. package/dist/types/src/common/space-id.d.ts.map +1 -1
  48. package/dist/types/src/db-host/automerge-data-source.d.ts +35 -0
  49. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  50. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  51. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  52. package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
  53. package/dist/types/src/db-host/data-service.d.ts +12 -6
  54. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  55. package/dist/types/src/db-host/database-root.d.ts +9 -3
  56. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  57. package/dist/types/src/db-host/documents-synchronizer.d.ts +30 -8
  58. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  59. package/dist/types/src/db-host/echo-host.d.ts +74 -28
  60. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  61. package/dist/types/src/db-host/index.d.ts +2 -0
  62. package/dist/types/src/db-host/index.d.ts.map +1 -1
  63. package/dist/types/src/db-host/invalidation-hint.d.ts +19 -0
  64. package/dist/types/src/db-host/invalidation-hint.d.ts.map +1 -0
  65. package/dist/types/src/db-host/local-queue-service.d.ts +19 -0
  66. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  67. package/dist/types/src/db-host/query-invalidation.test.d.ts +2 -0
  68. package/dist/types/src/db-host/query-invalidation.test.d.ts.map +1 -0
  69. package/dist/types/src/db-host/query-service.d.ts +18 -10
  70. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  71. package/dist/types/src/db-host/queue-data-source.d.ts +38 -0
  72. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  73. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  74. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  75. package/dist/types/src/db-host/space-state-manager.d.ts +3 -2
  76. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  77. package/dist/types/src/db-host/stub.d.ts +11 -0
  78. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  79. package/dist/types/src/edge/echo-edge-replicator.d.ts +7 -6
  80. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  81. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts +39 -0
  82. package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts.map +1 -0
  83. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts +2 -0
  84. package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts.map +1 -0
  85. package/dist/types/src/edge/index.d.ts +1 -0
  86. package/dist/types/src/edge/index.d.ts.map +1 -1
  87. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
  88. package/dist/types/src/filter/filter-match.d.ts +1 -1
  89. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  90. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  91. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  92. package/dist/types/src/pipeline/pipeline.d.ts +2 -2
  93. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  94. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
  95. package/dist/types/src/query/errors.d.ts +16 -20
  96. package/dist/types/src/query/errors.d.ts.map +1 -1
  97. package/dist/types/src/query/plan.d.ts +59 -8
  98. package/dist/types/src/query/plan.d.ts.map +1 -1
  99. package/dist/types/src/query/query-executor.d.ts +55 -12
  100. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  101. package/dist/types/src/query/query-planner.d.ts +10 -1
  102. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  103. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  104. package/dist/types/src/space/auth.d.ts +3 -3
  105. package/dist/types/src/space/auth.d.ts.map +1 -1
  106. package/dist/types/src/space/control-pipeline.d.ts +4 -3
  107. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  108. package/dist/types/src/space/space-manager.d.ts +7 -7
  109. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  110. package/dist/types/src/space/space-protocol.d.ts +9 -8
  111. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  112. package/dist/types/src/space/space.d.ts +5 -5
  113. package/dist/types/src/space/space.d.ts.map +1 -1
  114. package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
  115. package/dist/types/src/testing/index.d.ts +0 -2
  116. package/dist/types/src/testing/index.d.ts.map +1 -1
  117. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  118. package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -1
  119. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  120. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  121. package/dist/types/src/testing/test-replicator.d.ts +13 -13
  122. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  123. package/dist/types/src/util.d.ts.map +1 -1
  124. package/dist/types/tsconfig.tsbuildinfo +1 -1
  125. package/package.json +56 -54
  126. package/src/automerge/automerge-host-subduction.test.ts +263 -0
  127. package/src/automerge/automerge-host.test.ts +145 -32
  128. package/src/automerge/automerge-host.ts +442 -163
  129. package/src/automerge/automerge-repo-subduction.test.ts +1054 -0
  130. package/src/automerge/automerge-repo.test.ts +284 -77
  131. package/src/automerge/automerge-subduction.test.ts +330 -0
  132. package/src/automerge/collection-synchronizer.test.ts +96 -16
  133. package/src/automerge/collection-synchronizer.ts +125 -26
  134. package/src/automerge/echo-network-adapter.test.ts +2 -1
  135. package/src/automerge/echo-network-adapter.ts +44 -32
  136. package/src/automerge/echo-replicator.ts +30 -16
  137. package/src/automerge/errors.ts +10 -0
  138. package/src/automerge/handle-state.ts +62 -0
  139. package/src/automerge/heads-store.ts +42 -3
  140. package/src/automerge/index.ts +2 -0
  141. package/src/automerge/leveldb-storage-adapter.ts +70 -4
  142. package/src/automerge/mesh-echo-replicator-connection.ts +8 -8
  143. package/src/automerge/mesh-echo-replicator.ts +14 -11
  144. package/src/db-host/automerge-data-source.test.ts +299 -0
  145. package/src/db-host/automerge-data-source.ts +187 -0
  146. package/src/db-host/data-service.ts +27 -10
  147. package/src/db-host/database-root.ts +19 -7
  148. package/src/db-host/documents-synchronizer.test.ts +145 -14
  149. package/src/db-host/documents-synchronizer.ts +158 -59
  150. package/src/db-host/echo-host.ts +376 -90
  151. package/src/db-host/index.ts +2 -0
  152. package/src/db-host/invalidation-hint.ts +50 -0
  153. package/src/db-host/local-queue-service.ts +127 -0
  154. package/src/db-host/query-invalidation.test.ts +270 -0
  155. package/src/db-host/query-service.ts +108 -111
  156. package/src/db-host/queue-data-source.ts +160 -0
  157. package/src/db-host/queue-service.test.ts +145 -0
  158. package/src/db-host/space-state-manager.ts +11 -9
  159. package/src/db-host/stub.ts +33 -0
  160. package/src/edge/echo-edge-replicator.test.ts +9 -9
  161. package/src/edge/echo-edge-replicator.ts +50 -42
  162. package/src/edge/echo-edge-subduction-replicator.test.ts +158 -0
  163. package/src/edge/echo-edge-subduction-replicator.ts +377 -0
  164. package/src/edge/index.ts +1 -0
  165. package/src/filter/filter-match.test.ts +41 -25
  166. package/src/filter/filter-match.ts +75 -20
  167. package/src/metadata/metadata-store.ts +8 -5
  168. package/src/pipeline/pipeline-stress.test.ts +1 -3
  169. package/src/pipeline/pipeline.test.ts +0 -1
  170. package/src/pipeline/pipeline.ts +2 -3
  171. package/src/query/errors.ts +2 -2
  172. package/src/query/plan.ts +73 -9
  173. package/src/query/query-executor.ts +1064 -172
  174. package/src/query/query-planner.test.ts +854 -86
  175. package/src/query/query-planner.ts +444 -38
  176. package/src/space/admission-discovery-extension.ts +1 -1
  177. package/src/space/auth.ts +6 -6
  178. package/src/space/control-pipeline.test.ts +2 -2
  179. package/src/space/control-pipeline.ts +7 -7
  180. package/src/space/space-manager.ts +12 -17
  181. package/src/space/space-protocol.browser.test.ts +13 -12
  182. package/src/space/space-protocol.test.ts +13 -13
  183. package/src/space/space-protocol.ts +16 -15
  184. package/src/space/space.test.ts +2 -2
  185. package/src/space/space.ts +9 -10
  186. package/src/testing/index.ts +0 -2
  187. package/src/testing/test-agent-builder.ts +0 -1
  188. package/src/testing/test-network-adapter.ts +24 -3
  189. package/src/testing/test-replicator.ts +15 -20
  190. package/src/util.ts +7 -3
  191. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  192. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  193. package/dist/lib/browser/chunk-FR6GJ4S4.mjs.map +0 -7
  194. package/dist/lib/browser/chunk-WJJXJTNS.mjs.map +0 -7
  195. package/dist/lib/browser/index.mjs +0 -5492
  196. package/dist/lib/browser/index.mjs.map +0 -7
  197. package/dist/lib/browser/meta.json +0 -1
  198. package/dist/lib/browser/testing/index.mjs.map +0 -7
  199. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  200. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  201. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs +0 -2144
  202. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs.map +0 -7
  203. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs +0 -240
  204. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs.map +0 -7
  205. package/dist/lib/node-esm/filter/index.mjs +0 -13
  206. package/dist/lib/node-esm/filter/index.mjs.map +0 -7
  207. package/dist/lib/node-esm/index.mjs +0 -5492
  208. package/dist/lib/node-esm/index.mjs.map +0 -7
  209. package/dist/lib/node-esm/meta.json +0 -1
  210. package/dist/lib/node-esm/testing/index.mjs +0 -764
  211. package/dist/lib/node-esm/testing/index.mjs.map +0 -7
  212. package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
  213. package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
  214. package/dist/types/src/testing/test-data.d.ts +0 -18
  215. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  216. package/dist/types/src/testing/test-schema.d.ts +0 -37
  217. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  218. package/src/db-host/documents-iterator.ts +0 -74
  219. package/src/testing/test-data.ts +0 -127
  220. package/src/testing/test-schema.ts +0 -53
  221. /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
@@ -2,67 +2,77 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- import {
6
- type AnyDocumentId,
7
- type AutomergeUrl,
8
- type DocHandle,
9
- type DocumentId,
10
- type Repo,
11
- } from '@automerge/automerge-repo';
12
-
13
- import { type Context, LifecycleState, Resource } from '@dxos/context';
5
+ import { type AnyDocumentId, type AutomergeUrl, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
6
+ import * as SqlClient from '@effect/sql/SqlClient';
7
+ import * as Effect from 'effect/Effect';
8
+
9
+ import { asyncTimeout, DeferredTask, sleep } from '@dxos/async';
10
+ import { Context, LifecycleState, Resource } from '@dxos/context';
14
11
  import { todo } from '@dxos/debug';
15
12
  import { type DatabaseDirectory, SpaceDocVersion, createIdFromSpaceKey } from '@dxos/echo-protocol';
16
- import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
13
+ import { RuntimeProvider } from '@dxos/effect';
14
+ import { FeedStore } from '@dxos/feed';
15
+ import { IndexEngine, type IndexingResult } from '@dxos/index-core';
17
16
  import { invariant } from '@dxos/invariant';
18
- import { type PublicKey, type SpaceId } from '@dxos/keys';
17
+ import { type ObjectId, type PublicKey, type SpaceId } from '@dxos/keys';
19
18
  import { type LevelDB } from '@dxos/kv-store';
20
- import { IndexKind } from '@dxos/protocols/proto/dxos/echo/indexing';
19
+ import { log } from '@dxos/log';
20
+ import { type FeedProtocol } from '@dxos/protocols';
21
+ import type { SyncQueueRequest } from '@dxos/protocols/proto/dxos/client/services';
22
+ import type * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
21
23
  import { trace } from '@dxos/tracing';
22
24
 
23
25
  import {
24
26
  AutomergeHost,
27
+ type AutomergeReplicator,
25
28
  type CreateDocOptions,
26
29
  EchoDataMonitor,
27
30
  type EchoDataStats,
28
- type EchoReplicator,
29
- FIND_PARAMS,
30
31
  type LoadDocOptions,
31
32
  type PeerIdProvider,
32
33
  type RootDocumentSpaceKeyProvider,
33
34
  deriveCollectionIdFromSpaceId,
34
35
  } from '../automerge';
35
-
36
+ import { AutomergeDataSource } from './automerge-data-source';
36
37
  import { DataServiceImpl } from './data-service';
37
38
  import { type DatabaseRoot } from './database-root';
38
- import { createSelectedDocumentsIterator } from './documents-iterator';
39
+ import { hintFromIndexingResult } from './invalidation-hint';
40
+ import { LocalQueueServiceImpl } from './local-queue-service';
39
41
  import { QueryServiceImpl } from './query-service';
42
+ import { QueueDataSource } from './queue-data-source';
40
43
  import { SpaceStateManager } from './space-state-manager';
41
44
 
42
- export interface EchoHostIndexingConfig {
43
- /**
44
- * @default true
45
- */
46
- fullText: boolean;
45
+ /**
46
+ * Executes "PRAGMA quick_check;" on SQLite database on startup.
47
+ *
48
+ * NOTE: Keep DISABLED in production, "quick" check is O(dbSize) which took 6 seconds for my relatively-small profile.
49
+ *
50
+ */
51
+ const RUN_SQLITE_QUICK_CHECK_ON_STARTUP = false;
52
+
53
+ export type EchoHostProps = {
54
+ kv: LevelDB;
55
+ peerIdProvider?: PeerIdProvider;
56
+ getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
57
+
58
+ runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
47
59
 
48
60
  /**
61
+ * This peer is allowed to assign positions (global-order) to items appended to the queue.
49
62
  * @default false
50
63
  */
51
- vector: boolean;
52
- }
64
+ assignQueuePositions?: boolean;
53
65
 
54
- const DEFAULT_INDEXING_CONFIG: EchoHostIndexingConfig = {
55
- // TODO(dmaretskyi): Disabled by default since embedding generation is expensive.
56
- fullText: false,
57
- vector: false,
58
- };
59
-
60
- export type EchoHostParams = {
61
- kv: LevelDB;
62
- peerIdProvider?: PeerIdProvider;
63
- getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
66
+ /**
67
+ * Callback to run blocking queue sync.
68
+ */
69
+ syncQueue?: (ctx: Context, request: SyncQueueRequest) => Promise<void>;
64
70
 
65
- indexing?: Partial<EchoHostIndexingConfig>;
71
+ /**
72
+ * Enable Subduction sedimentree transport for Automerge document replication.
73
+ * @default false
74
+ */
75
+ useSubduction?: boolean;
66
76
  };
67
77
 
68
78
  /**
@@ -72,60 +82,71 @@ export type EchoHostParams = {
72
82
  * Can sync with pluggable data replicators.
73
83
  */
74
84
  export class EchoHost extends Resource {
75
- private readonly _indexMetadataStore: IndexMetadataStore;
76
- private readonly _indexer: Indexer;
77
85
  private readonly _automergeHost: AutomergeHost;
78
86
  private readonly _queryService: QueryServiceImpl;
79
87
  private readonly _dataService: DataServiceImpl;
80
88
  private readonly _spaceStateManager = new SpaceStateManager();
81
89
  private readonly _echoDataMonitor: EchoDataMonitor;
82
90
 
83
- constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
84
- super();
91
+ private readonly _automergeDataSource: AutomergeDataSource;
92
+ private readonly _indexEngine: IndexEngine;
93
+ private readonly _runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
94
+ private readonly _feedStore: FeedStore;
95
+ private readonly _queueDataSource: QueueDataSource;
85
96
 
86
- const indexingConfig = { ...DEFAULT_INDEXING_CONFIG, ...indexing };
97
+ private _updateIndexes!: DeferredTask;
98
+
99
+ private _queuesService: FeedProtocol.QueueService;
100
+
101
+ private _indexesUpToDate = false;
102
+
103
+ constructor({
104
+ kv,
105
+ peerIdProvider,
106
+ getSpaceKeyByRootDocumentId,
107
+ runtime,
108
+ assignQueuePositions = false,
109
+ syncQueue,
110
+ useSubduction,
111
+ }: EchoHostProps) {
112
+ super();
87
113
 
88
- this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
89
114
  this._echoDataMonitor = new EchoDataMonitor();
90
115
  this._automergeHost = new AutomergeHost({
91
116
  db: kv,
92
117
  dataMonitor: this._echoDataMonitor,
93
- indexMetadataStore: this._indexMetadataStore,
94
118
  peerIdProvider,
95
119
  getSpaceKeyByRootDocumentId,
120
+ useSubduction,
96
121
  });
97
122
 
98
- this._indexer = new Indexer({
99
- db: kv,
100
- indexStore: new IndexStore({ db: kv.sublevel('index-storage') }),
101
- metadataStore: this._indexMetadataStore,
102
- loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
103
- indexCooldownTime: process.env.NODE_ENV === 'test' ? 0 : undefined,
104
- });
105
- void this._indexer.setConfig({
106
- enabled: true,
107
- indexes: [
108
- //
109
- { kind: IndexKind.Kind.SCHEMA_MATCH },
110
- { kind: IndexKind.Kind.GRAPH },
111
-
112
- ...(indexingConfig.fullText ? [{ kind: IndexKind.Kind.FULL_TEXT }] : []),
113
- ...(indexingConfig.vector ? [{ kind: IndexKind.Kind.VECTOR }] : []),
114
- ],
123
+ this._runtime = runtime;
124
+ this._automergeDataSource = new AutomergeDataSource(this._automergeHost);
125
+
126
+ this._feedStore = new FeedStore({ assignPositions: assignQueuePositions, localActorId: crypto.randomUUID() });
127
+ this._queueDataSource = new QueueDataSource({
128
+ feedStore: this._feedStore,
129
+ runtime: this._runtime,
130
+ getSpaceIds: () => this._spaceStateManager.spaceIds,
115
131
  });
132
+ this._queuesService = new LocalQueueServiceImpl(runtime, this._feedStore, syncQueue);
133
+
134
+ // SQLite-based index engine for all queries.
135
+ this._indexEngine = new IndexEngine();
116
136
 
117
137
  this._queryService = new QueryServiceImpl({
118
138
  automergeHost: this._automergeHost,
119
- indexer: this._indexer,
139
+ indexEngine: this._indexEngine,
140
+ runtime: this._runtime,
120
141
  spaceStateManager: this._spaceStateManager,
121
142
  });
122
143
 
123
144
  this._dataService = new DataServiceImpl({
124
145
  automergeHost: this._automergeHost,
125
146
  spaceStateManager: this._spaceStateManager,
126
- updateIndexes: async () => {
127
- await this._indexer.updateIndexes();
128
- },
147
+ // Delegate to the public method so the closed-host early-out and
148
+ // cooperative loop apply uniformly to the RPC handler path.
149
+ updateIndexes: () => this.updateIndexes(),
129
150
  });
130
151
 
131
152
  trace.diagnostic<EchoStatsDiagnostic>({
@@ -169,6 +190,10 @@ export class EchoHost extends Resource {
169
190
  });
170
191
  }
171
192
 
193
+ get spaceIds(): SpaceId[] {
194
+ return this._spaceStateManager.spaceIds;
195
+ }
196
+
172
197
  get queryService(): QueryServiceImpl {
173
198
  return this._queryService;
174
199
  }
@@ -177,22 +202,56 @@ export class EchoHost extends Resource {
177
202
  return this._dataService;
178
203
  }
179
204
 
180
- /**
181
- * @deprecated To be abstracted away.
182
- */
183
- get automergeRepo(): Repo {
184
- return this._automergeHost.repo;
205
+ get queuesService(): FeedProtocol.QueueService {
206
+ return this._queuesService;
185
207
  }
186
208
 
187
209
  get roots(): ReadonlyMap<DocumentId, DatabaseRoot> {
188
210
  return this._spaceStateManager.roots;
189
211
  }
190
212
 
213
+ get feedStore(): FeedStore {
214
+ return this._feedStore;
215
+ }
216
+
217
+ /**
218
+ * Index engine for queries.
219
+ */
220
+ get indexEngine(): IndexEngine {
221
+ return this._indexEngine;
222
+ }
223
+
191
224
  protected override async _open(ctx: Context): Promise<void> {
192
- await this._automergeHost.open();
193
- await this._indexer.open(ctx);
225
+ log('echo-host: opening automerge host...');
226
+ await this._automergeHost.open(ctx);
227
+ log('echo-host: automerge host opened');
228
+
229
+ log('echo-host: opening query service...');
194
230
  await this._queryService.open(ctx);
231
+ log('echo-host: query service opened');
232
+
233
+ log('echo-host: opening space state manager...');
195
234
  await this._spaceStateManager.open(ctx);
235
+ log('echo-host: space state manager opened');
236
+
237
+ // Timeout needs to be outside effect runtime to catch the layer hanging on startup.
238
+ await asyncTimeout(
239
+ RuntimeProvider.runPromise(this._runtime)(testSqlite()),
240
+ 15_000,
241
+ new Error('SQLite quick check timed out'),
242
+ );
243
+
244
+ log('echo-host: running index engine migration...');
245
+ await RuntimeProvider.runPromise(this._runtime)(this._indexEngine.migrate());
246
+ log('echo-host: index engine migration done');
247
+ this._updateIndexes = new DeferredTask(this._ctx, this._runUpdateIndexes);
248
+
249
+ log('echo-host: running feed store migration...');
250
+ await RuntimeProvider.runPromise(this._runtime)(this._feedStore.migrate());
251
+ log('echo-host: feed store migration done');
252
+ this._feedStore.onNewBlocks.on(this._ctx, () => {
253
+ this._updateIndexes.schedule();
254
+ });
196
255
 
197
256
  this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
198
257
  if (e.previousRootId) {
@@ -204,57 +263,88 @@ export class EchoHost extends Resource {
204
263
  );
205
264
  });
206
265
  this._automergeHost.documentsSaved.on(this._ctx, () => {
207
- this._queryService.invalidateQueries();
266
+ this._updateIndexes.schedule();
208
267
  });
268
+ this._updateIndexes.schedule();
269
+ log('echo-host: open complete');
209
270
  }
210
271
 
211
272
  protected override async _close(ctx: Context): Promise<void> {
273
+ // Drain any in-flight indexer task before the Resource base disposes
274
+ // `this._ctx`. Without this, an in-flight `DataServiceImpl.updateIndexes`
275
+ // RPC handler's `do { await runBlocking() } while (!_indexesUpToDate)`
276
+ // loop can hit a disposed ctx on its next iteration and throw
277
+ // `ContextDisposedError` — which escapes as an unhandled rejection
278
+ // because the originating client `flush()` is fire-and-forget at the
279
+ // test layer. The cooperative `_indexesUpToDate = true` set inside
280
+ // `_runUpdateIndexes` lets the loop exit cleanly once the current
281
+ // iteration finishes.
282
+ await this._updateIndexes?.join();
283
+
212
284
  await this._queryService.close(ctx);
213
285
  await this._spaceStateManager.close(ctx);
214
- await this._indexer.close(ctx);
215
286
  await this._automergeHost.close();
216
287
  }
217
288
 
218
289
  /**
219
290
  * Flush all pending writes to the underlying storage.
220
291
  */
221
- async flush(): Promise<void> {
222
- await this._automergeHost.flush();
292
+ async flush(ctx: Context): Promise<void> {
293
+ await this._automergeHost.flush(ctx);
223
294
  }
224
295
 
225
296
  /**
226
297
  * Perform any pending index updates.
298
+ *
299
+ * Bails as a no-op when the host has been closed: a late `db.flush()` RPC
300
+ * (client still has an open service ref while the host is in/post-teardown)
301
+ * has nothing to update against. The pre-loop and post-iteration
302
+ * `_ctx.disposed` checks prevent `runBlocking` from being entered against a
303
+ * disposed context — which would throw `ContextDisposedError` and escape as
304
+ * an unhandled rejection at the fire-and-forget originating caller. Other
305
+ * `Resource` methods in this codebase (e.g. `LevelDBStorageAdapter.load`)
306
+ * follow the same closed-host early-out pattern.
227
307
  */
228
308
  async updateIndexes(): Promise<void> {
229
- await this._indexer.updateIndexes();
309
+ if (this._ctx.disposed) {
310
+ return;
311
+ }
312
+ do {
313
+ await this._updateIndexes.runBlocking();
314
+ if (this._ctx.disposed) {
315
+ return;
316
+ }
317
+ } while (!this._indexesUpToDate);
230
318
  }
231
319
 
232
320
  /**
233
321
  * Loads the document handle from the repo and waits for it to be ready.
322
+ *
323
+ * @returns `null` when the document is not available yet (e.g. storage-only load with no local chunks).
234
324
  */
235
- async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T>> {
325
+ async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T> | null> {
236
326
  return await this._automergeHost.loadDoc(ctx, documentId, opts);
237
327
  }
238
328
 
239
- async exportDoc(ctx: Context, id: AnyDocumentId): Promise<Uint8Array> {
240
- return await this._automergeHost.exportDoc(ctx, id);
329
+ async exportDoc(id: AnyDocumentId): Promise<Uint8Array> {
330
+ return await this._automergeHost.exportDoc(id);
241
331
  }
242
332
 
243
333
  /**
244
334
  * Create new persisted document.
245
335
  */
246
- createDoc<T>(initialValue?: T, opts?: CreateDocOptions): DocHandle<T> {
336
+ async createDoc<T>(initialValue?: T, opts?: CreateDocOptions): Promise<DocHandle<T>> {
247
337
  return this._automergeHost.createDoc(initialValue, opts);
248
338
  }
249
339
 
250
340
  /**
251
341
  * Create new space root.
252
342
  */
253
- async createSpaceRoot(spaceKey: PublicKey): Promise<DatabaseRoot> {
343
+ async createSpaceRoot(ctx: Context, spaceKey: PublicKey): Promise<DatabaseRoot> {
254
344
  invariant(this._lifecycleState === LifecycleState.OPEN);
255
345
  const spaceId = await createIdFromSpaceKey(spaceKey);
256
346
 
257
- const automergeRoot = this._automergeHost.createDoc<DatabaseDirectory>({
347
+ const automergeRoot = await this._automergeHost.createDoc<DatabaseDirectory>({
258
348
  version: SpaceDocVersion.CURRENT,
259
349
  access: { spaceKey: spaceKey.toHex() },
260
350
 
@@ -263,43 +353,239 @@ export class EchoHost extends Resource {
263
353
  links: {},
264
354
  });
265
355
 
266
- await this._automergeHost.flush({ documentIds: [automergeRoot.documentId] });
356
+ await this._automergeHost.flush(ctx, { documentIds: [automergeRoot.documentId] });
267
357
 
268
- return await this.openSpaceRoot(spaceId, automergeRoot.url);
358
+ return await this.openSpaceRoot(ctx, spaceId, automergeRoot.url);
269
359
  }
270
360
 
271
361
  // TODO(dmaretskyi): Change to document id.
272
- async openSpaceRoot(spaceId: SpaceId, automergeUrl: AutomergeUrl): Promise<DatabaseRoot> {
362
+ async openSpaceRoot(ctx: Context, spaceId: SpaceId, automergeUrl: AutomergeUrl): Promise<DatabaseRoot> {
273
363
  invariant(this._lifecycleState === LifecycleState.OPEN);
274
- const handle = await this._automergeHost.repo.find<DatabaseDirectory>(automergeUrl, FIND_PARAMS);
275
- await handle.whenReady();
364
+ const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(ctx, automergeUrl, {
365
+ fetchFromNetwork: true,
366
+ });
367
+ invariant(handle, 'Space root document must load before assignment.');
368
+ const query = this._automergeHost.findWithProgress<DatabaseDirectory>(handle.documentId);
276
369
 
277
- return this._spaceStateManager.assignRootToSpace(spaceId, handle);
370
+ return this._spaceStateManager.assignRootToSpace(spaceId, query);
278
371
  }
279
372
 
280
373
  // TODO(dmaretskyi): Change to document id.
281
- async closeSpaceRoot(automergeUrl: AutomergeUrl): Promise<void> {
374
+ async closeSpaceRoot(_automergeUrl: AutomergeUrl): Promise<void> {
282
375
  todo();
283
376
  }
284
377
 
285
378
  /**
286
379
  * Install data replicator.
287
380
  */
288
- async addReplicator(replicator: EchoReplicator): Promise<void> {
289
- await this._automergeHost.addReplicator(replicator);
381
+ async addReplicator(ctx: Context, replicator: AutomergeReplicator): Promise<void> {
382
+ await this._automergeHost.addReplicator(ctx, replicator);
290
383
  }
291
384
 
292
385
  /**
293
386
  * Remove data replicator.
294
387
  */
295
- async removeReplicator(replicator: EchoReplicator): Promise<void> {
388
+ async removeReplicator(replicator: AutomergeReplicator): Promise<void> {
296
389
  await this._automergeHost.removeReplicator(replicator);
297
390
  }
391
+
392
+ /**
393
+ * Run collection sync for the given space.
394
+ * Does not wait for the sync to complete.
395
+ */
396
+ async runCollectionSync(spaceId: SpaceId) {
397
+ const root = this._spaceStateManager.getRootBySpaceId(spaceId);
398
+ if (!root) {
399
+ throw new Error(`Space not found: ${spaceId}`);
400
+ }
401
+ this._automergeHost.refreshCollection(deriveCollectionIdFromSpaceId(spaceId, root.documentId));
402
+ }
403
+
404
+ /**
405
+ * Get all feeds and their blocks for a space.
406
+ * Used for space archive export.
407
+ */
408
+ async getAllFeedsForSpace(
409
+ spaceId: SpaceId,
410
+ ): Promise<Array<{ feedId: string; feedNamespace: string; blocks: FeedProtocol.Block[] }>> {
411
+ return RuntimeProvider.runPromise(this._runtime)(this._feedStore.getAllFeedsForSpace({ spaceId }));
412
+ }
413
+
414
+ private _runUpdateIndexes = async (): Promise<void> => {
415
+ if (this._ctx.disposed || !this.isOpen) {
416
+ // Signal the `updateIndexes` RPC handler's `do-while` loop to exit
417
+ // cooperatively. Without this, the loop sees `_indexesUpToDate === false`
418
+ // and calls `runBlocking` again, which throws on the disposed context.
419
+ this._indexesUpToDate = true;
420
+ return;
421
+ }
422
+
423
+ try {
424
+ const combinedResult = _makeEmptyMergedResult();
425
+
426
+ {
427
+ performance.mark('indexEngine.update.automerge:start');
428
+ const result = await this._indexEngine
429
+ .update(this._ctx, this._automergeDataSource, { spaceId: null, limit: 50 })
430
+ .pipe(RuntimeProvider.runPromise(this._runtime));
431
+ _mergeInto(combinedResult, result);
432
+ performance.measure('Index Automerge', {
433
+ start: 'indexEngine.update.automerge:start',
434
+ detail: {
435
+ devtools: {
436
+ dataType: 'track-entry',
437
+ track: 'Indexing',
438
+ trackGroup: 'ECHO', // Group related tracks together
439
+ color: 'tertiary-dark',
440
+ properties: [['count', result.updated]],
441
+ },
442
+ },
443
+ });
444
+ }
445
+ if (this._ctx.disposed || !this.isOpen) {
446
+ this._indexesUpToDate = true;
447
+ return;
448
+ }
449
+
450
+ {
451
+ performance.mark('indexEngine.update.queue:start');
452
+ const result = await this._indexEngine
453
+ .update(this._ctx, this._queueDataSource, { spaceId: null, limit: 50 })
454
+ .pipe(RuntimeProvider.runPromise(this._runtime));
455
+ _mergeInto(combinedResult, result);
456
+ performance.measure('Index Queues', {
457
+ start: 'indexEngine.update.queue:start',
458
+ detail: {
459
+ devtools: {
460
+ dataType: 'track-entry',
461
+ track: 'Indexing',
462
+ trackGroup: 'ECHO',
463
+ color: 'tertiary-dark',
464
+ properties: [['count', result.updated]],
465
+ },
466
+ },
467
+ });
468
+ }
469
+
470
+ log.verbose('indexEngine update completed', {
471
+ updated: combinedResult.updated,
472
+ done: combinedResult.done,
473
+ spaces: combinedResult.spaces.size,
474
+ queues: combinedResult.queues.size,
475
+ documents: combinedResult.documents.size,
476
+ types: combinedResult.types.size,
477
+ objects: combinedResult.objects.size,
478
+ });
479
+ await sleep(1);
480
+ if (!combinedResult.done) {
481
+ this._indexesUpToDate = false;
482
+ this._updateIndexes!.schedule();
483
+ } else {
484
+ this._indexesUpToDate = true;
485
+ }
486
+
487
+ // Invalidate queries after index update — the indexer is the sole invalidation source.
488
+ const hint = hintFromIndexingResult(combinedResult);
489
+ if (hint) {
490
+ this._queryService.invalidateQueries(hint);
491
+ }
492
+ } catch (err) {
493
+ if (this._ctx.disposed || !this.isOpen) {
494
+ this._indexesUpToDate = true;
495
+ return;
496
+ }
497
+ log.catch(err);
498
+ // Failsafe: prevent queries from freezing if the indexer faults.
499
+ this._queryService.invalidateQueries();
500
+ throw err;
501
+ }
502
+ };
298
503
  }
299
504
 
300
505
  export type { EchoDataStats };
301
506
 
507
+ type MutableIndexingAccumulator = {
508
+ updated: number;
509
+ done: boolean;
510
+ spaces: Set<SpaceId>;
511
+ queues: Set<ObjectId>;
512
+ documents: Set<string>;
513
+ types: Set<string>;
514
+ objects: Set<ObjectId>;
515
+ };
516
+
517
+ const _makeEmptyMergedResult = (): MutableIndexingAccumulator => ({
518
+ updated: 0,
519
+ done: true,
520
+ spaces: new Set(),
521
+ queues: new Set(),
522
+ documents: new Set(),
523
+ types: new Set(),
524
+ objects: new Set(),
525
+ });
526
+
527
+ const _mergeInto = (acc: MutableIndexingAccumulator, r: IndexingResult): void => {
528
+ acc.updated += r.updated;
529
+ acc.done = acc.done && r.done;
530
+ for (const s of r.spaces) {
531
+ acc.spaces.add(s);
532
+ }
533
+ for (const q of r.queues) {
534
+ acc.queues.add(q);
535
+ }
536
+ for (const d of r.documents) {
537
+ acc.documents.add(d);
538
+ }
539
+ for (const t of r.types) {
540
+ acc.types.add(t);
541
+ }
542
+ for (const o of r.objects) {
543
+ acc.objects.add(o);
544
+ }
545
+ };
546
+
302
547
  export type EchoStatsDiagnostic = {
303
548
  loadedDocsCount: number;
304
549
  dataStats: EchoDataStats;
305
550
  };
551
+
552
+ /**
553
+ * Sqlite health check on startup.
554
+ *
555
+ * Always runs a cheap schema-level probe (`PRAGMA schema_version` + a read from
556
+ * `sqlite_schema`). These do not walk user data, but they fail fast if the DB
557
+ * file header or schema root page is unreadable.
558
+ *
559
+ * Additionally runs `PRAGMA quick_check` when `RUN_SQLITE_QUICK_CHECK_ON_STARTUP`
560
+ * is enabled. That walks every page and is O(dbSize) — slow on OPFS.
561
+ */
562
+ const testSqlite = () =>
563
+ Effect.gen(function* () {
564
+ log('begin sqlite health check');
565
+ const sql = yield* SqlClient.SqlClient;
566
+ const databases = yield* sql<{ seq: number; name: string; file: string }>`PRAGMA database_list`;
567
+ log('SQLite databases', { databases });
568
+
569
+ // Cheap header/schema probe: forces SQLite to read the file header and
570
+ // schema root page. Catches catastrophic corruption (bad header,
571
+ // unreadable schema) without walking user data.
572
+ const [schemaVersion] = yield* sql<{ schema_version: number }>`PRAGMA schema_version`;
573
+ const [schemaCount] = yield* sql<{ n: number }>`SELECT count(*) AS n FROM sqlite_schema`;
574
+ log('sqlite schema probe passed', {
575
+ schemaVersion: schemaVersion.schema_version,
576
+ objectCount: schemaCount.n,
577
+ });
578
+
579
+ if (RUN_SQLITE_QUICK_CHECK_ON_STARTUP) {
580
+ // NOTE: This is slow on non-trivial databases: O(dbSize).
581
+ log('starting sqlite quick_check');
582
+ const [result] = yield* sql<{ quick_check: string }>`PRAGMA quick_check`;
583
+ if (result.quick_check !== 'ok') {
584
+ throw new Error('SQLite quick check failed');
585
+ }
586
+ log('sqlite quick_check passed');
587
+ } else {
588
+ log('sqlite quick_check skipped');
589
+ }
590
+ log('sqlite health check complete');
591
+ });
@@ -2,9 +2,11 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
+ export * from './automerge-data-source';
5
6
  export * from './data-service';
6
7
  export * from './documents-synchronizer';
7
8
  export * from './echo-host';
8
9
  export * from './database-root';
10
+ export * from './invalidation-hint';
9
11
  export * from './query-service';
10
12
  export * from './space-state-manager';