@dxos/echo-pipeline 0.8.2-staging.7ac8446 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-32WDI3LB.mjs → chunk-3XSXS5EX.mjs} +15 -21
- package/dist/lib/browser/chunk-3XSXS5EX.mjs.map +7 -0
- package/dist/lib/browser/chunk-CGS2ULMK.mjs +11 -0
- package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +7 -0
- package/dist/lib/browser/chunk-TQJTKNMS.mjs +126 -0
- package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +7 -0
- package/dist/lib/browser/filter/index.mjs +11 -0
- package/dist/lib/browser/filter/index.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1380 -516
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +202 -22
- package/dist/lib/browser/testing/index.mjs.map +4 -4
- package/dist/lib/node/chunk-HOPOFWAL.cjs +147 -0
- package/dist/lib/node/chunk-HOPOFWAL.cjs.map +7 -0
- package/dist/lib/node/chunk-Q7SFCCGT.cjs +33 -0
- package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +7 -0
- package/dist/lib/node/{chunk-TC2PRBEU.cjs → chunk-SG2PL5RH.cjs} +18 -24
- package/dist/lib/node/chunk-SG2PL5RH.cjs.map +7 -0
- package/dist/lib/node/filter/index.cjs +32 -0
- package/dist/lib/node/filter/index.cjs.map +7 -0
- package/dist/lib/node/index.cjs +1381 -525
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +207 -31
- package/dist/lib/node/testing/index.cjs.map +4 -4
- package/dist/lib/node-esm/{chunk-UKOLB3LW.mjs → chunk-3BZP75TJ.mjs} +15 -21
- package/dist/lib/node-esm/chunk-3BZP75TJ.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +11 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-RVK35BS7.mjs +126 -0
- package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +7 -0
- package/dist/lib/node-esm/filter/index.mjs +11 -0
- package/dist/lib/node-esm/filter/index.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +1380 -516
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +202 -22
- package/dist/lib/node-esm/testing/index.mjs.map +4 -4
- package/dist/types/src/automerge/automerge-host.d.ts +6 -4
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-data-monitor.d.ts +6 -6
- package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +4 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/heads-store.d.ts +2 -2
- package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/network-protocol.d.ts +1 -1
- package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
- package/dist/types/src/automerge/space-collection.d.ts +1 -1
- package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
- package/dist/types/src/common/feeds.d.ts.map +1 -1
- package/dist/types/src/common/space-id.d.ts.map +1 -1
- package/dist/types/src/db-host/automerge-metrics.d.ts +1 -1
- package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-root.d.ts +7 -7
- package/dist/types/src/db-host/database-root.d.ts.map +1 -1
- package/dist/types/src/db-host/documents-iterator.d.ts +1 -1
- package/dist/types/src/db-host/documents-iterator.d.ts.map +1 -1
- package/dist/types/src/db-host/documents-synchronizer.d.ts +4 -4
- package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
- package/dist/types/src/db-host/echo-host.d.ts +13 -2
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
- package/dist/types/src/db-host/index.d.ts +0 -1
- package/dist/types/src/db-host/index.d.ts.map +1 -1
- package/dist/types/src/db-host/query-service.d.ts +2 -0
- package/dist/types/src/db-host/query-service.d.ts.map +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts +4 -3
- package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
- package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
- package/dist/types/src/filter/filter-match.d.ts +13 -0
- package/dist/types/src/filter/filter-match.d.ts.map +1 -0
- package/dist/types/src/filter/filter-match.test.d.ts +2 -0
- package/dist/types/src/filter/filter-match.test.d.ts.map +1 -0
- package/dist/types/src/filter/index.d.ts +2 -0
- package/dist/types/src/filter/index.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +1 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
- package/dist/types/src/query/errors.d.ts +23 -0
- package/dist/types/src/query/errors.d.ts.map +1 -0
- package/dist/types/src/query/index.d.ts +5 -0
- package/dist/types/src/query/index.d.ts.map +1 -0
- package/dist/types/src/query/plan.d.ts +132 -0
- package/dist/types/src/query/plan.d.ts.map +1 -0
- package/dist/types/src/query/query-executor.d.ts +83 -0
- package/dist/types/src/query/query-executor.d.ts.map +1 -0
- package/dist/types/src/query/query-planner.d.ts +33 -0
- package/dist/types/src/query/query-planner.d.ts.map +1 -0
- package/dist/types/src/query/query-planner.test.d.ts +2 -0
- package/dist/types/src/query/query-planner.test.d.ts.map +1 -0
- package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
- package/dist/types/src/testing/index.d.ts +2 -0
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/test-data.d.ts +18 -0
- package/dist/types/src/testing/test-data.d.ts.map +1 -0
- package/dist/types/src/testing/test-network-adapter.d.ts +3 -2
- package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
- package/dist/types/src/testing/test-schema.d.ts +39 -0
- package/dist/types/src/testing/test-schema.d.ts.map +1 -0
- package/dist/types/src/util.d.ts +2 -2
- package/dist/types/src/util.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +43 -34
- package/src/automerge/automerge-host.test.ts +7 -7
- package/src/automerge/automerge-host.ts +58 -60
- package/src/automerge/automerge-repo.test.ts +65 -65
- package/src/automerge/collection-synchronizer.test.ts +1 -1
- package/src/automerge/collection-synchronizer.ts +11 -10
- package/src/automerge/echo-data-monitor.ts +21 -20
- package/src/automerge/echo-network-adapter.test.ts +1 -1
- package/src/automerge/echo-network-adapter.ts +25 -18
- package/src/automerge/heads-store.ts +4 -3
- package/src/automerge/leveldb-storage-adapter.ts +1 -1
- package/src/automerge/mesh-echo-replicator-connection.ts +6 -5
- package/src/automerge/mesh-echo-replicator.ts +2 -2
- package/src/automerge/network-protocol.ts +2 -1
- package/src/automerge/space-collection.ts +2 -1
- package/src/db-host/automerge-metrics.ts +2 -1
- package/src/db-host/data-service.ts +4 -3
- package/src/db-host/database-root.ts +17 -22
- package/src/db-host/documents-iterator.ts +9 -8
- package/src/db-host/documents-synchronizer.test.ts +2 -2
- package/src/db-host/documents-synchronizer.ts +20 -18
- package/src/db-host/echo-host.ts +44 -15
- package/src/db-host/index.ts +0 -1
- package/src/db-host/query-service.ts +43 -37
- package/src/db-host/space-state-manager.ts +14 -4
- package/src/edge/echo-edge-replicator.test.ts +3 -3
- package/src/edge/echo-edge-replicator.ts +9 -8
- package/src/edge/inflight-request-limiter.ts +4 -4
- package/src/filter/filter-match.test.ts +101 -0
- package/src/filter/filter-match.ts +174 -0
- package/src/filter/index.ts +5 -0
- package/src/index.ts +1 -0
- package/src/metadata/metadata-store.ts +13 -13
- package/src/pipeline/pipeline-stress.test.ts +9 -9
- package/src/pipeline/pipeline.ts +13 -13
- package/src/pipeline/timeframe-clock.ts +5 -5
- package/src/query/errors.ts +7 -0
- package/src/query/index.ts +8 -0
- package/src/query/plan.ts +179 -0
- package/src/query/query-executor.ts +648 -0
- package/src/query/query-planner.test.ts +613 -0
- package/src/query/query-planner.ts +470 -0
- package/src/space/admission-discovery-extension.ts +2 -2
- package/src/space/control-pipeline.ts +8 -8
- package/src/space/space-manager.ts +5 -4
- package/src/space/space-protocol.browser.test.ts +1 -0
- package/src/space/space-protocol.test.ts +1 -0
- package/src/space/space-protocol.ts +4 -4
- package/src/space/space.ts +5 -5
- package/src/testing/index.ts +2 -0
- package/src/testing/test-agent-builder.ts +6 -6
- package/src/testing/test-data.ts +127 -0
- package/src/testing/test-network-adapter.ts +15 -12
- package/src/testing/test-replicator.ts +2 -2
- package/src/testing/test-schema.ts +53 -0
- package/src/util.ts +7 -3
- package/dist/lib/browser/chunk-32WDI3LB.mjs.map +0 -7
- package/dist/lib/node/chunk-TC2PRBEU.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-UKOLB3LW.mjs.map +0 -7
- package/dist/types/src/db-host/query-state.d.ts +0 -41
- package/dist/types/src/db-host/query-state.d.ts.map +0 -1
- package/src/db-host/query-state.ts +0 -217
package/src/db-host/echo-host.ts
CHANGED
|
@@ -8,15 +8,16 @@ import {
|
|
|
8
8
|
type DocHandle,
|
|
9
9
|
type DocumentId,
|
|
10
10
|
type Repo,
|
|
11
|
-
} from '@
|
|
11
|
+
} from '@automerge/automerge-repo';
|
|
12
|
+
|
|
12
13
|
import { LifecycleState, Resource, type Context } from '@dxos/context';
|
|
13
14
|
import { todo } from '@dxos/debug';
|
|
14
|
-
import { createIdFromSpaceKey, SpaceDocVersion, type
|
|
15
|
+
import { createIdFromSpaceKey, SpaceDocVersion, type DatabaseDirectory } from '@dxos/echo-protocol';
|
|
15
16
|
import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
|
|
16
17
|
import { invariant } from '@dxos/invariant';
|
|
17
18
|
import { type PublicKey, type SpaceId } from '@dxos/keys';
|
|
18
19
|
import { type LevelDB } from '@dxos/kv-store';
|
|
19
|
-
import { IndexKind
|
|
20
|
+
import { IndexKind } from '@dxos/protocols/proto/dxos/echo/indexing';
|
|
20
21
|
import { trace } from '@dxos/tracing';
|
|
21
22
|
|
|
22
23
|
import { DataServiceImpl } from './data-service';
|
|
@@ -26,6 +27,7 @@ import { QueryServiceImpl } from './query-service';
|
|
|
26
27
|
import { SpaceStateManager } from './space-state-manager';
|
|
27
28
|
import {
|
|
28
29
|
AutomergeHost,
|
|
30
|
+
FIND_PARAMS,
|
|
29
31
|
EchoDataMonitor,
|
|
30
32
|
deriveCollectionIdFromSpaceId,
|
|
31
33
|
type LoadDocOptions,
|
|
@@ -36,15 +38,30 @@ import {
|
|
|
36
38
|
type RootDocumentSpaceKeyProvider,
|
|
37
39
|
} from '../automerge';
|
|
38
40
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
41
|
+
export interface EchoHostIndexingConfig {
|
|
42
|
+
/**
|
|
43
|
+
* @default true
|
|
44
|
+
*/
|
|
45
|
+
fullText: boolean;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @default false
|
|
49
|
+
*/
|
|
50
|
+
vector: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const DEFAULT_INDEXING_CONFIG: EchoHostIndexingConfig = {
|
|
54
|
+
// TODO(dmaretskyi): Disabled by default since embedding generation is expensive.
|
|
55
|
+
fullText: false,
|
|
56
|
+
vector: false,
|
|
42
57
|
};
|
|
43
58
|
|
|
44
59
|
export type EchoHostParams = {
|
|
45
60
|
kv: LevelDB;
|
|
46
61
|
peerIdProvider?: PeerIdProvider;
|
|
47
62
|
getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
|
|
63
|
+
|
|
64
|
+
indexing?: Partial<EchoHostIndexingConfig>;
|
|
48
65
|
};
|
|
49
66
|
|
|
50
67
|
/**
|
|
@@ -62,13 +79,13 @@ export class EchoHost extends Resource {
|
|
|
62
79
|
private readonly _spaceStateManager = new SpaceStateManager();
|
|
63
80
|
private readonly _echoDataMonitor: EchoDataMonitor;
|
|
64
81
|
|
|
65
|
-
constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
|
|
82
|
+
constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
|
|
66
83
|
super();
|
|
67
84
|
|
|
68
|
-
|
|
85
|
+
const indexingConfig = { ...DEFAULT_INDEXING_CONFIG, ...indexing };
|
|
69
86
|
|
|
87
|
+
this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
|
|
70
88
|
this._echoDataMonitor = new EchoDataMonitor();
|
|
71
|
-
|
|
72
89
|
this._automergeHost = new AutomergeHost({
|
|
73
90
|
db: kv,
|
|
74
91
|
dataMonitor: this._echoDataMonitor,
|
|
@@ -84,11 +101,22 @@ export class EchoHost extends Resource {
|
|
|
84
101
|
loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
|
|
85
102
|
indexCooldownTime: process.env.NODE_ENV === 'test' ? 0 : undefined,
|
|
86
103
|
});
|
|
87
|
-
this._indexer.setConfig(
|
|
104
|
+
void this._indexer.setConfig({
|
|
105
|
+
enabled: true,
|
|
106
|
+
indexes: [
|
|
107
|
+
//
|
|
108
|
+
{ kind: IndexKind.Kind.SCHEMA_MATCH },
|
|
109
|
+
{ kind: IndexKind.Kind.GRAPH },
|
|
110
|
+
|
|
111
|
+
...(indexingConfig.fullText ? [{ kind: IndexKind.Kind.FULL_TEXT }] : []),
|
|
112
|
+
...(indexingConfig.vector ? [{ kind: IndexKind.Kind.VECTOR }] : []),
|
|
113
|
+
],
|
|
114
|
+
});
|
|
88
115
|
|
|
89
116
|
this._queryService = new QueryServiceImpl({
|
|
90
117
|
automergeHost: this._automergeHost,
|
|
91
118
|
indexer: this._indexer,
|
|
119
|
+
spaceStateManager: this._spaceStateManager,
|
|
92
120
|
});
|
|
93
121
|
|
|
94
122
|
this._dataService = new DataServiceImpl({
|
|
@@ -179,8 +207,8 @@ export class EchoHost extends Resource {
|
|
|
179
207
|
}
|
|
180
208
|
|
|
181
209
|
protected override async _close(ctx: Context): Promise<void> {
|
|
182
|
-
await this._spaceStateManager.close();
|
|
183
210
|
await this._queryService.close(ctx);
|
|
211
|
+
await this._spaceStateManager.close(ctx);
|
|
184
212
|
await this._indexer.close(ctx);
|
|
185
213
|
await this._automergeHost.close();
|
|
186
214
|
}
|
|
@@ -188,14 +216,14 @@ export class EchoHost extends Resource {
|
|
|
188
216
|
/**
|
|
189
217
|
* Flush all pending writes to the underlying storage.
|
|
190
218
|
*/
|
|
191
|
-
async flush() {
|
|
219
|
+
async flush(): Promise<void> {
|
|
192
220
|
await this._automergeHost.repo.flush();
|
|
193
221
|
}
|
|
194
222
|
|
|
195
223
|
/**
|
|
196
224
|
* Perform any pending index updates.
|
|
197
225
|
*/
|
|
198
|
-
async updateIndexes() {
|
|
226
|
+
async updateIndexes(): Promise<void> {
|
|
199
227
|
await this._indexer.updateIndexes();
|
|
200
228
|
}
|
|
201
229
|
|
|
@@ -224,7 +252,7 @@ export class EchoHost extends Resource {
|
|
|
224
252
|
invariant(this._lifecycleState === LifecycleState.OPEN);
|
|
225
253
|
const spaceId = await createIdFromSpaceKey(spaceKey);
|
|
226
254
|
|
|
227
|
-
const automergeRoot = this._automergeHost.createDoc<
|
|
255
|
+
const automergeRoot = this._automergeHost.createDoc<DatabaseDirectory>({
|
|
228
256
|
version: SpaceDocVersion.CURRENT,
|
|
229
257
|
access: { spaceKey: spaceKey.toHex() },
|
|
230
258
|
|
|
@@ -241,7 +269,8 @@ export class EchoHost extends Resource {
|
|
|
241
269
|
// TODO(dmaretskyi): Change to document id.
|
|
242
270
|
async openSpaceRoot(spaceId: SpaceId, automergeUrl: AutomergeUrl): Promise<DatabaseRoot> {
|
|
243
271
|
invariant(this._lifecycleState === LifecycleState.OPEN);
|
|
244
|
-
const handle = this._automergeHost.repo.find<
|
|
272
|
+
const handle = await this._automergeHost.repo.find<DatabaseDirectory>(automergeUrl, FIND_PARAMS);
|
|
273
|
+
await handle.whenReady();
|
|
245
274
|
|
|
246
275
|
return this._spaceStateManager.assignRootToSpace(spaceId, handle);
|
|
247
276
|
}
|
package/src/db-host/index.ts
CHANGED
|
@@ -2,12 +2,15 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { getHeads } from '@automerge/automerge';
|
|
6
|
+
import { type DocHandle, type DocumentId } from '@automerge/automerge-repo';
|
|
7
|
+
import { Schema } from 'effect';
|
|
8
|
+
|
|
5
9
|
import { DeferredTask } from '@dxos/async';
|
|
6
|
-
import { getHeads } from '@dxos/automerge/automerge';
|
|
7
|
-
import { type DocHandle, type DocumentId } from '@dxos/automerge/automerge-repo';
|
|
8
10
|
import { Stream } from '@dxos/codec-protobuf/stream';
|
|
9
11
|
import { Context, Resource } from '@dxos/context';
|
|
10
|
-
import {
|
|
12
|
+
import { raise } from '@dxos/debug';
|
|
13
|
+
import { DatabaseDirectory, QueryAST } from '@dxos/echo-protocol';
|
|
11
14
|
import { type IdToHeads, type Indexer, type ObjectSnapshot } from '@dxos/indexing';
|
|
12
15
|
import { log } from '@dxos/log';
|
|
13
16
|
import { objectPointerCodec } from '@dxos/protocols';
|
|
@@ -20,19 +23,21 @@ import {
|
|
|
20
23
|
} from '@dxos/protocols/proto/dxos/echo/query';
|
|
21
24
|
import { trace } from '@dxos/tracing';
|
|
22
25
|
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
26
|
+
import type { SpaceStateManager } from './space-state-manager';
|
|
27
|
+
import { type AutomergeHost } from '../automerge';
|
|
28
|
+
import { QueryExecutor } from '../query';
|
|
25
29
|
|
|
26
30
|
export type QueryServiceParams = {
|
|
27
31
|
indexer: Indexer;
|
|
28
32
|
automergeHost: AutomergeHost;
|
|
33
|
+
spaceStateManager: SpaceStateManager;
|
|
29
34
|
};
|
|
30
35
|
|
|
31
36
|
/**
|
|
32
37
|
* Represents an active query (stream and query state connected to that stream).
|
|
33
38
|
*/
|
|
34
39
|
type ActiveQuery = {
|
|
35
|
-
|
|
40
|
+
executor: QueryExecutor;
|
|
36
41
|
sendResults: (results: QueryResult[]) => void;
|
|
37
42
|
close: () => Promise<void>;
|
|
38
43
|
};
|
|
@@ -44,9 +49,9 @@ export class QueryServiceImpl extends Resource implements QueryService {
|
|
|
44
49
|
await Promise.all(
|
|
45
50
|
Array.from(this._queries).map(async (query) => {
|
|
46
51
|
try {
|
|
47
|
-
const { changed } = await query.
|
|
52
|
+
const { changed } = await query.executor.execQuery();
|
|
48
53
|
if (changed) {
|
|
49
|
-
query.sendResults(query.
|
|
54
|
+
query.sendResults(query.executor.getResults());
|
|
50
55
|
}
|
|
51
56
|
} catch (err) {
|
|
52
57
|
log.catch(err);
|
|
@@ -65,37 +70,39 @@ export class QueryServiceImpl extends Resource implements QueryService {
|
|
|
65
70
|
fetch: () => {
|
|
66
71
|
return Array.from(this._queries).map((query) => {
|
|
67
72
|
return {
|
|
68
|
-
|
|
69
|
-
|
|
73
|
+
query: JSON.stringify(query.executor.query),
|
|
74
|
+
plan: JSON.stringify(query.executor.plan),
|
|
75
|
+
trace: JSON.stringify(query.executor.trace),
|
|
70
76
|
};
|
|
71
77
|
});
|
|
72
78
|
},
|
|
73
79
|
});
|
|
74
80
|
}
|
|
75
81
|
|
|
76
|
-
override async _open() {
|
|
82
|
+
override async _open(): Promise<void> {
|
|
77
83
|
this._params.indexer.updated.on(this._ctx, () => this._updateQueries.schedule());
|
|
78
84
|
}
|
|
79
85
|
|
|
80
|
-
override async _close() {
|
|
86
|
+
override async _close(): Promise<void> {
|
|
81
87
|
await Promise.all(Array.from(this._queries).map((query) => query.close()));
|
|
82
88
|
}
|
|
83
89
|
|
|
84
90
|
async setConfig(config: IndexConfig): Promise<void> {
|
|
85
|
-
|
|
86
|
-
log.warn('Indexer already initialized.');
|
|
87
|
-
return;
|
|
88
|
-
}
|
|
89
|
-
this._params.indexer.setConfig(config);
|
|
91
|
+
await this._params.indexer.setConfig(config);
|
|
90
92
|
}
|
|
91
93
|
|
|
92
94
|
execQuery(request: QueryRequest): Stream<QueryResponse> {
|
|
93
95
|
return new Stream<QueryResponse>(({ next, close, ctx }) => {
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
+
const parsedQuery = QueryAST.Query.pipe(Schema.decodeUnknownSync)(JSON.parse(request.query));
|
|
97
|
+
|
|
98
|
+
const queryEntry: ActiveQuery = {
|
|
99
|
+
executor: new QueryExecutor({
|
|
96
100
|
indexer: this._params.indexer,
|
|
97
101
|
automergeHost: this._params.automergeHost,
|
|
98
|
-
request,
|
|
102
|
+
queryId: request.queryId ?? raise(new Error('query id required')),
|
|
103
|
+
query: parsedQuery,
|
|
104
|
+
reactivity: request.reactivity,
|
|
105
|
+
spaceStateManager: this._params.spaceStateManager,
|
|
99
106
|
}),
|
|
100
107
|
sendResults: (results) => {
|
|
101
108
|
if (ctx.disposed) {
|
|
@@ -105,33 +112,32 @@ export class QueryServiceImpl extends Resource implements QueryService {
|
|
|
105
112
|
},
|
|
106
113
|
close: async () => {
|
|
107
114
|
close();
|
|
108
|
-
await
|
|
109
|
-
this._queries.delete(
|
|
115
|
+
await queryEntry.executor.close();
|
|
116
|
+
this._queries.delete(queryEntry);
|
|
110
117
|
},
|
|
111
118
|
};
|
|
112
|
-
this._queries.add(
|
|
119
|
+
this._queries.add(queryEntry);
|
|
113
120
|
|
|
114
121
|
queueMicrotask(async () => {
|
|
115
|
-
await
|
|
122
|
+
await queryEntry.executor.open();
|
|
116
123
|
|
|
117
124
|
try {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
log.catch(error);
|
|
125
|
+
await queryEntry.executor.execQuery();
|
|
126
|
+
// Always send first result set.
|
|
127
|
+
queryEntry.sendResults(queryEntry.executor.getResults());
|
|
128
|
+
} catch (error: any) {
|
|
129
|
+
close(error);
|
|
124
130
|
}
|
|
125
131
|
});
|
|
126
132
|
|
|
127
|
-
return
|
|
133
|
+
return queryEntry.close;
|
|
128
134
|
});
|
|
129
135
|
}
|
|
130
136
|
|
|
131
137
|
/**
|
|
132
138
|
* Re-index all loaded documents.
|
|
133
139
|
*/
|
|
134
|
-
async reindex() {
|
|
140
|
+
async reindex(): Promise<void> {
|
|
135
141
|
log.info('Reindexing all documents...');
|
|
136
142
|
const iterator = createDocumentsIterator(this._params.automergeHost);
|
|
137
143
|
const ids: IdToHeads = new Map();
|
|
@@ -162,13 +168,13 @@ const createDocumentsIterator = (automergeHost: AutomergeHost) =>
|
|
|
162
168
|
/** visited automerge handles */
|
|
163
169
|
const visited = new Set<string>();
|
|
164
170
|
|
|
165
|
-
async function* getObjectsFromHandle(handle: DocHandle<
|
|
166
|
-
if (visited.has(handle.documentId)) {
|
|
171
|
+
async function* getObjectsFromHandle(handle: DocHandle<DatabaseDirectory>): AsyncGenerator<ObjectSnapshot[]> {
|
|
172
|
+
if (visited.has(handle.documentId) || !handle.isReady()) {
|
|
167
173
|
return;
|
|
168
174
|
}
|
|
169
|
-
const doc = handle.
|
|
175
|
+
const doc = handle.doc()!;
|
|
170
176
|
|
|
171
|
-
const spaceKey =
|
|
177
|
+
const spaceKey = DatabaseDirectory.getSpaceKey(doc) ?? undefined;
|
|
172
178
|
|
|
173
179
|
if (doc.objects) {
|
|
174
180
|
yield Object.entries(doc.objects as { [key: string]: any }).map(([objectId, object]) => {
|
|
@@ -186,7 +192,7 @@ const createDocumentsIterator = (automergeHost: AutomergeHost) =>
|
|
|
186
192
|
if (visited.has(urlString)) {
|
|
187
193
|
continue;
|
|
188
194
|
}
|
|
189
|
-
const linkHandle = await automergeHost.loadDoc<
|
|
195
|
+
const linkHandle = await automergeHost.loadDoc<DatabaseDirectory>(Context.default(), urlString as DocumentId);
|
|
190
196
|
for await (const result of getObjectsFromHandle(linkHandle)) {
|
|
191
197
|
yield result;
|
|
192
198
|
}
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { interpretAsDocumentId, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
|
|
5
6
|
import isEqual from 'lodash.isequal';
|
|
6
7
|
|
|
7
8
|
import { Event, UpdateScheduler } from '@dxos/async';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
9
|
+
import { Resource, Context, LifecycleState } from '@dxos/context';
|
|
10
|
+
import { type DatabaseDirectory } from '@dxos/echo-protocol';
|
|
11
|
+
import { invariant } from '@dxos/invariant';
|
|
11
12
|
import { type SpaceId } from '@dxos/keys';
|
|
12
13
|
|
|
13
14
|
import { DatabaseRoot } from './database-root';
|
|
@@ -39,7 +40,16 @@ export class SpaceStateManager extends Resource {
|
|
|
39
40
|
return this._rootBySpace.get(spaceId);
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
|
|
43
|
+
getRootBySpaceId(spaceId: SpaceId): DatabaseRoot | undefined {
|
|
44
|
+
invariant(this._lifecycleState === LifecycleState.OPEN);
|
|
45
|
+
const documentId = this._rootBySpace.get(spaceId);
|
|
46
|
+
if (!documentId) {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
return this._roots.get(documentId);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async assignRootToSpace(spaceId: SpaceId, handle: DocHandle<DatabaseDirectory>): Promise<DatabaseRoot> {
|
|
43
53
|
let root: DatabaseRoot;
|
|
44
54
|
if (this._roots.has(handle.documentId)) {
|
|
45
55
|
root = this._roots.get(handle.documentId)!;
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { cbor } from '@automerge/automerge-repo';
|
|
5
6
|
import { getRandomPort } from 'get-port-please';
|
|
6
7
|
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
7
8
|
|
|
8
9
|
import { Event } from '@dxos/async';
|
|
9
|
-
import { cbor } from '@dxos/automerge/automerge-repo';
|
|
10
10
|
import { createEphemeralEdgeIdentity, EdgeClient, MessageSchema } from '@dxos/edge-client';
|
|
11
11
|
import { createTestEdgeWsServer } from '@dxos/edge-client/testing';
|
|
12
12
|
import { PublicKey, SpaceId } from '@dxos/keys';
|
|
@@ -33,12 +33,12 @@ describe('EchoEdgeReplicator', () => {
|
|
|
33
33
|
await connectionOpen.waitForCount(1);
|
|
34
34
|
|
|
35
35
|
const forbidden = createForbiddenMessage({ identityKey: client.identityKey, peerKey: client.peerKey }, spaceId);
|
|
36
|
-
server.sendMessage(forbidden);
|
|
36
|
+
await server.sendMessage(forbidden);
|
|
37
37
|
await connectionOpen.waitForCount(1);
|
|
38
38
|
|
|
39
39
|
// Double restart to check for race conditions.
|
|
40
40
|
client.setIdentity(await createEphemeralEdgeIdentity());
|
|
41
|
-
server.sendMessage(forbidden);
|
|
41
|
+
await server.sendMessage(forbidden);
|
|
42
42
|
await connectionOpen.waitForCount(1);
|
|
43
43
|
|
|
44
44
|
await replicator.disconnect();
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { cbor } from '@automerge/automerge-repo';
|
|
6
|
+
|
|
5
7
|
import { Mutex, scheduleTask, scheduleMicroTask } from '@dxos/async';
|
|
6
|
-
import { cbor } from '@dxos/automerge/automerge-repo';
|
|
7
8
|
import { Context, Resource } from '@dxos/context';
|
|
8
9
|
import { randomUUID } from '@dxos/crypto';
|
|
9
10
|
import type { CollectionId } from '@dxos/echo-protocol';
|
|
@@ -68,7 +69,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
|
|
|
68
69
|
);
|
|
69
70
|
}
|
|
70
71
|
|
|
71
|
-
private async _handleReconnect() {
|
|
72
|
+
private async _handleReconnect(): Promise<void> {
|
|
72
73
|
using _guard = await this._mutex.acquire();
|
|
73
74
|
|
|
74
75
|
const spaces = [...this._connectedSpaces];
|
|
@@ -94,7 +95,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
|
|
|
94
95
|
this._connections.clear();
|
|
95
96
|
}
|
|
96
97
|
|
|
97
|
-
async connectToSpace(spaceId: SpaceId) {
|
|
98
|
+
async connectToSpace(spaceId: SpaceId): Promise<void> {
|
|
98
99
|
using _guard = await this._mutex.acquire();
|
|
99
100
|
|
|
100
101
|
if (this._connectedSpaces.has(spaceId)) {
|
|
@@ -108,7 +109,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
|
|
|
108
109
|
}
|
|
109
110
|
}
|
|
110
111
|
|
|
111
|
-
async disconnectFromSpace(spaceId: SpaceId) {
|
|
112
|
+
async disconnectFromSpace(spaceId: SpaceId): Promise<void> {
|
|
112
113
|
using _guard = await this._mutex.acquire();
|
|
113
114
|
|
|
114
115
|
this._connectedSpaces.delete(spaceId);
|
|
@@ -120,7 +121,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
|
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
|
|
123
|
-
private async _openConnection(spaceId: SpaceId, reconnects: number = 0) {
|
|
124
|
+
private async _openConnection(spaceId: SpaceId, reconnects: number = 0): Promise<void> {
|
|
124
125
|
invariant(this._context);
|
|
125
126
|
invariant(!this._connections.has(spaceId));
|
|
126
127
|
|
|
@@ -310,7 +311,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
310
311
|
return spaceId === this._spaceId && params.collectionId.split(':').length === 3;
|
|
311
312
|
}
|
|
312
313
|
|
|
313
|
-
private _onMessage(message: RouterMessage) {
|
|
314
|
+
private _onMessage(message: RouterMessage): void {
|
|
314
315
|
if (message.serviceId !== this._targetServiceId) {
|
|
315
316
|
return;
|
|
316
317
|
}
|
|
@@ -327,7 +328,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
327
328
|
this._processMessage(payload);
|
|
328
329
|
}
|
|
329
330
|
|
|
330
|
-
private _processMessage(message: AutomergeProtocolMessage) {
|
|
331
|
+
private _processMessage(message: AutomergeProtocolMessage): void {
|
|
331
332
|
// There's a race between the credentials being replicated that are needed for access control and the data replication.
|
|
332
333
|
// AutomergeReplicator might return a Forbidden error if the credentials are not yet replicated.
|
|
333
334
|
// We restart the connection with some delay to account for that.
|
|
@@ -341,7 +342,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
341
342
|
this._readableStreamController.enqueue(message);
|
|
342
343
|
}
|
|
343
344
|
|
|
344
|
-
private async _sendMessage(message: AutomergeProtocolMessage) {
|
|
345
|
+
private async _sendMessage(message: AutomergeProtocolMessage): Promise<void> {
|
|
345
346
|
// Fix the peer id.
|
|
346
347
|
(message as any).targetId = this._targetServiceId as PeerId;
|
|
347
348
|
|
|
@@ -26,19 +26,19 @@ export class InflightRequestLimiter extends Resource {
|
|
|
26
26
|
super();
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
protected override async _open() {
|
|
29
|
+
protected override async _open(): Promise<void> {
|
|
30
30
|
this._inflightRequestBalance = 0;
|
|
31
31
|
this._requestBarrier.reset();
|
|
32
32
|
this._requestBarrier.wake();
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
protected override async _close() {
|
|
35
|
+
protected override async _close(): Promise<void> {
|
|
36
36
|
this._inflightRequestBalance = 0;
|
|
37
37
|
this._requestBarrier.throw(new Error('Rate limiter closed.'));
|
|
38
38
|
clearTimeout(this._resetBalanceTimeout);
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
public async rateLimit(message: AutomergeProtocolMessage) {
|
|
41
|
+
public async rateLimit(message: AutomergeProtocolMessage): Promise<void> {
|
|
42
42
|
if (message.type !== 'sync') {
|
|
43
43
|
return;
|
|
44
44
|
}
|
|
@@ -56,7 +56,7 @@ export class InflightRequestLimiter extends Resource {
|
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
public handleResponse(message: AutomergeProtocolMessage) {
|
|
59
|
+
public handleResponse(message: AutomergeProtocolMessage): void {
|
|
60
60
|
if (message.type !== 'sync') {
|
|
61
61
|
return;
|
|
62
62
|
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { ObjectStructure } from '@dxos/echo-protocol';
|
|
8
|
+
import { Expando, EXPANDO_TYPENAME, Filter, Ref } from '@dxos/echo-schema';
|
|
9
|
+
import { DXN, ObjectId, SpaceId } from '@dxos/keys';
|
|
10
|
+
|
|
11
|
+
import { filterMatchObject, type MatchedObject } from './filter-match';
|
|
12
|
+
|
|
13
|
+
describe('filterMatch', () => {
|
|
14
|
+
test('properties', () => {
|
|
15
|
+
expect(filterMatchObject(Filter.type(Expando, { title: 'test' }).ast, OBJECT_1)).to.be.true;
|
|
16
|
+
expect(filterMatchObject(Filter.type(Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
|
|
17
|
+
expect(filterMatchObject(Filter.type(Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
|
|
18
|
+
expect(filterMatchObject(Filter.type(Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test('and', () => {
|
|
22
|
+
const filter1 = Filter.type(Expando, { title: 'test' });
|
|
23
|
+
const filter2 = Filter.type(Expando, { value: 100 });
|
|
24
|
+
const filter3 = Filter.type(Expando, { complete: true });
|
|
25
|
+
|
|
26
|
+
expect(filterMatchObject(Filter.and(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test('or', () => {
|
|
30
|
+
const filter1 = Filter.type(Expando, { value: 200 });
|
|
31
|
+
const filter2 = Filter.type(Expando, { title: 'test' });
|
|
32
|
+
const filter3 = Filter.type(Expando, { complete: false });
|
|
33
|
+
|
|
34
|
+
expect(filterMatchObject(Filter.or(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('not', () => {
|
|
38
|
+
const filter1 = Filter.type(Expando, { title: 'test' });
|
|
39
|
+
const filter2 = Filter.not(filter1);
|
|
40
|
+
|
|
41
|
+
expect(filterMatchObject(filter1.ast, OBJECT_1)).to.be.true;
|
|
42
|
+
expect(filterMatchObject(filter2.ast, OBJECT_1)).to.be.false;
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test('complex', () => {
|
|
46
|
+
const filter1 = Filter.type(Expando, { title: 'bad' });
|
|
47
|
+
const filter2 = Filter.type(Expando, { value: 0 });
|
|
48
|
+
expect(filterMatchObject(Filter.not(Filter.or(filter1, filter2)).ast, OBJECT_1)).to.be.true;
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('ids', () => {
|
|
52
|
+
const filter = Filter.ids(OBJECT_1.id);
|
|
53
|
+
expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.true;
|
|
54
|
+
expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('everything', () => {
|
|
58
|
+
expect(filterMatchObject(Filter.everything().ast, OBJECT_1)).to.be.true;
|
|
59
|
+
expect(filterMatchObject(Filter.everything().ast, OBJECT_2)).to.be.true;
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('nothing', () => {
|
|
63
|
+
expect(filterMatchObject(Filter.nothing().ast, OBJECT_1)).to.be.false;
|
|
64
|
+
expect(filterMatchObject(Filter.nothing().ast, OBJECT_2)).to.be.false;
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('refs', () => {
|
|
68
|
+
const filter = Filter.type(Expando, { parent: Ref.fromDXN(DXN.fromLocalObjectId(OBJECT_1.id)) });
|
|
69
|
+
expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.false;
|
|
70
|
+
expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
|
|
71
|
+
expect(filterMatchObject(filter.ast, OBJECT_3)).to.be.true;
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const OBJECT_1: MatchedObject = {
|
|
76
|
+
id: ObjectId.make('01JVS9YYT5VMVJW0GGTM1YHCCH'),
|
|
77
|
+
spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
|
|
78
|
+
doc: ObjectStructure.makeObject({
|
|
79
|
+
type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
|
|
80
|
+
data: { title: 'test', value: 100, complete: true },
|
|
81
|
+
}),
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
const OBJECT_2: MatchedObject = {
|
|
85
|
+
id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
|
|
86
|
+
spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
|
|
87
|
+
doc: ObjectStructure.makeObject({
|
|
88
|
+
type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
|
|
89
|
+
data: { title: 'test', value: 100, complete: true },
|
|
90
|
+
deleted: true,
|
|
91
|
+
}),
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
const OBJECT_3: MatchedObject = {
|
|
95
|
+
id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
|
|
96
|
+
spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
|
|
97
|
+
doc: ObjectStructure.makeObject({
|
|
98
|
+
type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
|
|
99
|
+
data: { title: 'test', value: 100, complete: true, parent: { '/': DXN.fromLocalObjectId(OBJECT_1.id).toString() } },
|
|
100
|
+
}),
|
|
101
|
+
};
|