@dxos/index-core 0.0.0 → 0.8.4-main.03d5cd7b56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/neutral/index.mjs +790 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/types/src/index-engine.d.ts +112 -0
- package/dist/types/src/index-engine.d.ts.map +1 -0
- package/dist/types/src/index-engine.test.d.ts +2 -0
- package/dist/types/src/index-engine.test.d.ts.map +1 -0
- package/dist/types/src/index-tracker.d.ts +44 -0
- package/dist/types/src/index-tracker.d.ts.map +1 -0
- package/dist/types/src/index-tracker.test.d.ts +2 -0
- package/dist/types/src/index-tracker.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +8 -0
- package/dist/types/src/index.d.ts.map +1 -0
- package/dist/types/src/indexes/fts-index.d.ts +64 -0
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -0
- package/dist/types/src/indexes/fts-index.test.d.ts +2 -0
- package/dist/types/src/indexes/fts-index.test.d.ts.map +1 -0
- package/dist/types/src/indexes/fts5.test.d.ts +2 -0
- package/dist/types/src/indexes/fts5.test.d.ts.map +1 -0
- package/dist/types/src/indexes/index.d.ts +5 -0
- package/dist/types/src/indexes/index.d.ts.map +1 -0
- package/dist/types/src/indexes/interface.d.ts +56 -0
- package/dist/types/src/indexes/interface.d.ts.map +1 -0
- package/dist/types/src/indexes/object-meta-index.d.ts +94 -0
- package/dist/types/src/indexes/object-meta-index.d.ts.map +1 -0
- package/dist/types/src/indexes/object-meta-index.test.d.ts +2 -0
- package/dist/types/src/indexes/object-meta-index.test.d.ts.map +1 -0
- package/dist/types/src/indexes/reverse-ref-index.d.ts +37 -0
- package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -0
- package/dist/types/src/indexes/reverse-ref-index.test.d.ts +2 -0
- package/dist/types/src/indexes/reverse-ref-index.test.d.ts.map +1 -0
- package/dist/types/src/utils.d.ts +17 -0
- package/dist/types/src/utils.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +22 -18
- package/src/index-engine.test.ts +172 -9
- package/src/index-engine.ts +161 -29
- package/src/index-tracker.ts +9 -0
- package/src/index.ts +10 -3
- package/src/indexes/fts-index.test.ts +153 -3
- package/src/indexes/fts-index.ts +66 -10
- package/src/indexes/interface.ts +10 -0
- package/src/indexes/object-meta-index.test.ts +361 -3
- package/src/indexes/object-meta-index.ts +304 -17
- package/src/indexes/reverse-ref-index.test.ts +16 -2
- package/src/indexes/reverse-ref-index.ts +0 -1
- package/src/utils.ts +1 -1
package/package.json
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/index-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.4-main.03d5cd7b56",
|
|
4
4
|
"description": "Indexing core.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "https://github.com/dxos/dxos"
|
|
10
|
+
},
|
|
7
11
|
"license": "MIT",
|
|
8
12
|
"author": "info@dxos.org",
|
|
9
13
|
"sideEffects": false,
|
|
@@ -12,8 +16,7 @@
|
|
|
12
16
|
".": {
|
|
13
17
|
"source": "./src/index.ts",
|
|
14
18
|
"types": "./dist/types/src/index.d.ts",
|
|
15
|
-
"
|
|
16
|
-
"node": "./dist/lib/node-esm/index.mjs"
|
|
19
|
+
"default": "./dist/lib/neutral/index.mjs"
|
|
17
20
|
}
|
|
18
21
|
},
|
|
19
22
|
"files": [
|
|
@@ -21,23 +24,24 @@
|
|
|
21
24
|
"src"
|
|
22
25
|
],
|
|
23
26
|
"dependencies": {
|
|
24
|
-
"@
|
|
25
|
-
"@
|
|
26
|
-
"@
|
|
27
|
-
"@
|
|
28
|
-
"
|
|
29
|
-
"@dxos/
|
|
30
|
-
"@dxos/
|
|
31
|
-
"@dxos/
|
|
32
|
-
"@dxos/
|
|
33
|
-
"@
|
|
34
|
-
"@effect
|
|
35
|
-
"@
|
|
36
|
-
"@
|
|
37
|
-
"
|
|
27
|
+
"@effect/cli": "0.73.2",
|
|
28
|
+
"@effect/experimental": "0.58.0",
|
|
29
|
+
"@effect/platform": "0.94.4",
|
|
30
|
+
"@effect/sql": "0.49.0",
|
|
31
|
+
"effect": "3.20.0",
|
|
32
|
+
"@dxos/async": "0.8.4-main.03d5cd7b56",
|
|
33
|
+
"@dxos/context": "0.8.4-main.03d5cd7b56",
|
|
34
|
+
"@dxos/debug": "0.8.4-main.03d5cd7b56",
|
|
35
|
+
"@dxos/echo-protocol": "0.8.4-main.03d5cd7b56",
|
|
36
|
+
"@dxos/invariant": "0.8.4-main.03d5cd7b56",
|
|
37
|
+
"@dxos/effect": "0.8.4-main.03d5cd7b56",
|
|
38
|
+
"@dxos/keys": "0.8.4-main.03d5cd7b56",
|
|
39
|
+
"@dxos/log": "0.8.4-main.03d5cd7b56",
|
|
40
|
+
"@dxos/sql-sqlite": "0.8.4-main.03d5cd7b56",
|
|
41
|
+
"@dxos/echo": "0.8.4-main.03d5cd7b56"
|
|
38
42
|
},
|
|
39
43
|
"devDependencies": {
|
|
40
|
-
"@effect/sql-sqlite-node": "0.
|
|
44
|
+
"@effect/sql-sqlite-node": "0.50.1"
|
|
41
45
|
},
|
|
42
46
|
"publishConfig": {
|
|
43
47
|
"access": "public"
|
package/src/index-engine.test.ts
CHANGED
|
@@ -8,11 +8,13 @@ import { describe, expect, it } from '@effect/vitest';
|
|
|
8
8
|
import * as Effect from 'effect/Effect';
|
|
9
9
|
import * as Layer from 'effect/Layer';
|
|
10
10
|
|
|
11
|
+
import { Context } from '@dxos/context';
|
|
11
12
|
import { ATTR_TYPE } from '@dxos/echo/internal';
|
|
12
13
|
import { invariant } from '@dxos/invariant';
|
|
13
14
|
import { DXN, ObjectId, SpaceId } from '@dxos/keys';
|
|
15
|
+
import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
|
|
14
16
|
|
|
15
|
-
import { type DataSourceCursor, type IndexDataSource, IndexEngine } from './index-engine';
|
|
17
|
+
import { type DataSourceCursor, type IndexDataSource, IndexEngine, type IndexingResult } from './index-engine';
|
|
16
18
|
import { type IndexCursor, IndexTracker } from './index-tracker';
|
|
17
19
|
import { FtsIndex, type IndexerObject, ObjectMetaIndex, ReverseRefIndex } from './indexes';
|
|
18
20
|
|
|
@@ -20,11 +22,13 @@ const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
|
|
|
20
22
|
const TYPE_A = DXN.parse('dxn:type:test.com/type/TypeA:0.1.0').toString();
|
|
21
23
|
const TYPE_B = DXN.parse('dxn:type:test.com/type/TypeB:0.1.0').toString();
|
|
22
24
|
|
|
23
|
-
const TestLayer =
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
const TestLayer = SqlTransaction.layer.pipe(
|
|
26
|
+
Layer.provideMerge(
|
|
27
|
+
SqliteClient.layer({
|
|
28
|
+
filename: ':memory:',
|
|
29
|
+
}),
|
|
30
|
+
),
|
|
31
|
+
Layer.provideMerge(Reactivity.layer),
|
|
28
32
|
);
|
|
29
33
|
|
|
30
34
|
class MockIndexDataSource implements IndexDataSource {
|
|
@@ -46,6 +50,7 @@ class MockIndexDataSource implements IndexDataSource {
|
|
|
46
50
|
}
|
|
47
51
|
|
|
48
52
|
getChangedObjects(
|
|
53
|
+
_ctx: Context,
|
|
49
54
|
cursors: IndexCursor[],
|
|
50
55
|
opts?: { limit?: number },
|
|
51
56
|
): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }> {
|
|
@@ -114,7 +119,9 @@ describe('IndexEngine', () => {
|
|
|
114
119
|
spaceId,
|
|
115
120
|
documentId: 'doc-1',
|
|
116
121
|
queueId: null,
|
|
122
|
+
queueNamespace: null,
|
|
117
123
|
recordId: null,
|
|
124
|
+
updatedAt: Date.now(),
|
|
118
125
|
data: {
|
|
119
126
|
id: ObjectId.random(),
|
|
120
127
|
[ATTR_TYPE]: TYPE_DEFAULT,
|
|
@@ -125,7 +132,7 @@ describe('IndexEngine', () => {
|
|
|
125
132
|
dataSource.push([obj1]);
|
|
126
133
|
|
|
127
134
|
// First update.
|
|
128
|
-
const { updated } = yield* engine.update(dataSource, { spaceId: null });
|
|
135
|
+
const { updated } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
129
136
|
// Updates objectMeta, FTS, and reverseRef indexes.
|
|
130
137
|
expect(updated).toBe(2);
|
|
131
138
|
|
|
@@ -150,13 +157,15 @@ describe('IndexEngine', () => {
|
|
|
150
157
|
spaceId,
|
|
151
158
|
documentId: obj1.documentId,
|
|
152
159
|
queueId: null,
|
|
160
|
+
queueNamespace: null,
|
|
153
161
|
recordId: null,
|
|
162
|
+
updatedAt: Date.now(),
|
|
154
163
|
data: { id: obj1.data.id, [ATTR_TYPE]: obj1.data[ATTR_TYPE], title: 'Hello World' },
|
|
155
164
|
};
|
|
156
165
|
dataSource.push([obj1Updated]);
|
|
157
166
|
|
|
158
167
|
// Second update.
|
|
159
|
-
const { updated: updated2 } = yield* engine.update(dataSource, { spaceId: null });
|
|
168
|
+
const { updated: updated2 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
160
169
|
expect(updated2).toBe(2);
|
|
161
170
|
|
|
162
171
|
// Verify update.
|
|
@@ -188,8 +197,10 @@ describe('IndexEngine', () => {
|
|
|
188
197
|
{
|
|
189
198
|
spaceId,
|
|
190
199
|
queueId: null,
|
|
200
|
+
queueNamespace: null,
|
|
191
201
|
documentId: 'd1',
|
|
192
202
|
recordId: null,
|
|
203
|
+
updatedAt: Date.now(),
|
|
193
204
|
data: {
|
|
194
205
|
id: ObjectId.random(),
|
|
195
206
|
[ATTR_TYPE]: TYPE_A,
|
|
@@ -199,8 +210,10 @@ describe('IndexEngine', () => {
|
|
|
199
210
|
{
|
|
200
211
|
spaceId,
|
|
201
212
|
queueId: null,
|
|
213
|
+
queueNamespace: null,
|
|
202
214
|
documentId: 'd2',
|
|
203
215
|
recordId: null,
|
|
216
|
+
updatedAt: Date.now(),
|
|
204
217
|
data: {
|
|
205
218
|
id: ObjectId.random(),
|
|
206
219
|
[ATTR_TYPE]: TYPE_A,
|
|
@@ -210,8 +223,10 @@ describe('IndexEngine', () => {
|
|
|
210
223
|
{
|
|
211
224
|
spaceId,
|
|
212
225
|
queueId: null,
|
|
226
|
+
queueNamespace: null,
|
|
213
227
|
documentId: 'd3',
|
|
214
228
|
recordId: null,
|
|
229
|
+
updatedAt: Date.now(),
|
|
215
230
|
data: {
|
|
216
231
|
id: ObjectId.random(),
|
|
217
232
|
[ATTR_TYPE]: TYPE_B,
|
|
@@ -222,7 +237,7 @@ describe('IndexEngine', () => {
|
|
|
222
237
|
|
|
223
238
|
dataSource.push(objects);
|
|
224
239
|
|
|
225
|
-
yield* engine.update(dataSource, { spaceId: null });
|
|
240
|
+
yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
226
241
|
|
|
227
242
|
const resultsA = yield* metaIndex.query({ spaceId: spaceId.toString(), typeDxn: TYPE_A });
|
|
228
243
|
expect(resultsA).toHaveLength(2);
|
|
@@ -239,4 +254,152 @@ describe('IndexEngine', () => {
|
|
|
239
254
|
expect(ftsResults).toHaveLength(2);
|
|
240
255
|
}, Effect.provide(TestLayer)),
|
|
241
256
|
);
|
|
257
|
+
|
|
258
|
+
it.effect(
|
|
259
|
+
'done is true only when all sub-indexes have no remaining work',
|
|
260
|
+
Effect.fnUntraced(function* () {
|
|
261
|
+
const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
|
|
262
|
+
|
|
263
|
+
const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
|
|
264
|
+
const dataSource = new MockIndexDataSource();
|
|
265
|
+
const spaceId = SpaceId.random();
|
|
266
|
+
|
|
267
|
+
// First update with no data — both sub-indexes report done immediately.
|
|
268
|
+
const { updated: updated0, done: done0 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
269
|
+
expect(updated0).toBe(0);
|
|
270
|
+
expect(done0).toBe(true);
|
|
271
|
+
|
|
272
|
+
// Push an object so sub-indexes have work to do.
|
|
273
|
+
dataSource.push([
|
|
274
|
+
{
|
|
275
|
+
spaceId,
|
|
276
|
+
queueId: null,
|
|
277
|
+
queueNamespace: null,
|
|
278
|
+
documentId: 'doc-done-test',
|
|
279
|
+
recordId: null,
|
|
280
|
+
updatedAt: Date.now(),
|
|
281
|
+
data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_DEFAULT, title: 'Done test' },
|
|
282
|
+
},
|
|
283
|
+
]);
|
|
284
|
+
|
|
285
|
+
// Update with pending data — sub-indexes process objects, done is false.
|
|
286
|
+
const { updated: updated1, done: done1 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
287
|
+
expect(updated1).toBeGreaterThan(0);
|
|
288
|
+
expect(done1).toBe(false);
|
|
289
|
+
|
|
290
|
+
// Second update with no new data — all sub-indexes caught up, done is true.
|
|
291
|
+
const { updated: updated2, done: done2 } = yield* engine.update(Context.default(), dataSource, {
|
|
292
|
+
spaceId: null,
|
|
293
|
+
});
|
|
294
|
+
expect(updated2).toBe(0);
|
|
295
|
+
expect(done2).toBe(true);
|
|
296
|
+
}, Effect.provide(TestLayer)),
|
|
297
|
+
);
|
|
298
|
+
|
|
299
|
+
it.effect(
|
|
300
|
+
'IndexingResult contains correct sets for a batch with multiple objects across spaces',
|
|
301
|
+
Effect.fnUntraced(function* () {
|
|
302
|
+
const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
|
|
303
|
+
const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
|
|
304
|
+
const dataSource = new MockIndexDataSource();
|
|
305
|
+
const spaceId1 = SpaceId.random();
|
|
306
|
+
const spaceId2 = SpaceId.random();
|
|
307
|
+
const id1 = ObjectId.random();
|
|
308
|
+
const id2 = ObjectId.random();
|
|
309
|
+
|
|
310
|
+
const obj1: IndexerObject = {
|
|
311
|
+
spaceId: spaceId1,
|
|
312
|
+
documentId: 'doc-result-1',
|
|
313
|
+
queueId: null,
|
|
314
|
+
queueNamespace: null,
|
|
315
|
+
recordId: null,
|
|
316
|
+
updatedAt: Date.now(),
|
|
317
|
+
data: { id: id1, [ATTR_TYPE]: TYPE_A, title: 'Doc in space1' },
|
|
318
|
+
};
|
|
319
|
+
const obj2: IndexerObject = {
|
|
320
|
+
spaceId: spaceId2,
|
|
321
|
+
documentId: 'doc-result-2',
|
|
322
|
+
queueId: null,
|
|
323
|
+
queueNamespace: null,
|
|
324
|
+
recordId: null,
|
|
325
|
+
updatedAt: Date.now(),
|
|
326
|
+
data: { id: id2, [ATTR_TYPE]: TYPE_B, title: 'Doc in space2' },
|
|
327
|
+
};
|
|
328
|
+
|
|
329
|
+
dataSource.push([obj1, obj2]);
|
|
330
|
+
|
|
331
|
+
const result: IndexingResult = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
332
|
+
|
|
333
|
+
expect(result.updated).toBeGreaterThan(0);
|
|
334
|
+
expect(result.done).toBe(false);
|
|
335
|
+
|
|
336
|
+
// Spaces: both spaceIds should be present.
|
|
337
|
+
expect(result.spaces.has(spaceId1)).toBe(true);
|
|
338
|
+
expect(result.spaces.has(spaceId2)).toBe(true);
|
|
339
|
+
|
|
340
|
+
// Documents: both doc IDs should be present.
|
|
341
|
+
expect(result.documents.has('doc-result-1')).toBe(true);
|
|
342
|
+
expect(result.documents.has('doc-result-2')).toBe(true);
|
|
343
|
+
|
|
344
|
+
// Types: both TYPE_A and TYPE_B.
|
|
345
|
+
expect(result.types.has(TYPE_A)).toBe(true);
|
|
346
|
+
expect(result.types.has(TYPE_B)).toBe(true);
|
|
347
|
+
|
|
348
|
+
// Object ids.
|
|
349
|
+
expect(result.objects.has(id1)).toBe(true);
|
|
350
|
+
expect(result.objects.has(id2)).toBe(true);
|
|
351
|
+
}, Effect.provide(TestLayer)),
|
|
352
|
+
);
|
|
353
|
+
|
|
354
|
+
it.effect(
|
|
355
|
+
'IndexingResult includes typename for deleted objects',
|
|
356
|
+
Effect.fnUntraced(function* () {
|
|
357
|
+
const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
|
|
358
|
+
const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
|
|
359
|
+
const dataSource = new MockIndexDataSource();
|
|
360
|
+
const spaceId = SpaceId.random();
|
|
361
|
+
|
|
362
|
+
const deletedObj: IndexerObject = {
|
|
363
|
+
spaceId,
|
|
364
|
+
documentId: 'doc-deleted',
|
|
365
|
+
queueId: null,
|
|
366
|
+
queueNamespace: null,
|
|
367
|
+
recordId: null,
|
|
368
|
+
updatedAt: Date.now(),
|
|
369
|
+
data: {
|
|
370
|
+
id: ObjectId.random(),
|
|
371
|
+
[ATTR_TYPE]: TYPE_DEFAULT,
|
|
372
|
+
'@deleted': true,
|
|
373
|
+
},
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
dataSource.push([deletedObj]);
|
|
377
|
+
|
|
378
|
+
const result: IndexingResult = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
379
|
+
|
|
380
|
+
expect(result.updated).toBeGreaterThan(0);
|
|
381
|
+
// Deleted objects should still contribute their typename to the hint.
|
|
382
|
+
expect(result.types.has(TYPE_DEFAULT)).toBe(true);
|
|
383
|
+
expect(result.spaces.has(spaceId)).toBe(true);
|
|
384
|
+
}, Effect.provide(TestLayer)),
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
it.effect(
|
|
388
|
+
'IndexingResult is empty when no objects are indexed',
|
|
389
|
+
Effect.fnUntraced(function* () {
|
|
390
|
+
const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
|
|
391
|
+
const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
|
|
392
|
+
const dataSource = new MockIndexDataSource();
|
|
393
|
+
|
|
394
|
+
const result: IndexingResult = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
395
|
+
|
|
396
|
+
expect(result.updated).toBe(0);
|
|
397
|
+
expect(result.done).toBe(true);
|
|
398
|
+
expect(result.spaces.size).toBe(0);
|
|
399
|
+
expect(result.queues.size).toBe(0);
|
|
400
|
+
expect(result.documents.size).toBe(0);
|
|
401
|
+
expect(result.types.size).toBe(0);
|
|
402
|
+
expect(result.objects.size).toBe(0);
|
|
403
|
+
}, Effect.provide(TestLayer)),
|
|
404
|
+
);
|
|
242
405
|
});
|
package/src/index-engine.ts
CHANGED
|
@@ -2,16 +2,20 @@
|
|
|
2
2
|
// Copyright 2026 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import * as SqlClient from '@effect/sql/SqlClient';
|
|
5
|
+
import type * as SqlClient from '@effect/sql/SqlClient';
|
|
6
6
|
import type * as SqlError from '@effect/sql/SqlError';
|
|
7
7
|
import * as Effect from 'effect/Effect';
|
|
8
8
|
|
|
9
|
-
import type
|
|
9
|
+
import { type Context } from '@dxos/context';
|
|
10
|
+
import { ATTR_TYPE } from '@dxos/echo/internal';
|
|
11
|
+
import type { ObjectId, SpaceId } from '@dxos/keys';
|
|
12
|
+
import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
|
|
10
13
|
|
|
11
14
|
import { type IndexCursor, IndexTracker } from './index-tracker';
|
|
12
15
|
import {
|
|
13
16
|
FtsIndex,
|
|
14
17
|
type FtsQuery,
|
|
18
|
+
type FtsQueryResult,
|
|
15
19
|
type Index,
|
|
16
20
|
type IndexerObject,
|
|
17
21
|
type ObjectMeta,
|
|
@@ -20,6 +24,59 @@ import {
|
|
|
20
24
|
type ReverseRefQuery,
|
|
21
25
|
} from './indexes';
|
|
22
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Result of a single indexing pass over a data source.
|
|
29
|
+
* Carries enough metadata for callers to build targeted invalidation hints.
|
|
30
|
+
*/
|
|
31
|
+
export type IndexingResult = {
|
|
32
|
+
updated: number;
|
|
33
|
+
done: boolean;
|
|
34
|
+
spaces: ReadonlySet<SpaceId>;
|
|
35
|
+
queues: ReadonlySet<ObjectId>;
|
|
36
|
+
documents: ReadonlySet<string>;
|
|
37
|
+
types: ReadonlySet<string>;
|
|
38
|
+
objects: ReadonlySet<ObjectId>;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
type MutableIndexingResult = {
|
|
42
|
+
updated: number;
|
|
43
|
+
done: boolean;
|
|
44
|
+
spaces: Set<SpaceId>;
|
|
45
|
+
queues: Set<ObjectId>;
|
|
46
|
+
documents: Set<string>;
|
|
47
|
+
types: Set<string>;
|
|
48
|
+
objects: Set<ObjectId>;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const makeEmptyIndexingResult = (): MutableIndexingResult => ({
|
|
52
|
+
updated: 0,
|
|
53
|
+
done: true,
|
|
54
|
+
spaces: new Set(),
|
|
55
|
+
queues: new Set(),
|
|
56
|
+
documents: new Set(),
|
|
57
|
+
types: new Set(),
|
|
58
|
+
objects: new Set(),
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
const accumulateIndexingResult = (acc: MutableIndexingResult, objects: readonly IndexerObject[]) => {
|
|
62
|
+
for (const obj of objects) {
|
|
63
|
+
acc.spaces.add(obj.spaceId);
|
|
64
|
+
if (obj.queueId) {
|
|
65
|
+
acc.queues.add(obj.queueId);
|
|
66
|
+
}
|
|
67
|
+
if (obj.documentId) {
|
|
68
|
+
acc.documents.add(obj.documentId);
|
|
69
|
+
}
|
|
70
|
+
const t = (obj.data as Record<string, unknown>)[ATTR_TYPE];
|
|
71
|
+
if (t) {
|
|
72
|
+
acc.types.add(String(t));
|
|
73
|
+
}
|
|
74
|
+
if (obj.data.id) {
|
|
75
|
+
acc.objects.add(obj.data.id as ObjectId);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
|
|
23
80
|
/**
|
|
24
81
|
* Cursor into indexable data-source.
|
|
25
82
|
*/
|
|
@@ -27,7 +84,7 @@ export interface DataSourceCursor {
|
|
|
27
84
|
spaceId: SpaceId | null;
|
|
28
85
|
|
|
29
86
|
/**
|
|
30
|
-
* documentId or
|
|
87
|
+
* documentId or queueNamespace.
|
|
31
88
|
*/
|
|
32
89
|
resourceId: string | null;
|
|
33
90
|
|
|
@@ -41,6 +98,7 @@ export interface IndexDataSource {
|
|
|
41
98
|
readonly sourceName: string; // e.g. queue, automerge, etc.
|
|
42
99
|
|
|
43
100
|
getChangedObjects(
|
|
101
|
+
ctx: Context,
|
|
44
102
|
cursors: DataSourceCursor[],
|
|
45
103
|
opts?: { limit?: number },
|
|
46
104
|
): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }>;
|
|
@@ -76,9 +134,9 @@ export class IndexEngine {
|
|
|
76
134
|
}
|
|
77
135
|
|
|
78
136
|
/**
|
|
79
|
-
* Query text index and return full object metadata.
|
|
137
|
+
* Query text index and return full object metadata with rank.
|
|
80
138
|
*/
|
|
81
|
-
queryText(query: FtsQuery): Effect.Effect<readonly
|
|
139
|
+
queryText(query: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
82
140
|
return Effect.gen(this, function* () {
|
|
83
141
|
return yield* this.#ftsIndex.query(query);
|
|
84
142
|
});
|
|
@@ -89,6 +147,14 @@ export class IndexEngine {
|
|
|
89
147
|
return this.#reverseRefIndex.query(query);
|
|
90
148
|
}
|
|
91
149
|
|
|
150
|
+
queryAll(query: {
|
|
151
|
+
spaceIds: readonly SpaceId[];
|
|
152
|
+
includeAllQueues?: boolean;
|
|
153
|
+
queueIds?: readonly string[] | null;
|
|
154
|
+
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
155
|
+
return this.#objectMetaIndex.queryAll(query);
|
|
156
|
+
}
|
|
157
|
+
|
|
92
158
|
/**
|
|
93
159
|
* Query snapshots by recordIds.
|
|
94
160
|
* Used to load queue objects from indexed snapshots.
|
|
@@ -103,32 +169,90 @@ export class IndexEngine {
|
|
|
103
169
|
return this.#objectMetaIndex.query(query);
|
|
104
170
|
}
|
|
105
171
|
|
|
172
|
+
/**
|
|
173
|
+
* Query children by parent object ids.
|
|
174
|
+
*/
|
|
175
|
+
queryChildren(query: {
|
|
176
|
+
spaceId: SpaceId[];
|
|
177
|
+
parentIds: ObjectId[];
|
|
178
|
+
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
179
|
+
return this.#objectMetaIndex.queryChildren(query);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
queryTypes(query: {
|
|
183
|
+
spaceIds: readonly SpaceId[];
|
|
184
|
+
typeDxns: readonly ObjectMeta['typeDxn'][];
|
|
185
|
+
inverted?: boolean;
|
|
186
|
+
includeAllQueues?: boolean;
|
|
187
|
+
queueIds?: readonly string[] | null;
|
|
188
|
+
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
189
|
+
return this.#objectMetaIndex.queryTypes(query);
|
|
190
|
+
}
|
|
191
|
+
queryByTimeRange(query: {
|
|
192
|
+
spaceIds: readonly string[];
|
|
193
|
+
updatedAfter?: number;
|
|
194
|
+
updatedBefore?: number;
|
|
195
|
+
createdAfter?: number;
|
|
196
|
+
createdBefore?: number;
|
|
197
|
+
includeAllQueues?: boolean;
|
|
198
|
+
queueIds?: readonly string[] | null;
|
|
199
|
+
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
200
|
+
return this.#objectMetaIndex.queryByTimeRange(query);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
queryRelations(query: {
|
|
204
|
+
endpoint: 'source' | 'target';
|
|
205
|
+
anchorDxns: readonly string[];
|
|
206
|
+
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
207
|
+
return this.#objectMetaIndex.queryRelations(query);
|
|
208
|
+
}
|
|
209
|
+
lookupByRecordIds(recordIds: number[]): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
210
|
+
return this.#objectMetaIndex.lookupByRecordIds(recordIds);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
lookupByObjectId(query: {
|
|
214
|
+
objectId: string;
|
|
215
|
+
spaceId: string;
|
|
216
|
+
queueId: string;
|
|
217
|
+
}): Effect.Effect<ObjectMeta | null, SqlError.SqlError, SqlClient.SqlClient> {
|
|
218
|
+
return this.#objectMetaIndex.lookupByObjectId(query);
|
|
219
|
+
}
|
|
220
|
+
|
|
106
221
|
update(
|
|
222
|
+
ctx: Context,
|
|
107
223
|
dataSource: IndexDataSource,
|
|
108
224
|
opts: { spaceId: SpaceId | null; limit?: number },
|
|
109
|
-
): Effect.Effect<
|
|
225
|
+
): Effect.Effect<IndexingResult, SqlError.SqlError, SqlTransaction.SqlTransaction | SqlClient.SqlClient> {
|
|
110
226
|
return Effect.gen(this, function* () {
|
|
111
|
-
|
|
227
|
+
const result = makeEmptyIndexingResult();
|
|
112
228
|
|
|
113
|
-
const {
|
|
114
|
-
|
|
229
|
+
const {
|
|
230
|
+
updated: updatedFtsIndex,
|
|
231
|
+
done: doneFtsIndex,
|
|
232
|
+
objects: ftsObjects,
|
|
233
|
+
} = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
234
|
+
indexName: 'fts5',
|
|
115
235
|
spaceId: opts.spaceId,
|
|
116
236
|
limit: opts.limit,
|
|
117
237
|
});
|
|
118
|
-
updated += updatedFtsIndex;
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
this.#reverseRefIndex,
|
|
122
|
-
dataSource,
|
|
123
|
-
{
|
|
124
|
-
indexName: 'reverseRef',
|
|
125
|
-
spaceId: opts.spaceId,
|
|
126
|
-
limit: opts.limit,
|
|
127
|
-
},
|
|
128
|
-
);
|
|
129
|
-
updated += updatedReverseRefIndex;
|
|
238
|
+
result.updated += updatedFtsIndex;
|
|
239
|
+
result.done = result.done && doneFtsIndex;
|
|
240
|
+
accumulateIndexingResult(result, ftsObjects);
|
|
130
241
|
|
|
131
|
-
|
|
242
|
+
const {
|
|
243
|
+
updated: updatedReverseRefIndex,
|
|
244
|
+
done: doneReverseRefIndex,
|
|
245
|
+
objects: reverseRefObjects,
|
|
246
|
+
} = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
|
|
247
|
+
indexName: 'reverseRef',
|
|
248
|
+
spaceId: opts.spaceId,
|
|
249
|
+
limit: opts.limit,
|
|
250
|
+
});
|
|
251
|
+
result.updated += updatedReverseRefIndex;
|
|
252
|
+
result.done = result.done && doneReverseRefIndex;
|
|
253
|
+
accumulateIndexingResult(result, reverseRefObjects);
|
|
254
|
+
|
|
255
|
+
return result as IndexingResult;
|
|
132
256
|
}).pipe(Effect.withSpan('IndexEngine.update'));
|
|
133
257
|
}
|
|
134
258
|
|
|
@@ -142,13 +266,19 @@ export class IndexEngine {
|
|
|
142
266
|
* 5. Updates the dependent index.
|
|
143
267
|
*/
|
|
144
268
|
#update(
|
|
269
|
+
ctx: Context,
|
|
145
270
|
index: Index,
|
|
146
271
|
source: IndexDataSource,
|
|
147
272
|
opts: { indexName: string; spaceId: SpaceId | null; limit?: number },
|
|
148
|
-
): Effect.Effect<
|
|
273
|
+
): Effect.Effect<
|
|
274
|
+
{ updated: number; done: boolean; objects: readonly IndexerObject[] },
|
|
275
|
+
SqlError.SqlError,
|
|
276
|
+
SqlTransaction.SqlTransaction | SqlClient.SqlClient
|
|
277
|
+
> {
|
|
149
278
|
return Effect.gen(this, function* () {
|
|
150
|
-
const
|
|
151
|
-
|
|
279
|
+
const sqlTransaction = yield* SqlTransaction.SqlTransaction;
|
|
280
|
+
|
|
281
|
+
return yield* sqlTransaction.withTransaction(
|
|
152
282
|
Effect.gen(this, function* () {
|
|
153
283
|
const cursors = yield* this.#tracker.queryCursors({
|
|
154
284
|
indexName: opts.indexName,
|
|
@@ -156,9 +286,11 @@ export class IndexEngine {
|
|
|
156
286
|
// Pass undefined to get all cursors when spaceId is null.
|
|
157
287
|
spaceId: opts.spaceId ?? undefined,
|
|
158
288
|
});
|
|
159
|
-
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(cursors, {
|
|
289
|
+
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
|
|
290
|
+
limit: opts.limit,
|
|
291
|
+
});
|
|
160
292
|
if (objects.length === 0) {
|
|
161
|
-
return { updated: 0, done: true };
|
|
293
|
+
return { updated: 0, done: true, objects: [] as readonly IndexerObject[] };
|
|
162
294
|
}
|
|
163
295
|
|
|
164
296
|
// Ensure objects exist in ObjectMetaIndex.
|
|
@@ -179,9 +311,9 @@ export class IndexEngine {
|
|
|
179
311
|
}),
|
|
180
312
|
),
|
|
181
313
|
);
|
|
182
|
-
return { updated: objects.length, done: false };
|
|
314
|
+
return { updated: objects.length, done: false, objects };
|
|
183
315
|
}),
|
|
184
316
|
);
|
|
185
|
-
}).pipe(Effect.withSpan('IndexEngine.#
|
|
317
|
+
}).pipe(Effect.withSpan('IndexEngine.#update'));
|
|
186
318
|
}
|
|
187
319
|
}
|
package/src/index-tracker.ts
CHANGED
|
@@ -35,6 +35,11 @@ export const IndexCursor = Schema.Struct({
|
|
|
35
35
|
});
|
|
36
36
|
export interface IndexCursor extends Schema.Schema.Type<typeof IndexCursor> {}
|
|
37
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Deprecated index names that are no longer used. Will be cleaned up on migration.
|
|
40
|
+
*/
|
|
41
|
+
const DEPRECATED_INDEX_NAMES = ['fts'];
|
|
42
|
+
|
|
38
43
|
export class IndexTracker {
|
|
39
44
|
migrate = Effect.fn('IndexTracker.migrate')(function* () {
|
|
40
45
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -49,6 +54,10 @@ export class IndexTracker {
|
|
|
49
54
|
cursor,
|
|
50
55
|
PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
|
|
51
56
|
)`;
|
|
57
|
+
|
|
58
|
+
yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
|
|
59
|
+
return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
|
|
60
|
+
});
|
|
52
61
|
});
|
|
53
62
|
|
|
54
63
|
queryCursors = Effect.fn('IndexTracker.queryCursors')(
|
package/src/index.ts
CHANGED
|
@@ -2,9 +2,16 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
export {
|
|
5
|
+
export {
|
|
6
|
+
IndexEngine,
|
|
7
|
+
type IndexDataSource,
|
|
8
|
+
type DataSourceCursor,
|
|
9
|
+
type IndexEngineParams,
|
|
10
|
+
type IndexingResult,
|
|
11
|
+
} from './index-engine';
|
|
6
12
|
export { IndexTracker, type IndexCursor } from './index-tracker';
|
|
7
13
|
export { type IndexerObject, type Index } from './indexes/interface';
|
|
8
|
-
export { FtsIndex } from './indexes/fts-index';
|
|
14
|
+
export { FtsIndex, type FtsQuery } from './indexes/fts-index';
|
|
9
15
|
export { ObjectMetaIndex, type ObjectMeta } from './indexes/object-meta-index';
|
|
10
|
-
export { ReverseRefIndex, type ReverseRef } from './indexes/reverse-ref-index';
|
|
16
|
+
export { ReverseRefIndex, type ReverseRef, type ReverseRefQuery } from './indexes/reverse-ref-index';
|
|
17
|
+
export { EscapedPropPath, type ObjectPropPath } from './utils';
|