@dxos/index-core 0.0.0 → 0.8.4-main.03d5cd7b56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/lib/neutral/index.mjs +790 -0
  2. package/dist/lib/neutral/index.mjs.map +7 -0
  3. package/dist/lib/neutral/meta.json +1 -0
  4. package/dist/types/src/index-engine.d.ts +112 -0
  5. package/dist/types/src/index-engine.d.ts.map +1 -0
  6. package/dist/types/src/index-engine.test.d.ts +2 -0
  7. package/dist/types/src/index-engine.test.d.ts.map +1 -0
  8. package/dist/types/src/index-tracker.d.ts +44 -0
  9. package/dist/types/src/index-tracker.d.ts.map +1 -0
  10. package/dist/types/src/index-tracker.test.d.ts +2 -0
  11. package/dist/types/src/index-tracker.test.d.ts.map +1 -0
  12. package/dist/types/src/index.d.ts +8 -0
  13. package/dist/types/src/index.d.ts.map +1 -0
  14. package/dist/types/src/indexes/fts-index.d.ts +64 -0
  15. package/dist/types/src/indexes/fts-index.d.ts.map +1 -0
  16. package/dist/types/src/indexes/fts-index.test.d.ts +2 -0
  17. package/dist/types/src/indexes/fts-index.test.d.ts.map +1 -0
  18. package/dist/types/src/indexes/fts5.test.d.ts +2 -0
  19. package/dist/types/src/indexes/fts5.test.d.ts.map +1 -0
  20. package/dist/types/src/indexes/index.d.ts +5 -0
  21. package/dist/types/src/indexes/index.d.ts.map +1 -0
  22. package/dist/types/src/indexes/interface.d.ts +56 -0
  23. package/dist/types/src/indexes/interface.d.ts.map +1 -0
  24. package/dist/types/src/indexes/object-meta-index.d.ts +94 -0
  25. package/dist/types/src/indexes/object-meta-index.d.ts.map +1 -0
  26. package/dist/types/src/indexes/object-meta-index.test.d.ts +2 -0
  27. package/dist/types/src/indexes/object-meta-index.test.d.ts.map +1 -0
  28. package/dist/types/src/indexes/reverse-ref-index.d.ts +37 -0
  29. package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -0
  30. package/dist/types/src/indexes/reverse-ref-index.test.d.ts +2 -0
  31. package/dist/types/src/indexes/reverse-ref-index.test.d.ts.map +1 -0
  32. package/dist/types/src/utils.d.ts +17 -0
  33. package/dist/types/src/utils.d.ts.map +1 -0
  34. package/dist/types/tsconfig.tsbuildinfo +1 -0
  35. package/package.json +22 -18
  36. package/src/index-engine.test.ts +172 -9
  37. package/src/index-engine.ts +161 -29
  38. package/src/index-tracker.ts +9 -0
  39. package/src/index.ts +10 -3
  40. package/src/indexes/fts-index.test.ts +153 -3
  41. package/src/indexes/fts-index.ts +66 -10
  42. package/src/indexes/interface.ts +10 -0
  43. package/src/indexes/object-meta-index.test.ts +361 -3
  44. package/src/indexes/object-meta-index.ts +304 -17
  45. package/src/indexes/reverse-ref-index.test.ts +16 -2
  46. package/src/indexes/reverse-ref-index.ts +0 -1
  47. package/src/utils.ts +1 -1
package/package.json CHANGED
@@ -1,9 +1,13 @@
1
1
  {
2
2
  "name": "@dxos/index-core",
3
- "version": "0.0.0",
3
+ "version": "0.8.4-main.03d5cd7b56",
4
4
  "description": "Indexing core.",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "https://github.com/dxos/dxos"
10
+ },
7
11
  "license": "MIT",
8
12
  "author": "info@dxos.org",
9
13
  "sideEffects": false,
@@ -12,8 +16,7 @@
12
16
  ".": {
13
17
  "source": "./src/index.ts",
14
18
  "types": "./dist/types/src/index.d.ts",
15
- "browser": "./dist/lib/browser/index.mjs",
16
- "node": "./dist/lib/node-esm/index.mjs"
19
+ "default": "./dist/lib/neutral/index.mjs"
17
20
  }
18
21
  },
19
22
  "files": [
@@ -21,23 +24,24 @@
21
24
  "src"
22
25
  ],
23
26
  "dependencies": {
24
- "@dxos/async": "",
25
- "@dxos/context": "",
26
- "@dxos/debug": "",
27
- "@dxos/echo": "",
28
- "@dxos/echo-protocol": "",
29
- "@dxos/effect": "",
30
- "@dxos/invariant": "",
31
- "@dxos/keys": "",
32
- "@dxos/log": "",
33
- "@effect/cli": "0.72.1",
34
- "@effect/experimental": "0.57.11",
35
- "@effect/platform": "0.93.6",
36
- "@effect/sql": "0.48.6",
37
- "effect": "3.19.11"
27
+ "@effect/cli": "0.73.2",
28
+ "@effect/experimental": "0.58.0",
29
+ "@effect/platform": "0.94.4",
30
+ "@effect/sql": "0.49.0",
31
+ "effect": "3.20.0",
32
+ "@dxos/async": "0.8.4-main.03d5cd7b56",
33
+ "@dxos/context": "0.8.4-main.03d5cd7b56",
34
+ "@dxos/debug": "0.8.4-main.03d5cd7b56",
35
+ "@dxos/echo-protocol": "0.8.4-main.03d5cd7b56",
36
+ "@dxos/invariant": "0.8.4-main.03d5cd7b56",
37
+ "@dxos/effect": "0.8.4-main.03d5cd7b56",
38
+ "@dxos/keys": "0.8.4-main.03d5cd7b56",
39
+ "@dxos/log": "0.8.4-main.03d5cd7b56",
40
+ "@dxos/sql-sqlite": "0.8.4-main.03d5cd7b56",
41
+ "@dxos/echo": "0.8.4-main.03d5cd7b56"
38
42
  },
39
43
  "devDependencies": {
40
- "@effect/sql-sqlite-node": "0.49.1"
44
+ "@effect/sql-sqlite-node": "0.50.1"
41
45
  },
42
46
  "publishConfig": {
43
47
  "access": "public"
@@ -8,11 +8,13 @@ import { describe, expect, it } from '@effect/vitest';
8
8
  import * as Effect from 'effect/Effect';
9
9
  import * as Layer from 'effect/Layer';
10
10
 
11
+ import { Context } from '@dxos/context';
11
12
  import { ATTR_TYPE } from '@dxos/echo/internal';
12
13
  import { invariant } from '@dxos/invariant';
13
14
  import { DXN, ObjectId, SpaceId } from '@dxos/keys';
15
+ import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
14
16
 
15
- import { type DataSourceCursor, type IndexDataSource, IndexEngine } from './index-engine';
17
+ import { type DataSourceCursor, type IndexDataSource, IndexEngine, type IndexingResult } from './index-engine';
16
18
  import { type IndexCursor, IndexTracker } from './index-tracker';
17
19
  import { FtsIndex, type IndexerObject, ObjectMetaIndex, ReverseRefIndex } from './indexes';
18
20
 
@@ -20,11 +22,13 @@ const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
20
22
  const TYPE_A = DXN.parse('dxn:type:test.com/type/TypeA:0.1.0').toString();
21
23
  const TYPE_B = DXN.parse('dxn:type:test.com/type/TypeB:0.1.0').toString();
22
24
 
23
- const TestLayer = Layer.merge(
24
- SqliteClient.layer({
25
- filename: ':memory:',
26
- }),
27
- Reactivity.layer,
25
+ const TestLayer = SqlTransaction.layer.pipe(
26
+ Layer.provideMerge(
27
+ SqliteClient.layer({
28
+ filename: ':memory:',
29
+ }),
30
+ ),
31
+ Layer.provideMerge(Reactivity.layer),
28
32
  );
29
33
 
30
34
  class MockIndexDataSource implements IndexDataSource {
@@ -46,6 +50,7 @@ class MockIndexDataSource implements IndexDataSource {
46
50
  }
47
51
 
48
52
  getChangedObjects(
53
+ _ctx: Context,
49
54
  cursors: IndexCursor[],
50
55
  opts?: { limit?: number },
51
56
  ): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }> {
@@ -114,7 +119,9 @@ describe('IndexEngine', () => {
114
119
  spaceId,
115
120
  documentId: 'doc-1',
116
121
  queueId: null,
122
+ queueNamespace: null,
117
123
  recordId: null,
124
+ updatedAt: Date.now(),
118
125
  data: {
119
126
  id: ObjectId.random(),
120
127
  [ATTR_TYPE]: TYPE_DEFAULT,
@@ -125,7 +132,7 @@ describe('IndexEngine', () => {
125
132
  dataSource.push([obj1]);
126
133
 
127
134
  // First update.
128
- const { updated } = yield* engine.update(dataSource, { spaceId: null });
135
+ const { updated } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
129
136
  // Updates objectMeta, FTS, and reverseRef indexes.
130
137
  expect(updated).toBe(2);
131
138
 
@@ -150,13 +157,15 @@ describe('IndexEngine', () => {
150
157
  spaceId,
151
158
  documentId: obj1.documentId,
152
159
  queueId: null,
160
+ queueNamespace: null,
153
161
  recordId: null,
162
+ updatedAt: Date.now(),
154
163
  data: { id: obj1.data.id, [ATTR_TYPE]: obj1.data[ATTR_TYPE], title: 'Hello World' },
155
164
  };
156
165
  dataSource.push([obj1Updated]);
157
166
 
158
167
  // Second update.
159
- const { updated: updated2 } = yield* engine.update(dataSource, { spaceId: null });
168
+ const { updated: updated2 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
160
169
  expect(updated2).toBe(2);
161
170
 
162
171
  // Verify update.
@@ -188,8 +197,10 @@ describe('IndexEngine', () => {
188
197
  {
189
198
  spaceId,
190
199
  queueId: null,
200
+ queueNamespace: null,
191
201
  documentId: 'd1',
192
202
  recordId: null,
203
+ updatedAt: Date.now(),
193
204
  data: {
194
205
  id: ObjectId.random(),
195
206
  [ATTR_TYPE]: TYPE_A,
@@ -199,8 +210,10 @@ describe('IndexEngine', () => {
199
210
  {
200
211
  spaceId,
201
212
  queueId: null,
213
+ queueNamespace: null,
202
214
  documentId: 'd2',
203
215
  recordId: null,
216
+ updatedAt: Date.now(),
204
217
  data: {
205
218
  id: ObjectId.random(),
206
219
  [ATTR_TYPE]: TYPE_A,
@@ -210,8 +223,10 @@ describe('IndexEngine', () => {
210
223
  {
211
224
  spaceId,
212
225
  queueId: null,
226
+ queueNamespace: null,
213
227
  documentId: 'd3',
214
228
  recordId: null,
229
+ updatedAt: Date.now(),
215
230
  data: {
216
231
  id: ObjectId.random(),
217
232
  [ATTR_TYPE]: TYPE_B,
@@ -222,7 +237,7 @@ describe('IndexEngine', () => {
222
237
 
223
238
  dataSource.push(objects);
224
239
 
225
- yield* engine.update(dataSource, { spaceId: null });
240
+ yield* engine.update(Context.default(), dataSource, { spaceId: null });
226
241
 
227
242
  const resultsA = yield* metaIndex.query({ spaceId: spaceId.toString(), typeDxn: TYPE_A });
228
243
  expect(resultsA).toHaveLength(2);
@@ -239,4 +254,152 @@ describe('IndexEngine', () => {
239
254
  expect(ftsResults).toHaveLength(2);
240
255
  }, Effect.provide(TestLayer)),
241
256
  );
257
+
258
+ it.effect(
259
+ 'done is true only when all sub-indexes have no remaining work',
260
+ Effect.fnUntraced(function* () {
261
+ const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
262
+
263
+ const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
264
+ const dataSource = new MockIndexDataSource();
265
+ const spaceId = SpaceId.random();
266
+
267
+ // First update with no data — both sub-indexes report done immediately.
268
+ const { updated: updated0, done: done0 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
269
+ expect(updated0).toBe(0);
270
+ expect(done0).toBe(true);
271
+
272
+ // Push an object so sub-indexes have work to do.
273
+ dataSource.push([
274
+ {
275
+ spaceId,
276
+ queueId: null,
277
+ queueNamespace: null,
278
+ documentId: 'doc-done-test',
279
+ recordId: null,
280
+ updatedAt: Date.now(),
281
+ data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_DEFAULT, title: 'Done test' },
282
+ },
283
+ ]);
284
+
285
+ // Update with pending data — sub-indexes process objects, done is false.
286
+ const { updated: updated1, done: done1 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
287
+ expect(updated1).toBeGreaterThan(0);
288
+ expect(done1).toBe(false);
289
+
290
+ // Second update with no new data — all sub-indexes caught up, done is true.
291
+ const { updated: updated2, done: done2 } = yield* engine.update(Context.default(), dataSource, {
292
+ spaceId: null,
293
+ });
294
+ expect(updated2).toBe(0);
295
+ expect(done2).toBe(true);
296
+ }, Effect.provide(TestLayer)),
297
+ );
298
+
299
+ it.effect(
300
+ 'IndexingResult contains correct sets for a batch with multiple objects across spaces',
301
+ Effect.fnUntraced(function* () {
302
+ const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
303
+ const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
304
+ const dataSource = new MockIndexDataSource();
305
+ const spaceId1 = SpaceId.random();
306
+ const spaceId2 = SpaceId.random();
307
+ const id1 = ObjectId.random();
308
+ const id2 = ObjectId.random();
309
+
310
+ const obj1: IndexerObject = {
311
+ spaceId: spaceId1,
312
+ documentId: 'doc-result-1',
313
+ queueId: null,
314
+ queueNamespace: null,
315
+ recordId: null,
316
+ updatedAt: Date.now(),
317
+ data: { id: id1, [ATTR_TYPE]: TYPE_A, title: 'Doc in space1' },
318
+ };
319
+ const obj2: IndexerObject = {
320
+ spaceId: spaceId2,
321
+ documentId: 'doc-result-2',
322
+ queueId: null,
323
+ queueNamespace: null,
324
+ recordId: null,
325
+ updatedAt: Date.now(),
326
+ data: { id: id2, [ATTR_TYPE]: TYPE_B, title: 'Doc in space2' },
327
+ };
328
+
329
+ dataSource.push([obj1, obj2]);
330
+
331
+ const result: IndexingResult = yield* engine.update(Context.default(), dataSource, { spaceId: null });
332
+
333
+ expect(result.updated).toBeGreaterThan(0);
334
+ expect(result.done).toBe(false);
335
+
336
+ // Spaces: both spaceIds should be present.
337
+ expect(result.spaces.has(spaceId1)).toBe(true);
338
+ expect(result.spaces.has(spaceId2)).toBe(true);
339
+
340
+ // Documents: both doc IDs should be present.
341
+ expect(result.documents.has('doc-result-1')).toBe(true);
342
+ expect(result.documents.has('doc-result-2')).toBe(true);
343
+
344
+ // Types: both TYPE_A and TYPE_B.
345
+ expect(result.types.has(TYPE_A)).toBe(true);
346
+ expect(result.types.has(TYPE_B)).toBe(true);
347
+
348
+ // Object ids.
349
+ expect(result.objects.has(id1)).toBe(true);
350
+ expect(result.objects.has(id2)).toBe(true);
351
+ }, Effect.provide(TestLayer)),
352
+ );
353
+
354
+ it.effect(
355
+ 'IndexingResult includes typename for deleted objects',
356
+ Effect.fnUntraced(function* () {
357
+ const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
358
+ const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
359
+ const dataSource = new MockIndexDataSource();
360
+ const spaceId = SpaceId.random();
361
+
362
+ const deletedObj: IndexerObject = {
363
+ spaceId,
364
+ documentId: 'doc-deleted',
365
+ queueId: null,
366
+ queueNamespace: null,
367
+ recordId: null,
368
+ updatedAt: Date.now(),
369
+ data: {
370
+ id: ObjectId.random(),
371
+ [ATTR_TYPE]: TYPE_DEFAULT,
372
+ '@deleted': true,
373
+ },
374
+ };
375
+
376
+ dataSource.push([deletedObj]);
377
+
378
+ const result: IndexingResult = yield* engine.update(Context.default(), dataSource, { spaceId: null });
379
+
380
+ expect(result.updated).toBeGreaterThan(0);
381
+ // Deleted objects should still contribute their typename to the hint.
382
+ expect(result.types.has(TYPE_DEFAULT)).toBe(true);
383
+ expect(result.spaces.has(spaceId)).toBe(true);
384
+ }, Effect.provide(TestLayer)),
385
+ );
386
+
387
+ it.effect(
388
+ 'IndexingResult is empty when no objects are indexed',
389
+ Effect.fnUntraced(function* () {
390
+ const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
391
+ const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
392
+ const dataSource = new MockIndexDataSource();
393
+
394
+ const result: IndexingResult = yield* engine.update(Context.default(), dataSource, { spaceId: null });
395
+
396
+ expect(result.updated).toBe(0);
397
+ expect(result.done).toBe(true);
398
+ expect(result.spaces.size).toBe(0);
399
+ expect(result.queues.size).toBe(0);
400
+ expect(result.documents.size).toBe(0);
401
+ expect(result.types.size).toBe(0);
402
+ expect(result.objects.size).toBe(0);
403
+ }, Effect.provide(TestLayer)),
404
+ );
242
405
  });
@@ -2,16 +2,20 @@
2
2
  // Copyright 2026 DXOS.org
3
3
  //
4
4
 
5
- import * as SqlClient from '@effect/sql/SqlClient';
5
+ import type * as SqlClient from '@effect/sql/SqlClient';
6
6
  import type * as SqlError from '@effect/sql/SqlError';
7
7
  import * as Effect from 'effect/Effect';
8
8
 
9
- import type { SpaceId } from '@dxos/keys';
9
+ import { type Context } from '@dxos/context';
10
+ import { ATTR_TYPE } from '@dxos/echo/internal';
11
+ import type { ObjectId, SpaceId } from '@dxos/keys';
12
+ import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
10
13
 
11
14
  import { type IndexCursor, IndexTracker } from './index-tracker';
12
15
  import {
13
16
  FtsIndex,
14
17
  type FtsQuery,
18
+ type FtsQueryResult,
15
19
  type Index,
16
20
  type IndexerObject,
17
21
  type ObjectMeta,
@@ -20,6 +24,59 @@ import {
20
24
  type ReverseRefQuery,
21
25
  } from './indexes';
22
26
 
27
+ /**
28
+ * Result of a single indexing pass over a data source.
29
+ * Carries enough metadata for callers to build targeted invalidation hints.
30
+ */
31
+ export type IndexingResult = {
32
+ updated: number;
33
+ done: boolean;
34
+ spaces: ReadonlySet<SpaceId>;
35
+ queues: ReadonlySet<ObjectId>;
36
+ documents: ReadonlySet<string>;
37
+ types: ReadonlySet<string>;
38
+ objects: ReadonlySet<ObjectId>;
39
+ };
40
+
41
+ type MutableIndexingResult = {
42
+ updated: number;
43
+ done: boolean;
44
+ spaces: Set<SpaceId>;
45
+ queues: Set<ObjectId>;
46
+ documents: Set<string>;
47
+ types: Set<string>;
48
+ objects: Set<ObjectId>;
49
+ };
50
+
51
+ const makeEmptyIndexingResult = (): MutableIndexingResult => ({
52
+ updated: 0,
53
+ done: true,
54
+ spaces: new Set(),
55
+ queues: new Set(),
56
+ documents: new Set(),
57
+ types: new Set(),
58
+ objects: new Set(),
59
+ });
60
+
61
+ const accumulateIndexingResult = (acc: MutableIndexingResult, objects: readonly IndexerObject[]) => {
62
+ for (const obj of objects) {
63
+ acc.spaces.add(obj.spaceId);
64
+ if (obj.queueId) {
65
+ acc.queues.add(obj.queueId);
66
+ }
67
+ if (obj.documentId) {
68
+ acc.documents.add(obj.documentId);
69
+ }
70
+ const t = (obj.data as Record<string, unknown>)[ATTR_TYPE];
71
+ if (t) {
72
+ acc.types.add(String(t));
73
+ }
74
+ if (obj.data.id) {
75
+ acc.objects.add(obj.data.id as ObjectId);
76
+ }
77
+ }
78
+ };
79
+
23
80
  /**
24
81
  * Cursor into indexable data-source.
25
82
  */
@@ -27,7 +84,7 @@ export interface DataSourceCursor {
27
84
  spaceId: SpaceId | null;
28
85
 
29
86
  /**
30
- * documentId or queueId.
87
+ * documentId or queueNamespace.
31
88
  */
32
89
  resourceId: string | null;
33
90
 
@@ -41,6 +98,7 @@ export interface IndexDataSource {
41
98
  readonly sourceName: string; // e.g. queue, automerge, etc.
42
99
 
43
100
  getChangedObjects(
101
+ ctx: Context,
44
102
  cursors: DataSourceCursor[],
45
103
  opts?: { limit?: number },
46
104
  ): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }>;
@@ -76,9 +134,9 @@ export class IndexEngine {
76
134
  }
77
135
 
78
136
  /**
79
- * Query text index and return full object metadata.
137
+ * Query text index and return full object metadata with rank.
80
138
  */
81
- queryText(query: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
139
+ queryText(query: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
82
140
  return Effect.gen(this, function* () {
83
141
  return yield* this.#ftsIndex.query(query);
84
142
  });
@@ -89,6 +147,14 @@ export class IndexEngine {
89
147
  return this.#reverseRefIndex.query(query);
90
148
  }
91
149
 
150
+ queryAll(query: {
151
+ spaceIds: readonly SpaceId[];
152
+ includeAllQueues?: boolean;
153
+ queueIds?: readonly string[] | null;
154
+ }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
155
+ return this.#objectMetaIndex.queryAll(query);
156
+ }
157
+
92
158
  /**
93
159
  * Query snapshots by recordIds.
94
160
  * Used to load queue objects from indexed snapshots.
@@ -103,32 +169,90 @@ export class IndexEngine {
103
169
  return this.#objectMetaIndex.query(query);
104
170
  }
105
171
 
172
+ /**
173
+ * Query children by parent object ids.
174
+ */
175
+ queryChildren(query: {
176
+ spaceId: SpaceId[];
177
+ parentIds: ObjectId[];
178
+ }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
179
+ return this.#objectMetaIndex.queryChildren(query);
180
+ }
181
+
182
+ queryTypes(query: {
183
+ spaceIds: readonly SpaceId[];
184
+ typeDxns: readonly ObjectMeta['typeDxn'][];
185
+ inverted?: boolean;
186
+ includeAllQueues?: boolean;
187
+ queueIds?: readonly string[] | null;
188
+ }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
189
+ return this.#objectMetaIndex.queryTypes(query);
190
+ }
191
+ queryByTimeRange(query: {
192
+ spaceIds: readonly string[];
193
+ updatedAfter?: number;
194
+ updatedBefore?: number;
195
+ createdAfter?: number;
196
+ createdBefore?: number;
197
+ includeAllQueues?: boolean;
198
+ queueIds?: readonly string[] | null;
199
+ }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
200
+ return this.#objectMetaIndex.queryByTimeRange(query);
201
+ }
202
+
203
+ queryRelations(query: {
204
+ endpoint: 'source' | 'target';
205
+ anchorDxns: readonly string[];
206
+ }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
207
+ return this.#objectMetaIndex.queryRelations(query);
208
+ }
209
+ lookupByRecordIds(recordIds: number[]): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
210
+ return this.#objectMetaIndex.lookupByRecordIds(recordIds);
211
+ }
212
+
213
+ lookupByObjectId(query: {
214
+ objectId: string;
215
+ spaceId: string;
216
+ queueId: string;
217
+ }): Effect.Effect<ObjectMeta | null, SqlError.SqlError, SqlClient.SqlClient> {
218
+ return this.#objectMetaIndex.lookupByObjectId(query);
219
+ }
220
+
106
221
  update(
222
+ ctx: Context,
107
223
  dataSource: IndexDataSource,
108
224
  opts: { spaceId: SpaceId | null; limit?: number },
109
- ): Effect.Effect<{ updated: number; done: boolean }, SqlError.SqlError, SqlClient.SqlClient> {
225
+ ): Effect.Effect<IndexingResult, SqlError.SqlError, SqlTransaction.SqlTransaction | SqlClient.SqlClient> {
110
226
  return Effect.gen(this, function* () {
111
- let updated = 0;
227
+ const result = makeEmptyIndexingResult();
112
228
 
113
- const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(this.#ftsIndex, dataSource, {
114
- indexName: 'fts',
229
+ const {
230
+ updated: updatedFtsIndex,
231
+ done: doneFtsIndex,
232
+ objects: ftsObjects,
233
+ } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
234
+ indexName: 'fts5',
115
235
  spaceId: opts.spaceId,
116
236
  limit: opts.limit,
117
237
  });
118
- updated += updatedFtsIndex;
119
-
120
- const { updated: updatedReverseRefIndex, done: doneReverseRefIndex } = yield* this.#update(
121
- this.#reverseRefIndex,
122
- dataSource,
123
- {
124
- indexName: 'reverseRef',
125
- spaceId: opts.spaceId,
126
- limit: opts.limit,
127
- },
128
- );
129
- updated += updatedReverseRefIndex;
238
+ result.updated += updatedFtsIndex;
239
+ result.done = result.done && doneFtsIndex;
240
+ accumulateIndexingResult(result, ftsObjects);
130
241
 
131
- return { updated, done: doneFtsIndex && doneReverseRefIndex };
242
+ const {
243
+ updated: updatedReverseRefIndex,
244
+ done: doneReverseRefIndex,
245
+ objects: reverseRefObjects,
246
+ } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
247
+ indexName: 'reverseRef',
248
+ spaceId: opts.spaceId,
249
+ limit: opts.limit,
250
+ });
251
+ result.updated += updatedReverseRefIndex;
252
+ result.done = result.done && doneReverseRefIndex;
253
+ accumulateIndexingResult(result, reverseRefObjects);
254
+
255
+ return result as IndexingResult;
132
256
  }).pipe(Effect.withSpan('IndexEngine.update'));
133
257
  }
134
258
 
@@ -142,13 +266,19 @@ export class IndexEngine {
142
266
  * 5. Updates the dependent index.
143
267
  */
144
268
  #update(
269
+ ctx: Context,
145
270
  index: Index,
146
271
  source: IndexDataSource,
147
272
  opts: { indexName: string; spaceId: SpaceId | null; limit?: number },
148
- ): Effect.Effect<{ updated: number; done: boolean }, SqlError.SqlError, SqlClient.SqlClient> {
273
+ ): Effect.Effect<
274
+ { updated: number; done: boolean; objects: readonly IndexerObject[] },
275
+ SqlError.SqlError,
276
+ SqlTransaction.SqlTransaction | SqlClient.SqlClient
277
+ > {
149
278
  return Effect.gen(this, function* () {
150
- const sql = yield* SqlClient.SqlClient;
151
- return yield* sql.withTransaction(
279
+ const sqlTransaction = yield* SqlTransaction.SqlTransaction;
280
+
281
+ return yield* sqlTransaction.withTransaction(
152
282
  Effect.gen(this, function* () {
153
283
  const cursors = yield* this.#tracker.queryCursors({
154
284
  indexName: opts.indexName,
@@ -156,9 +286,11 @@ export class IndexEngine {
156
286
  // Pass undefined to get all cursors when spaceId is null.
157
287
  spaceId: opts.spaceId ?? undefined,
158
288
  });
159
- const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(cursors, { limit: opts.limit });
289
+ const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
290
+ limit: opts.limit,
291
+ });
160
292
  if (objects.length === 0) {
161
- return { updated: 0, done: true };
293
+ return { updated: 0, done: true, objects: [] as readonly IndexerObject[] };
162
294
  }
163
295
 
164
296
  // Ensure objects exist in ObjectMetaIndex.
@@ -179,9 +311,9 @@ export class IndexEngine {
179
311
  }),
180
312
  ),
181
313
  );
182
- return { updated: objects.length, done: false };
314
+ return { updated: objects.length, done: false, objects };
183
315
  }),
184
316
  );
185
- }).pipe(Effect.withSpan('IndexEngine.#updateDependentIndex'));
317
+ }).pipe(Effect.withSpan('IndexEngine.#update'));
186
318
  }
187
319
  }
@@ -35,6 +35,11 @@ export const IndexCursor = Schema.Struct({
35
35
  });
36
36
  export interface IndexCursor extends Schema.Schema.Type<typeof IndexCursor> {}
37
37
 
38
+ /**
39
+ * Deprecated index names that are no longer used. Will be cleaned up on migration.
40
+ */
41
+ const DEPRECATED_INDEX_NAMES = ['fts'];
42
+
38
43
  export class IndexTracker {
39
44
  migrate = Effect.fn('IndexTracker.migrate')(function* () {
40
45
  const sql = yield* SqlClient.SqlClient;
@@ -49,6 +54,10 @@ export class IndexTracker {
49
54
  cursor,
50
55
  PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
51
56
  )`;
57
+
58
+ yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
59
+ return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
60
+ });
52
61
  });
53
62
 
54
63
  queryCursors = Effect.fn('IndexTracker.queryCursors')(
package/src/index.ts CHANGED
@@ -2,9 +2,16 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- export { IndexEngine, type IndexDataSource, type DataSourceCursor, type IndexEngineParams } from './index-engine';
5
+ export {
6
+ IndexEngine,
7
+ type IndexDataSource,
8
+ type DataSourceCursor,
9
+ type IndexEngineParams,
10
+ type IndexingResult,
11
+ } from './index-engine';
6
12
  export { IndexTracker, type IndexCursor } from './index-tracker';
7
13
  export { type IndexerObject, type Index } from './indexes/interface';
8
- export { FtsIndex } from './indexes/fts-index';
14
+ export { FtsIndex, type FtsQuery } from './indexes/fts-index';
9
15
  export { ObjectMetaIndex, type ObjectMeta } from './indexes/object-meta-index';
10
- export { ReverseRefIndex, type ReverseRef } from './indexes/reverse-ref-index';
16
+ export { ReverseRefIndex, type ReverseRef, type ReverseRefQuery } from './indexes/reverse-ref-index';
17
+ export { EscapedPropPath, type ObjectPropPath } from './utils';