@dxos/index-core 0.10.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,837 +0,0 @@
1
- // src/index-engine.ts
2
- import * as Effect5 from "effect/Effect";
3
- import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
4
- import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
5
-
6
- // src/index-tracker.ts
7
- import * as SqlClient from "@effect/sql/SqlClient";
8
- import * as Effect from "effect/Effect";
9
- import * as Schema from "effect/Schema";
10
- import { SpaceId } from "@dxos/keys";
11
- var IndexCursor = Schema.Struct({
12
- /**
13
- * Name of the index owning this cursor.
14
- */
15
- indexName: Schema.String,
16
- /**
17
- * Space id.
18
- */
19
- spaceId: Schema.NullOr(SpaceId),
20
- /**
21
- * Source name.
22
- * 'automerge' / 'queue' / 'index' (for secondary indexes)
23
- */
24
- sourceName: Schema.String,
25
- /**
26
- * Document id or queue id.
27
- * doc_id, queue_id, '' <empty string> (if indexing entire namespace)
28
- */
29
- resourceId: Schema.NullOr(Schema.String),
30
- /**
31
- * Heads, queue position, version.
32
- */
33
- cursor: Schema.Union(Schema.Number, Schema.String)
34
- });
35
- var DEPRECATED_INDEX_NAMES = [
36
- "fts"
37
- ];
38
- var IndexTracker = class {
39
- migrate = Effect.fn("IndexTracker.migrate")(function* () {
40
- const sql = yield* SqlClient.SqlClient;
41
- yield* sql`CREATE TABLE IF NOT EXISTS indexCursor (
42
- indexName TEXT NOT NULL,
43
- spaceId TEXT NOT NULL DEFAULT '',
44
- sourceName TEXT NOT NULL,
45
- resourceId TEXT NOT NULL DEFAULT '',
46
- cursor,
47
- PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
48
- )`;
49
- yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
50
- return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
51
- });
52
- });
53
- queryCursors = Effect.fn("IndexTracker.queryCursors")((query) => Effect.gen(function* () {
54
- const sql = yield* SqlClient.SqlClient;
55
- const spaceIdParam = query.spaceId === void 0 ? null : query.spaceId ?? "";
56
- const sourceNameParam = query.sourceName === void 0 ? null : query.sourceName;
57
- const resourceIdParam = query.resourceId === void 0 ? null : query.resourceId ?? "";
58
- const rows = yield* sql`
59
- SELECT * FROM indexCursor
60
- WHERE indexName = ${query.indexName}
61
- AND (${spaceIdParam} IS NULL OR spaceId = ${spaceIdParam})
62
- AND (${sourceNameParam} IS NULL OR sourceName = ${sourceNameParam})
63
- AND (${resourceIdParam} IS NULL OR resourceId = ${resourceIdParam})
64
- `;
65
- return rows.map((row) => ({
66
- indexName: row.indexName,
67
- spaceId: row.spaceId === "" ? null : Schema.decodeSync(SpaceId)(row.spaceId),
68
- sourceName: row.sourceName,
69
- resourceId: row.resourceId === "" ? null : row.resourceId,
70
- cursor: row.cursor
71
- }));
72
- }));
73
- updateCursors = Effect.fn("IndexTracker.updateCursors")((cursors) => Effect.gen(function* () {
74
- const sql = yield* SqlClient.SqlClient;
75
- yield* Effect.forEach(cursors, (cursor) => {
76
- const spaceId = cursor.spaceId ?? "";
77
- const resourceId = cursor.resourceId ?? "";
78
- return sql`
79
- INSERT INTO indexCursor (indexName, spaceId, sourceName, resourceId, cursor)
80
- VALUES (${cursor.indexName}, ${spaceId}, ${cursor.sourceName}, ${resourceId}, ${cursor.cursor})
81
- ON CONFLICT(indexName, spaceId, sourceName, resourceId) DO UPDATE SET cursor = excluded.cursor
82
- `;
83
- }, {
84
- discard: true
85
- });
86
- }));
87
- };
88
-
89
- // src/indexes/fts-index.ts
90
- import * as SqlClient3 from "@effect/sql/SqlClient";
91
- import * as Effect2 from "effect/Effect";
92
- var SQL_CHUNK_SIZE = 500;
93
- var escapeFts5Query = (text) => {
94
- return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
95
- };
96
- var FtsIndex = class {
97
- migrate = Effect2.fn("FtsIndex.migrate")(function* () {
98
- const sql = yield* SqlClient3.SqlClient;
99
- yield* sql`CREATE VIRTUAL TABLE IF NOT EXISTS ftsIndex USING fts5(snapshot, tokenize='trigram')`;
100
- });
101
- query({ query, spaceId, includeAllQueues, queueIds }) {
102
- return Effect2.gen(function* () {
103
- const trimmed = query.trim();
104
- if (trimmed.length === 0) {
105
- return [];
106
- }
107
- const sql = yield* SqlClient3.SqlClient;
108
- const terms = trimmed.split(/\s+/).filter(Boolean);
109
- const minTermLength = Math.min(...terms.map((t) => t.length));
110
- const useBm25 = minTermLength >= 3;
111
- const conditions = minTermLength < 3 ? terms.map((term) => sql`f.snapshot LIKE ${"%" + term + "%"}`) : [
112
- sql`f.snapshot MATCH ${escapeFts5Query(trimmed)}`
113
- ];
114
- const sourceConditions = [];
115
- if (spaceId && spaceId.length > 0) {
116
- if (includeAllQueues) {
117
- sourceConditions.push(sql`m.spaceId IN ${sql.in(spaceId)}`);
118
- } else {
119
- sourceConditions.push(sql`(m.spaceId IN ${sql.in(spaceId)} AND m.queueId = '')`);
120
- }
121
- }
122
- if (queueIds && queueIds.length > 0) {
123
- sourceConditions.push(sql`m.queueId IN ${sql.in(queueIds)}`);
124
- }
125
- if (sourceConditions.length > 0) {
126
- conditions.push(sql`(${sql.or(sourceConditions)})`);
127
- }
128
- if (useBm25) {
129
- const rows = yield* sql`
130
- SELECT m.*, -bm25(ftsIndex) AS rank
131
- FROM ftsIndex AS f
132
- JOIN objectMeta AS m ON f.rowid = m.recordId
133
- WHERE ${sql.and(conditions)}
134
- ORDER BY rank DESC
135
- `;
136
- return rows;
137
- } else {
138
- const rows = yield* sql`
139
- SELECT m.*
140
- FROM ftsIndex AS f
141
- JOIN objectMeta AS m ON f.rowid = m.recordId
142
- WHERE ${sql.and(conditions)}
143
- `;
144
- return rows.map((row) => ({
145
- ...row,
146
- rank: 1
147
- }));
148
- }
149
- });
150
- }
151
- /**
152
- * Query snapshots by recordIds.
153
- * Returns the parsed JSON snapshots for queue objects.
154
- * RecordIds not present in the FTS index are silently omitted from the result.
155
- */
156
- querySnapshotsJSON(recordIds) {
157
- return Effect2.gen(function* () {
158
- if (recordIds.length === 0) {
159
- return [];
160
- }
161
- const sql = yield* SqlClient3.SqlClient;
162
- const chunks = [];
163
- for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
164
- chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
165
- }
166
- const allResults = [];
167
- for (const chunk of chunks) {
168
- const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
169
- for (const r of rows) {
170
- allResults.push({
171
- recordId: r.rowid,
172
- snapshot: JSON.parse(r.snapshot)
173
- });
174
- }
175
- }
176
- return allResults;
177
- });
178
- }
179
- update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
180
- const sql = yield* SqlClient3.SqlClient;
181
- yield* Effect2.forEach(objects, (object) => Effect2.gen(function* () {
182
- const { recordId, data } = object;
183
- if (recordId === null) {
184
- return yield* Effect2.die(new Error("FtsIndex.update requires recordId to be set"));
185
- }
186
- const snapshot = JSON.stringify(data);
187
- const existing = yield* sql`SELECT rowid FROM ftsIndex WHERE rowid = ${recordId}`;
188
- if (existing.length > 0) {
189
- yield* sql`DELETE FROM ftsIndex WHERE rowid = ${recordId}`;
190
- }
191
- yield* sql`INSERT INTO ftsIndex (rowid, snapshot) VALUES (${recordId}, ${snapshot})`;
192
- }), {
193
- discard: true
194
- });
195
- }));
196
- };
197
-
198
- // src/indexes/entity-meta-index.ts
199
- import * as SqlClient5 from "@effect/sql/SqlClient";
200
- import * as Effect3 from "effect/Effect";
201
- import * as Schema2 from "effect/Schema";
202
- import { ATTR_DELETED, ATTR_PARENT, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from "@dxos/echo/internal";
203
- import { DXN, EID, EntityId, SpaceId as SpaceId2, URI } from "@dxos/keys";
204
- var _normalizeTypeUri = (typeDXN) => {
205
- if (!typeDXN.startsWith("echo:")) {
206
- return typeDXN;
207
- }
208
- const eid = EID.tryParse(typeDXN);
209
- if (!eid) {
210
- return typeDXN;
211
- }
212
- const entityId = EID.getEntityId(eid);
213
- return entityId ? EID.make({
214
- entityId
215
- }) : typeDXN;
216
- };
217
- var _escapeLikePrefix = (prefix) => {
218
- const escaped = prefix.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
219
- return `${escaped}:%`;
220
- };
221
- var EntityMeta = Schema2.Struct({
222
- recordId: Schema2.Number,
223
- objectId: EntityId,
224
- /** Empty string for non-queue objects. */
225
- queueId: Schema2.String,
226
- /** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
227
- queueNamespace: Schema2.String,
228
- spaceId: SpaceId2,
229
- documentId: Schema2.String,
230
- entityKind: Schema2.String,
231
- /**
232
- * Type identifier URI for the object — typename DXN for non-stored schemas,
233
- * schema-as-object EID for stored (dynamic) schemas. Mirrors the value
234
- * written into the object's `system.type`.
235
- */
236
- typeDXN: URI.Schema,
237
- deleted: Schema2.Boolean,
238
- source: Schema2.NullOr(EID.Schema),
239
- target: Schema2.NullOr(EID.Schema),
240
- /** Parent object id (nullable). */
241
- parent: Schema2.NullOr(EID.Schema),
242
- /** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
243
- version: Schema2.Number,
244
- /** Unix ms timestamp when the object was first indexed. */
245
- createdAt: Schema2.NullOr(Schema2.Number),
246
- /** Unix ms timestamp when the object was last re-indexed. */
247
- updatedAt: Schema2.NullOr(Schema2.Number)
248
- });
249
- var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
250
- const conditions = [];
251
- if (spaceIds.length > 0) {
252
- if (includeAllQueues) {
253
- conditions.push(sql`${sql.in("spaceId", spaceIds)}`);
254
- } else {
255
- conditions.push(sql`(${sql.in("spaceId", spaceIds)} AND queueId = '')`);
256
- }
257
- }
258
- if (queueIds && queueIds.length > 0) {
259
- conditions.push(sql`${sql.in("queueId", queueIds)}`);
260
- }
261
- if (conditions.length === 0) {
262
- return sql`1 = 0`;
263
- }
264
- return sql.or(conditions);
265
- };
266
- var EntityMetaIndex = class {
267
- migrate = Effect3.fn("EntityMetaIndex.runMigrations")(function* () {
268
- const sql = yield* SqlClient5.SqlClient;
269
- yield* sql`CREATE TABLE IF NOT EXISTS objectMeta (
270
- recordId INTEGER PRIMARY KEY AUTOINCREMENT,
271
- objectId TEXT NOT NULL,
272
- queueId TEXT NOT NULL DEFAULT '',
273
- queueNamespace TEXT NOT NULL DEFAULT '',
274
- spaceId TEXT NOT NULL,
275
- documentId TEXT NOT NULL DEFAULT '',
276
- entityKind TEXT NOT NULL,
277
- typeDXN TEXT NOT NULL,
278
- deleted INTEGER NOT NULL,
279
- source TEXT,
280
- target TEXT,
281
- parent TEXT,
282
- version INTEGER NOT NULL,
283
- createdAt INTEGER,
284
- updatedAt INTEGER
285
- )`;
286
- yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
287
- yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
288
- yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
289
- yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
290
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
291
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
292
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
293
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
294
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
295
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
296
- });
297
- query = Effect3.fn("EntityMetaIndex.query")((query) => Effect3.gen(function* () {
298
- const sql = yield* SqlClient5.SqlClient;
299
- const normalizedTypeDXN = _normalizeTypeUri(query.typeDXN);
300
- const parsedDxn = DXN.isDXN(normalizedTypeDXN) ? normalizedTypeDXN : void 0;
301
- const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
302
- const rows = hasNoVersion ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${normalizedTypeDXN} OR typeDXN LIKE ${_escapeLikePrefix(normalizedTypeDXN)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${normalizedTypeDXN}`;
303
- return rows.map((row) => ({
304
- ...row,
305
- deleted: !!row.deleted
306
- }));
307
- }));
308
- queryAll = Effect3.fn("EntityMetaIndex.queryAll")((query) => Effect3.gen(function* () {
309
- if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
310
- return [];
311
- }
312
- const sql = yield* SqlClient5.SqlClient;
313
- const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
314
- const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
315
- return rows.map((row) => ({
316
- ...row,
317
- deleted: !!row.deleted
318
- }));
319
- }));
320
- queryTypes = Effect3.fn("EntityMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false, includeAllQueues = false, queueIds = null }) => Effect3.gen(function* () {
321
- if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) {
322
- return [];
323
- }
324
- if (typeDxns.length === 0) {
325
- if (!inverted) {
326
- return [];
327
- }
328
- const sql2 = yield* SqlClient5.SqlClient;
329
- const sourceCondition2 = buildSourceCondition(sql2, spaceIds, includeAllQueues, queueIds);
330
- const rows2 = yield* sql2`SELECT * FROM objectMeta WHERE ${sourceCondition2}`;
331
- return rows2.map((row) => ({
332
- ...row,
333
- deleted: !!row.deleted
334
- }));
335
- }
336
- const sql = yield* SqlClient5.SqlClient;
337
- const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
338
- const typeWhere = sql.or(typeDxns.map((typeDXN) => {
339
- const normalized = _normalizeTypeUri(typeDXN);
340
- const parsedDxn = DXN.isDXN(normalized) ? normalized : void 0;
341
- const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
342
- const exactMatch = sql`typeDXN = ${normalized}`;
343
- return hasNoVersion ? sql.or([
344
- exactMatch,
345
- sql`typeDXN LIKE ${_escapeLikePrefix(normalized)} ESCAPE '\\'`
346
- ]) : exactMatch;
347
- }));
348
- const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
349
- return rows.map((row) => ({
350
- ...row,
351
- deleted: !!row.deleted
352
- }));
353
- }));
354
- queryRelations = Effect3.fn("EntityMetaIndex.queryRelations")(({ endpoint, anchorDxns }) => Effect3.gen(function* () {
355
- if (anchorDxns.length === 0) {
356
- return [];
357
- }
358
- const sql = yield* SqlClient5.SqlClient;
359
- const column = endpoint === "source" ? "source" : "target";
360
- const rows = yield* sql`SELECT * FROM objectMeta WHERE entityKind = 'relation' AND ${sql.in(column, anchorDxns)}`;
361
- return rows.map((row) => ({
362
- ...row,
363
- deleted: !!row.deleted
364
- }));
365
- }));
366
- // TODO(dmaretskyi): Update recordId on objects so that we don't need to look it up separately.
367
- update = Effect3.fn("EntityMetaIndex.update")((objects) => Effect3.gen(function* () {
368
- const sql = yield* SqlClient5.SqlClient;
369
- yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
370
- const { spaceId, queueId, queueNamespace, documentId, data } = object;
371
- const castData = data;
372
- const objectId = castData.id;
373
- let existing;
374
- if (documentId) {
375
- existing = yield* sql`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
376
- } else if (queueId) {
377
- existing = yield* sql`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
378
- } else {
379
- existing = [];
380
- }
381
- const result = yield* sql`SELECT MAX(version) as v FROM objectMeta`;
382
- const [{ v }] = result;
383
- const version = (v ?? 0) + 1;
384
- const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
385
- const typeDXN = URI.make(castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type");
386
- const deleted = castData[ATTR_DELETED] ? 1 : 0;
387
- const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
388
- const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
389
- const parent = castData[ATTR_PARENT] ?? null;
390
- const updatedAtTimestamp = object.updatedAt;
391
- const createdAtTimestamp = object.createdAt ?? updatedAtTimestamp;
392
- if (existing.length > 0) {
393
- yield* sql`
394
- UPDATE objectMeta SET
395
- version = ${version},
396
- queueNamespace = ${queueNamespace ?? ""},
397
- entityKind = ${entityKind},
398
- typeDXN = ${typeDXN},
399
- deleted = ${deleted},
400
- source = ${source},
401
- target = ${target},
402
- parent = ${parent},
403
- updatedAt = ${updatedAtTimestamp}
404
- WHERE recordId = ${existing[0].recordId}
405
- `;
406
- } else {
407
- yield* sql`
408
- INSERT INTO objectMeta (
409
- objectId, queueId, queueNamespace, spaceId, documentId,
410
- entityKind, typeDXN, deleted, source, target, parent, version,
411
- createdAt, updatedAt
412
- ) VALUES (
413
- ${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
414
- ${entityKind}, ${typeDXN}, ${deleted},
415
- ${source}, ${target}, ${parent}, ${version},
416
- ${createdAtTimestamp}, ${updatedAtTimestamp}
417
- )
418
- `;
419
- }
420
- }), {
421
- discard: true
422
- });
423
- }));
424
- /**
425
- * Look up `recordIds` for objects that are already stored in the EntityMetaIndex.
426
- * Mutates the objects in place.
427
- */
428
- lookupRecordIds = Effect3.fn("EntityMetaIndex.lookupRecordIds")((objects) => Effect3.gen(function* () {
429
- const sql = yield* SqlClient5.SqlClient;
430
- for (const object of objects) {
431
- const { spaceId, queueId, documentId, data } = object;
432
- const objectId = data.id;
433
- let result;
434
- if (documentId) {
435
- result = yield* sql`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
436
- } else if (queueId) {
437
- result = yield* sql`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
438
- } else {
439
- result = [];
440
- }
441
- if (result.length === 0) {
442
- return yield* Effect3.die(new Error(`Object not found in EntityMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`));
443
- }
444
- object.recordId = result[0].recordId;
445
- }
446
- }));
447
- /**
448
- * Look up object metadata by recordIds.
449
- */
450
- lookupByRecordIds = Effect3.fn("EntityMetaIndex.lookupByRecordIds")((recordIds) => Effect3.gen(function* () {
451
- if (recordIds.length === 0) {
452
- return [];
453
- }
454
- const sql = yield* SqlClient5.SqlClient;
455
- const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("recordId", recordIds)}`;
456
- return rows.map((row) => ({
457
- ...row,
458
- deleted: !!row.deleted
459
- }));
460
- }));
461
- /**
462
- * Look up object metadata by object id across one or more spaces (space db and queue items).
463
- */
464
- queryObjectIds = Effect3.fn("EntityMetaIndex.queryObjectIds")((query) => Effect3.gen(function* () {
465
- if (query.spaceIds.length === 0 || query.objectIds.length === 0) {
466
- return [];
467
- }
468
- const sql = yield* SqlClient5.SqlClient;
469
- const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceIds)} AND ${sql.in("objectId", query.objectIds)}`;
470
- return rows.map((row) => ({
471
- ...row,
472
- deleted: !!row.deleted
473
- }));
474
- }));
475
- /**
476
- * Look up object metadata by objectId, spaceId, and queueId.
477
- */
478
- lookupByObjectId = Effect3.fn("EntityMetaIndex.lookupByObjectId")((query) => Effect3.gen(function* () {
479
- const sql = yield* SqlClient5.SqlClient;
480
- const rows = yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
481
- if (rows.length === 0) {
482
- return null;
483
- }
484
- return {
485
- ...rows[0],
486
- deleted: !!rows[0].deleted
487
- };
488
- }));
489
- /**
490
- * Query objects by timestamp range.
491
- */
492
- queryByTimeRange = Effect3.fn("EntityMetaIndex.queryByTimeRange")((query) => Effect3.gen(function* () {
493
- if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
494
- return [];
495
- }
496
- const sql = yield* SqlClient5.SqlClient;
497
- const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
498
- const timeConditions = [];
499
- if (query.updatedAfter != null) {
500
- timeConditions.push(sql`updatedAt >= ${query.updatedAfter}`);
501
- }
502
- if (query.updatedBefore != null) {
503
- timeConditions.push(sql`updatedAt <= ${query.updatedBefore}`);
504
- }
505
- if (query.createdAfter != null) {
506
- timeConditions.push(sql`createdAt >= ${query.createdAfter}`);
507
- }
508
- if (query.createdBefore != null) {
509
- timeConditions.push(sql`createdAt <= ${query.createdBefore}`);
510
- }
511
- const rows = timeConditions.length > 0 ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${sql.and(timeConditions)}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
512
- return rows.map((row) => ({
513
- ...row,
514
- deleted: !!row.deleted
515
- }));
516
- }));
517
- /**
518
- * Query children by parent object ids.
519
- * Matches both:
520
- * - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
521
- * - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
522
- * DXN uses the feed's object id as its queue id — see `Feed.getQueueUri`).
523
- */
524
- queryChildren = Effect3.fn("EntityMetaIndex.queryChildren")((query) => Effect3.gen(function* () {
525
- if (query.parentIds.length === 0) {
526
- return [];
527
- }
528
- const sql = yield* SqlClient5.SqlClient;
529
- const parentDzns = query.parentIds.map((id) => EID.make({
530
- entityId: id
531
- }));
532
- const parentDxns = parentDzns;
533
- const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`;
534
- return rows.map((row) => ({
535
- ...row,
536
- deleted: !!row.deleted
537
- }));
538
- }));
539
- };
540
-
541
- // src/indexes/reverse-ref-index.ts
542
- import * as SqlClient7 from "@effect/sql/SqlClient";
543
- import * as Effect4 from "effect/Effect";
544
- import * as Schema4 from "effect/Schema";
545
- import { EncodedReference, isEncodedReference } from "@dxos/echo-protocol";
546
- import { EID as EID2 } from "@dxos/keys";
547
-
548
- // src/utils.ts
549
- import * as Schema3 from "effect/Schema";
550
- import { invariant } from "@dxos/invariant";
551
- var __dxlog_file = "/__w/dxos/dxos/packages/core/echo/index-core/src/utils.ts";
552
- var EscapedPropPath = class extends Schema3.String.annotations({
553
- title: "EscapedPropPath"
554
- }) {
555
- static escape(path) {
556
- return path.map((p) => p.toString().replaceAll("\\", "\\\\").replaceAll(".", "\\.")).join(".");
557
- }
558
- static unescape(path) {
559
- const parts = [];
560
- let current = "";
561
- for (let i = 0; i < path.length; i++) {
562
- if (path[i] === "\\") {
563
- invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
564
- current = current + path[i + 1];
565
- i++;
566
- } else if (path[i] === ".") {
567
- parts.push(current);
568
- current = "";
569
- } else {
570
- current += path[i];
571
- }
572
- }
573
- parts.push(current);
574
- return parts;
575
- }
576
- };
577
-
578
- // src/indexes/reverse-ref-index.ts
579
- var extractReferences = (data) => {
580
- const refs = [];
581
- const visit = (path, value) => {
582
- if (isEncodedReference(value)) {
583
- const uri = EncodedReference.toURI(value);
584
- const parsedEchoUri = EID2.tryParse(uri);
585
- const echoUri = parsedEchoUri ? EID2.getEntityId(parsedEchoUri) : void 0;
586
- if (!echoUri || !parsedEchoUri) {
587
- return;
588
- }
589
- refs.push({
590
- path,
591
- targetDXN: EID2.toLocal(parsedEchoUri)
592
- });
593
- } else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
594
- for (const [key, v] of Object.entries(value)) {
595
- visit([
596
- ...path,
597
- key
598
- ], v);
599
- }
600
- } else if (Array.isArray(value)) {
601
- for (let i = 0; i < value.length; i++) {
602
- visit([
603
- ...path,
604
- String(i)
605
- ], value[i]);
606
- }
607
- }
608
- };
609
- visit([], data);
610
- return refs;
611
- };
612
- var ReverseRef = Schema4.Struct({
613
- recordId: Schema4.Number,
614
- targetDXN: EID2.Schema,
615
- /**
616
- * Escaped property path within an object.
617
- *
618
- * Escaping rules:
619
- *
620
- * - '.' -> '\.'
621
- * - '\' -> '\\'
622
- * - contact with .
623
- */
624
- propPath: Schema4.String
625
- });
626
- var ReverseRefIndex = class {
627
- migrate = Effect4.fn("ReverseRefIndex.migrate")(function* () {
628
- const sql = yield* SqlClient7.SqlClient;
629
- yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
630
- recordId INTEGER NOT NULL,
631
- targetDXN TEXT NOT NULL,
632
- propPath TEXT NOT NULL,
633
- PRIMARY KEY (recordId, targetDXN, propPath)
634
- )`;
635
- yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
636
- });
637
- /**
638
- * Query all references pointing to a target DXN.
639
- */
640
- query = Effect4.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect4.gen(function* () {
641
- const sql = yield* SqlClient7.SqlClient;
642
- const normalized = EID2.toLocal(targetDXN);
643
- const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDXN = ${normalized}`;
644
- return rows;
645
- }));
646
- update = Effect4.fn("ReverseRefIndex.update")((objects) => Effect4.gen(function* () {
647
- const sql = yield* SqlClient7.SqlClient;
648
- yield* Effect4.forEach(objects, (object) => Effect4.gen(function* () {
649
- const { recordId, data } = object;
650
- if (recordId === null) {
651
- yield* Effect4.die(new Error("ReverseRefIndex.update requires recordId to be set"));
652
- }
653
- yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
654
- const refs = extractReferences(data);
655
- yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, {
656
- discard: true
657
- });
658
- }), {
659
- discard: true
660
- });
661
- }));
662
- };
663
-
664
- // src/index-engine.ts
665
- var makeEmptyIndexingResult = () => ({
666
- updated: 0,
667
- done: true,
668
- spaces: /* @__PURE__ */ new Set(),
669
- queues: /* @__PURE__ */ new Set(),
670
- documents: /* @__PURE__ */ new Set(),
671
- types: /* @__PURE__ */ new Set(),
672
- objects: /* @__PURE__ */ new Set()
673
- });
674
- var accumulateIndexingResult = (acc, objects) => {
675
- for (const obj of objects) {
676
- acc.spaces.add(obj.spaceId);
677
- if (obj.queueId) {
678
- acc.queues.add(obj.queueId);
679
- }
680
- if (obj.documentId) {
681
- acc.documents.add(obj.documentId);
682
- }
683
- const t = obj.data[ATTR_TYPE2];
684
- if (t) {
685
- acc.types.add(String(t));
686
- }
687
- if (obj.data.id) {
688
- acc.objects.add(obj.data.id);
689
- }
690
- }
691
- };
692
- var IndexEngine = class {
693
- #tracker;
694
- #objectMetaIndex;
695
- #ftsIndex;
696
- #reverseRefIndex;
697
- constructor(params) {
698
- this.#tracker = params?.tracker ?? new IndexTracker();
699
- this.#objectMetaIndex = params?.objectMetaIndex ?? new EntityMetaIndex();
700
- this.#ftsIndex = params?.ftsIndex ?? new FtsIndex();
701
- this.#reverseRefIndex = params?.reverseRefIndex ?? new ReverseRefIndex();
702
- }
703
- migrate() {
704
- return Effect5.gen(this, function* () {
705
- yield* this.#tracker.migrate();
706
- yield* this.#objectMetaIndex.migrate();
707
- yield* this.#ftsIndex.migrate();
708
- yield* this.#reverseRefIndex.migrate();
709
- });
710
- }
711
- /**
712
- * Query text index and return full object metadata with rank.
713
- */
714
- queryText(query) {
715
- return Effect5.gen(this, function* () {
716
- return yield* this.#ftsIndex.query(query);
717
- });
718
- }
719
- queryReverseRef(query) {
720
- return this.#reverseRefIndex.query(query);
721
- }
722
- queryAll(query) {
723
- return this.#objectMetaIndex.queryAll(query);
724
- }
725
- /**
726
- * Query snapshots by recordIds.
727
- * Used to load queue objects from indexed snapshots.
728
- */
729
- querySnapshotsJSON(recordIds) {
730
- return this.#ftsIndex.querySnapshotsJSON(recordIds);
731
- }
732
- queryType(query) {
733
- return this.#objectMetaIndex.query(query);
734
- }
735
- /**
736
- * Query children by parent object ids.
737
- */
738
- queryChildren(query) {
739
- return this.#objectMetaIndex.queryChildren(query);
740
- }
741
- queryTypes(query) {
742
- return this.#objectMetaIndex.queryTypes(query);
743
- }
744
- queryByTimeRange(query) {
745
- return this.#objectMetaIndex.queryByTimeRange(query);
746
- }
747
- queryRelations(query) {
748
- return this.#objectMetaIndex.queryRelations(query);
749
- }
750
- lookupByRecordIds(recordIds) {
751
- return this.#objectMetaIndex.lookupByRecordIds(recordIds);
752
- }
753
- lookupByObjectId(query) {
754
- return this.#objectMetaIndex.lookupByObjectId(query);
755
- }
756
- queryObjectIds(query) {
757
- return this.#objectMetaIndex.queryObjectIds(query);
758
- }
759
- update(ctx, dataSource, opts) {
760
- return Effect5.gen(this, function* () {
761
- const result = makeEmptyIndexingResult();
762
- const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
763
- indexName: "fts5",
764
- spaceId: opts.spaceId,
765
- limit: opts.limit
766
- });
767
- result.updated += updatedFtsIndex;
768
- result.done = result.done && doneFtsIndex;
769
- accumulateIndexingResult(result, ftsObjects);
770
- const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
771
- indexName: "reverseRef",
772
- spaceId: opts.spaceId,
773
- limit: opts.limit
774
- });
775
- result.updated += updatedReverseRefIndex;
776
- result.done = result.done && doneReverseRefIndex;
777
- accumulateIndexingResult(result, reverseRefObjects);
778
- return result;
779
- }).pipe(Effect5.withSpan("IndexEngine.update"));
780
- }
781
- /**
782
- * Update a dependent index that requires recordId enrichment.
783
- * This method:
784
- * 1. Gets changed objects from the source.
785
- * 2. Ensures those objects exist in EntityMetaIndex.
786
- * 3. Looks up recordIds for those objects.
787
- * 4. Enriches objects with recordIds.
788
- * 5. Updates the dependent index.
789
- */
790
- #update(ctx, index, source, opts) {
791
- return Effect5.gen(this, function* () {
792
- const sqlTransaction = yield* SqlTransaction.SqlTransaction;
793
- const cursors = yield* this.#tracker.queryCursors({
794
- indexName: opts.indexName,
795
- sourceName: source.sourceName,
796
- // Pass undefined to get all cursors when spaceId is null.
797
- spaceId: opts.spaceId ?? void 0
798
- });
799
- const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
800
- limit: opts.limit
801
- });
802
- if (objects.length === 0) {
803
- return {
804
- updated: 0,
805
- done: true,
806
- objects: []
807
- };
808
- }
809
- return yield* sqlTransaction.withTransaction(Effect5.gen(this, function* () {
810
- yield* this.#objectMetaIndex.update(objects);
811
- yield* this.#objectMetaIndex.lookupRecordIds(objects);
812
- yield* index.update(objects);
813
- yield* this.#tracker.updateCursors(updatedCursors.map((_) => ({
814
- indexName: opts.indexName,
815
- spaceId: _.spaceId,
816
- sourceName: source.sourceName,
817
- resourceId: _.resourceId,
818
- cursor: _.cursor
819
- })));
820
- return {
821
- updated: objects.length,
822
- done: false,
823
- objects
824
- };
825
- }));
826
- }).pipe(Effect5.withSpan("IndexEngine.#update"));
827
- }
828
- };
829
- export {
830
- EntityMetaIndex,
831
- EscapedPropPath,
832
- FtsIndex,
833
- IndexEngine,
834
- IndexTracker,
835
- ReverseRefIndex
836
- };
837
- //# sourceMappingURL=index.mjs.map