@dxos/index-core 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,725 @@
1
+ import * as Effect from "effect/Effect";
2
+ import { ATTR_DELETED, ATTR_PARENT, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from "@dxos/echo/internal";
3
+ import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
4
+ import * as SqlClient from "@effect/sql/SqlClient";
5
+ import * as Schema from "effect/Schema";
6
+ import { DXN, EID, EntityId, SpaceId, URI } from "@dxos/keys";
7
+ import { EncodedReference, isEncodedReference } from "@dxos/echo-protocol";
8
+ import { invariant } from "@dxos/invariant";
9
+ Schema.Struct({
10
+ /**
11
+ * Name of the index owning this cursor.
12
+ */
13
+ indexName: Schema.String,
14
+ /**
15
+ * Space id.
16
+ */
17
+ spaceId: Schema.NullOr(SpaceId),
18
+ /**
19
+ * Source name.
20
+ * 'automerge' / 'queue' / 'index' (for secondary indexes)
21
+ */
22
+ sourceName: Schema.String,
23
+ /**
24
+ * Document id or queue id.
25
+ * doc_id, queue_id, '' <empty string> (if indexing entire namespace)
26
+ */
27
+ resourceId: Schema.NullOr(Schema.String),
28
+ /**
29
+ * Heads, queue position, version.
30
+ */
31
+ cursor: Schema.Union(Schema.Number, Schema.String)
32
+ });
33
+ /**
34
+ * Deprecated index names that are no longer used. Will be cleaned up on migration.
35
+ */
36
+ var DEPRECATED_INDEX_NAMES = ["fts"];
37
+ var IndexTracker = class {
38
+ migrate = Effect.fn("IndexTracker.migrate")(function* () {
39
+ const sql = yield* SqlClient.SqlClient;
40
+ yield* sql`CREATE TABLE IF NOT EXISTS indexCursor (
41
+ indexName TEXT NOT NULL,
42
+ spaceId TEXT NOT NULL DEFAULT '',
43
+ sourceName TEXT NOT NULL,
44
+ resourceId TEXT NOT NULL DEFAULT '',
45
+ cursor,
46
+ PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
47
+ )`;
48
+ yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
49
+ return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
50
+ });
51
+ });
52
+ queryCursors = Effect.fn("IndexTracker.queryCursors")((query) => Effect.gen(function* () {
53
+ const sql = yield* SqlClient.SqlClient;
54
+ const spaceIdParam = query.spaceId === void 0 ? null : query.spaceId ?? "";
55
+ const sourceNameParam = query.sourceName === void 0 ? null : query.sourceName;
56
+ const resourceIdParam = query.resourceId === void 0 ? null : query.resourceId ?? "";
57
+ return (yield* sql`
58
+ SELECT * FROM indexCursor
59
+ WHERE indexName = ${query.indexName}
60
+ AND (${spaceIdParam} IS NULL OR spaceId = ${spaceIdParam})
61
+ AND (${sourceNameParam} IS NULL OR sourceName = ${sourceNameParam})
62
+ AND (${resourceIdParam} IS NULL OR resourceId = ${resourceIdParam})
63
+ `).map((row) => ({
64
+ indexName: row.indexName,
65
+ spaceId: row.spaceId === "" ? null : Schema.decodeSync(SpaceId)(row.spaceId),
66
+ sourceName: row.sourceName,
67
+ resourceId: row.resourceId === "" ? null : row.resourceId,
68
+ cursor: row.cursor
69
+ }));
70
+ }));
71
+ updateCursors = Effect.fn("IndexTracker.updateCursors")((cursors) => Effect.gen(function* () {
72
+ const sql = yield* SqlClient.SqlClient;
73
+ yield* Effect.forEach(cursors, (cursor) => {
74
+ const spaceId = cursor.spaceId ?? "";
75
+ const resourceId = cursor.resourceId ?? "";
76
+ return sql`
77
+ INSERT INTO indexCursor (indexName, spaceId, sourceName, resourceId, cursor)
78
+ VALUES (${cursor.indexName}, ${spaceId}, ${cursor.sourceName}, ${resourceId}, ${cursor.cursor})
79
+ ON CONFLICT(indexName, spaceId, sourceName, resourceId) DO UPDATE SET cursor = excluded.cursor
80
+ `;
81
+ }, { discard: true });
82
+ }));
83
+ };
84
+ //#endregion
85
+ //#region src/indexes/fts-index.ts
86
+ var SQL_CHUNK_SIZE = 500;
87
+ /**
88
+ * Escapes user input for safe FTS5 queries.
89
+ *
90
+ * FTS5 has special syntax characters that can cause errors or unexpected behavior:
91
+ * - `*` suffix for prefix matching (e.g., `prog*` matches "program", "programming")
92
+ * - `"..."` for phrase queries
93
+ * - `.` for column specification
94
+ * - `AND`, `OR`, `NOT` boolean operators
95
+ * - `+`, `-` for required/excluded terms
96
+ *
97
+ * This function wraps each whitespace-separated term in double quotes, treating all
98
+ * characters as literals. Double quotes within terms are escaped by doubling (`""`).
99
+ *
100
+ * Example: `prog* AND test.` becomes `"prog*" "AND" "test."`.
101
+ */
102
+ var escapeFts5Query = (text) => {
103
+ return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, "\"\"")}"`).join(" ");
104
+ };
105
+ var FtsIndex = class {
106
+ migrate = Effect.fn("FtsIndex.migrate")(function* () {
107
+ yield* (yield* SqlClient.SqlClient)`CREATE VIRTUAL TABLE IF NOT EXISTS ftsIndex USING fts5(snapshot, tokenize='trigram')`;
108
+ });
109
+ query({ query, spaceId, includeAllQueues, queueIds }) {
110
+ return Effect.gen(function* () {
111
+ const trimmed = query.trim();
112
+ if (trimmed.length === 0) return [];
113
+ const sql = yield* SqlClient.SqlClient;
114
+ const terms = trimmed.split(/\s+/).filter(Boolean);
115
+ const minTermLength = Math.min(...terms.map((t) => t.length));
116
+ const useBm25 = minTermLength >= 3;
117
+ const conditions = minTermLength < 3 ? terms.map((term) => sql`f.snapshot LIKE ${"%" + term + "%"}`) : [sql`f.snapshot MATCH ${escapeFts5Query(trimmed)}`];
118
+ const sourceConditions = [];
119
+ if (spaceId && spaceId.length > 0) if (includeAllQueues) sourceConditions.push(sql`m.spaceId IN ${sql.in(spaceId)}`);
120
+ else sourceConditions.push(sql`(m.spaceId IN ${sql.in(spaceId)} AND m.queueId = '')`);
121
+ if (queueIds && queueIds.length > 0) sourceConditions.push(sql`m.queueId IN ${sql.in(queueIds)}`);
122
+ if (sourceConditions.length > 0) conditions.push(sql`(${sql.or(sourceConditions)})`);
123
+ if (useBm25) return yield* sql`
124
+ SELECT m.*, -bm25(ftsIndex) AS rank
125
+ FROM ftsIndex AS f
126
+ JOIN objectMeta AS m ON f.rowid = m.recordId
127
+ WHERE ${sql.and(conditions)}
128
+ ORDER BY rank DESC
129
+ `;
130
+ else return (yield* sql`
131
+ SELECT m.*
132
+ FROM ftsIndex AS f
133
+ JOIN objectMeta AS m ON f.rowid = m.recordId
134
+ WHERE ${sql.and(conditions)}
135
+ `).map((row) => ({
136
+ ...row,
137
+ rank: 1
138
+ }));
139
+ });
140
+ }
141
+ /**
142
+ * Query snapshots by recordIds.
143
+ * Returns the parsed JSON snapshots for queue objects.
144
+ * RecordIds not present in the FTS index are silently omitted from the result.
145
+ */
146
+ querySnapshotsJSON(recordIds) {
147
+ return Effect.gen(function* () {
148
+ if (recordIds.length === 0) return [];
149
+ const sql = yield* SqlClient.SqlClient;
150
+ const chunks = [];
151
+ for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
152
+ const allResults = [];
153
+ for (const chunk of chunks) {
154
+ const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
155
+ for (const r of rows) allResults.push({
156
+ recordId: r.rowid,
157
+ snapshot: JSON.parse(r.snapshot)
158
+ });
159
+ }
160
+ return allResults;
161
+ });
162
+ }
163
+ update = Effect.fn("FtsIndex.update")((objects) => Effect.gen(function* () {
164
+ const sql = yield* SqlClient.SqlClient;
165
+ yield* Effect.forEach(objects, (object) => Effect.gen(function* () {
166
+ const { recordId, data } = object;
167
+ if (recordId === null) return yield* Effect.die(/* @__PURE__ */ new Error("FtsIndex.update requires recordId to be set"));
168
+ const existing = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid = ${recordId}`;
169
+ const merged = data[ATTR_TYPE] === void 0 && existing.length > 0 ? {
170
+ ...JSON.parse(existing[0].snapshot),
171
+ ...data
172
+ } : data;
173
+ const snapshot = JSON.stringify(merged);
174
+ if (existing.length > 0) yield* sql`DELETE FROM ftsIndex WHERE rowid = ${recordId}`;
175
+ yield* sql`INSERT INTO ftsIndex (rowid, snapshot) VALUES (${recordId}, ${snapshot})`;
176
+ }), { discard: true });
177
+ }));
178
+ };
179
+ //#endregion
180
+ //#region src/indexes/entity-meta-index.ts
181
+ /**
182
+ * Normalizes an echo: EID to the local (unqualified) form so SQL comparisons are consistent.
183
+ * Rows are indexed in the canonical `echo:///<entityId>` form; a space-qualified
184
+ * `echo://<space>/<entityId>` would otherwise miss every row for that type.
185
+ */
186
+ var _normalizeTypeUri = (typeDXN) => {
187
+ if (!typeDXN.startsWith("echo:")) return typeDXN;
188
+ const eid = EID.tryParse(typeDXN);
189
+ if (!eid) return typeDXN;
190
+ const entityId = EID.getEntityId(eid);
191
+ return entityId ? EID.make({ entityId }) : typeDXN;
192
+ };
193
+ /**
194
+ * Every stored `typeDXN` form equivalent to a normalized type identifier. Rows written before
195
+ * `echo:///<id>` became the canonical local EID hold the single-slash `echo:/<id>` form; both
196
+ * address the same stored schema, so queries must match either without requiring a reindex.
197
+ */
198
+ var _typeUriEquivalents = (normalized) => {
199
+ if (normalized.startsWith("echo:///")) return [normalized, `echo:/${normalized.slice(8)}`];
200
+ return [normalized];
201
+ };
202
+ var _escapeLikePrefix = (prefix) => {
203
+ return `${prefix.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_")}:%`;
204
+ };
205
+ Schema.Struct({
206
+ recordId: Schema.Number,
207
+ objectId: EntityId,
208
+ /** Empty string for non-queue objects. */
209
+ queueId: Schema.String,
210
+ /** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
211
+ queueNamespace: Schema.String,
212
+ spaceId: SpaceId,
213
+ documentId: Schema.String,
214
+ entityKind: Schema.String,
215
+ /**
216
+ * Type identifier URI for the object — typename DXN for non-stored schemas,
217
+ * schema-as-object EID for stored (dynamic) schemas. Mirrors the value
218
+ * written into the object's `system.type`.
219
+ */
220
+ typeDXN: URI.Schema,
221
+ deleted: Schema.Boolean,
222
+ source: Schema.NullOr(EID.Schema),
223
+ target: Schema.NullOr(EID.Schema),
224
+ /** Parent object id (nullable). */
225
+ parent: Schema.NullOr(EID.Schema),
226
+ /** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
227
+ version: Schema.Number,
228
+ /** Unix ms timestamp when the object was first indexed. */
229
+ createdAt: Schema.NullOr(Schema.Number),
230
+ /** Unix ms timestamp when the object was last re-indexed. */
231
+ updatedAt: Schema.NullOr(Schema.Number)
232
+ });
233
+ /**
234
+ * Builds a SQL condition for filtering by space and queue source.
235
+ * When `includeAllQueues` is false and no `queueIds`, only non-queue objects are returned.
236
+ */
237
+ var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
238
+ const conditions = [];
239
+ if (spaceIds.length > 0) if (includeAllQueues) conditions.push(sql`${sql.in("spaceId", spaceIds)}`);
240
+ else conditions.push(sql`(${sql.in("spaceId", spaceIds)} AND queueId = '')`);
241
+ if (queueIds && queueIds.length > 0) conditions.push(sql`${sql.in("queueId", queueIds)}`);
242
+ if (conditions.length === 0) return sql`1 = 0`;
243
+ return sql.or(conditions);
244
+ };
245
+ var EntityMetaIndex = class {
246
+ migrate = Effect.fn("EntityMetaIndex.runMigrations")(function* () {
247
+ const sql = yield* SqlClient.SqlClient;
248
+ yield* sql`CREATE TABLE IF NOT EXISTS objectMeta (
249
+ recordId INTEGER PRIMARY KEY AUTOINCREMENT,
250
+ objectId TEXT NOT NULL,
251
+ queueId TEXT NOT NULL DEFAULT '',
252
+ queueNamespace TEXT NOT NULL DEFAULT '',
253
+ spaceId TEXT NOT NULL,
254
+ documentId TEXT NOT NULL DEFAULT '',
255
+ entityKind TEXT NOT NULL,
256
+ typeDXN TEXT NOT NULL,
257
+ deleted INTEGER NOT NULL,
258
+ source TEXT,
259
+ target TEXT,
260
+ parent TEXT,
261
+ version INTEGER NOT NULL,
262
+ createdAt INTEGER,
263
+ updatedAt INTEGER
264
+ )`;
265
+ yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect.void);
266
+ yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect.void);
267
+ yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect.void);
268
+ yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect.void);
269
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
270
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
271
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
272
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
273
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
274
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
275
+ });
276
+ query = Effect.fn("EntityMetaIndex.query")((query) => Effect.gen(function* () {
277
+ const sql = yield* SqlClient.SqlClient;
278
+ const normalizedTypeDXN = _normalizeTypeUri(query.typeDXN);
279
+ const parsedDxn = DXN.isDXN(normalizedTypeDXN) ? normalizedTypeDXN : void 0;
280
+ const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
281
+ const forms = _typeUriEquivalents(normalizedTypeDXN);
282
+ const exactMatch = sql.or(forms.map((form) => sql`typeDXN = ${form}`));
283
+ const likeMatch = sql.or(forms.map((form) => sql`typeDXN LIKE ${_escapeLikePrefix(form)} ESCAPE '\\'`));
284
+ return (hasNoVersion ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (${exactMatch} OR ${likeMatch})` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND ${exactMatch}`).map((row) => ({
285
+ ...row,
286
+ deleted: !!row.deleted
287
+ }));
288
+ }));
289
+ queryAll = Effect.fn("EntityMetaIndex.queryAll")((query) => Effect.gen(function* () {
290
+ if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) return [];
291
+ const sql = yield* SqlClient.SqlClient;
292
+ return (yield* sql`SELECT * FROM objectMeta WHERE ${buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null)}`).map((row) => ({
293
+ ...row,
294
+ deleted: !!row.deleted
295
+ }));
296
+ }));
297
+ queryTypes = Effect.fn("EntityMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false, includeAllQueues = false, queueIds = null }) => Effect.gen(function* () {
298
+ if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) return [];
299
+ if (typeDxns.length === 0) {
300
+ if (!inverted) return [];
301
+ const sql = yield* SqlClient.SqlClient;
302
+ return (yield* sql`SELECT * FROM objectMeta WHERE ${buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds)}`).map((row) => ({
303
+ ...row,
304
+ deleted: !!row.deleted
305
+ }));
306
+ }
307
+ const sql = yield* SqlClient.SqlClient;
308
+ const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
309
+ const typeWhere = sql.or(typeDxns.map((typeDXN) => {
310
+ const normalized = _normalizeTypeUri(typeDXN);
311
+ const parsedDxn = DXN.isDXN(normalized) ? normalized : void 0;
312
+ const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
313
+ const forms = _typeUriEquivalents(normalized);
314
+ const exactMatch = sql.or(forms.map((form) => sql`typeDXN = ${form}`));
315
+ return hasNoVersion ? sql.or([exactMatch, sql.or(forms.map((form) => sql`typeDXN LIKE ${_escapeLikePrefix(form)} ESCAPE '\\'`))]) : exactMatch;
316
+ }));
317
+ return (inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`).map((row) => ({
318
+ ...row,
319
+ deleted: !!row.deleted
320
+ }));
321
+ }));
322
+ queryRelations = Effect.fn("EntityMetaIndex.queryRelations")(({ endpoint, anchorDxns }) => Effect.gen(function* () {
323
+ if (anchorDxns.length === 0) return [];
324
+ const sql = yield* SqlClient.SqlClient;
325
+ const column = endpoint === "source" ? "source" : "target";
326
+ return (yield* sql`SELECT * FROM objectMeta WHERE entityKind = 'relation' AND ${sql.in(column, anchorDxns)}`).map((row) => ({
327
+ ...row,
328
+ deleted: !!row.deleted
329
+ }));
330
+ }));
331
+ update = Effect.fn("EntityMetaIndex.update")((objects) => Effect.gen(function* () {
332
+ const sql = yield* SqlClient.SqlClient;
333
+ yield* Effect.forEach(objects, (object) => Effect.gen(function* () {
334
+ const { spaceId, queueId, queueNamespace, documentId, data } = object;
335
+ const castData = data;
336
+ const objectId = castData.id;
337
+ let existing;
338
+ if (documentId) existing = yield* sql`SELECT recordId, entityKind, typeDXN, source, target, parent FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
339
+ else if (queueId) existing = yield* sql`SELECT recordId, entityKind, typeDXN, source, target, parent FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
340
+ else existing = [];
341
+ const [{ v }] = yield* sql`SELECT MAX(version) as v FROM objectMeta`;
342
+ const version = (v ?? 0) + 1;
343
+ const priorRow = existing.length > 0 ? existing[0] : void 0;
344
+ const preserveBody = castData[ATTR_TYPE] === void 0 && priorRow !== void 0;
345
+ const entityKind = preserveBody ? priorRow.entityKind : castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
346
+ const typeDXN = preserveBody ? priorRow.typeDXN : URI.make(_normalizeTypeUri(castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type"));
347
+ const deleted = castData[ATTR_DELETED] ? 1 : 0;
348
+ const source = preserveBody ? priorRow.source : entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
349
+ const target = preserveBody ? priorRow.target : entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
350
+ const parent = preserveBody ? priorRow.parent : castData[ATTR_PARENT] ?? null;
351
+ const updatedAtTimestamp = object.updatedAt;
352
+ const createdAtTimestamp = object.createdAt ?? updatedAtTimestamp;
353
+ if (existing.length > 0) yield* sql`
354
+ UPDATE objectMeta SET
355
+ version = ${version},
356
+ queueNamespace = ${queueNamespace ?? ""},
357
+ entityKind = ${entityKind},
358
+ typeDXN = ${typeDXN},
359
+ deleted = ${deleted},
360
+ source = ${source},
361
+ target = ${target},
362
+ parent = ${parent},
363
+ updatedAt = ${updatedAtTimestamp}
364
+ WHERE recordId = ${existing[0].recordId}
365
+ `;
366
+ else yield* sql`
367
+ INSERT INTO objectMeta (
368
+ objectId, queueId, queueNamespace, spaceId, documentId,
369
+ entityKind, typeDXN, deleted, source, target, parent, version,
370
+ createdAt, updatedAt
371
+ ) VALUES (
372
+ ${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
373
+ ${entityKind}, ${typeDXN}, ${deleted},
374
+ ${source}, ${target}, ${parent}, ${version},
375
+ ${createdAtTimestamp}, ${updatedAtTimestamp}
376
+ )
377
+ `;
378
+ }), { discard: true });
379
+ }));
380
+ /**
381
+ * Look up `recordIds` for objects that are already stored in the EntityMetaIndex.
382
+ * Mutates the objects in place.
383
+ */
384
+ lookupRecordIds = Effect.fn("EntityMetaIndex.lookupRecordIds")((objects) => Effect.gen(function* () {
385
+ const sql = yield* SqlClient.SqlClient;
386
+ for (const object of objects) {
387
+ const { spaceId, queueId, documentId, data } = object;
388
+ const objectId = data.id;
389
+ let result;
390
+ if (documentId) result = yield* sql`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
391
+ else if (queueId) result = yield* sql`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
392
+ else result = [];
393
+ if (result.length === 0) return yield* Effect.die(/* @__PURE__ */ new Error(`Object not found in EntityMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`));
394
+ object.recordId = result[0].recordId;
395
+ }
396
+ }));
397
+ /**
398
+ * Look up object metadata by recordIds.
399
+ */
400
+ lookupByRecordIds = Effect.fn("EntityMetaIndex.lookupByRecordIds")((recordIds) => Effect.gen(function* () {
401
+ if (recordIds.length === 0) return [];
402
+ const sql = yield* SqlClient.SqlClient;
403
+ return (yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("recordId", recordIds)}`).map((row) => ({
404
+ ...row,
405
+ deleted: !!row.deleted
406
+ }));
407
+ }));
408
+ /**
409
+ * Look up object metadata by object id across one or more spaces (space db and queue items).
410
+ */
411
+ queryObjectIds = Effect.fn("EntityMetaIndex.queryObjectIds")((query) => Effect.gen(function* () {
412
+ if (query.spaceIds.length === 0 || query.objectIds.length === 0) return [];
413
+ const sql = yield* SqlClient.SqlClient;
414
+ return (yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceIds)} AND ${sql.in("objectId", query.objectIds)}`).map((row) => ({
415
+ ...row,
416
+ deleted: !!row.deleted
417
+ }));
418
+ }));
419
+ /**
420
+ * Look up object metadata by objectId, spaceId, and queueId.
421
+ */
422
+ lookupByObjectId = Effect.fn("EntityMetaIndex.lookupByObjectId")((query) => Effect.gen(function* () {
423
+ const rows = yield* (yield* SqlClient.SqlClient)`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
424
+ if (rows.length === 0) return null;
425
+ return {
426
+ ...rows[0],
427
+ deleted: !!rows[0].deleted
428
+ };
429
+ }));
430
+ /**
431
+ * Query objects by timestamp range.
432
+ */
433
+ queryByTimeRange = Effect.fn("EntityMetaIndex.queryByTimeRange")((query) => Effect.gen(function* () {
434
+ if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) return [];
435
+ const sql = yield* SqlClient.SqlClient;
436
+ const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
437
+ const timeConditions = [];
438
+ if (query.updatedAfter != null) timeConditions.push(sql`updatedAt >= ${query.updatedAfter}`);
439
+ if (query.updatedBefore != null) timeConditions.push(sql`updatedAt <= ${query.updatedBefore}`);
440
+ if (query.createdAfter != null) timeConditions.push(sql`createdAt >= ${query.createdAfter}`);
441
+ if (query.createdBefore != null) timeConditions.push(sql`createdAt <= ${query.createdBefore}`);
442
+ return (timeConditions.length > 0 ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${sql.and(timeConditions)}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`).map((row) => ({
443
+ ...row,
444
+ deleted: !!row.deleted
445
+ }));
446
+ }));
447
+ /**
448
+ * Query children by parent object ids.
449
+ * Matches both:
450
+ * - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
451
+ * - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
452
+ * DXN uses the feed's object id as its queue id — see `Feed.getFeedUri`).
453
+ */
454
+ queryChildren = Effect.fn("EntityMetaIndex.queryChildren")((query) => Effect.gen(function* () {
455
+ if (query.parentIds.length === 0) return [];
456
+ const sql = yield* SqlClient.SqlClient;
457
+ const parentDxns = query.parentIds.map((id) => EID.make({ entityId: id }));
458
+ return (yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`).map((row) => ({
459
+ ...row,
460
+ deleted: !!row.deleted
461
+ }));
462
+ }));
463
+ };
464
+ //#endregion
465
+ //#region src/utils.ts
466
+ var __dxlog_file = "/__w/dxos/dxos/packages/core/echo/index-core/src/utils.ts";
467
+ /**
468
+ * Escaped property path within an object.
469
+ *
470
+ * Escaping rules:
471
+ *
472
+ * - '.' -> '\.'
473
+ * - '\' -> '\\'
474
+ * - contact with .
475
+ */
476
+ var EscapedPropPath = class extends Schema.String.annotations({ title: "EscapedPropPath" }) {
477
+ static escape(path) {
478
+ return path.map((p) => p.toString().replaceAll("\\", "\\\\").replaceAll(".", "\\.")).join(".");
479
+ }
480
+ static unescape(path) {
481
+ const parts = [];
482
+ let current = "";
483
+ for (let i = 0; i < path.length; i++) if (path[i] === "\\") {
484
+ invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
485
+ "~LogMeta": "~LogMeta",
486
+ F: __dxlog_file,
487
+ L: 34,
488
+ S: this,
489
+ A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"]
490
+ });
491
+ current = current + path[i + 1];
492
+ i++;
493
+ } else if (path[i] === ".") {
494
+ parts.push(current);
495
+ current = "";
496
+ } else current += path[i];
497
+ parts.push(current);
498
+ return parts;
499
+ }
500
+ };
501
+ //#endregion
502
+ //#region src/indexes/reverse-ref-index.ts
503
+ /**
504
+ * Extracts all outgoing references from an object's data.
505
+ */
506
+ var extractReferences = (data) => {
507
+ const refs = [];
508
+ const visit = (path, value) => {
509
+ if (isEncodedReference(value)) {
510
+ const uri = EncodedReference.toURI(value);
511
+ const parsedEchoUri = EID.tryParse(uri);
512
+ if (!(parsedEchoUri ? EID.getEntityId(parsedEchoUri) : void 0) || !parsedEchoUri) return;
513
+ refs.push({
514
+ path,
515
+ targetDXN: EID.toLocal(parsedEchoUri)
516
+ });
517
+ } else if (typeof value === "object" && value !== null && !Array.isArray(value)) for (const [key, v] of Object.entries(value)) visit([...path, key], v);
518
+ else if (Array.isArray(value)) for (let i = 0; i < value.length; i++) visit([...path, String(i)], value[i]);
519
+ };
520
+ visit([], data);
521
+ return refs;
522
+ };
523
+ Schema.Struct({
524
+ recordId: Schema.Number,
525
+ targetDXN: EID.Schema,
526
+ /**
527
+ * Escaped property path within an object.
528
+ *
529
+ * Escaping rules:
530
+ *
531
+ * - '.' -> '\.'
532
+ * - '\' -> '\\'
533
+ * - contact with .
534
+ */
535
+ propPath: Schema.String
536
+ });
537
+ /**
538
+ * Indexes reverse references - tracks which objects reference which targets.
539
+ * Only indexes references, not relations.
540
+ */
541
+ var ReverseRefIndex = class {
542
+ migrate = Effect.fn("ReverseRefIndex.migrate")(function* () {
543
+ const sql = yield* SqlClient.SqlClient;
544
+ yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
545
+ recordId INTEGER NOT NULL,
546
+ targetDXN TEXT NOT NULL,
547
+ propPath TEXT NOT NULL,
548
+ PRIMARY KEY (recordId, targetDXN, propPath)
549
+ )`;
550
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
551
+ });
552
+ /**
553
+ * Query all references pointing to a target DXN.
554
+ */
555
+ query = Effect.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect.gen(function* () {
556
+ return yield* (yield* SqlClient.SqlClient)`SELECT * FROM reverseRef WHERE targetDXN = ${EID.toLocal(targetDXN)}`;
557
+ }));
558
+ update = Effect.fn("ReverseRefIndex.update")((objects) => Effect.gen(function* () {
559
+ const sql = yield* SqlClient.SqlClient;
560
+ yield* Effect.forEach(objects, (object) => Effect.gen(function* () {
561
+ const { recordId, data } = object;
562
+ if (recordId === null) yield* Effect.die(/* @__PURE__ */ new Error("ReverseRefIndex.update requires recordId to be set"));
563
+ yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
564
+ const refs = extractReferences(data);
565
+ yield* Effect.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, { discard: true });
566
+ }), { discard: true });
567
+ }));
568
+ };
569
+ //#endregion
570
+ //#region src/index-engine.ts
571
+ var makeEmptyIndexingResult = () => ({
572
+ updated: 0,
573
+ done: true,
574
+ spaces: /* @__PURE__ */ new Set(),
575
+ queues: /* @__PURE__ */ new Set(),
576
+ documents: /* @__PURE__ */ new Set(),
577
+ types: /* @__PURE__ */ new Set(),
578
+ objects: /* @__PURE__ */ new Set()
579
+ });
580
+ var accumulateIndexingResult = (acc, objects) => {
581
+ for (const obj of objects) {
582
+ acc.spaces.add(obj.spaceId);
583
+ if (obj.queueId) acc.queues.add(obj.queueId);
584
+ if (obj.documentId) acc.documents.add(obj.documentId);
585
+ const t = obj.data[ATTR_TYPE];
586
+ if (t) acc.types.add(String(t));
587
+ if (obj.data.id) acc.objects.add(obj.data.id);
588
+ }
589
+ };
590
+ var IndexEngine = class {
591
+ #tracker;
592
+ #objectMetaIndex;
593
+ #ftsIndex;
594
+ #reverseRefIndex;
595
+ constructor(params) {
596
+ this.#tracker = params?.tracker ?? new IndexTracker();
597
+ this.#objectMetaIndex = params?.objectMetaIndex ?? new EntityMetaIndex();
598
+ this.#ftsIndex = params?.ftsIndex ?? new FtsIndex();
599
+ this.#reverseRefIndex = params?.reverseRefIndex ?? new ReverseRefIndex();
600
+ }
601
+ migrate() {
602
+ return Effect.gen(this, function* () {
603
+ yield* this.#tracker.migrate();
604
+ yield* this.#objectMetaIndex.migrate();
605
+ yield* this.#ftsIndex.migrate();
606
+ yield* this.#reverseRefIndex.migrate();
607
+ });
608
+ }
609
+ /**
610
+ * Query text index and return full object metadata with rank.
611
+ */
612
+ queryText(query) {
613
+ return Effect.gen(this, function* () {
614
+ return yield* this.#ftsIndex.query(query);
615
+ });
616
+ }
617
+ queryReverseRef(query) {
618
+ return this.#reverseRefIndex.query(query);
619
+ }
620
+ queryAll(query) {
621
+ return this.#objectMetaIndex.queryAll(query);
622
+ }
623
+ /**
624
+ * Query snapshots by recordIds.
625
+ * Used to load queue objects from indexed snapshots.
626
+ */
627
+ querySnapshotsJSON(recordIds) {
628
+ return this.#ftsIndex.querySnapshotsJSON(recordIds);
629
+ }
630
+ queryType(query) {
631
+ return this.#objectMetaIndex.query(query);
632
+ }
633
+ /**
634
+ * Query children by parent object ids.
635
+ */
636
+ queryChildren(query) {
637
+ return this.#objectMetaIndex.queryChildren(query);
638
+ }
639
+ queryTypes(query) {
640
+ return this.#objectMetaIndex.queryTypes(query);
641
+ }
642
+ queryByTimeRange(query) {
643
+ return this.#objectMetaIndex.queryByTimeRange(query);
644
+ }
645
+ queryRelations(query) {
646
+ return this.#objectMetaIndex.queryRelations(query);
647
+ }
648
+ lookupByRecordIds(recordIds) {
649
+ return this.#objectMetaIndex.lookupByRecordIds(recordIds);
650
+ }
651
+ lookupByObjectId(query) {
652
+ return this.#objectMetaIndex.lookupByObjectId(query);
653
+ }
654
+ queryObjectIds(query) {
655
+ return this.#objectMetaIndex.queryObjectIds(query);
656
+ }
657
+ update(ctx, dataSource, opts) {
658
+ return Effect.gen(this, function* () {
659
+ const result = makeEmptyIndexingResult();
660
+ const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
661
+ indexName: "fts5",
662
+ spaceId: opts.spaceId,
663
+ limit: opts.limit
664
+ });
665
+ result.updated += updatedFtsIndex;
666
+ result.done = result.done && doneFtsIndex;
667
+ accumulateIndexingResult(result, ftsObjects);
668
+ const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
669
+ indexName: "reverseRef",
670
+ spaceId: opts.spaceId,
671
+ limit: opts.limit
672
+ });
673
+ result.updated += updatedReverseRefIndex;
674
+ result.done = result.done && doneReverseRefIndex;
675
+ accumulateIndexingResult(result, reverseRefObjects);
676
+ return result;
677
+ }).pipe(Effect.withSpan("IndexEngine.update"));
678
+ }
679
+ /**
680
+ * Update a dependent index that requires recordId enrichment.
681
+ * This method:
682
+ * 1. Gets changed objects from the source.
683
+ * 2. Ensures those objects exist in EntityMetaIndex.
684
+ * 3. Looks up recordIds for those objects.
685
+ * 4. Enriches objects with recordIds.
686
+ * 5. Updates the dependent index.
687
+ */
688
+ #update(ctx, index, source, opts) {
689
+ return Effect.gen(this, function* () {
690
+ const sqlTransaction = yield* SqlTransaction.SqlTransaction;
691
+ const cursors = yield* this.#tracker.queryCursors({
692
+ indexName: opts.indexName,
693
+ sourceName: source.sourceName,
694
+ spaceId: opts.spaceId ?? void 0
695
+ });
696
+ const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, { limit: opts.limit });
697
+ if (objects.length === 0) return {
698
+ updated: 0,
699
+ done: true,
700
+ objects: []
701
+ };
702
+ return yield* sqlTransaction.withTransaction(Effect.gen(this, function* () {
703
+ yield* this.#objectMetaIndex.update(objects);
704
+ yield* this.#objectMetaIndex.lookupRecordIds(objects);
705
+ yield* index.update(objects);
706
+ yield* this.#tracker.updateCursors(updatedCursors.map((_) => ({
707
+ indexName: opts.indexName,
708
+ spaceId: _.spaceId,
709
+ sourceName: source.sourceName,
710
+ resourceId: _.resourceId,
711
+ cursor: _.cursor
712
+ })));
713
+ return {
714
+ updated: objects.length,
715
+ done: false,
716
+ objects
717
+ };
718
+ }));
719
+ }).pipe(Effect.withSpan("IndexEngine.#update"));
720
+ }
721
+ };
722
+ //#endregion
723
+ export { EntityMetaIndex, EscapedPropPath, FtsIndex, IndexEngine, IndexTracker, ReverseRefIndex };
724
+
725
+ //# sourceMappingURL=index.mjs.map