@dxos/index-core 0.8.4-main.fcfe5033a5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/neutral/index.mjs +190 -101
- package/dist/lib/neutral/index.mjs.map +4 -4
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/types/src/index-engine.d.ts +31 -17
- package/dist/types/src/index-engine.d.ts.map +1 -1
- package/dist/types/src/index-tracker.d.ts +3 -3
- package/dist/types/src/index.d.ts +3 -3
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/indexes/entity-meta-index.d.ts +113 -0
- package/dist/types/src/indexes/entity-meta-index.d.ts.map +1 -0
- package/dist/types/src/indexes/entity-meta-index.test.d.ts +2 -0
- package/dist/types/src/indexes/entity-meta-index.test.d.ts.map +1 -0
- package/dist/types/src/indexes/fts-index.d.ts +7 -6
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/src/indexes/index.d.ts +1 -1
- package/dist/types/src/indexes/interface.d.ts +15 -4
- package/dist/types/src/indexes/interface.d.ts.map +1 -1
- package/dist/types/src/indexes/reverse-ref-index.d.ts +3 -2
- package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -1
- package/dist/types/src/utils.d.ts +3 -3
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +13 -18
- package/src/index-engine.test.ts +138 -16
- package/src/index-engine.ts +123 -58
- package/src/index.ts +9 -3
- package/src/indexes/{object-meta-index.test.ts → entity-meta-index.test.ts} +114 -53
- package/src/indexes/{object-meta-index.ts → entity-meta-index.ts} +140 -71
- package/src/indexes/fts-index.test.ts +188 -34
- package/src/indexes/fts-index.ts +32 -15
- package/src/indexes/index.ts +1 -1
- package/src/indexes/interface.ts +16 -4
- package/src/indexes/reverse-ref-index.test.ts +57 -43
- package/src/indexes/reverse-ref-index.ts +22 -14
- package/src/utils.ts +5 -5
- package/dist/types/src/indexes/object-meta-index.d.ts +0 -88
- package/dist/types/src/indexes/object-meta-index.d.ts.map +0 -1
- package/dist/types/src/indexes/object-meta-index.test.d.ts +0 -2
- package/dist/types/src/indexes/object-meta-index.test.d.ts.map +0 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/index-engine.ts
|
|
2
2
|
import * as Effect5 from "effect/Effect";
|
|
3
|
+
import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
|
|
3
4
|
import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
|
|
4
5
|
|
|
5
6
|
// src/index-tracker.ts
|
|
@@ -88,6 +89,7 @@ var IndexTracker = class {
|
|
|
88
89
|
// src/indexes/fts-index.ts
|
|
89
90
|
import * as SqlClient3 from "@effect/sql/SqlClient";
|
|
90
91
|
import * as Effect2 from "effect/Effect";
|
|
92
|
+
var SQL_CHUNK_SIZE = 500;
|
|
91
93
|
var escapeFts5Query = (text) => {
|
|
92
94
|
return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
|
|
93
95
|
};
|
|
@@ -149,6 +151,7 @@ var FtsIndex = class {
|
|
|
149
151
|
/**
|
|
150
152
|
* Query snapshots by recordIds.
|
|
151
153
|
* Returns the parsed JSON snapshots for queue objects.
|
|
154
|
+
* RecordIds not present in the FTS index are silently omitted from the result.
|
|
152
155
|
*/
|
|
153
156
|
querySnapshotsJSON(recordIds) {
|
|
154
157
|
return Effect2.gen(function* () {
|
|
@@ -156,11 +159,21 @@ var FtsIndex = class {
|
|
|
156
159
|
return [];
|
|
157
160
|
}
|
|
158
161
|
const sql = yield* SqlClient3.SqlClient;
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
162
|
+
const chunks = [];
|
|
163
|
+
for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
|
|
164
|
+
chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
|
|
165
|
+
}
|
|
166
|
+
const allResults = [];
|
|
167
|
+
for (const chunk of chunks) {
|
|
168
|
+
const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
|
|
169
|
+
for (const r of rows) {
|
|
170
|
+
allResults.push({
|
|
171
|
+
recordId: r.rowid,
|
|
172
|
+
snapshot: JSON.parse(r.snapshot)
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return allResults;
|
|
164
177
|
});
|
|
165
178
|
}
|
|
166
179
|
update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
|
|
@@ -182,30 +195,50 @@ var FtsIndex = class {
|
|
|
182
195
|
}));
|
|
183
196
|
};
|
|
184
197
|
|
|
185
|
-
// src/indexes/
|
|
198
|
+
// src/indexes/entity-meta-index.ts
|
|
186
199
|
import * as SqlClient5 from "@effect/sql/SqlClient";
|
|
187
200
|
import * as Effect3 from "effect/Effect";
|
|
188
201
|
import * as Schema2 from "effect/Schema";
|
|
189
202
|
import { ATTR_DELETED, ATTR_PARENT, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from "@dxos/echo/internal";
|
|
190
|
-
import { DXN } from "@dxos/keys";
|
|
203
|
+
import { DXN, EID, EntityId, SpaceId as SpaceId2, URI } from "@dxos/keys";
|
|
204
|
+
var _normalizeTypeUri = (typeDXN) => {
|
|
205
|
+
if (!typeDXN.startsWith("echo:")) {
|
|
206
|
+
return typeDXN;
|
|
207
|
+
}
|
|
208
|
+
const eid = EID.tryParse(typeDXN);
|
|
209
|
+
if (!eid) {
|
|
210
|
+
return typeDXN;
|
|
211
|
+
}
|
|
212
|
+
const entityId = EID.getEntityId(eid);
|
|
213
|
+
return entityId ? EID.make({
|
|
214
|
+
entityId
|
|
215
|
+
}) : typeDXN;
|
|
216
|
+
};
|
|
191
217
|
var _escapeLikePrefix = (prefix) => {
|
|
192
218
|
const escaped = prefix.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
|
193
219
|
return `${escaped}:%`;
|
|
194
220
|
};
|
|
195
|
-
var
|
|
221
|
+
var EntityMeta = Schema2.Struct({
|
|
196
222
|
recordId: Schema2.Number,
|
|
197
|
-
objectId:
|
|
223
|
+
objectId: EntityId,
|
|
224
|
+
/** Empty string for non-queue objects. */
|
|
198
225
|
queueId: Schema2.String,
|
|
199
|
-
|
|
226
|
+
/** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
|
|
227
|
+
queueNamespace: Schema2.String,
|
|
228
|
+
spaceId: SpaceId2,
|
|
200
229
|
documentId: Schema2.String,
|
|
201
230
|
entityKind: Schema2.String,
|
|
202
|
-
/**
|
|
203
|
-
|
|
231
|
+
/**
|
|
232
|
+
* Type identifier URI for the object — typename DXN for non-stored schemas,
|
|
233
|
+
* schema-as-object EID for stored (dynamic) schemas. Mirrors the value
|
|
234
|
+
* written into the object's `system.type`.
|
|
235
|
+
*/
|
|
236
|
+
typeDXN: URI.Schema,
|
|
204
237
|
deleted: Schema2.Boolean,
|
|
205
|
-
source: Schema2.NullOr(
|
|
206
|
-
target: Schema2.NullOr(
|
|
238
|
+
source: Schema2.NullOr(EID.Schema),
|
|
239
|
+
target: Schema2.NullOr(EID.Schema),
|
|
207
240
|
/** Parent object id (nullable). */
|
|
208
|
-
parent: Schema2.NullOr(
|
|
241
|
+
parent: Schema2.NullOr(EID.Schema),
|
|
209
242
|
/** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
|
|
210
243
|
version: Schema2.Number,
|
|
211
244
|
/** Unix ms timestamp when the object was first indexed. */
|
|
@@ -230,17 +263,18 @@ var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
|
|
|
230
263
|
}
|
|
231
264
|
return sql.or(conditions);
|
|
232
265
|
};
|
|
233
|
-
var
|
|
234
|
-
migrate = Effect3.fn("
|
|
266
|
+
var EntityMetaIndex = class {
|
|
267
|
+
migrate = Effect3.fn("EntityMetaIndex.runMigrations")(function* () {
|
|
235
268
|
const sql = yield* SqlClient5.SqlClient;
|
|
236
269
|
yield* sql`CREATE TABLE IF NOT EXISTS objectMeta (
|
|
237
270
|
recordId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
238
271
|
objectId TEXT NOT NULL,
|
|
239
272
|
queueId TEXT NOT NULL DEFAULT '',
|
|
273
|
+
queueNamespace TEXT NOT NULL DEFAULT '',
|
|
240
274
|
spaceId TEXT NOT NULL,
|
|
241
275
|
documentId TEXT NOT NULL DEFAULT '',
|
|
242
276
|
entityKind TEXT NOT NULL,
|
|
243
|
-
|
|
277
|
+
typeDXN TEXT NOT NULL,
|
|
244
278
|
deleted INTEGER NOT NULL,
|
|
245
279
|
source TEXT,
|
|
246
280
|
target TEXT,
|
|
@@ -252,23 +286,26 @@ var ObjectMetaIndex = class {
|
|
|
252
286
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
|
|
253
287
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
|
|
254
288
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
|
|
289
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
|
|
255
290
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
|
|
256
|
-
yield* sql`CREATE INDEX IF NOT EXISTS
|
|
291
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
|
|
257
292
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
|
|
258
293
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
|
|
259
294
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
|
|
260
295
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
|
|
261
296
|
});
|
|
262
|
-
query = Effect3.fn("
|
|
297
|
+
query = Effect3.fn("EntityMetaIndex.query")((query) => Effect3.gen(function* () {
|
|
263
298
|
const sql = yield* SqlClient5.SqlClient;
|
|
264
|
-
const
|
|
265
|
-
const
|
|
299
|
+
const normalizedTypeDXN = _normalizeTypeUri(query.typeDXN);
|
|
300
|
+
const parsedDxn = DXN.isDXN(normalizedTypeDXN) ? normalizedTypeDXN : void 0;
|
|
301
|
+
const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
|
|
302
|
+
const rows = hasNoVersion ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${normalizedTypeDXN} OR typeDXN LIKE ${_escapeLikePrefix(normalizedTypeDXN)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${normalizedTypeDXN}`;
|
|
266
303
|
return rows.map((row) => ({
|
|
267
304
|
...row,
|
|
268
305
|
deleted: !!row.deleted
|
|
269
306
|
}));
|
|
270
307
|
}));
|
|
271
|
-
queryAll = Effect3.fn("
|
|
308
|
+
queryAll = Effect3.fn("EntityMetaIndex.queryAll")((query) => Effect3.gen(function* () {
|
|
272
309
|
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
273
310
|
return [];
|
|
274
311
|
}
|
|
@@ -280,7 +317,7 @@ var ObjectMetaIndex = class {
|
|
|
280
317
|
deleted: !!row.deleted
|
|
281
318
|
}));
|
|
282
319
|
}));
|
|
283
|
-
queryTypes = Effect3.fn("
|
|
320
|
+
queryTypes = Effect3.fn("EntityMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false, includeAllQueues = false, queueIds = null }) => Effect3.gen(function* () {
|
|
284
321
|
if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) {
|
|
285
322
|
return [];
|
|
286
323
|
}
|
|
@@ -298,12 +335,15 @@ var ObjectMetaIndex = class {
|
|
|
298
335
|
}
|
|
299
336
|
const sql = yield* SqlClient5.SqlClient;
|
|
300
337
|
const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
|
|
301
|
-
const typeWhere = sql.or(typeDxns.map((
|
|
302
|
-
const
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
338
|
+
const typeWhere = sql.or(typeDxns.map((typeDXN) => {
|
|
339
|
+
const normalized = _normalizeTypeUri(typeDXN);
|
|
340
|
+
const parsedDxn = DXN.isDXN(normalized) ? normalized : void 0;
|
|
341
|
+
const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
|
|
342
|
+
const exactMatch = sql`typeDXN = ${normalized}`;
|
|
343
|
+
return hasNoVersion ? sql.or([
|
|
344
|
+
exactMatch,
|
|
345
|
+
sql`typeDXN LIKE ${_escapeLikePrefix(normalized)} ESCAPE '\\'`
|
|
346
|
+
]) : exactMatch;
|
|
307
347
|
}));
|
|
308
348
|
const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
|
|
309
349
|
return rows.map((row) => ({
|
|
@@ -311,7 +351,7 @@ var ObjectMetaIndex = class {
|
|
|
311
351
|
deleted: !!row.deleted
|
|
312
352
|
}));
|
|
313
353
|
}));
|
|
314
|
-
queryRelations = Effect3.fn("
|
|
354
|
+
queryRelations = Effect3.fn("EntityMetaIndex.queryRelations")(({ endpoint, anchorDxns }) => Effect3.gen(function* () {
|
|
315
355
|
if (anchorDxns.length === 0) {
|
|
316
356
|
return [];
|
|
317
357
|
}
|
|
@@ -324,10 +364,10 @@ var ObjectMetaIndex = class {
|
|
|
324
364
|
}));
|
|
325
365
|
}));
|
|
326
366
|
// TODO(dmaretskyi): Update recordId on objects so that we don't need to look it up separately.
|
|
327
|
-
update = Effect3.fn("
|
|
367
|
+
update = Effect3.fn("EntityMetaIndex.update")((objects) => Effect3.gen(function* () {
|
|
328
368
|
const sql = yield* SqlClient5.SqlClient;
|
|
329
369
|
yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
|
|
330
|
-
const { spaceId, queueId, documentId, data } = object;
|
|
370
|
+
const { spaceId, queueId, queueNamespace, documentId, data } = object;
|
|
331
371
|
const castData = data;
|
|
332
372
|
const objectId = castData.id;
|
|
333
373
|
let existing;
|
|
@@ -342,36 +382,38 @@ var ObjectMetaIndex = class {
|
|
|
342
382
|
const [{ v }] = result;
|
|
343
383
|
const version = (v ?? 0) + 1;
|
|
344
384
|
const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
|
|
345
|
-
const
|
|
385
|
+
const typeDXN = URI.make(castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type");
|
|
346
386
|
const deleted = castData[ATTR_DELETED] ? 1 : 0;
|
|
347
387
|
const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
|
|
348
388
|
const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
|
|
349
389
|
const parent = castData[ATTR_PARENT] ?? null;
|
|
350
|
-
const
|
|
390
|
+
const updatedAtTimestamp = object.updatedAt;
|
|
391
|
+
const createdAtTimestamp = object.createdAt ?? updatedAtTimestamp;
|
|
351
392
|
if (existing.length > 0) {
|
|
352
393
|
yield* sql`
|
|
353
394
|
UPDATE objectMeta SET
|
|
354
395
|
version = ${version},
|
|
396
|
+
queueNamespace = ${queueNamespace ?? ""},
|
|
355
397
|
entityKind = ${entityKind},
|
|
356
|
-
|
|
398
|
+
typeDXN = ${typeDXN},
|
|
357
399
|
deleted = ${deleted},
|
|
358
400
|
source = ${source},
|
|
359
401
|
target = ${target},
|
|
360
402
|
parent = ${parent},
|
|
361
|
-
updatedAt = ${
|
|
403
|
+
updatedAt = ${updatedAtTimestamp}
|
|
362
404
|
WHERE recordId = ${existing[0].recordId}
|
|
363
405
|
`;
|
|
364
406
|
} else {
|
|
365
407
|
yield* sql`
|
|
366
408
|
INSERT INTO objectMeta (
|
|
367
|
-
objectId, queueId, spaceId, documentId,
|
|
368
|
-
entityKind,
|
|
409
|
+
objectId, queueId, queueNamespace, spaceId, documentId,
|
|
410
|
+
entityKind, typeDXN, deleted, source, target, parent, version,
|
|
369
411
|
createdAt, updatedAt
|
|
370
412
|
) VALUES (
|
|
371
|
-
${objectId}, ${queueId ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
372
|
-
${entityKind}, ${
|
|
413
|
+
${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
414
|
+
${entityKind}, ${typeDXN}, ${deleted},
|
|
373
415
|
${source}, ${target}, ${parent}, ${version},
|
|
374
|
-
${
|
|
416
|
+
${createdAtTimestamp}, ${updatedAtTimestamp}
|
|
375
417
|
)
|
|
376
418
|
`;
|
|
377
419
|
}
|
|
@@ -380,10 +422,10 @@ var ObjectMetaIndex = class {
|
|
|
380
422
|
});
|
|
381
423
|
}));
|
|
382
424
|
/**
|
|
383
|
-
* Look up `recordIds` for objects that are already stored in the
|
|
425
|
+
* Look up `recordIds` for objects that are already stored in the EntityMetaIndex.
|
|
384
426
|
* Mutates the objects in place.
|
|
385
427
|
*/
|
|
386
|
-
lookupRecordIds = Effect3.fn("
|
|
428
|
+
lookupRecordIds = Effect3.fn("EntityMetaIndex.lookupRecordIds")((objects) => Effect3.gen(function* () {
|
|
387
429
|
const sql = yield* SqlClient5.SqlClient;
|
|
388
430
|
for (const object of objects) {
|
|
389
431
|
const { spaceId, queueId, documentId, data } = object;
|
|
@@ -397,7 +439,7 @@ var ObjectMetaIndex = class {
|
|
|
397
439
|
result = [];
|
|
398
440
|
}
|
|
399
441
|
if (result.length === 0) {
|
|
400
|
-
return yield* Effect3.die(new Error(`Object not found in
|
|
442
|
+
return yield* Effect3.die(new Error(`Object not found in EntityMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`));
|
|
401
443
|
}
|
|
402
444
|
object.recordId = result[0].recordId;
|
|
403
445
|
}
|
|
@@ -405,7 +447,7 @@ var ObjectMetaIndex = class {
|
|
|
405
447
|
/**
|
|
406
448
|
* Look up object metadata by recordIds.
|
|
407
449
|
*/
|
|
408
|
-
lookupByRecordIds = Effect3.fn("
|
|
450
|
+
lookupByRecordIds = Effect3.fn("EntityMetaIndex.lookupByRecordIds")((recordIds) => Effect3.gen(function* () {
|
|
409
451
|
if (recordIds.length === 0) {
|
|
410
452
|
return [];
|
|
411
453
|
}
|
|
@@ -417,9 +459,23 @@ var ObjectMetaIndex = class {
|
|
|
417
459
|
}));
|
|
418
460
|
}));
|
|
419
461
|
/**
|
|
462
|
+
* Look up object metadata by object id across one or more spaces (space db and queue items).
|
|
463
|
+
*/
|
|
464
|
+
queryObjectIds = Effect3.fn("EntityMetaIndex.queryObjectIds")((query) => Effect3.gen(function* () {
|
|
465
|
+
if (query.spaceIds.length === 0 || query.objectIds.length === 0) {
|
|
466
|
+
return [];
|
|
467
|
+
}
|
|
468
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
469
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceIds)} AND ${sql.in("objectId", query.objectIds)}`;
|
|
470
|
+
return rows.map((row) => ({
|
|
471
|
+
...row,
|
|
472
|
+
deleted: !!row.deleted
|
|
473
|
+
}));
|
|
474
|
+
}));
|
|
475
|
+
/**
|
|
420
476
|
* Look up object metadata by objectId, spaceId, and queueId.
|
|
421
477
|
*/
|
|
422
|
-
lookupByObjectId = Effect3.fn("
|
|
478
|
+
lookupByObjectId = Effect3.fn("EntityMetaIndex.lookupByObjectId")((query) => Effect3.gen(function* () {
|
|
423
479
|
const sql = yield* SqlClient5.SqlClient;
|
|
424
480
|
const rows = yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
|
|
425
481
|
if (rows.length === 0) {
|
|
@@ -433,7 +489,7 @@ var ObjectMetaIndex = class {
|
|
|
433
489
|
/**
|
|
434
490
|
* Query objects by timestamp range.
|
|
435
491
|
*/
|
|
436
|
-
queryByTimeRange = Effect3.fn("
|
|
492
|
+
queryByTimeRange = Effect3.fn("EntityMetaIndex.queryByTimeRange")((query) => Effect3.gen(function* () {
|
|
437
493
|
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
438
494
|
return [];
|
|
439
495
|
}
|
|
@@ -460,13 +516,21 @@ var ObjectMetaIndex = class {
|
|
|
460
516
|
}));
|
|
461
517
|
/**
|
|
462
518
|
* Query children by parent object ids.
|
|
519
|
+
* Matches both:
|
|
520
|
+
* - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
|
|
521
|
+
* - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
|
|
522
|
+
* DXN uses the feed's object id as its queue id — see `Feed.getQueueUri`).
|
|
463
523
|
*/
|
|
464
|
-
queryChildren = Effect3.fn("
|
|
524
|
+
queryChildren = Effect3.fn("EntityMetaIndex.queryChildren")((query) => Effect3.gen(function* () {
|
|
465
525
|
if (query.parentIds.length === 0) {
|
|
466
526
|
return [];
|
|
467
527
|
}
|
|
468
528
|
const sql = yield* SqlClient5.SqlClient;
|
|
469
|
-
const
|
|
529
|
+
const parentDzns = query.parentIds.map((id) => EID.make({
|
|
530
|
+
entityId: id
|
|
531
|
+
}));
|
|
532
|
+
const parentDxns = parentDzns;
|
|
533
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`;
|
|
470
534
|
return rows.map((row) => ({
|
|
471
535
|
...row,
|
|
472
536
|
deleted: !!row.deleted
|
|
@@ -479,6 +543,7 @@ import * as SqlClient7 from "@effect/sql/SqlClient";
|
|
|
479
543
|
import * as Effect4 from "effect/Effect";
|
|
480
544
|
import * as Schema4 from "effect/Schema";
|
|
481
545
|
import { EncodedReference, isEncodedReference } from "@dxos/echo-protocol";
|
|
546
|
+
import { EID as EID2 } from "@dxos/keys";
|
|
482
547
|
|
|
483
548
|
// src/utils.ts
|
|
484
549
|
import * as Schema3 from "effect/Schema";
|
|
@@ -495,15 +560,7 @@ var EscapedPropPath = class extends Schema3.String.annotations({
|
|
|
495
560
|
let current = "";
|
|
496
561
|
for (let i = 0; i < path.length; i++) {
|
|
497
562
|
if (path[i] === "\\") {
|
|
498
|
-
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
|
|
499
|
-
F: __dxlog_file,
|
|
500
|
-
L: 34,
|
|
501
|
-
S: this,
|
|
502
|
-
A: [
|
|
503
|
-
"i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')",
|
|
504
|
-
"'Malformed escaping.'"
|
|
505
|
-
]
|
|
506
|
-
});
|
|
563
|
+
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
|
|
507
564
|
current = current + path[i + 1];
|
|
508
565
|
i++;
|
|
509
566
|
} else if (path[i] === ".") {
|
|
@@ -523,14 +580,15 @@ var extractReferences = (data) => {
|
|
|
523
580
|
const refs = [];
|
|
524
581
|
const visit = (path, value) => {
|
|
525
582
|
if (isEncodedReference(value)) {
|
|
526
|
-
const
|
|
527
|
-
const
|
|
528
|
-
|
|
583
|
+
const uri = EncodedReference.toURI(value);
|
|
584
|
+
const parsedEchoUri = EID2.tryParse(uri);
|
|
585
|
+
const echoUri = parsedEchoUri ? EID2.getEntityId(parsedEchoUri) : void 0;
|
|
586
|
+
if (!echoUri || !parsedEchoUri) {
|
|
529
587
|
return;
|
|
530
588
|
}
|
|
531
589
|
refs.push({
|
|
532
590
|
path,
|
|
533
|
-
|
|
591
|
+
targetDXN: EID2.toLocal(parsedEchoUri)
|
|
534
592
|
});
|
|
535
593
|
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
536
594
|
for (const [key, v] of Object.entries(value)) {
|
|
@@ -553,7 +611,7 @@ var extractReferences = (data) => {
|
|
|
553
611
|
};
|
|
554
612
|
var ReverseRef = Schema4.Struct({
|
|
555
613
|
recordId: Schema4.Number,
|
|
556
|
-
|
|
614
|
+
targetDXN: EID2.Schema,
|
|
557
615
|
/**
|
|
558
616
|
* Escaped property path within an object.
|
|
559
617
|
*
|
|
@@ -570,18 +628,19 @@ var ReverseRefIndex = class {
|
|
|
570
628
|
const sql = yield* SqlClient7.SqlClient;
|
|
571
629
|
yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
|
|
572
630
|
recordId INTEGER NOT NULL,
|
|
573
|
-
|
|
631
|
+
targetDXN TEXT NOT NULL,
|
|
574
632
|
propPath TEXT NOT NULL,
|
|
575
|
-
PRIMARY KEY (recordId,
|
|
633
|
+
PRIMARY KEY (recordId, targetDXN, propPath)
|
|
576
634
|
)`;
|
|
577
|
-
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(
|
|
635
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
|
|
578
636
|
});
|
|
579
637
|
/**
|
|
580
638
|
* Query all references pointing to a target DXN.
|
|
581
639
|
*/
|
|
582
|
-
query = Effect4.fn("ReverseRefIndex.query")(({
|
|
640
|
+
query = Effect4.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect4.gen(function* () {
|
|
583
641
|
const sql = yield* SqlClient7.SqlClient;
|
|
584
|
-
const
|
|
642
|
+
const normalized = EID2.toLocal(targetDXN);
|
|
643
|
+
const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDXN = ${normalized}`;
|
|
585
644
|
return rows;
|
|
586
645
|
}));
|
|
587
646
|
update = Effect4.fn("ReverseRefIndex.update")((objects) => Effect4.gen(function* () {
|
|
@@ -593,7 +652,7 @@ var ReverseRefIndex = class {
|
|
|
593
652
|
}
|
|
594
653
|
yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
|
|
595
654
|
const refs = extractReferences(data);
|
|
596
|
-
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId,
|
|
655
|
+
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, {
|
|
597
656
|
discard: true
|
|
598
657
|
});
|
|
599
658
|
}), {
|
|
@@ -603,6 +662,33 @@ var ReverseRefIndex = class {
|
|
|
603
662
|
};
|
|
604
663
|
|
|
605
664
|
// src/index-engine.ts
|
|
665
|
+
var makeEmptyIndexingResult = () => ({
|
|
666
|
+
updated: 0,
|
|
667
|
+
done: true,
|
|
668
|
+
spaces: /* @__PURE__ */ new Set(),
|
|
669
|
+
queues: /* @__PURE__ */ new Set(),
|
|
670
|
+
documents: /* @__PURE__ */ new Set(),
|
|
671
|
+
types: /* @__PURE__ */ new Set(),
|
|
672
|
+
objects: /* @__PURE__ */ new Set()
|
|
673
|
+
});
|
|
674
|
+
var accumulateIndexingResult = (acc, objects) => {
|
|
675
|
+
for (const obj of objects) {
|
|
676
|
+
acc.spaces.add(obj.spaceId);
|
|
677
|
+
if (obj.queueId) {
|
|
678
|
+
acc.queues.add(obj.queueId);
|
|
679
|
+
}
|
|
680
|
+
if (obj.documentId) {
|
|
681
|
+
acc.documents.add(obj.documentId);
|
|
682
|
+
}
|
|
683
|
+
const t = obj.data[ATTR_TYPE2];
|
|
684
|
+
if (t) {
|
|
685
|
+
acc.types.add(String(t));
|
|
686
|
+
}
|
|
687
|
+
if (obj.data.id) {
|
|
688
|
+
acc.objects.add(obj.data.id);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
};
|
|
606
692
|
var IndexEngine = class {
|
|
607
693
|
#tracker;
|
|
608
694
|
#objectMetaIndex;
|
|
@@ -610,7 +696,7 @@ var IndexEngine = class {
|
|
|
610
696
|
#reverseRefIndex;
|
|
611
697
|
constructor(params) {
|
|
612
698
|
this.#tracker = params?.tracker ?? new IndexTracker();
|
|
613
|
-
this.#objectMetaIndex = params?.objectMetaIndex ?? new
|
|
699
|
+
this.#objectMetaIndex = params?.objectMetaIndex ?? new EntityMetaIndex();
|
|
614
700
|
this.#ftsIndex = params?.ftsIndex ?? new FtsIndex();
|
|
615
701
|
this.#reverseRefIndex = params?.reverseRefIndex ?? new ReverseRefIndex();
|
|
616
702
|
}
|
|
@@ -667,35 +753,36 @@ var IndexEngine = class {
|
|
|
667
753
|
lookupByObjectId(query) {
|
|
668
754
|
return this.#objectMetaIndex.lookupByObjectId(query);
|
|
669
755
|
}
|
|
756
|
+
queryObjectIds(query) {
|
|
757
|
+
return this.#objectMetaIndex.queryObjectIds(query);
|
|
758
|
+
}
|
|
670
759
|
update(ctx, dataSource, opts) {
|
|
671
760
|
return Effect5.gen(this, function* () {
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
761
|
+
const result = makeEmptyIndexingResult();
|
|
762
|
+
const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
675
763
|
indexName: "fts5",
|
|
676
764
|
spaceId: opts.spaceId,
|
|
677
765
|
limit: opts.limit
|
|
678
766
|
});
|
|
679
|
-
updated += updatedFtsIndex;
|
|
680
|
-
done = done && doneFtsIndex;
|
|
681
|
-
|
|
767
|
+
result.updated += updatedFtsIndex;
|
|
768
|
+
result.done = result.done && doneFtsIndex;
|
|
769
|
+
accumulateIndexingResult(result, ftsObjects);
|
|
770
|
+
const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
|
|
682
771
|
indexName: "reverseRef",
|
|
683
772
|
spaceId: opts.spaceId,
|
|
684
773
|
limit: opts.limit
|
|
685
774
|
});
|
|
686
|
-
updated += updatedReverseRefIndex;
|
|
687
|
-
done = done && doneReverseRefIndex;
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
done
|
|
691
|
-
};
|
|
775
|
+
result.updated += updatedReverseRefIndex;
|
|
776
|
+
result.done = result.done && doneReverseRefIndex;
|
|
777
|
+
accumulateIndexingResult(result, reverseRefObjects);
|
|
778
|
+
return result;
|
|
692
779
|
}).pipe(Effect5.withSpan("IndexEngine.update"));
|
|
693
780
|
}
|
|
694
781
|
/**
|
|
695
782
|
* Update a dependent index that requires recordId enrichment.
|
|
696
783
|
* This method:
|
|
697
784
|
* 1. Gets changed objects from the source.
|
|
698
|
-
* 2. Ensures those objects exist in
|
|
785
|
+
* 2. Ensures those objects exist in EntityMetaIndex.
|
|
699
786
|
* 3. Looks up recordIds for those objects.
|
|
700
787
|
* 4. Enriches objects with recordIds.
|
|
701
788
|
* 5. Updates the dependent index.
|
|
@@ -703,22 +790,23 @@ var IndexEngine = class {
|
|
|
703
790
|
#update(ctx, index, source, opts) {
|
|
704
791
|
return Effect5.gen(this, function* () {
|
|
705
792
|
const sqlTransaction = yield* SqlTransaction.SqlTransaction;
|
|
793
|
+
const cursors = yield* this.#tracker.queryCursors({
|
|
794
|
+
indexName: opts.indexName,
|
|
795
|
+
sourceName: source.sourceName,
|
|
796
|
+
// Pass undefined to get all cursors when spaceId is null.
|
|
797
|
+
spaceId: opts.spaceId ?? void 0
|
|
798
|
+
});
|
|
799
|
+
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
|
|
800
|
+
limit: opts.limit
|
|
801
|
+
});
|
|
802
|
+
if (objects.length === 0) {
|
|
803
|
+
return {
|
|
804
|
+
updated: 0,
|
|
805
|
+
done: true,
|
|
806
|
+
objects: []
|
|
807
|
+
};
|
|
808
|
+
}
|
|
706
809
|
return yield* sqlTransaction.withTransaction(Effect5.gen(this, function* () {
|
|
707
|
-
const cursors = yield* this.#tracker.queryCursors({
|
|
708
|
-
indexName: opts.indexName,
|
|
709
|
-
sourceName: source.sourceName,
|
|
710
|
-
// Pass undefined to get all cursors when spaceId is null.
|
|
711
|
-
spaceId: opts.spaceId ?? void 0
|
|
712
|
-
});
|
|
713
|
-
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
|
|
714
|
-
limit: opts.limit
|
|
715
|
-
});
|
|
716
|
-
if (objects.length === 0) {
|
|
717
|
-
return {
|
|
718
|
-
updated: 0,
|
|
719
|
-
done: true
|
|
720
|
-
};
|
|
721
|
-
}
|
|
722
810
|
yield* this.#objectMetaIndex.update(objects);
|
|
723
811
|
yield* this.#objectMetaIndex.lookupRecordIds(objects);
|
|
724
812
|
yield* index.update(objects);
|
|
@@ -731,18 +819,19 @@ var IndexEngine = class {
|
|
|
731
819
|
})));
|
|
732
820
|
return {
|
|
733
821
|
updated: objects.length,
|
|
734
|
-
done: false
|
|
822
|
+
done: false,
|
|
823
|
+
objects
|
|
735
824
|
};
|
|
736
825
|
}));
|
|
737
826
|
}).pipe(Effect5.withSpan("IndexEngine.#update"));
|
|
738
827
|
}
|
|
739
828
|
};
|
|
740
829
|
export {
|
|
830
|
+
EntityMetaIndex,
|
|
741
831
|
EscapedPropPath,
|
|
742
832
|
FtsIndex,
|
|
743
833
|
IndexEngine,
|
|
744
834
|
IndexTracker,
|
|
745
|
-
ObjectMetaIndex,
|
|
746
835
|
ReverseRefIndex
|
|
747
836
|
};
|
|
748
837
|
//# sourceMappingURL=index.mjs.map
|