@dxos/index-core 0.8.4-main.d05539e30a → 0.8.4-main.d9fc60f731
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/neutral/index.mjs +59 -40
- package/dist/lib/neutral/index.mjs.map +4 -4
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/types/src/index-engine.d.ts +15 -15
- package/dist/types/src/index.d.ts +2 -2
- package/dist/types/src/indexes/{object-meta-index.d.ts → entity-meta-index.d.ts} +38 -26
- package/dist/types/src/indexes/entity-meta-index.d.ts.map +1 -0
- package/dist/types/src/indexes/entity-meta-index.test.d.ts +2 -0
- package/dist/types/src/indexes/entity-meta-index.test.d.ts.map +1 -0
- package/dist/types/src/indexes/fts-index.d.ts +5 -5
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/src/indexes/index.d.ts +1 -1
- package/dist/types/src/indexes/interface.d.ts +4 -4
- package/dist/types/src/indexes/reverse-ref-index.d.ts +3 -2
- package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -1
- package/dist/types/src/utils.d.ts +3 -3
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +12 -17
- package/src/index-engine.test.ts +18 -18
- package/src/index-engine.ts +24 -24
- package/src/index.ts +2 -2
- package/src/indexes/{object-meta-index.test.ts → entity-meta-index.test.ts} +47 -47
- package/src/indexes/{object-meta-index.ts → entity-meta-index.ts} +76 -60
- package/src/indexes/fts-index.test.ts +45 -45
- package/src/indexes/fts-index.ts +7 -7
- package/src/indexes/index.ts +1 -1
- package/src/indexes/interface.ts +4 -4
- package/src/indexes/reverse-ref-index.test.ts +33 -33
- package/src/indexes/reverse-ref-index.ts +10 -8
- package/src/utils.ts +5 -5
- package/dist/types/src/indexes/object-meta-index.d.ts.map +0 -1
- package/dist/types/src/indexes/object-meta-index.test.d.ts +0 -2
- package/dist/types/src/indexes/object-meta-index.test.d.ts.map +0 -1
|
@@ -195,32 +195,37 @@ var FtsIndex = class {
|
|
|
195
195
|
}));
|
|
196
196
|
};
|
|
197
197
|
|
|
198
|
-
// src/indexes/
|
|
198
|
+
// src/indexes/entity-meta-index.ts
|
|
199
199
|
import * as SqlClient5 from "@effect/sql/SqlClient";
|
|
200
200
|
import * as Effect3 from "effect/Effect";
|
|
201
201
|
import * as Schema2 from "effect/Schema";
|
|
202
202
|
import { ATTR_DELETED, ATTR_PARENT, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from "@dxos/echo/internal";
|
|
203
|
-
import { DXN } from "@dxos/keys";
|
|
203
|
+
import { DXN, EID, EntityId, SpaceId as SpaceId2, URI } from "@dxos/keys";
|
|
204
204
|
var _escapeLikePrefix = (prefix) => {
|
|
205
205
|
const escaped = prefix.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
|
206
206
|
return `${escaped}:%`;
|
|
207
207
|
};
|
|
208
|
-
var
|
|
208
|
+
var EntityMeta = Schema2.Struct({
|
|
209
209
|
recordId: Schema2.Number,
|
|
210
|
-
objectId:
|
|
210
|
+
objectId: EntityId,
|
|
211
|
+
/** Empty string for non-queue objects. */
|
|
211
212
|
queueId: Schema2.String,
|
|
212
213
|
/** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
|
|
213
214
|
queueNamespace: Schema2.String,
|
|
214
|
-
spaceId:
|
|
215
|
+
spaceId: SpaceId2,
|
|
215
216
|
documentId: Schema2.String,
|
|
216
217
|
entityKind: Schema2.String,
|
|
217
|
-
/**
|
|
218
|
-
|
|
218
|
+
/**
|
|
219
|
+
* Type identifier URI for the object — typename DXN for non-stored schemas,
|
|
220
|
+
* schema-as-object EID for stored (dynamic) schemas. Mirrors the value
|
|
221
|
+
* written into the object's `system.type`.
|
|
222
|
+
*/
|
|
223
|
+
typeDXN: URI.Schema,
|
|
219
224
|
deleted: Schema2.Boolean,
|
|
220
|
-
source: Schema2.NullOr(
|
|
221
|
-
target: Schema2.NullOr(
|
|
225
|
+
source: Schema2.NullOr(EID.Schema),
|
|
226
|
+
target: Schema2.NullOr(EID.Schema),
|
|
222
227
|
/** Parent object id (nullable). */
|
|
223
|
-
parent: Schema2.NullOr(
|
|
228
|
+
parent: Schema2.NullOr(EID.Schema),
|
|
224
229
|
/** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
|
|
225
230
|
version: Schema2.Number,
|
|
226
231
|
/** Unix ms timestamp when the object was first indexed. */
|
|
@@ -245,8 +250,8 @@ var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
|
|
|
245
250
|
}
|
|
246
251
|
return sql.or(conditions);
|
|
247
252
|
};
|
|
248
|
-
var
|
|
249
|
-
migrate = Effect3.fn("
|
|
253
|
+
var EntityMetaIndex = class {
|
|
254
|
+
migrate = Effect3.fn("EntityMetaIndex.runMigrations")(function* () {
|
|
250
255
|
const sql = yield* SqlClient5.SqlClient;
|
|
251
256
|
yield* sql`CREATE TABLE IF NOT EXISTS objectMeta (
|
|
252
257
|
recordId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -276,16 +281,18 @@ var ObjectMetaIndex = class {
|
|
|
276
281
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
|
|
277
282
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
|
|
278
283
|
});
|
|
279
|
-
query = Effect3.fn("
|
|
284
|
+
query = Effect3.fn("EntityMetaIndex.query")((query) => Effect3.gen(function* () {
|
|
280
285
|
const sql = yield* SqlClient5.SqlClient;
|
|
281
|
-
const
|
|
282
|
-
const
|
|
286
|
+
const parsedDxn = DXN.isDXN(query.typeDXN) ? query.typeDXN : void 0;
|
|
287
|
+
const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
|
|
288
|
+
const legacyTypeDXN = parsedDxn ? `dxn:type:${parsedDxn.slice(4)}` : void 0;
|
|
289
|
+
const rows = hasNoVersion ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${query.typeDXN} OR typeDXN LIKE ${_escapeLikePrefix(query.typeDXN)} ESCAPE '\\' ${legacyTypeDXN ? sql`OR typeDXN = ${legacyTypeDXN}` : sql``})` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${query.typeDXN} ${legacyTypeDXN ? sql`OR typeDXN = ${legacyTypeDXN}` : sql``})`;
|
|
283
290
|
return rows.map((row) => ({
|
|
284
291
|
...row,
|
|
285
292
|
deleted: !!row.deleted
|
|
286
293
|
}));
|
|
287
294
|
}));
|
|
288
|
-
queryAll = Effect3.fn("
|
|
295
|
+
queryAll = Effect3.fn("EntityMetaIndex.queryAll")((query) => Effect3.gen(function* () {
|
|
289
296
|
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
290
297
|
return [];
|
|
291
298
|
}
|
|
@@ -297,7 +304,7 @@ var ObjectMetaIndex = class {
|
|
|
297
304
|
deleted: !!row.deleted
|
|
298
305
|
}));
|
|
299
306
|
}));
|
|
300
|
-
queryTypes = Effect3.fn("
|
|
307
|
+
queryTypes = Effect3.fn("EntityMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false, includeAllQueues = false, queueIds = null }) => Effect3.gen(function* () {
|
|
301
308
|
if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) {
|
|
302
309
|
return [];
|
|
303
310
|
}
|
|
@@ -316,11 +323,17 @@ var ObjectMetaIndex = class {
|
|
|
316
323
|
const sql = yield* SqlClient5.SqlClient;
|
|
317
324
|
const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
|
|
318
325
|
const typeWhere = sql.or(typeDxns.map((typeDXN) => {
|
|
319
|
-
const
|
|
320
|
-
|
|
326
|
+
const parsedDxn = DXN.isDXN(typeDXN) ? typeDXN : void 0;
|
|
327
|
+
const hasNoVersion = parsedDxn !== void 0 && DXN.getVersion(parsedDxn) === void 0;
|
|
328
|
+
const legacyTypeDXN = parsedDxn ? `dxn:type:${parsedDxn.slice(4)}` : void 0;
|
|
329
|
+
const exactMatch = legacyTypeDXN ? sql.or([
|
|
321
330
|
sql`typeDXN = ${typeDXN}`,
|
|
322
|
-
sql`typeDXN
|
|
331
|
+
sql`typeDXN = ${legacyTypeDXN}`
|
|
323
332
|
]) : sql`typeDXN = ${typeDXN}`;
|
|
333
|
+
return hasNoVersion ? sql.or([
|
|
334
|
+
exactMatch,
|
|
335
|
+
sql`typeDXN LIKE ${_escapeLikePrefix(typeDXN)} ESCAPE '\\'`
|
|
336
|
+
]) : exactMatch;
|
|
324
337
|
}));
|
|
325
338
|
const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
|
|
326
339
|
return rows.map((row) => ({
|
|
@@ -328,7 +341,7 @@ var ObjectMetaIndex = class {
|
|
|
328
341
|
deleted: !!row.deleted
|
|
329
342
|
}));
|
|
330
343
|
}));
|
|
331
|
-
queryRelations = Effect3.fn("
|
|
344
|
+
queryRelations = Effect3.fn("EntityMetaIndex.queryRelations")(({ endpoint, anchorDxns }) => Effect3.gen(function* () {
|
|
332
345
|
if (anchorDxns.length === 0) {
|
|
333
346
|
return [];
|
|
334
347
|
}
|
|
@@ -341,7 +354,7 @@ var ObjectMetaIndex = class {
|
|
|
341
354
|
}));
|
|
342
355
|
}));
|
|
343
356
|
// TODO(dmaretskyi): Update recordId on objects so that we don't need to look it up separately.
|
|
344
|
-
update = Effect3.fn("
|
|
357
|
+
update = Effect3.fn("EntityMetaIndex.update")((objects) => Effect3.gen(function* () {
|
|
345
358
|
const sql = yield* SqlClient5.SqlClient;
|
|
346
359
|
yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
|
|
347
360
|
const { spaceId, queueId, queueNamespace, documentId, data } = object;
|
|
@@ -359,7 +372,8 @@ var ObjectMetaIndex = class {
|
|
|
359
372
|
const [{ v }] = result;
|
|
360
373
|
const version = (v ?? 0) + 1;
|
|
361
374
|
const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
|
|
362
|
-
const
|
|
375
|
+
const rawTypeDXN = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type";
|
|
376
|
+
const typeDXN = DXN.tryMake(rawTypeDXN) ?? rawTypeDXN;
|
|
363
377
|
const deleted = castData[ATTR_DELETED] ? 1 : 0;
|
|
364
378
|
const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
|
|
365
379
|
const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
|
|
@@ -398,10 +412,10 @@ var ObjectMetaIndex = class {
|
|
|
398
412
|
});
|
|
399
413
|
}));
|
|
400
414
|
/**
|
|
401
|
-
* Look up `recordIds` for objects that are already stored in the
|
|
415
|
+
* Look up `recordIds` for objects that are already stored in the EntityMetaIndex.
|
|
402
416
|
* Mutates the objects in place.
|
|
403
417
|
*/
|
|
404
|
-
lookupRecordIds = Effect3.fn("
|
|
418
|
+
lookupRecordIds = Effect3.fn("EntityMetaIndex.lookupRecordIds")((objects) => Effect3.gen(function* () {
|
|
405
419
|
const sql = yield* SqlClient5.SqlClient;
|
|
406
420
|
for (const object of objects) {
|
|
407
421
|
const { spaceId, queueId, documentId, data } = object;
|
|
@@ -415,7 +429,7 @@ var ObjectMetaIndex = class {
|
|
|
415
429
|
result = [];
|
|
416
430
|
}
|
|
417
431
|
if (result.length === 0) {
|
|
418
|
-
return yield* Effect3.die(new Error(`Object not found in
|
|
432
|
+
return yield* Effect3.die(new Error(`Object not found in EntityMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`));
|
|
419
433
|
}
|
|
420
434
|
object.recordId = result[0].recordId;
|
|
421
435
|
}
|
|
@@ -423,7 +437,7 @@ var ObjectMetaIndex = class {
|
|
|
423
437
|
/**
|
|
424
438
|
* Look up object metadata by recordIds.
|
|
425
439
|
*/
|
|
426
|
-
lookupByRecordIds = Effect3.fn("
|
|
440
|
+
lookupByRecordIds = Effect3.fn("EntityMetaIndex.lookupByRecordIds")((recordIds) => Effect3.gen(function* () {
|
|
427
441
|
if (recordIds.length === 0) {
|
|
428
442
|
return [];
|
|
429
443
|
}
|
|
@@ -437,7 +451,7 @@ var ObjectMetaIndex = class {
|
|
|
437
451
|
/**
|
|
438
452
|
* Look up object metadata by objectId, spaceId, and queueId.
|
|
439
453
|
*/
|
|
440
|
-
lookupByObjectId = Effect3.fn("
|
|
454
|
+
lookupByObjectId = Effect3.fn("EntityMetaIndex.lookupByObjectId")((query) => Effect3.gen(function* () {
|
|
441
455
|
const sql = yield* SqlClient5.SqlClient;
|
|
442
456
|
const rows = yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
|
|
443
457
|
if (rows.length === 0) {
|
|
@@ -451,7 +465,7 @@ var ObjectMetaIndex = class {
|
|
|
451
465
|
/**
|
|
452
466
|
* Query objects by timestamp range.
|
|
453
467
|
*/
|
|
454
|
-
queryByTimeRange = Effect3.fn("
|
|
468
|
+
queryByTimeRange = Effect3.fn("EntityMetaIndex.queryByTimeRange")((query) => Effect3.gen(function* () {
|
|
455
469
|
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
456
470
|
return [];
|
|
457
471
|
}
|
|
@@ -481,14 +495,17 @@ var ObjectMetaIndex = class {
|
|
|
481
495
|
* Matches both:
|
|
482
496
|
* - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
|
|
483
497
|
* - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
|
|
484
|
-
* DXN uses the feed's object id as its queue id — see `Feed.
|
|
498
|
+
* DXN uses the feed's object id as its queue id — see `Feed.getQueueUri`).
|
|
485
499
|
*/
|
|
486
|
-
queryChildren = Effect3.fn("
|
|
500
|
+
queryChildren = Effect3.fn("EntityMetaIndex.queryChildren")((query) => Effect3.gen(function* () {
|
|
487
501
|
if (query.parentIds.length === 0) {
|
|
488
502
|
return [];
|
|
489
503
|
}
|
|
490
504
|
const sql = yield* SqlClient5.SqlClient;
|
|
491
|
-
const
|
|
505
|
+
const parentDzns = query.parentIds.map((id) => EID.make({
|
|
506
|
+
entityId: id
|
|
507
|
+
}));
|
|
508
|
+
const parentDxns = parentDzns;
|
|
492
509
|
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`;
|
|
493
510
|
return rows.map((row) => ({
|
|
494
511
|
...row,
|
|
@@ -502,6 +519,7 @@ import * as SqlClient7 from "@effect/sql/SqlClient";
|
|
|
502
519
|
import * as Effect4 from "effect/Effect";
|
|
503
520
|
import * as Schema4 from "effect/Schema";
|
|
504
521
|
import { EncodedReference, isEncodedReference } from "@dxos/echo-protocol";
|
|
522
|
+
import { EID as EID2 } from "@dxos/keys";
|
|
505
523
|
|
|
506
524
|
// src/utils.ts
|
|
507
525
|
import * as Schema3 from "effect/Schema";
|
|
@@ -538,14 +556,15 @@ var extractReferences = (data) => {
|
|
|
538
556
|
const refs = [];
|
|
539
557
|
const visit = (path, value) => {
|
|
540
558
|
if (isEncodedReference(value)) {
|
|
541
|
-
const
|
|
542
|
-
const
|
|
543
|
-
|
|
559
|
+
const uri = EncodedReference.toURI(value);
|
|
560
|
+
const parsedEchoUri = EID2.tryParse(uri);
|
|
561
|
+
const echoUri = parsedEchoUri ? EID2.getEntityId(parsedEchoUri) : void 0;
|
|
562
|
+
if (!echoUri || !parsedEchoUri) {
|
|
544
563
|
return;
|
|
545
564
|
}
|
|
546
565
|
refs.push({
|
|
547
566
|
path,
|
|
548
|
-
targetDXN:
|
|
567
|
+
targetDXN: parsedEchoUri
|
|
549
568
|
});
|
|
550
569
|
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
551
570
|
for (const [key, v] of Object.entries(value)) {
|
|
@@ -568,7 +587,7 @@ var extractReferences = (data) => {
|
|
|
568
587
|
};
|
|
569
588
|
var ReverseRef = Schema4.Struct({
|
|
570
589
|
recordId: Schema4.Number,
|
|
571
|
-
targetDXN:
|
|
590
|
+
targetDXN: EID2.Schema,
|
|
572
591
|
/**
|
|
573
592
|
* Escaped property path within an object.
|
|
574
593
|
*
|
|
@@ -652,7 +671,7 @@ var IndexEngine = class {
|
|
|
652
671
|
#reverseRefIndex;
|
|
653
672
|
constructor(params) {
|
|
654
673
|
this.#tracker = params?.tracker ?? new IndexTracker();
|
|
655
|
-
this.#objectMetaIndex = params?.objectMetaIndex ?? new
|
|
674
|
+
this.#objectMetaIndex = params?.objectMetaIndex ?? new EntityMetaIndex();
|
|
656
675
|
this.#ftsIndex = params?.ftsIndex ?? new FtsIndex();
|
|
657
676
|
this.#reverseRefIndex = params?.reverseRefIndex ?? new ReverseRefIndex();
|
|
658
677
|
}
|
|
@@ -735,7 +754,7 @@ var IndexEngine = class {
|
|
|
735
754
|
* Update a dependent index that requires recordId enrichment.
|
|
736
755
|
* This method:
|
|
737
756
|
* 1. Gets changed objects from the source.
|
|
738
|
-
* 2. Ensures those objects exist in
|
|
757
|
+
* 2. Ensures those objects exist in EntityMetaIndex.
|
|
739
758
|
* 3. Looks up recordIds for those objects.
|
|
740
759
|
* 4. Enriches objects with recordIds.
|
|
741
760
|
* 5. Updates the dependent index.
|
|
@@ -780,11 +799,11 @@ var IndexEngine = class {
|
|
|
780
799
|
}
|
|
781
800
|
};
|
|
782
801
|
export {
|
|
802
|
+
EntityMetaIndex,
|
|
783
803
|
EscapedPropPath,
|
|
784
804
|
FtsIndex,
|
|
785
805
|
IndexEngine,
|
|
786
806
|
IndexTracker,
|
|
787
|
-
ObjectMetaIndex,
|
|
788
807
|
ReverseRefIndex
|
|
789
808
|
};
|
|
790
809
|
//# sourceMappingURL=index.mjs.map
|