@dxos/index-core 0.8.4-main.422d1c7879 → 0.8.4-main.4f23b4e393
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/neutral/index.mjs +90 -53
- package/dist/lib/neutral/index.mjs.map +3 -3
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/types/src/index-engine.d.ts +16 -6
- package/dist/types/src/index-engine.d.ts.map +1 -1
- package/dist/types/src/index-tracker.d.ts +3 -3
- package/dist/types/src/index.d.ts +1 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/indexes/fts-index.d.ts +2 -1
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/src/indexes/interface.d.ts +5 -0
- package/dist/types/src/indexes/interface.d.ts.map +1 -1
- package/dist/types/src/indexes/object-meta-index.d.ts +8 -6
- package/dist/types/src/indexes/object-meta-index.d.ts.map +1 -1
- package/dist/types/src/indexes/reverse-ref-index.d.ts +2 -2
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +11 -11
- package/src/index-engine.test.ts +118 -5
- package/src/index-engine.ts +83 -29
- package/src/index.ts +7 -1
- package/src/indexes/fts-index.test.ts +132 -0
- package/src/indexes/fts-index.ts +25 -8
- package/src/indexes/interface.ts +5 -0
- package/src/indexes/object-meta-index.test.ts +59 -10
- package/src/indexes/object-meta-index.ts +30 -21
- package/src/indexes/reverse-ref-index.test.ts +22 -15
- package/src/indexes/reverse-ref-index.ts +11 -11
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/index-engine.ts
|
|
2
2
|
import * as Effect5 from "effect/Effect";
|
|
3
|
+
import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
|
|
3
4
|
import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
|
|
4
5
|
|
|
5
6
|
// src/index-tracker.ts
|
|
@@ -88,6 +89,7 @@ var IndexTracker = class {
|
|
|
88
89
|
// src/indexes/fts-index.ts
|
|
89
90
|
import * as SqlClient3 from "@effect/sql/SqlClient";
|
|
90
91
|
import * as Effect2 from "effect/Effect";
|
|
92
|
+
var SQL_CHUNK_SIZE = 500;
|
|
91
93
|
var escapeFts5Query = (text) => {
|
|
92
94
|
return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
|
|
93
95
|
};
|
|
@@ -149,6 +151,7 @@ var FtsIndex = class {
|
|
|
149
151
|
/**
|
|
150
152
|
* Query snapshots by recordIds.
|
|
151
153
|
* Returns the parsed JSON snapshots for queue objects.
|
|
154
|
+
* RecordIds not present in the FTS index are silently omitted from the result.
|
|
152
155
|
*/
|
|
153
156
|
querySnapshotsJSON(recordIds) {
|
|
154
157
|
return Effect2.gen(function* () {
|
|
@@ -156,11 +159,21 @@ var FtsIndex = class {
|
|
|
156
159
|
return [];
|
|
157
160
|
}
|
|
158
161
|
const sql = yield* SqlClient3.SqlClient;
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
162
|
+
const chunks = [];
|
|
163
|
+
for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
|
|
164
|
+
chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
|
|
165
|
+
}
|
|
166
|
+
const allResults = [];
|
|
167
|
+
for (const chunk of chunks) {
|
|
168
|
+
const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
|
|
169
|
+
for (const r of rows) {
|
|
170
|
+
allResults.push({
|
|
171
|
+
recordId: r.rowid,
|
|
172
|
+
snapshot: JSON.parse(r.snapshot)
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return allResults;
|
|
164
177
|
});
|
|
165
178
|
}
|
|
166
179
|
update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
|
|
@@ -196,11 +209,13 @@ var ObjectMeta = Schema2.Struct({
|
|
|
196
209
|
recordId: Schema2.Number,
|
|
197
210
|
objectId: Schema2.String,
|
|
198
211
|
queueId: Schema2.String,
|
|
212
|
+
/** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
|
|
213
|
+
queueNamespace: Schema2.String,
|
|
199
214
|
spaceId: Schema2.String,
|
|
200
215
|
documentId: Schema2.String,
|
|
201
216
|
entityKind: Schema2.String,
|
|
202
217
|
/** The versioned DXN of the type of the object. */
|
|
203
|
-
|
|
218
|
+
typeDXN: Schema2.String,
|
|
204
219
|
deleted: Schema2.Boolean,
|
|
205
220
|
source: Schema2.NullOr(Schema2.String),
|
|
206
221
|
target: Schema2.NullOr(Schema2.String),
|
|
@@ -237,10 +252,11 @@ var ObjectMetaIndex = class {
|
|
|
237
252
|
recordId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
238
253
|
objectId TEXT NOT NULL,
|
|
239
254
|
queueId TEXT NOT NULL DEFAULT '',
|
|
255
|
+
queueNamespace TEXT NOT NULL DEFAULT '',
|
|
240
256
|
spaceId TEXT NOT NULL,
|
|
241
257
|
documentId TEXT NOT NULL DEFAULT '',
|
|
242
258
|
entityKind TEXT NOT NULL,
|
|
243
|
-
|
|
259
|
+
typeDXN TEXT NOT NULL,
|
|
244
260
|
deleted INTEGER NOT NULL,
|
|
245
261
|
source TEXT,
|
|
246
262
|
target TEXT,
|
|
@@ -252,8 +268,9 @@ var ObjectMetaIndex = class {
|
|
|
252
268
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
|
|
253
269
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
|
|
254
270
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
|
|
271
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
|
|
255
272
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
|
|
256
|
-
yield* sql`CREATE INDEX IF NOT EXISTS
|
|
273
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
|
|
257
274
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
|
|
258
275
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
|
|
259
276
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
|
|
@@ -261,8 +278,8 @@ var ObjectMetaIndex = class {
|
|
|
261
278
|
});
|
|
262
279
|
query = Effect3.fn("ObjectMetaIndex.query")((query) => Effect3.gen(function* () {
|
|
263
280
|
const sql = yield* SqlClient5.SqlClient;
|
|
264
|
-
const parsedType = DXN.tryParse(query.
|
|
265
|
-
const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (
|
|
281
|
+
const parsedType = DXN.tryParse(query.typeDXN)?.asTypeDXN();
|
|
282
|
+
const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${query.typeDXN} OR typeDXN LIKE ${_escapeLikePrefix(query.typeDXN)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${query.typeDXN}`;
|
|
266
283
|
return rows.map((row) => ({
|
|
267
284
|
...row,
|
|
268
285
|
deleted: !!row.deleted
|
|
@@ -298,12 +315,12 @@ var ObjectMetaIndex = class {
|
|
|
298
315
|
}
|
|
299
316
|
const sql = yield* SqlClient5.SqlClient;
|
|
300
317
|
const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
|
|
301
|
-
const typeWhere = sql.or(typeDxns.map((
|
|
302
|
-
const parsedType = DXN.tryParse(
|
|
318
|
+
const typeWhere = sql.or(typeDxns.map((typeDXN) => {
|
|
319
|
+
const parsedType = DXN.tryParse(typeDXN)?.asTypeDXN();
|
|
303
320
|
return parsedType && parsedType.version === void 0 ? sql.or([
|
|
304
|
-
sql`
|
|
305
|
-
sql`
|
|
306
|
-
]) : sql`
|
|
321
|
+
sql`typeDXN = ${typeDXN}`,
|
|
322
|
+
sql`typeDXN LIKE ${_escapeLikePrefix(typeDXN)} ESCAPE '\\'`
|
|
323
|
+
]) : sql`typeDXN = ${typeDXN}`;
|
|
307
324
|
}));
|
|
308
325
|
const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
|
|
309
326
|
return rows.map((row) => ({
|
|
@@ -327,7 +344,7 @@ var ObjectMetaIndex = class {
|
|
|
327
344
|
update = Effect3.fn("ObjectMetaIndex.update")((objects) => Effect3.gen(function* () {
|
|
328
345
|
const sql = yield* SqlClient5.SqlClient;
|
|
329
346
|
yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
|
|
330
|
-
const { spaceId, queueId, documentId, data } = object;
|
|
347
|
+
const { spaceId, queueId, queueNamespace, documentId, data } = object;
|
|
331
348
|
const castData = data;
|
|
332
349
|
const objectId = castData.id;
|
|
333
350
|
let existing;
|
|
@@ -342,7 +359,7 @@ var ObjectMetaIndex = class {
|
|
|
342
359
|
const [{ v }] = result;
|
|
343
360
|
const version = (v ?? 0) + 1;
|
|
344
361
|
const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
|
|
345
|
-
const
|
|
362
|
+
const typeDXN = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type";
|
|
346
363
|
const deleted = castData[ATTR_DELETED] ? 1 : 0;
|
|
347
364
|
const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
|
|
348
365
|
const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
|
|
@@ -352,8 +369,9 @@ var ObjectMetaIndex = class {
|
|
|
352
369
|
yield* sql`
|
|
353
370
|
UPDATE objectMeta SET
|
|
354
371
|
version = ${version},
|
|
372
|
+
queueNamespace = ${queueNamespace ?? ""},
|
|
355
373
|
entityKind = ${entityKind},
|
|
356
|
-
|
|
374
|
+
typeDXN = ${typeDXN},
|
|
357
375
|
deleted = ${deleted},
|
|
358
376
|
source = ${source},
|
|
359
377
|
target = ${target},
|
|
@@ -364,12 +382,12 @@ var ObjectMetaIndex = class {
|
|
|
364
382
|
} else {
|
|
365
383
|
yield* sql`
|
|
366
384
|
INSERT INTO objectMeta (
|
|
367
|
-
objectId, queueId, spaceId, documentId,
|
|
368
|
-
entityKind,
|
|
385
|
+
objectId, queueId, queueNamespace, spaceId, documentId,
|
|
386
|
+
entityKind, typeDXN, deleted, source, target, parent, version,
|
|
369
387
|
createdAt, updatedAt
|
|
370
388
|
) VALUES (
|
|
371
|
-
${objectId}, ${queueId ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
372
|
-
${entityKind}, ${
|
|
389
|
+
${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
390
|
+
${entityKind}, ${typeDXN}, ${deleted},
|
|
373
391
|
${source}, ${target}, ${parent}, ${version},
|
|
374
392
|
${sourceTimestamp}, ${sourceTimestamp}
|
|
375
393
|
)
|
|
@@ -500,15 +518,7 @@ var EscapedPropPath = class extends Schema3.String.annotations({
|
|
|
500
518
|
let current = "";
|
|
501
519
|
for (let i = 0; i < path.length; i++) {
|
|
502
520
|
if (path[i] === "\\") {
|
|
503
|
-
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
|
|
504
|
-
F: __dxlog_file,
|
|
505
|
-
L: 34,
|
|
506
|
-
S: this,
|
|
507
|
-
A: [
|
|
508
|
-
"i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')",
|
|
509
|
-
"'Malformed escaping.'"
|
|
510
|
-
]
|
|
511
|
-
});
|
|
521
|
+
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
|
|
512
522
|
current = current + path[i + 1];
|
|
513
523
|
i++;
|
|
514
524
|
} else if (path[i] === ".") {
|
|
@@ -535,7 +545,7 @@ var extractReferences = (data) => {
|
|
|
535
545
|
}
|
|
536
546
|
refs.push({
|
|
537
547
|
path,
|
|
538
|
-
|
|
548
|
+
targetDXN: dxn.toString()
|
|
539
549
|
});
|
|
540
550
|
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
541
551
|
for (const [key, v] of Object.entries(value)) {
|
|
@@ -558,7 +568,7 @@ var extractReferences = (data) => {
|
|
|
558
568
|
};
|
|
559
569
|
var ReverseRef = Schema4.Struct({
|
|
560
570
|
recordId: Schema4.Number,
|
|
561
|
-
|
|
571
|
+
targetDXN: Schema4.String,
|
|
562
572
|
/**
|
|
563
573
|
* Escaped property path within an object.
|
|
564
574
|
*
|
|
@@ -575,18 +585,18 @@ var ReverseRefIndex = class {
|
|
|
575
585
|
const sql = yield* SqlClient7.SqlClient;
|
|
576
586
|
yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
|
|
577
587
|
recordId INTEGER NOT NULL,
|
|
578
|
-
|
|
588
|
+
targetDXN TEXT NOT NULL,
|
|
579
589
|
propPath TEXT NOT NULL,
|
|
580
|
-
PRIMARY KEY (recordId,
|
|
590
|
+
PRIMARY KEY (recordId, targetDXN, propPath)
|
|
581
591
|
)`;
|
|
582
|
-
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(
|
|
592
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
|
|
583
593
|
});
|
|
584
594
|
/**
|
|
585
595
|
* Query all references pointing to a target DXN.
|
|
586
596
|
*/
|
|
587
|
-
query = Effect4.fn("ReverseRefIndex.query")(({
|
|
597
|
+
query = Effect4.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect4.gen(function* () {
|
|
588
598
|
const sql = yield* SqlClient7.SqlClient;
|
|
589
|
-
const rows = yield* sql`SELECT * FROM reverseRef WHERE
|
|
599
|
+
const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDXN = ${targetDXN}`;
|
|
590
600
|
return rows;
|
|
591
601
|
}));
|
|
592
602
|
update = Effect4.fn("ReverseRefIndex.update")((objects) => Effect4.gen(function* () {
|
|
@@ -598,7 +608,7 @@ var ReverseRefIndex = class {
|
|
|
598
608
|
}
|
|
599
609
|
yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
|
|
600
610
|
const refs = extractReferences(data);
|
|
601
|
-
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId,
|
|
611
|
+
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, {
|
|
602
612
|
discard: true
|
|
603
613
|
});
|
|
604
614
|
}), {
|
|
@@ -608,6 +618,33 @@ var ReverseRefIndex = class {
|
|
|
608
618
|
};
|
|
609
619
|
|
|
610
620
|
// src/index-engine.ts
|
|
621
|
+
var makeEmptyIndexingResult = () => ({
|
|
622
|
+
updated: 0,
|
|
623
|
+
done: true,
|
|
624
|
+
spaces: /* @__PURE__ */ new Set(),
|
|
625
|
+
queues: /* @__PURE__ */ new Set(),
|
|
626
|
+
documents: /* @__PURE__ */ new Set(),
|
|
627
|
+
types: /* @__PURE__ */ new Set(),
|
|
628
|
+
objects: /* @__PURE__ */ new Set()
|
|
629
|
+
});
|
|
630
|
+
var accumulateIndexingResult = (acc, objects) => {
|
|
631
|
+
for (const obj of objects) {
|
|
632
|
+
acc.spaces.add(obj.spaceId);
|
|
633
|
+
if (obj.queueId) {
|
|
634
|
+
acc.queues.add(obj.queueId);
|
|
635
|
+
}
|
|
636
|
+
if (obj.documentId) {
|
|
637
|
+
acc.documents.add(obj.documentId);
|
|
638
|
+
}
|
|
639
|
+
const t = obj.data[ATTR_TYPE2];
|
|
640
|
+
if (t) {
|
|
641
|
+
acc.types.add(String(t));
|
|
642
|
+
}
|
|
643
|
+
if (obj.data.id) {
|
|
644
|
+
acc.objects.add(obj.data.id);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
};
|
|
611
648
|
var IndexEngine = class {
|
|
612
649
|
#tracker;
|
|
613
650
|
#objectMetaIndex;
|
|
@@ -674,26 +711,24 @@ var IndexEngine = class {
|
|
|
674
711
|
}
|
|
675
712
|
update(ctx, dataSource, opts) {
|
|
676
713
|
return Effect5.gen(this, function* () {
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
714
|
+
const result = makeEmptyIndexingResult();
|
|
715
|
+
const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
680
716
|
indexName: "fts5",
|
|
681
717
|
spaceId: opts.spaceId,
|
|
682
718
|
limit: opts.limit
|
|
683
719
|
});
|
|
684
|
-
updated += updatedFtsIndex;
|
|
685
|
-
done = done && doneFtsIndex;
|
|
686
|
-
|
|
720
|
+
result.updated += updatedFtsIndex;
|
|
721
|
+
result.done = result.done && doneFtsIndex;
|
|
722
|
+
accumulateIndexingResult(result, ftsObjects);
|
|
723
|
+
const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
|
|
687
724
|
indexName: "reverseRef",
|
|
688
725
|
spaceId: opts.spaceId,
|
|
689
726
|
limit: opts.limit
|
|
690
727
|
});
|
|
691
|
-
updated += updatedReverseRefIndex;
|
|
692
|
-
done = done && doneReverseRefIndex;
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
done
|
|
696
|
-
};
|
|
728
|
+
result.updated += updatedReverseRefIndex;
|
|
729
|
+
result.done = result.done && doneReverseRefIndex;
|
|
730
|
+
accumulateIndexingResult(result, reverseRefObjects);
|
|
731
|
+
return result;
|
|
697
732
|
}).pipe(Effect5.withSpan("IndexEngine.update"));
|
|
698
733
|
}
|
|
699
734
|
/**
|
|
@@ -721,7 +756,8 @@ var IndexEngine = class {
|
|
|
721
756
|
if (objects.length === 0) {
|
|
722
757
|
return {
|
|
723
758
|
updated: 0,
|
|
724
|
-
done: true
|
|
759
|
+
done: true,
|
|
760
|
+
objects: []
|
|
725
761
|
};
|
|
726
762
|
}
|
|
727
763
|
yield* this.#objectMetaIndex.update(objects);
|
|
@@ -736,7 +772,8 @@ var IndexEngine = class {
|
|
|
736
772
|
})));
|
|
737
773
|
return {
|
|
738
774
|
updated: objects.length,
|
|
739
|
-
done: false
|
|
775
|
+
done: false,
|
|
776
|
+
objects
|
|
740
777
|
};
|
|
741
778
|
}));
|
|
742
779
|
}).pipe(Effect5.withSpan("IndexEngine.#update"));
|