@dxos/index-core 0.8.4-main.422d1c7879 → 0.8.4-main.4f23b4e393

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  // src/index-engine.ts
2
2
  import * as Effect5 from "effect/Effect";
3
+ import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
3
4
  import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
4
5
 
5
6
  // src/index-tracker.ts
@@ -88,6 +89,7 @@ var IndexTracker = class {
88
89
  // src/indexes/fts-index.ts
89
90
  import * as SqlClient3 from "@effect/sql/SqlClient";
90
91
  import * as Effect2 from "effect/Effect";
92
+ var SQL_CHUNK_SIZE = 500;
91
93
  var escapeFts5Query = (text) => {
92
94
  return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
93
95
  };
@@ -149,6 +151,7 @@ var FtsIndex = class {
149
151
  /**
150
152
  * Query snapshots by recordIds.
151
153
  * Returns the parsed JSON snapshots for queue objects.
154
+ * RecordIds not present in the FTS index are silently omitted from the result.
152
155
  */
153
156
  querySnapshotsJSON(recordIds) {
154
157
  return Effect2.gen(function* () {
@@ -156,11 +159,21 @@ var FtsIndex = class {
156
159
  return [];
157
160
  }
158
161
  const sql = yield* SqlClient3.SqlClient;
159
- const results = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(recordIds)}`;
160
- return results.map((r) => ({
161
- recordId: r.rowid,
162
- snapshot: JSON.parse(r.snapshot)
163
- }));
162
+ const chunks = [];
163
+ for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
164
+ chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
165
+ }
166
+ const allResults = [];
167
+ for (const chunk of chunks) {
168
+ const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
169
+ for (const r of rows) {
170
+ allResults.push({
171
+ recordId: r.rowid,
172
+ snapshot: JSON.parse(r.snapshot)
173
+ });
174
+ }
175
+ }
176
+ return allResults;
164
177
  });
165
178
  }
166
179
  update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
@@ -196,11 +209,13 @@ var ObjectMeta = Schema2.Struct({
196
209
  recordId: Schema2.Number,
197
210
  objectId: Schema2.String,
198
211
  queueId: Schema2.String,
212
+ /** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
213
+ queueNamespace: Schema2.String,
199
214
  spaceId: Schema2.String,
200
215
  documentId: Schema2.String,
201
216
  entityKind: Schema2.String,
202
217
  /** The versioned DXN of the type of the object. */
203
- typeDxn: Schema2.String,
218
+ typeDXN: Schema2.String,
204
219
  deleted: Schema2.Boolean,
205
220
  source: Schema2.NullOr(Schema2.String),
206
221
  target: Schema2.NullOr(Schema2.String),
@@ -237,10 +252,11 @@ var ObjectMetaIndex = class {
237
252
  recordId INTEGER PRIMARY KEY AUTOINCREMENT,
238
253
  objectId TEXT NOT NULL,
239
254
  queueId TEXT NOT NULL DEFAULT '',
255
+ queueNamespace TEXT NOT NULL DEFAULT '',
240
256
  spaceId TEXT NOT NULL,
241
257
  documentId TEXT NOT NULL DEFAULT '',
242
258
  entityKind TEXT NOT NULL,
243
- typeDxn TEXT NOT NULL,
259
+ typeDXN TEXT NOT NULL,
244
260
  deleted INTEGER NOT NULL,
245
261
  source TEXT,
246
262
  target TEXT,
@@ -252,8 +268,9 @@ var ObjectMetaIndex = class {
252
268
  yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
253
269
  yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
254
270
  yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
271
+ yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
255
272
  yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
256
- yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDxn ON objectMeta(spaceId, typeDxn)`;
273
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
257
274
  yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
258
275
  yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
259
276
  yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
@@ -261,8 +278,8 @@ var ObjectMetaIndex = class {
261
278
  });
262
279
  query = Effect3.fn("ObjectMetaIndex.query")((query) => Effect3.gen(function* () {
263
280
  const sql = yield* SqlClient5.SqlClient;
264
- const parsedType = DXN.tryParse(query.typeDxn)?.asTypeDXN();
265
- const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDxn = ${query.typeDxn} OR typeDxn LIKE ${_escapeLikePrefix(query.typeDxn)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDxn = ${query.typeDxn}`;
281
+ const parsedType = DXN.tryParse(query.typeDXN)?.asTypeDXN();
282
+ const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${query.typeDXN} OR typeDXN LIKE ${_escapeLikePrefix(query.typeDXN)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${query.typeDXN}`;
266
283
  return rows.map((row) => ({
267
284
  ...row,
268
285
  deleted: !!row.deleted
@@ -298,12 +315,12 @@ var ObjectMetaIndex = class {
298
315
  }
299
316
  const sql = yield* SqlClient5.SqlClient;
300
317
  const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
301
- const typeWhere = sql.or(typeDxns.map((typeDxn) => {
302
- const parsedType = DXN.tryParse(typeDxn)?.asTypeDXN();
318
+ const typeWhere = sql.or(typeDxns.map((typeDXN) => {
319
+ const parsedType = DXN.tryParse(typeDXN)?.asTypeDXN();
303
320
  return parsedType && parsedType.version === void 0 ? sql.or([
304
- sql`typeDxn = ${typeDxn}`,
305
- sql`typeDxn LIKE ${_escapeLikePrefix(typeDxn)} ESCAPE '\\'`
306
- ]) : sql`typeDxn = ${typeDxn}`;
321
+ sql`typeDXN = ${typeDXN}`,
322
+ sql`typeDXN LIKE ${_escapeLikePrefix(typeDXN)} ESCAPE '\\'`
323
+ ]) : sql`typeDXN = ${typeDXN}`;
307
324
  }));
308
325
  const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
309
326
  return rows.map((row) => ({
@@ -327,7 +344,7 @@ var ObjectMetaIndex = class {
327
344
  update = Effect3.fn("ObjectMetaIndex.update")((objects) => Effect3.gen(function* () {
328
345
  const sql = yield* SqlClient5.SqlClient;
329
346
  yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
330
- const { spaceId, queueId, documentId, data } = object;
347
+ const { spaceId, queueId, queueNamespace, documentId, data } = object;
331
348
  const castData = data;
332
349
  const objectId = castData.id;
333
350
  let existing;
@@ -342,7 +359,7 @@ var ObjectMetaIndex = class {
342
359
  const [{ v }] = result;
343
360
  const version = (v ?? 0) + 1;
344
361
  const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
345
- const typeDxn = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type";
362
+ const typeDXN = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type";
346
363
  const deleted = castData[ATTR_DELETED] ? 1 : 0;
347
364
  const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
348
365
  const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
@@ -352,8 +369,9 @@ var ObjectMetaIndex = class {
352
369
  yield* sql`
353
370
  UPDATE objectMeta SET
354
371
  version = ${version},
372
+ queueNamespace = ${queueNamespace ?? ""},
355
373
  entityKind = ${entityKind},
356
- typeDxn = ${typeDxn},
374
+ typeDXN = ${typeDXN},
357
375
  deleted = ${deleted},
358
376
  source = ${source},
359
377
  target = ${target},
@@ -364,12 +382,12 @@ var ObjectMetaIndex = class {
364
382
  } else {
365
383
  yield* sql`
366
384
  INSERT INTO objectMeta (
367
- objectId, queueId, spaceId, documentId,
368
- entityKind, typeDxn, deleted, source, target, parent, version,
385
+ objectId, queueId, queueNamespace, spaceId, documentId,
386
+ entityKind, typeDXN, deleted, source, target, parent, version,
369
387
  createdAt, updatedAt
370
388
  ) VALUES (
371
- ${objectId}, ${queueId ?? ""}, ${spaceId}, ${documentId ?? ""},
372
- ${entityKind}, ${typeDxn}, ${deleted},
389
+ ${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
390
+ ${entityKind}, ${typeDXN}, ${deleted},
373
391
  ${source}, ${target}, ${parent}, ${version},
374
392
  ${sourceTimestamp}, ${sourceTimestamp}
375
393
  )
@@ -500,15 +518,7 @@ var EscapedPropPath = class extends Schema3.String.annotations({
500
518
  let current = "";
501
519
  for (let i = 0; i < path.length; i++) {
502
520
  if (path[i] === "\\") {
503
- invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
504
- F: __dxlog_file,
505
- L: 34,
506
- S: this,
507
- A: [
508
- "i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')",
509
- "'Malformed escaping.'"
510
- ]
511
- });
521
+ invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
512
522
  current = current + path[i + 1];
513
523
  i++;
514
524
  } else if (path[i] === ".") {
@@ -535,7 +545,7 @@ var extractReferences = (data) => {
535
545
  }
536
546
  refs.push({
537
547
  path,
538
- targetDxn: dxn.toString()
548
+ targetDXN: dxn.toString()
539
549
  });
540
550
  } else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
541
551
  for (const [key, v] of Object.entries(value)) {
@@ -558,7 +568,7 @@ var extractReferences = (data) => {
558
568
  };
559
569
  var ReverseRef = Schema4.Struct({
560
570
  recordId: Schema4.Number,
561
- targetDxn: Schema4.String,
571
+ targetDXN: Schema4.String,
562
572
  /**
563
573
  * Escaped property path within an object.
564
574
  *
@@ -575,18 +585,18 @@ var ReverseRefIndex = class {
575
585
  const sql = yield* SqlClient7.SqlClient;
576
586
  yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
577
587
  recordId INTEGER NOT NULL,
578
- targetDxn TEXT NOT NULL,
588
+ targetDXN TEXT NOT NULL,
579
589
  propPath TEXT NOT NULL,
580
- PRIMARY KEY (recordId, targetDxn, propPath)
590
+ PRIMARY KEY (recordId, targetDXN, propPath)
581
591
  )`;
582
- yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDxn)`;
592
+ yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
583
593
  });
584
594
  /**
585
595
  * Query all references pointing to a target DXN.
586
596
  */
587
- query = Effect4.fn("ReverseRefIndex.query")(({ targetDxn }) => Effect4.gen(function* () {
597
+ query = Effect4.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect4.gen(function* () {
588
598
  const sql = yield* SqlClient7.SqlClient;
589
- const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDxn = ${targetDxn}`;
599
+ const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDXN = ${targetDXN}`;
590
600
  return rows;
591
601
  }));
592
602
  update = Effect4.fn("ReverseRefIndex.update")((objects) => Effect4.gen(function* () {
@@ -598,7 +608,7 @@ var ReverseRefIndex = class {
598
608
  }
599
609
  yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
600
610
  const refs = extractReferences(data);
601
- yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDxn, propPath) VALUES (${recordId}, ${ref.targetDxn}, ${EscapedPropPath.escape(ref.path)})`, {
611
+ yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, {
602
612
  discard: true
603
613
  });
604
614
  }), {
@@ -608,6 +618,33 @@ var ReverseRefIndex = class {
608
618
  };
609
619
 
610
620
  // src/index-engine.ts
621
+ var makeEmptyIndexingResult = () => ({
622
+ updated: 0,
623
+ done: true,
624
+ spaces: /* @__PURE__ */ new Set(),
625
+ queues: /* @__PURE__ */ new Set(),
626
+ documents: /* @__PURE__ */ new Set(),
627
+ types: /* @__PURE__ */ new Set(),
628
+ objects: /* @__PURE__ */ new Set()
629
+ });
630
+ var accumulateIndexingResult = (acc, objects) => {
631
+ for (const obj of objects) {
632
+ acc.spaces.add(obj.spaceId);
633
+ if (obj.queueId) {
634
+ acc.queues.add(obj.queueId);
635
+ }
636
+ if (obj.documentId) {
637
+ acc.documents.add(obj.documentId);
638
+ }
639
+ const t = obj.data[ATTR_TYPE2];
640
+ if (t) {
641
+ acc.types.add(String(t));
642
+ }
643
+ if (obj.data.id) {
644
+ acc.objects.add(obj.data.id);
645
+ }
646
+ }
647
+ };
611
648
  var IndexEngine = class {
612
649
  #tracker;
613
650
  #objectMetaIndex;
@@ -674,26 +711,24 @@ var IndexEngine = class {
674
711
  }
675
712
  update(ctx, dataSource, opts) {
676
713
  return Effect5.gen(this, function* () {
677
- let updated = 0;
678
- let done = true;
679
- const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
714
+ const result = makeEmptyIndexingResult();
715
+ const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
680
716
  indexName: "fts5",
681
717
  spaceId: opts.spaceId,
682
718
  limit: opts.limit
683
719
  });
684
- updated += updatedFtsIndex;
685
- done = done && doneFtsIndex;
686
- const { updated: updatedReverseRefIndex, done: doneReverseRefIndex } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
720
+ result.updated += updatedFtsIndex;
721
+ result.done = result.done && doneFtsIndex;
722
+ accumulateIndexingResult(result, ftsObjects);
723
+ const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
687
724
  indexName: "reverseRef",
688
725
  spaceId: opts.spaceId,
689
726
  limit: opts.limit
690
727
  });
691
- updated += updatedReverseRefIndex;
692
- done = done && doneReverseRefIndex;
693
- return {
694
- updated,
695
- done
696
- };
728
+ result.updated += updatedReverseRefIndex;
729
+ result.done = result.done && doneReverseRefIndex;
730
+ accumulateIndexingResult(result, reverseRefObjects);
731
+ return result;
697
732
  }).pipe(Effect5.withSpan("IndexEngine.update"));
698
733
  }
699
734
  /**
@@ -721,7 +756,8 @@ var IndexEngine = class {
721
756
  if (objects.length === 0) {
722
757
  return {
723
758
  updated: 0,
724
- done: true
759
+ done: true,
760
+ objects: []
725
761
  };
726
762
  }
727
763
  yield* this.#objectMetaIndex.update(objects);
@@ -736,7 +772,8 @@ var IndexEngine = class {
736
772
  })));
737
773
  return {
738
774
  updated: objects.length,
739
- done: false
775
+ done: false,
776
+ objects
740
777
  };
741
778
  }));
742
779
  }).pipe(Effect5.withSpan("IndexEngine.#update"));