@dxos/index-core 0.8.4-main.2244d791bb → 0.8.4-main.3fbcb4aa9b

package/dist/lib/neutral/index.mjs

@@ -1,5 +1,6 @@
 // src/index-engine.ts
 import * as Effect5 from "effect/Effect";
+import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
 import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
 
 // src/index-tracker.ts
@@ -31,6 +32,9 @@ var IndexCursor = Schema.Struct({
   */
   cursor: Schema.Union(Schema.Number, Schema.String)
 });
+var DEPRECATED_INDEX_NAMES = [
+  "fts"
+];
 var IndexTracker = class {
   migrate = Effect.fn("IndexTracker.migrate")(function* () {
     const sql = yield* SqlClient.SqlClient;
@@ -42,6 +46,9 @@ var IndexTracker = class {
       cursor,
       PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
     )`;
+    yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
+      return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
+    });
   });
   queryCursors = Effect.fn("IndexTracker.queryCursors")((query) => Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
@@ -82,6 +89,7 @@ var IndexTracker = class {
 // src/indexes/fts-index.ts
 import * as SqlClient3 from "@effect/sql/SqlClient";
 import * as Effect2 from "effect/Effect";
+var SQL_CHUNK_SIZE = 500;
 var escapeFts5Query = (text) => {
   return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
 };
@@ -143,6 +151,7 @@ var FtsIndex = class {
   /**
   * Query snapshots by recordIds.
   * Returns the parsed JSON snapshots for queue objects.
+  * RecordIds not present in the FTS index are silently omitted from the result.
   */
   querySnapshotsJSON(recordIds) {
     return Effect2.gen(function* () {
@@ -150,11 +159,21 @@ var FtsIndex = class {
         return [];
       }
       const sql = yield* SqlClient3.SqlClient;
-      const results = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(recordIds)}`;
-      return results.map((r) => ({
-        recordId: r.rowid,
-        snapshot: JSON.parse(r.snapshot)
-      }));
+      const chunks = [];
+      for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
+        chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
+      }
+      const allResults = [];
+      for (const chunk of chunks) {
+        const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
+        for (const r of rows) {
+          allResults.push({
+            recordId: r.rowid,
+            snapshot: JSON.parse(r.snapshot)
+          });
+        }
+      }
+      return allResults;
     });
   }
   update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
@@ -190,6 +209,8 @@ var ObjectMeta = Schema2.Struct({
   recordId: Schema2.Number,
   objectId: Schema2.String,
   queueId: Schema2.String,
+  /** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
+  queueNamespace: Schema2.String,
   spaceId: Schema2.String,
   documentId: Schema2.String,
   entityKind: Schema2.String,
@@ -201,8 +222,29 @@ var ObjectMeta = Schema2.Struct({
   /** Parent object id (nullable). */
   parent: Schema2.NullOr(Schema2.String),
   /** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
-  version: Schema2.Number
+  version: Schema2.Number,
+  /** Unix ms timestamp when the object was first indexed. */
+  createdAt: Schema2.NullOr(Schema2.Number),
+  /** Unix ms timestamp when the object was last re-indexed. */
+  updatedAt: Schema2.NullOr(Schema2.Number)
 });
+var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
+  const conditions = [];
+  if (spaceIds.length > 0) {
+    if (includeAllQueues) {
+      conditions.push(sql`${sql.in("spaceId", spaceIds)}`);
+    } else {
+      conditions.push(sql`(${sql.in("spaceId", spaceIds)} AND queueId = '')`);
+    }
+  }
+  if (queueIds && queueIds.length > 0) {
+    conditions.push(sql`${sql.in("queueId", queueIds)}`);
+  }
+  if (conditions.length === 0) {
+    return sql`1 = 0`;
+  }
+  return sql.or(conditions);
+};
 var ObjectMetaIndex = class {
   migrate = Effect3.fn("ObjectMetaIndex.runMigrations")(function* () {
     const sql = yield* SqlClient5.SqlClient;
@@ -210,6 +252,7 @@ var ObjectMetaIndex = class {
       recordId INTEGER PRIMARY KEY AUTOINCREMENT,
       objectId TEXT NOT NULL,
       queueId TEXT NOT NULL DEFAULT '',
+      queueNamespace TEXT NOT NULL DEFAULT '',
       spaceId TEXT NOT NULL,
       documentId TEXT NOT NULL DEFAULT '',
       entityKind TEXT NOT NULL,
@@ -218,13 +261,20 @@ var ObjectMetaIndex = class {
       source TEXT,
       target TEXT,
       parent TEXT,
-      version INTEGER NOT NULL
+      version INTEGER NOT NULL,
+      createdAt INTEGER,
+      updatedAt INTEGER
     )`;
     yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
+    yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
+    yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
+    yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDxn ON objectMeta(spaceId, typeDxn)`;
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
   });
   query = Effect3.fn("ObjectMetaIndex.query")((query) => Effect3.gen(function* () {
     const sql = yield* SqlClient5.SqlClient;
@@ -236,18 +286,19 @@ var ObjectMetaIndex = class {
     }));
   }));
   queryAll = Effect3.fn("ObjectMetaIndex.queryAll")((query) => Effect3.gen(function* () {
-    if (query.spaceIds.length === 0) {
+    if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
       return [];
     }
     const sql = yield* SqlClient5.SqlClient;
-    const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceIds)}`;
+    const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
+    const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
     return rows.map((row) => ({
       ...row,
       deleted: !!row.deleted
     }));
   }));
-  queryTypes = Effect3.fn("ObjectMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false }) => Effect3.gen(function* () {
-    if (spaceIds.length === 0) {
+  queryTypes = Effect3.fn("ObjectMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false, includeAllQueues = false, queueIds = null }) => Effect3.gen(function* () {
+    if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) {
       return [];
     }
     if (typeDxns.length === 0) {
@@ -255,14 +306,15 @@ var ObjectMetaIndex = class {
         return [];
       }
       const sql2 = yield* SqlClient5.SqlClient;
-      const rows2 = yield* sql2`SELECT * FROM objectMeta WHERE ${sql2.in("spaceId", spaceIds)}`;
+      const sourceCondition2 = buildSourceCondition(sql2, spaceIds, includeAllQueues, queueIds);
+      const rows2 = yield* sql2`SELECT * FROM objectMeta WHERE ${sourceCondition2}`;
       return rows2.map((row) => ({
         ...row,
         deleted: !!row.deleted
       }));
     }
     const sql = yield* SqlClient5.SqlClient;
-    const spaceWhere = sql.in("spaceId", spaceIds);
+    const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
     const typeWhere = sql.or(typeDxns.map((typeDxn) => {
       const parsedType = DXN.tryParse(typeDxn)?.asTypeDXN();
       return parsedType && parsedType.version === void 0 ? sql.or([
@@ -270,7 +322,7 @@ var ObjectMetaIndex = class {
         sql`typeDxn LIKE ${_escapeLikePrefix(typeDxn)} ESCAPE '\\'`
       ]) : sql`typeDxn = ${typeDxn}`;
     }));
-    const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${spaceWhere} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${spaceWhere} AND ${typeWhere}`;
+    const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
     return rows.map((row) => ({
       ...row,
       deleted: !!row.deleted
@@ -292,7 +344,7 @@ var ObjectMetaIndex = class {
   update = Effect3.fn("ObjectMetaIndex.update")((objects) => Effect3.gen(function* () {
     const sql = yield* SqlClient5.SqlClient;
     yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
-      const { spaceId, queueId, documentId, data } = object;
+      const { spaceId, queueId, queueNamespace, documentId, data } = object;
       const castData = data;
       const objectId = castData.id;
       let existing;
@@ -312,27 +364,32 @@ var ObjectMetaIndex = class {
       const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
       const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
       const parent = castData[ATTR_PARENT] ?? null;
+      const sourceTimestamp = object.updatedAt;
       if (existing.length > 0) {
         yield* sql`
                   UPDATE objectMeta SET
                     version = ${version},
+                    queueNamespace = ${queueNamespace ?? ""},
                     entityKind = ${entityKind},
                     typeDxn = ${typeDxn},
                     deleted = ${deleted},
                     source = ${source},
                     target = ${target},
-                    parent = ${parent}
+                    parent = ${parent},
+                    updatedAt = ${sourceTimestamp}
                   WHERE recordId = ${existing[0].recordId}
                 `;
       } else {
         yield* sql`
                   INSERT INTO objectMeta (
-                    objectId, queueId, spaceId, documentId, 
-                    entityKind, typeDxn, deleted, source, target, parent, version
+                    objectId, queueId, queueNamespace, spaceId, documentId,
+                    entityKind, typeDxn, deleted, source, target, parent, version,
+                    createdAt, updatedAt
                   ) VALUES (
-                    ${objectId}, ${queueId ?? ""}, ${spaceId}, ${documentId ?? ""}, 
-                    ${entityKind}, ${typeDxn}, ${deleted}, 
-                    ${source}, ${target}, ${parent}, ${version}
+                    ${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
+                    ${entityKind}, ${typeDxn}, ${deleted},
+                    ${source}, ${target}, ${parent}, ${version},
+                    ${sourceTimestamp}, ${sourceTimestamp}
                   )
                 `;
       }
@@ -378,14 +435,61 @@ var ObjectMetaIndex = class {
     }));
   }));
   /**
+  * Look up object metadata by objectId, spaceId, and queueId.
+  */
+  lookupByObjectId = Effect3.fn("ObjectMetaIndex.lookupByObjectId")((query) => Effect3.gen(function* () {
+    const sql = yield* SqlClient5.SqlClient;
+    const rows = yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
+    if (rows.length === 0) {
+      return null;
+    }
+    return {
+      ...rows[0],
+      deleted: !!rows[0].deleted
+    };
+  }));
+  /**
+  * Query objects by timestamp range.
+  */
+  queryByTimeRange = Effect3.fn("ObjectMetaIndex.queryByTimeRange")((query) => Effect3.gen(function* () {
+    if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
+      return [];
+    }
+    const sql = yield* SqlClient5.SqlClient;
+    const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
+    const timeConditions = [];
+    if (query.updatedAfter != null) {
+      timeConditions.push(sql`updatedAt >= ${query.updatedAfter}`);
+    }
+    if (query.updatedBefore != null) {
+      timeConditions.push(sql`updatedAt <= ${query.updatedBefore}`);
+    }
+    if (query.createdAfter != null) {
+      timeConditions.push(sql`createdAt >= ${query.createdAfter}`);
+    }
+    if (query.createdBefore != null) {
+      timeConditions.push(sql`createdAt <= ${query.createdBefore}`);
+    }
+    const rows = timeConditions.length > 0 ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${sql.and(timeConditions)}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
+    return rows.map((row) => ({
+      ...row,
+      deleted: !!row.deleted
+    }));
+  }));
+  /**
   * Query children by parent object ids.
+  * Matches both:
+  * - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
+  * - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
+  *   DXN uses the feed's object id as its queue id — see `Feed.getQueueDxn`).
   */
   queryChildren = Effect3.fn("ObjectMetaIndex.queryChildren")((query) => Effect3.gen(function* () {
     if (query.parentIds.length === 0) {
       return [];
     }
     const sql = yield* SqlClient5.SqlClient;
-    const rows = yield* sql`SELECT * FROM objectMeta WHERE spaceId IN ${sql.in(query.spaceId)} AND parent IN ${sql.in(query.parentIds.map((id) => DXN.fromLocalObjectId(id).toString()))}`;
+    const parentDxns = query.parentIds.map((id) => DXN.fromLocalObjectId(id).toString());
+    const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`;
     return rows.map((row) => ({
       ...row,
       deleted: !!row.deleted
@@ -414,15 +518,7 @@ var EscapedPropPath = class extends Schema3.String.annotations({
     let current = "";
     for (let i = 0; i < path.length; i++) {
       if (path[i] === "\\") {
-        invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
-          F: __dxlog_file,
-          L: 34,
-          S: this,
-          A: [
-            "i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')",
-            "'Malformed escaping.'"
-          ]
-        });
+        invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
         current = current + path[i + 1];
         i++;
       } else if (path[i] === ".") {
@@ -522,6 +618,33 @@ var ReverseRefIndex = class {
 };
 
 // src/index-engine.ts
+var makeEmptyIndexingResult = () => ({
+  updated: 0,
+  done: true,
+  spaces: /* @__PURE__ */ new Set(),
+  queues: /* @__PURE__ */ new Set(),
+  documents: /* @__PURE__ */ new Set(),
+  types: /* @__PURE__ */ new Set(),
+  objects: /* @__PURE__ */ new Set()
+});
+var accumulateIndexingResult = (acc, objects) => {
+  for (const obj of objects) {
+    acc.spaces.add(obj.spaceId);
+    if (obj.queueId) {
+      acc.queues.add(obj.queueId);
+    }
+    if (obj.documentId) {
+      acc.documents.add(obj.documentId);
+    }
+    const t = obj.data[ATTR_TYPE2];
+    if (t) {
+      acc.types.add(String(t));
+    }
+    if (obj.data.id) {
+      acc.objects.add(obj.data.id);
+    }
+  }
+};
 var IndexEngine = class {
   #tracker;
   #objectMetaIndex;
@@ -574,31 +697,38 @@ var IndexEngine = class {
   queryTypes(query) {
     return this.#objectMetaIndex.queryTypes(query);
   }
+  queryByTimeRange(query) {
+    return this.#objectMetaIndex.queryByTimeRange(query);
+  }
   queryRelations(query) {
     return this.#objectMetaIndex.queryRelations(query);
   }
   lookupByRecordIds(recordIds) {
     return this.#objectMetaIndex.lookupByRecordIds(recordIds);
   }
-  update(dataSource, opts) {
+  lookupByObjectId(query) {
+    return this.#objectMetaIndex.lookupByObjectId(query);
+  }
+  update(ctx, dataSource, opts) {
     return Effect5.gen(this, function* () {
-      let updated = 0;
-      const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(this.#ftsIndex, dataSource, {
-        indexName: "fts",
+      const result = makeEmptyIndexingResult();
+      const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
+        indexName: "fts5",
         spaceId: opts.spaceId,
         limit: opts.limit
       });
-      updated += updatedFtsIndex;
-      const { updated: updatedReverseRefIndex, done: doneReverseRefIndex } = yield* this.#update(this.#reverseRefIndex, dataSource, {
+      result.updated += updatedFtsIndex;
+      result.done = result.done && doneFtsIndex;
+      accumulateIndexingResult(result, ftsObjects);
+      const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
         indexName: "reverseRef",
         spaceId: opts.spaceId,
         limit: opts.limit
       });
-      updated += updatedReverseRefIndex;
-      return {
-        updated,
-        done: doneFtsIndex && doneReverseRefIndex
-      };
+      result.updated += updatedReverseRefIndex;
+      result.done = result.done && doneReverseRefIndex;
+      accumulateIndexingResult(result, reverseRefObjects);
+      return result;
     }).pipe(Effect5.withSpan("IndexEngine.update"));
   }
   /**
@@ -610,7 +740,7 @@ var IndexEngine = class {
   * 4. Enriches objects with recordIds.
   * 5. Updates the dependent index.
   */
-  #update(index, source, opts) {
+  #update(ctx, index, source, opts) {
     return Effect5.gen(this, function* () {
       const sqlTransaction = yield* SqlTransaction.SqlTransaction;
       return yield* sqlTransaction.withTransaction(Effect5.gen(this, function* () {
@@ -620,13 +750,14 @@ var IndexEngine = class {
           // Pass undefined to get all cursors when spaceId is null.
           spaceId: opts.spaceId ?? void 0
         });
-        const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(cursors, {
+        const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
           limit: opts.limit
         });
         if (objects.length === 0) {
           return {
             updated: 0,
-            done: true
+            done: true,
+            objects: []
           };
         }
         yield* this.#objectMetaIndex.update(objects);
@@ -641,7 +772,8 @@ var IndexEngine = class {
         })));
         return {
           updated: objects.length,
-          done: false
+          done: false,
+          objects
         };
       }));
     }).pipe(Effect5.withSpan("IndexEngine.#update"));