@dxos/index-core 0.8.4-main.fcfe5033a5 → 0.9.0

package/src/indexes/fts-index.test.ts

@@ -10,14 +10,14 @@ import * as Effect from 'effect/Effect';
 import * as Layer from 'effect/Layer';
 
 import { ATTR_TYPE } from '@dxos/echo/internal';
-import { DXN, ObjectId, SpaceId } from '@dxos/keys';
+import { DXN, EntityId, SpaceId } from '@dxos/keys';
 
+import { EntityMetaIndex } from './entity-meta-index';
 import { FtsIndex } from './fts-index';
 import type { IndexerObject } from './interface';
-import { ObjectMetaIndex } from './object-meta-index';
 
-const TYPE_PERSON = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
-const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
+const TYPE_PERSON = DXN.make('com.example.type.person', '0.1.0');
+const TYPE_DEFAULT = DXN.make('com.example.type.Type', '0.1.0');
 
 const TestLayer = Layer.merge(
   SqliteClient.layer({
@@ -46,7 +46,7 @@ describe('FtsIndex', () => {
     'should insert snapshots and query them via MATCH',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
@@ -55,11 +55,13 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-1',
           recordId: null,
+          createdAt: null,
           updatedAt: Date.now(),
           data: {
-            id: ObjectId.random(),
+            id: EntityId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
             title: 'Hello Effect',
             body: 'This is a message about Effect and SQL.',
@@ -89,23 +91,25 @@ describe('FtsIndex', () => {
     'should upsert objects on update',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
       const spaceId = SpaceId.random();
-      const objectId = ObjectId.random();
+      const objectId = EntityId.random();
 
       // Initial insert.
       const obj1: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-1',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
           id: objectId,
-          [ATTR_TYPE]: DXN.parse('dxn:type:com.example.type.person:0.1.0').toString(),
+          [ATTR_TYPE]: TYPE_PERSON,
           title: 'Original Title',
         },
       };
@@ -120,8 +124,10 @@ describe('FtsIndex', () => {
       const obj2: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-1',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
           id: objectId,
@@ -149,7 +155,7 @@ describe('FtsIndex', () => {
     'should handle non-sequential recordIds',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
@@ -158,11 +164,13 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-100',
           recordId: null,
+          createdAt: null,
           updatedAt: Date.now(),
           data: {
-            id: ObjectId.random(),
+            id: EntityId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
             title: 'Alpha Document',
           },
@@ -170,11 +178,13 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-200',
           recordId: null,
+          createdAt: null,
           updatedAt: Date.now(),
           data: {
-            id: ObjectId.random(),
+            id: EntityId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
             title: 'Beta Document',
           },
@@ -182,11 +192,13 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-1000',
           recordId: null,
+          createdAt: null,
           updatedAt: Date.now(),
           data: {
-            id: ObjectId.random(),
+            id: EntityId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
             title: 'Gamma Document',
           },
@@ -221,7 +233,7 @@ describe('FtsIndex', () => {
     'should query from one space only',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
@@ -231,11 +243,13 @@ describe('FtsIndex', () => {
       const obj1: IndexerObject = {
         spaceId: space1,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s1',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Space One Content',
         },
@@ -244,11 +258,13 @@ describe('FtsIndex', () => {
       const obj2: IndexerObject = {
         spaceId: space2,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s2',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Space Two Content',
         },
@@ -294,7 +310,7 @@ describe('FtsIndex', () => {
     'partial word matches',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
@@ -303,11 +319,13 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-1',
           recordId: null,
+          createdAt: null,
           updatedAt: Date.now(),
           data: {
-            id: ObjectId.random(),
+            id: EntityId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
             title: 'Programming in TypeScript',
             body: 'Learn about functional programming patterns.',
@@ -316,11 +334,13 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-2',
           recordId: null,
+          createdAt: null,
           updatedAt: Date.now(),
           data: {
-            id: ObjectId.random(),
+            id: EntityId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
             title: 'Database Design',
             body: 'Understanding program architecture.',
@@ -375,22 +395,24 @@ describe('FtsIndex', () => {
     'should query from specific queues',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
       const spaceId = SpaceId.random();
-      const queue1 = ObjectId.random();
-      const queue2 = ObjectId.random();
+      const queue1 = EntityId.random();
+      const queue2 = EntityId.random();
 
       const spaceObj: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-space',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Space Content',
         },
@@ -399,11 +421,13 @@ describe('FtsIndex', () => {
       const queue1Obj: IndexerObject = {
         spaceId,
         queueId: queue1,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Queue One Content',
         },
@@ -412,11 +436,13 @@ describe('FtsIndex', () => {
       const queue2Obj: IndexerObject = {
         spaceId,
         queueId: queue2,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Queue Two Content',
         },
@@ -451,21 +477,23 @@ describe('FtsIndex', () => {
     'should query with includeAllQueues',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
       const spaceId = SpaceId.random();
-      const queueId = ObjectId.random();
+      const queueId = EntityId.random();
 
       const spaceObj: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-space',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Space Content',
         },
@@ -474,11 +502,13 @@ describe('FtsIndex', () => {
       const queueObj: IndexerObject = {
         spaceId,
         queueId,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Queue Content',
         },
@@ -513,22 +543,24 @@ describe('FtsIndex', () => {
     'should OR space and queue constraints',
     Effect.fnUntraced(function* () {
       const index = new FtsIndex();
-      const metaIndex = new ObjectMetaIndex();
+      const metaIndex = new EntityMetaIndex();
       yield* index.migrate();
       yield* metaIndex.migrate();
 
       const space1 = SpaceId.random();
       const space2 = SpaceId.random();
-      const queueInSpace2 = ObjectId.random();
+      const queueInSpace2 = EntityId.random();
 
       const space1Obj: IndexerObject = {
         spaceId: space1,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s1',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Space One Content',
         },
@@ -537,11 +569,13 @@ describe('FtsIndex', () => {
       const space2Obj: IndexerObject = {
         spaceId: space2,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s2',
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Space Two Content',
         },
@@ -550,11 +584,13 @@ describe('FtsIndex', () => {
       const queueObj: IndexerObject = {
         spaceId: space2,
         queueId: queueInSpace2,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        createdAt: null,
         updatedAt: Date.now(),
         data: {
-          id: ObjectId.random(),
+          id: EntityId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
           title: 'Queue Content',
         },
@@ -579,4 +615,122 @@ describe('FtsIndex', () => {
       expect(objectIds).not.toContain(space2Obj.data.id);
     }, Effect.provide(TestLayer)),
   );
+
+  describe('querySnapshotsJSON', () => {
+    it.effect(
+      'returns snapshots for all present recordIds',
+      Effect.fnUntraced(function* () {
+        const index = new FtsIndex();
+        const metaIndex = new EntityMetaIndex();
+        yield* index.migrate();
+        yield* metaIndex.migrate();
+
+        const spaceId = SpaceId.random();
+        const objects: IndexerObject[] = [
+          {
+            spaceId,
+            queueId: EntityId.random(),
+            queueNamespace: 'data',
+            documentId: null,
+            recordId: null,
+            createdAt: null,
+            updatedAt: Date.now(),
+            data: { id: EntityId.random(), [ATTR_TYPE]: TYPE_PERSON, value: 'alpha' },
+          },
+          {
+            spaceId,
+            queueId: EntityId.random(),
+            queueNamespace: 'data',
+            documentId: null,
+            recordId: null,
+            createdAt: null,
+            updatedAt: Date.now(),
+            data: { id: EntityId.random(), [ATTR_TYPE]: TYPE_PERSON, value: 'beta' },
+          },
+        ];
+
+        yield* metaIndex.update(objects);
+        yield* metaIndex.lookupRecordIds(objects);
+        yield* index.update(objects);
+
+        const recordIds = objects.map((o) => o.recordId!);
+        const snapshots = yield* index.querySnapshotsJSON(recordIds);
+
+        expect(snapshots).toHaveLength(2);
+        const snapshotMap = new Map(snapshots.map((s) => [s.recordId, s.snapshot]));
+        expect((snapshotMap.get(objects[0].recordId!) as any).value).toBe('alpha');
+        expect((snapshotMap.get(objects[1].recordId!) as any).value).toBe('beta');
+      }, Effect.provide(TestLayer)),
+    );
+
+    it.effect(
+      'omits stale recordIds not present in FTS index',
+      Effect.fnUntraced(function* () {
+        const index = new FtsIndex();
+        const metaIndex = new EntityMetaIndex();
+        yield* index.migrate();
+        yield* metaIndex.migrate();
+
+        const spaceId = SpaceId.random();
+        const object: IndexerObject = {
+          spaceId,
+          queueId: EntityId.random(),
+          queueNamespace: 'data',
+          documentId: null,
+          recordId: null,
+          createdAt: null,
+          updatedAt: Date.now(),
+          data: { id: EntityId.random(), [ATTR_TYPE]: TYPE_PERSON, value: 'present' },
+        };
+
+        yield* metaIndex.update([object]);
+        yield* metaIndex.lookupRecordIds([object]);
+        yield* index.update([object]);
+
+        // Query with the real id plus a stale/non-existent id.
+        const staleId = 99999;
+        const snapshots = yield* index.querySnapshotsJSON([object.recordId!, staleId]);
+
+        expect(snapshots).toHaveLength(1);
+        expect(snapshots[0].recordId).toBe(object.recordId!);
+        expect((snapshots[0].snapshot as any).value).toBe('present');
+      }, Effect.provide(TestLayer)),
+    );
+
+    it.effect(
+      'handles more than 999 recordIds without exceeding SQLite variable limit',
+      Effect.fnUntraced(function* () {
+        const index = new FtsIndex();
+        const metaIndex = new EntityMetaIndex();
+        yield* index.migrate();
+        yield* metaIndex.migrate();
+
+        const spaceId = SpaceId.random();
+        const count = 1100;
+        const objects: IndexerObject[] = Array.from({ length: count }, (_, i) => ({
+          spaceId,
+          queueId: EntityId.random(),
+          queueNamespace: 'data',
+          documentId: null,
+          recordId: null,
+          createdAt: null,
+          updatedAt: Date.now(),
+          data: { id: EntityId.random(), [ATTR_TYPE]: TYPE_PERSON, index: i },
+        }));
+
+        yield* metaIndex.update(objects);
+        yield* metaIndex.lookupRecordIds(objects);
+        yield* index.update(objects);
+
+        const recordIds = objects.map((o) => o.recordId!);
+        const snapshots = yield* index.querySnapshotsJSON(recordIds);
+
+        expect(snapshots).toHaveLength(count);
+        const returnedIds = new Set(snapshots.map((s) => s.recordId));
+        for (const id of recordIds) {
+          expect(returnedIds.has(id)).toBe(true);
+        }
+      }, Effect.provide(TestLayer)),
+    );
+  });
 });

package/src/indexes/fts-index.ts

@@ -8,10 +8,14 @@ import type * as Statement from '@effect/sql/Statement';
 import * as Effect from 'effect/Effect';
 
 import type { Obj } from '@dxos/echo';
-import type { ObjectId, SpaceId } from '@dxos/keys';
+import type { EntityId, SpaceId } from '@dxos/keys';
 
+import type { EntityMeta } from './entity-meta-index';
 import type { Index, IndexerObject } from './interface';
-import type { ObjectMeta } from './object-meta-index';
+
+// SQLite bound-variable limit (SQLITE_LIMIT_VARIABLE_NUMBER) is 999 in most builds.
+// Use 500 as a safe chunk size for IN (...) clauses.
+const SQL_CHUNK_SIZE = 500;
 
 /**
  * The space and queue constrains are combined together using a logical OR.
@@ -35,13 +39,13 @@ export interface FtsQuery {
   /**
    * Queue IDs to search within.
    */
-  queueIds: readonly ObjectId[] | null;
+  queueIds: readonly EntityId[] | null;
 }
 
 /**
  * Result of FTS query including the indexed snapshot data.
  */
-export interface FtsResult extends ObjectMeta {
+export interface FtsResult extends EntityMeta {
   /**
    * The indexed snapshot data (JSON string).
    * Used to load queue objects without going through document loading.
@@ -52,7 +56,7 @@ export interface FtsResult extends ObjectMeta {
 /**
  * Result of FTS query with rank.
  */
-export interface FtsQueryResult extends ObjectMeta {
+export interface FtsQueryResult extends EntityMeta {
   /**
    * Relevance rank from FTS5.
    * Higher values indicate better matches.
@@ -157,7 +161,7 @@ export class FtsIndex implements Index {
         // BM25 returns negative values, negate to get higher = better match.
         // Order by rank descending so best matches come first.
         // Note: bm25() requires the actual table name, not an alias.
-        const rows = yield* sql<ObjectMeta & { rank: number }>`
+        const rows = yield* sql<EntityMeta & { rank: number }>`
           SELECT m.*, -bm25(ftsIndex) AS rank 
           FROM ftsIndex AS f 
           JOIN objectMeta AS m ON f.rowid = m.recordId 
@@ -167,7 +171,7 @@ export class FtsIndex implements Index {
         return rows;
       } else {
         // LIKE fallback - no ranking available, default to 1.
-        const rows = yield* sql<ObjectMeta>`
+        const rows = yield* sql<EntityMeta>`
           SELECT m.* 
           FROM ftsIndex AS f 
           JOIN objectMeta AS m ON f.rowid = m.recordId 
@@ -181,6 +185,7 @@ export class FtsIndex implements Index {
   /**
    * Query snapshots by recordIds.
    * Returns the parsed JSON snapshots for queue objects.
+   * RecordIds not present in the FTS index are silently omitted from the result.
    */
   querySnapshotsJSON(
     recordIds: number[],
@@ -190,14 +195,26 @@ export class FtsIndex implements Index {
         return [];
       }
       const sql = yield* SqlClient.SqlClient;
-      const results = yield* sql<{
-        rowid: number;
-        snapshot: string;
-      }>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(recordIds)}`;
-      return results.map((r) => ({
-        recordId: r.rowid,
-        snapshot: JSON.parse(r.snapshot),
-      }));
+
+      // Chunk to avoid SQLite bound-variable limit (SQLITE_LIMIT_VARIABLE_NUMBER,
+      // typically 999 in wasm builds). 500 gives a safe margin.
+      const chunks: number[][] = [];
+      for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
+        chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
+      }
+
+      const allResults: { recordId: number; snapshot: Obj.JSON }[] = [];
+      for (const chunk of chunks) {
+        const rows = yield* sql<{
+          rowid: number;
+          snapshot: string;
+        }>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
+        for (const r of rows) {
+          allResults.push({ recordId: r.rowid, snapshot: JSON.parse(r.snapshot) });
+        }
+      }
+
+      return allResults;
     });
   }
 

package/src/indexes/index.ts

@@ -3,6 +3,6 @@
 //
 
 export * from './fts-index';
-export * from './object-meta-index';
+export * from './entity-meta-index';
 export * from './reverse-ref-index';
 export * from './interface';

package/src/indexes/interface.ts

@@ -7,7 +7,7 @@ import type * as SqlError from '@effect/sql/SqlError';
 import type * as Effect from 'effect/Effect';
 
 import type { Obj } from '@dxos/echo';
-import type { ObjectId, SpaceId } from '@dxos/keys';
+import type { EntityId, SpaceId } from '@dxos/keys';
 
 /**
  * Data describing objects returned from sources to the indexer.
@@ -18,7 +18,12 @@ export interface IndexerObject {
    * Queue id if object is from the queue.
    * If null, `documentId` must be set.
    */
-  queueId: ObjectId | null;
+  queueId: EntityId | null;
+  /**
+   * Queue subspace namespace (e.g. 'data', 'trace') the object lives in.
+   * Set together with `queueId`; null for non-queue objects.
+   */
+  queueNamespace: string | null;
   /**
    * Document id if object is from the automerge document.
    * If null, `queueId` must be set.
@@ -27,8 +32,8 @@ export interface IndexerObject {
 
   /**
    * Record id from the objectMeta index.
-   * `Null` before the object is stored in the ObjectMetaIndex.
-   * Enriched by the IndexEngine after the object is stored in the ObjectMetaIndex.
+   * `Null` before the object is stored in the EntityMetaIndex.
+   * Enriched by the IndexEngine after the object is stored in the EntityMetaIndex.
    */
   recordId: number | null;
 
@@ -37,6 +42,13 @@ export interface IndexerObject {
    */
   data: Obj.JSON;
 
+  /**
+   * Unix ms timestamp when this object was first created.
+   * Sourced from system.createdAt in the automerge document; null for legacy objects
+   * created before this field was introduced.
+   */
+  createdAt: number | null;
+
   /**
    * Timestamp of the last update of the object.
    */