@dxos/index-core 0.0.0 → 0.8.4-main.03d5cd7b56

package/src/indexes/fts-index.test.ts

@@ -3,8 +3,8 @@
 //
 
 import * as Reactivity from '@effect/experimental/Reactivity';
-import * as SqlClient from '@effect/sql/SqlClient';
 import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
+import * as SqlClient from '@effect/sql/SqlClient';
 import { describe, expect, it } from '@effect/vitest';
 import * as Effect from 'effect/Effect';
 import * as Layer from 'effect/Layer';
@@ -16,7 +16,7 @@ import { FtsIndex } from './fts-index';
 import type { IndexerObject } from './interface';
 import { ObjectMetaIndex } from './object-meta-index';
 
-const TYPE_PERSON = DXN.parse('dxn:type:example.com/type/Person:0.1.0').toString();
+const TYPE_PERSON = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
 const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
 
 const TestLayer = Layer.merge(
@@ -55,8 +55,10 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-1',
           recordId: null,
+          updatedAt: Date.now(),
           data: {
             id: ObjectId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
@@ -99,11 +101,13 @@ describe('FtsIndex', () => {
       const obj1: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-1',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: objectId,
-          [ATTR_TYPE]: DXN.parse('dxn:type:example.com/type/Person:0.1.0').toString(),
+          [ATTR_TYPE]: DXN.parse('dxn:type:com.example.type.person:0.1.0').toString(),
           title: 'Original Title',
         },
       };
@@ -118,8 +122,10 @@ describe('FtsIndex', () => {
       const obj2: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-1',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: objectId,
           [ATTR_TYPE]: TYPE_DEFAULT,
@@ -155,8 +161,10 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-100',
           recordId: null,
+          updatedAt: Date.now(),
           data: {
             id: ObjectId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
@@ -166,8 +174,10 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-200',
           recordId: null,
+          updatedAt: Date.now(),
           data: {
             id: ObjectId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
@@ -177,8 +187,10 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-1000',
           recordId: null,
+          updatedAt: Date.now(),
           data: {
             id: ObjectId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
@@ -225,8 +237,10 @@ describe('FtsIndex', () => {
       const obj1: IndexerObject = {
         spaceId: space1,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s1',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -237,8 +251,10 @@ describe('FtsIndex', () => {
       const obj2: IndexerObject = {
         spaceId: space2,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s2',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -295,8 +311,10 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-1',
           recordId: null,
+          updatedAt: Date.now(),
           data: {
             id: ObjectId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
@@ -307,8 +325,10 @@ describe('FtsIndex', () => {
         {
           spaceId,
           queueId: null,
+          queueNamespace: null,
           documentId: 'doc-2',
           recordId: null,
+          updatedAt: Date.now(),
           data: {
             id: ObjectId.random(),
             [ATTR_TYPE]: TYPE_PERSON,
@@ -376,8 +396,10 @@ describe('FtsIndex', () => {
       const spaceObj: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-space',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -388,8 +410,10 @@ describe('FtsIndex', () => {
       const queue1Obj: IndexerObject = {
         spaceId,
         queueId: queue1,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -400,8 +424,10 @@ describe('FtsIndex', () => {
       const queue2Obj: IndexerObject = {
         spaceId,
         queueId: queue2,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -448,8 +474,10 @@ describe('FtsIndex', () => {
       const spaceObj: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-space',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -460,8 +488,10 @@ describe('FtsIndex', () => {
       const queueObj: IndexerObject = {
         spaceId,
         queueId,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -509,8 +539,10 @@ describe('FtsIndex', () => {
       const space1Obj: IndexerObject = {
         spaceId: space1,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s1',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -521,8 +553,10 @@ describe('FtsIndex', () => {
       const space2Obj: IndexerObject = {
         spaceId: space2,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-s2',
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -533,8 +567,10 @@ describe('FtsIndex', () => {
       const queueObj: IndexerObject = {
         spaceId: space2,
         queueId: queueInSpace2,
+        queueNamespace: 'data',
         documentId: null,
         recordId: null,
+        updatedAt: Date.now(),
         data: {
           id: ObjectId.random(),
           [ATTR_TYPE]: TYPE_PERSON,
@@ -561,4 +597,118 @@ describe('FtsIndex', () => {
       expect(objectIds).not.toContain(space2Obj.data.id);
     }, Effect.provide(TestLayer)),
   );
+
+  describe('querySnapshotsJSON', () => {
+    it.effect(
+      'returns snapshots for all present recordIds',
+      Effect.fnUntraced(function* () {
+        const index = new FtsIndex();
+        const metaIndex = new ObjectMetaIndex();
+        yield* index.migrate();
+        yield* metaIndex.migrate();
+
+        const spaceId = SpaceId.random();
+        const objects: IndexerObject[] = [
+          {
+            spaceId,
+            queueId: ObjectId.random(),
+            queueNamespace: 'data',
+            documentId: null,
+            recordId: null,
+            updatedAt: Date.now(),
+            data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_PERSON, value: 'alpha' },
+          },
+          {
+            spaceId,
+            queueId: ObjectId.random(),
+            queueNamespace: 'data',
+            documentId: null,
+            recordId: null,
+            updatedAt: Date.now(),
+            data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_PERSON, value: 'beta' },
+          },
+        ];
+
+        yield* metaIndex.update(objects);
+        yield* metaIndex.lookupRecordIds(objects);
+        yield* index.update(objects);
+
+        const recordIds = objects.map((o) => o.recordId!);
+        const snapshots = yield* index.querySnapshotsJSON(recordIds);
+
+        expect(snapshots).toHaveLength(2);
+        const snapshotMap = new Map(snapshots.map((s) => [s.recordId, s.snapshot]));
+        expect((snapshotMap.get(objects[0].recordId!) as any).value).toBe('alpha');
+        expect((snapshotMap.get(objects[1].recordId!) as any).value).toBe('beta');
+      }, Effect.provide(TestLayer)),
+    );
+
+    it.effect(
+      'omits stale recordIds not present in FTS index',
+      Effect.fnUntraced(function* () {
+        const index = new FtsIndex();
+        const metaIndex = new ObjectMetaIndex();
+        yield* index.migrate();
+        yield* metaIndex.migrate();
+
+        const spaceId = SpaceId.random();
+        const object: IndexerObject = {
+          spaceId,
+          queueId: ObjectId.random(),
+          queueNamespace: 'data',
+          documentId: null,
+          recordId: null,
+          updatedAt: Date.now(),
+          data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_PERSON, value: 'present' },
+        };
+
+        yield* metaIndex.update([object]);
+        yield* metaIndex.lookupRecordIds([object]);
+        yield* index.update([object]);
+
+        // Query with the real id plus a stale/non-existent id.
+        const staleId = 99999;
+        const snapshots = yield* index.querySnapshotsJSON([object.recordId!, staleId]);
+
+        expect(snapshots).toHaveLength(1);
+        expect(snapshots[0].recordId).toBe(object.recordId!);
+        expect((snapshots[0].snapshot as any).value).toBe('present');
+      }, Effect.provide(TestLayer)),
+    );
+
+    it.effect(
+      'handles more than 999 recordIds without exceeding SQLite variable limit',
+      Effect.fnUntraced(function* () {
+        const index = new FtsIndex();
+        const metaIndex = new ObjectMetaIndex();
+        yield* index.migrate();
+        yield* metaIndex.migrate();
+
+        const spaceId = SpaceId.random();
+        const count = 1100;
+        const objects: IndexerObject[] = Array.from({ length: count }, (_, i) => ({
+          spaceId,
+          queueId: ObjectId.random(),
+          queueNamespace: 'data',
+          documentId: null,
+          recordId: null,
+          updatedAt: Date.now(),
+          data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_PERSON, index: i },
+        }));
+
+        yield* metaIndex.update(objects);
+        yield* metaIndex.lookupRecordIds(objects);
+        yield* index.update(objects);
+
+        const recordIds = objects.map((o) => o.recordId!);
+        const snapshots = yield* index.querySnapshotsJSON(recordIds);
+
+        expect(snapshots).toHaveLength(count);
+        const returnedIds = new Set(snapshots.map((s) => s.recordId));
+        for (const id of recordIds) {
+          expect(returnedIds.has(id)).toBe(true);
+        }
+      }, Effect.provide(TestLayer)),
+    );
+  });
 });

package/src/indexes/fts-index.ts

@@ -13,6 +13,10 @@ import type { ObjectId, SpaceId } from '@dxos/keys';
 import type { Index, IndexerObject } from './interface';
 import type { ObjectMeta } from './object-meta-index';
 
+// SQLite bound-variable limit (SQLITE_LIMIT_VARIABLE_NUMBER) is 999 in most builds.
+// Use 500 as a safe chunk size for IN (...) clauses.
+const SQL_CHUNK_SIZE = 500;
+
 /**
  * The space and queue constrains are combined together using a logical OR.
  */
@@ -49,6 +53,18 @@ export interface FtsResult extends ObjectMeta {
   snapshot: string;
 }
 
+/**
+ * Result of FTS query with rank.
+ */
+export interface FtsQueryResult extends ObjectMeta {
+  /**
+   * Relevance rank from FTS5.
+   * Higher values indicate better matches.
+   * Uses BM25 algorithm when available, falls back to 1 for non-BM25 queries.
+   */
+  rank: number;
+}
+
 /**
  * Escapes user input for safe FTS5 queries.
  *
@@ -92,7 +108,7 @@ export class FtsIndex implements Index {
     spaceId,
     includeAllQueues,
     queueIds,
-  }: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+  }: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
     return Effect.gen(function* () {
       const trimmed = query.trim();
       if (trimmed.length === 0) {
@@ -106,6 +122,11 @@ export class FtsIndex implements Index {
       const terms = trimmed.split(/\s+/).filter(Boolean);
       const minTermLength = Math.min(...terms.map((t) => t.length));
 
+      // Use BM25 ranking for FTS5 MATCH queries, fall back to rank 1 for LIKE queries.
+      // BM25 returns negative values where lower (more negative) means better match,
+      // so we negate it to get higher = better.
+      const useBm25 = minTermLength >= 3;
+
       const conditions =
         minTermLength < 3
           ? // LIKE fallback - scan the entire table, AND all terms.
@@ -135,13 +156,36 @@ export class FtsIndex implements Index {
         conditions.push(sql`(${sql.or(sourceConditions)})`);
       }
 
-      return yield* sql<ObjectMeta>`SELECT m.* FROM ftsIndex AS f JOIN objectMeta AS m ON f.rowid = m.recordId WHERE ${sql.and(conditions)}`;
+      if (useBm25) {
+        // Use BM25 ranking for FTS5 MATCH queries.
+        // BM25 returns negative values, negate to get higher = better match.
+        // Order by rank descending so best matches come first.
+        // Note: bm25() requires the actual table name, not an alias.
+        const rows = yield* sql<ObjectMeta & { rank: number }>`
+          SELECT m.*, -bm25(ftsIndex) AS rank 
+          FROM ftsIndex AS f 
+          JOIN objectMeta AS m ON f.rowid = m.recordId 
+          WHERE ${sql.and(conditions)}
+          ORDER BY rank DESC
+        `;
+        return rows;
+      } else {
+        // LIKE fallback - no ranking available, default to 1.
+        const rows = yield* sql<ObjectMeta>`
+          SELECT m.* 
+          FROM ftsIndex AS f 
+          JOIN objectMeta AS m ON f.rowid = m.recordId 
+          WHERE ${sql.and(conditions)}
+        `;
+        return rows.map((row) => ({ ...row, rank: 1 }));
+      }
     });
   }
 
   /**
    * Query snapshots by recordIds.
    * Returns the parsed JSON snapshots for queue objects.
+   * RecordIds not present in the FTS index are silently omitted from the result.
    */
   querySnapshotsJSON(
     recordIds: number[],
@@ -151,14 +195,26 @@ export class FtsIndex implements Index {
         return [];
       }
       const sql = yield* SqlClient.SqlClient;
-      const results = yield* sql<{
-        rowid: number;
-        snapshot: string;
-      }>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(recordIds)}`;
-      return results.map((r) => ({
-        recordId: r.rowid,
-        snapshot: JSON.parse(r.snapshot),
-      }));
+
+      // Chunk to avoid SQLite bound-variable limit (SQLITE_LIMIT_VARIABLE_NUMBER,
+      // typically 999 in wasm builds). 500 gives a safe margin.
+      const chunks: number[][] = [];
+      for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
+        chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
+      }
+
+      const allResults: { recordId: number; snapshot: Obj.JSON }[] = [];
+      for (const chunk of chunks) {
+        const rows = yield* sql<{
+          rowid: number;
+          snapshot: string;
+        }>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
+        for (const r of rows) {
+          allResults.push({ recordId: r.rowid, snapshot: JSON.parse(r.snapshot) });
+        }
+      }
+
+      return allResults;
     });
   }
 

package/src/indexes/interface.ts

@@ -19,6 +19,11 @@ export interface IndexerObject {
    * If null, `documentId` must be set.
    */
   queueId: ObjectId | null;
+  /**
+   * Queue subspace namespace (e.g. 'data', 'trace') the object lives in.
+   * Set together with `queueId`; null for non-queue objects.
+   */
+  queueNamespace: string | null;
   /**
    * Document id if object is from the automerge document.
    * If null, `queueId` must be set.
@@ -36,6 +41,11 @@ export interface IndexerObject {
    * JSON data of the object.
    */
   data: Obj.JSON;
+
+  /**
+   * Timestamp of the last update of the object.
+   */
+  updatedAt: number;
 }
 
 /**