@dxos/index-core 0.0.0 → 0.8.4-main.1068cf700f

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@dxos/index-core",
-  "version": "0.0.0",
+  "version": "0.8.4-main.1068cf700f",
   "description": "Indexing core.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "info@dxos.org",
   "sideEffects": false,
@@ -12,8 +16,7 @@
     ".": {
       "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "default": "./dist/lib/neutral/index.mjs"
     }
   },
   "files": [
@@ -21,23 +24,24 @@
     "src"
   ],
   "dependencies": {
-    "@dxos/async": "",
-    "@dxos/context": "",
-    "@dxos/debug": "",
-    "@dxos/echo": "",
-    "@dxos/echo-protocol": "",
-    "@dxos/effect": "",
-    "@dxos/invariant": "",
-    "@dxos/keys": "",
-    "@dxos/log": "",
-    "@effect/cli": "0.72.1",
-    "@effect/experimental": "0.57.11",
-    "@effect/platform": "0.93.6",
-    "@effect/sql": "0.48.6",
-    "effect": "3.19.11"
+    "@effect/cli": "0.73.2",
+    "@effect/experimental": "0.58.0",
+    "@effect/platform": "0.94.4",
+    "@effect/sql": "0.49.0",
+    "effect": "3.19.16",
+    "@dxos/context": "0.8.4-main.1068cf700f",
+    "@dxos/async": "0.8.4-main.1068cf700f",
+    "@dxos/echo": "0.8.4-main.1068cf700f",
+    "@dxos/debug": "0.8.4-main.1068cf700f",
+    "@dxos/effect": "0.8.4-main.1068cf700f",
+    "@dxos/echo-protocol": "0.8.4-main.1068cf700f",
+    "@dxos/invariant": "0.8.4-main.1068cf700f",
+    "@dxos/log": "0.8.4-main.1068cf700f",
+    "@dxos/sql-sqlite": "0.8.4-main.1068cf700f",
+    "@dxos/keys": "0.8.4-main.1068cf700f"
   },
   "devDependencies": {
-    "@effect/sql-sqlite-node": "0.49.1"
+    "@effect/sql-sqlite-node": "0.50.1"
   },
   "publishConfig": {
     "access": "public"

package/src/index-engine.test.ts

@@ -11,6 +11,7 @@ import * as Layer from 'effect/Layer';
 import { ATTR_TYPE } from '@dxos/echo/internal';
 import { invariant } from '@dxos/invariant';
 import { DXN, ObjectId, SpaceId } from '@dxos/keys';
+import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 
 import { type DataSourceCursor, type IndexDataSource, IndexEngine } from './index-engine';
 import { type IndexCursor, IndexTracker } from './index-tracker';
@@ -20,11 +21,13 @@ const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
 const TYPE_A = DXN.parse('dxn:type:test.com/type/TypeA:0.1.0').toString();
 const TYPE_B = DXN.parse('dxn:type:test.com/type/TypeB:0.1.0').toString();
 
-const TestLayer = Layer.merge(
-  SqliteClient.layer({
-    filename: ':memory:',
-  }),
-  Reactivity.layer,
+const TestLayer = SqlTransaction.layer.pipe(
+  Layer.provideMerge(
+    SqliteClient.layer({
+      filename: ':memory:',
+    }),
+  ),
+  Layer.provideMerge(Reactivity.layer),
 );
 
 class MockIndexDataSource implements IndexDataSource {

package/src/index-engine.ts

@@ -2,16 +2,18 @@
 // Copyright 2026 DXOS.org
 //
 
-import * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlClient from '@effect/sql/SqlClient';
 import type * as SqlError from '@effect/sql/SqlError';
 import * as Effect from 'effect/Effect';
 
-import type { SpaceId } from '@dxos/keys';
+import type { ObjectId, SpaceId } from '@dxos/keys';
+import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 
 import { type IndexCursor, IndexTracker } from './index-tracker';
 import {
   FtsIndex,
   type FtsQuery,
+  type FtsQueryResult,
   type Index,
   type IndexerObject,
   type ObjectMeta,
@@ -76,9 +78,9 @@ export class IndexEngine {
   }
 
   /**
-   * Query text index and return full object metadata.
+   * Query text index and return full object metadata with rank.
    */
-  queryText(query: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+  queryText(query: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
     return Effect.gen(this, function* () {
       return yield* this.#ftsIndex.query(query);
     });
@@ -89,6 +91,12 @@ export class IndexEngine {
     return this.#reverseRefIndex.query(query);
   }
 
+  queryAll(query: {
+    spaceIds: readonly SpaceId[];
+  }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+    return this.#objectMetaIndex.queryAll(query);
+  }
+
   /**
    * Query snapshots by recordIds.
    * Used to load queue objects from indexed snapshots.
@@ -103,10 +111,41 @@ export class IndexEngine {
     return this.#objectMetaIndex.query(query);
   }
 
+  /**
+   * Query children by parent object ids.
+   */
+  queryChildren(query: {
+    spaceId: SpaceId[];
+    parentIds: ObjectId[];
+  }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+    return this.#objectMetaIndex.queryChildren(query);
+  }
+
+  queryTypes(query: {
+    spaceIds: readonly SpaceId[];
+    typeDxns: readonly ObjectMeta['typeDxn'][];
+    inverted?: boolean;
+  }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+    return this.#objectMetaIndex.queryTypes(query);
+  }
+  queryRelations(query: {
+    endpoint: 'source' | 'target';
+    anchorDxns: readonly string[];
+  }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+    return this.#objectMetaIndex.queryRelations(query);
+  }
+  lookupByRecordIds(recordIds: number[]): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+    return this.#objectMetaIndex.lookupByRecordIds(recordIds);
+  }
+
   update(
     dataSource: IndexDataSource,
     opts: { spaceId: SpaceId | null; limit?: number },
-  ): Effect.Effect<{ updated: number; done: boolean }, SqlError.SqlError, SqlClient.SqlClient> {
+  ): Effect.Effect<
+    { updated: number; done: boolean },
+    SqlError.SqlError,
+    SqlTransaction.SqlTransaction | SqlClient.SqlClient
+  > {
     return Effect.gen(this, function* () {
       let updated = 0;
 
@@ -145,10 +184,15 @@ export class IndexEngine {
     index: Index,
     source: IndexDataSource,
     opts: { indexName: string; spaceId: SpaceId | null; limit?: number },
-  ): Effect.Effect<{ updated: number; done: boolean }, SqlError.SqlError, SqlClient.SqlClient> {
+  ): Effect.Effect<
+    { updated: number; done: boolean },
+    SqlError.SqlError,
+    SqlTransaction.SqlTransaction | SqlClient.SqlClient
+  > {
     return Effect.gen(this, function* () {
-      const sql = yield* SqlClient.SqlClient;
-      return yield* sql.withTransaction(
+      const sqlTransaction = yield* SqlTransaction.SqlTransaction;
+
+      return yield* sqlTransaction.withTransaction(
         Effect.gen(this, function* () {
           const cursors = yield* this.#tracker.queryCursors({
             indexName: opts.indexName,
@@ -182,6 +226,6 @@ export class IndexEngine {
           return { updated: objects.length, done: false };
         }),
       );
-    }).pipe(Effect.withSpan('IndexEngine.#updateDependentIndex'));
+    }).pipe(Effect.withSpan('IndexEngine.#update'));
   }
 }

package/src/index.ts

@@ -5,6 +5,7 @@
 export { IndexEngine, type IndexDataSource, type DataSourceCursor, type IndexEngineParams } from './index-engine';
 export { IndexTracker, type IndexCursor } from './index-tracker';
 export { type IndexerObject, type Index } from './indexes/interface';
-export { FtsIndex } from './indexes/fts-index';
+export { FtsIndex, type FtsQuery } from './indexes/fts-index';
 export { ObjectMetaIndex, type ObjectMeta } from './indexes/object-meta-index';
-export { ReverseRefIndex, type ReverseRef } from './indexes/reverse-ref-index';
+export { ReverseRefIndex, type ReverseRef, type ReverseRefQuery } from './indexes/reverse-ref-index';
+export { EscapedPropPath, type ObjectPropPath } from './utils';

package/src/indexes/fts-index.ts

@@ -49,6 +49,18 @@ export interface FtsResult extends ObjectMeta {
   snapshot: string;
 }
 
+/**
+ * Result of FTS query with rank.
+ */
+export interface FtsQueryResult extends ObjectMeta {
+  /**
+   * Relevance rank from FTS5.
+   * Higher values indicate better matches.
+   * Uses BM25 algorithm when available, falls back to 1 for non-BM25 queries.
+   */
+  rank: number;
+}
+
 /**
  * Escapes user input for safe FTS5 queries.
  *
@@ -92,7 +104,7 @@ export class FtsIndex implements Index {
     spaceId,
     includeAllQueues,
     queueIds,
-  }: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+  }: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
     return Effect.gen(function* () {
       const trimmed = query.trim();
       if (trimmed.length === 0) {
@@ -106,6 +118,11 @@ export class FtsIndex implements Index {
       const terms = trimmed.split(/\s+/).filter(Boolean);
       const minTermLength = Math.min(...terms.map((t) => t.length));
 
+      // Use BM25 ranking for FTS5 MATCH queries, fall back to rank 1 for LIKE queries.
+      // BM25 returns negative values where lower (more negative) means better match,
+      // so we negate it to get higher = better.
+      const useBm25 = minTermLength >= 3;
+
       const conditions =
         minTermLength < 3
           ? // LIKE fallback - scan the entire table, AND all terms.
@@ -135,7 +152,29 @@ export class FtsIndex implements Index {
         conditions.push(sql`(${sql.or(sourceConditions)})`);
       }
 
-      return yield* sql<ObjectMeta>`SELECT m.* FROM ftsIndex AS f JOIN objectMeta AS m ON f.rowid = m.recordId WHERE ${sql.and(conditions)}`;
+      if (useBm25) {
+        // Use BM25 ranking for FTS5 MATCH queries.
+        // BM25 returns negative values, negate to get higher = better match.
+        // Order by rank descending so best matches come first.
+        // Note: bm25() requires the actual table name, not an alias.
+        const rows = yield* sql<ObjectMeta & { rank: number }>`
+          SELECT m.*, -bm25(ftsIndex) AS rank 
+          FROM ftsIndex AS f 
+          JOIN objectMeta AS m ON f.rowid = m.recordId 
+          WHERE ${sql.and(conditions)}
+          ORDER BY rank DESC
+        `;
+        return rows;
+      } else {
+        // LIKE fallback - no ranking available, default to 1.
+        const rows = yield* sql<ObjectMeta>`
+          SELECT m.* 
+          FROM ftsIndex AS f 
+          JOIN objectMeta AS m ON f.rowid = m.recordId 
+          WHERE ${sql.and(conditions)}
+        `;
+        return rows.map((row) => ({ ...row, rank: 1 }));
+      }
     });
   }
 

package/src/indexes/object-meta-index.test.ts

@@ -15,8 +15,12 @@ import type { IndexerObject } from './interface';
 import { ObjectMetaIndex } from './object-meta-index';
 
 const TYPE_PERSON = DXN.parse('dxn:type:example.com/type/Person:0.1.0').toString();
+const TYPE_PERSON_VERSIONLESS = DXN.parse('dxn:type:example.com/type/Person').toString();
 const TYPE_RELATION = DXN.parse('dxn:type:example.com/type/Relation:0.1.0').toString();
 const TYPE_RELATION_UPDATED = DXN.parse('dxn:type:example.com/type/RelationUpdated:0.1.0').toString();
+const TYPE_WITH_UNDERSCORE = DXN.parse('dxn:type:example.com/type/Person_Extra:0.1.0').toString();
+const TYPE_WITH_UNDERSCORE_VERSIONLESS = DXN.parse('dxn:type:example.com/type/Person_Extra').toString();
+const TYPE_UNDERSCORE_FALSE_POSITIVE = DXN.parse('dxn:type:example.com/type/PersonAExtra:0.1.0').toString();
 
 const TestLayer = Layer.merge(
   SqliteClient.layer({
@@ -26,6 +30,86 @@ const TestLayer = Layer.merge(
 );
 
 describe('ObjectMetaIndex', () => {
+  it.effect('should match versioned types when queried by versionless type', () =>
+    Effect.gen(function* () {
+      const index = new ObjectMetaIndex();
+      yield* index.migrate();
+
+      const spaceId = SpaceId.random();
+      const objectId = ObjectId.random();
+
+      const item: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: objectId,
+          [ATTR_TYPE]: TYPE_PERSON,
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      yield* index.update([item]);
+
+      const results = yield* index.query({ spaceId, typeDxn: TYPE_PERSON_VERSIONLESS });
+      expect(results.map((_) => _.objectId)).toEqual([objectId]);
+
+      const otherTypeResults = yield* index.query({
+        spaceId,
+        typeDxn: DXN.parse('dxn:type:example.com/type/Other').toString(),
+      });
+      expect(otherTypeResults).toEqual([]);
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
+  it.effect('should not treat LIKE wildcards in versionless type queries', () =>
+    Effect.gen(function* () {
+      const index = new ObjectMetaIndex();
+      yield* index.migrate();
+
+      const spaceId = SpaceId.random();
+      const objectIdMatch = ObjectId.random();
+      const objectIdFalsePositive = ObjectId.random();
+
+      const match: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: objectIdMatch,
+          [ATTR_TYPE]: TYPE_WITH_UNDERSCORE,
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      // Would match prior implementation because '_' is a LIKE wildcard.
+      const falsePositive: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: objectIdFalsePositive,
+          [ATTR_TYPE]: TYPE_UNDERSCORE_FALSE_POSITIVE,
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      yield* index.update([match, falsePositive]);
+
+      const queryResults = yield* index.query({ spaceId, typeDxn: TYPE_WITH_UNDERSCORE_VERSIONLESS });
+      expect(queryResults.map((_) => _.objectId)).toEqual([objectIdMatch]);
+
+      const queryTypesResults = yield* index.queryTypes({
+        spaceIds: [spaceId],
+        typeDxns: [TYPE_WITH_UNDERSCORE_VERSIONLESS],
+      });
+      expect(queryTypesResults.map((_) => _.objectId)).toEqual([objectIdMatch]);
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
   it.effect('should store and update object metadata', () =>
     Effect.gen(function* () {
       const index = new ObjectMetaIndex();
@@ -116,4 +200,99 @@ describe('ObjectMetaIndex', () => {
       expect(oldTypeResults.length).toBe(0);
     }).pipe(Effect.provide(TestLayer)),
   );
+
+  it.effect('should support queryAll/queryTypes/queryRelations', () =>
+    Effect.gen(function* () {
+      const index = new ObjectMetaIndex();
+      yield* index.migrate();
+
+      const spaceId = SpaceId.random();
+      const objectId1 = ObjectId.random();
+      const objectId2 = ObjectId.random();
+      const relationId = ObjectId.random();
+
+      const item1: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: objectId1,
+          [ATTR_TYPE]: TYPE_PERSON,
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      const item2: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: objectId2,
+          [ATTR_TYPE]: TYPE_RELATION,
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      const relation: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: relationId,
+          [ATTR_TYPE]: TYPE_RELATION,
+          [ATTR_RELATION_SOURCE]: DXN.fromLocalObjectId(objectId1).toString(),
+          [ATTR_RELATION_TARGET]: DXN.fromLocalObjectId(objectId2).toString(),
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      yield* index.update([item1, item2, relation]);
+
+      const all = yield* index.queryAll({ spaceIds: [spaceId] });
+      expect(all.map((_) => _.objectId).sort()).toEqual([objectId1, objectId2, relationId].sort());
+
+      const types = yield* index.queryTypes({ spaceIds: [spaceId], typeDxns: [TYPE_PERSON, TYPE_RELATION] });
+      expect(types.map((_) => _.objectId).sort()).toEqual([objectId1, objectId2, relationId].sort());
+
+      const onlyPerson = yield* index.queryTypes({ spaceIds: [spaceId], typeDxns: [TYPE_PERSON] });
+      expect(onlyPerson.map((_) => _.objectId)).toEqual([objectId1]);
+
+      const onlyPersonVersionless = yield* index.queryTypes({
+        spaceIds: [spaceId],
+        typeDxns: [TYPE_PERSON_VERSIONLESS],
+      });
+      expect(onlyPersonVersionless.map((_) => _.objectId)).toEqual([objectId1]);
+
+      const notPerson = yield* index.queryTypes({ spaceIds: [spaceId], typeDxns: [TYPE_PERSON], inverted: true });
+      expect(notPerson.map((_) => _.objectId).sort()).toEqual([objectId2, relationId].sort());
+
+      const notPersonVersionless = yield* index.queryTypes({
+        spaceIds: [spaceId],
+        typeDxns: [TYPE_PERSON_VERSIONLESS],
+        inverted: true,
+      });
+      expect(notPersonVersionless.map((_) => _.objectId).sort()).toEqual([objectId2, relationId].sort());
+
+      const emptyTypes = yield* index.queryTypes({ spaceIds: [spaceId], typeDxns: [] });
+      expect(emptyTypes).toEqual([]);
+
+      const notEmptyTypes = yield* index.queryTypes({ spaceIds: [spaceId], typeDxns: [], inverted: true });
+      expect(notEmptyTypes.map((_) => _.objectId).sort()).toEqual([objectId1, objectId2, relationId].sort());
+
+      const bySource = yield* index.queryRelations({
+        endpoint: 'source',
+        anchorDxns: [DXN.fromLocalObjectId(objectId1).toString()],
+      });
+      expect(bySource.map((_) => _.objectId)).toEqual([relationId]);
+
+      const byTarget = yield* index.queryRelations({
+        endpoint: 'target',
+        anchorDxns: [DXN.fromLocalObjectId(objectId2).toString()],
+      });
+      expect(byTarget.map((_) => _.objectId)).toEqual([relationId]);
+    }).pipe(Effect.provide(TestLayer)),
+  );
 });