@dxos/index-core 0.0.0 → 0.8.4-main.59c2e9b

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@dxos/index-core",
-  "version": "0.0.0",
+  "version": "0.8.4-main.59c2e9b",
   "description": "Indexing core.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "info@dxos.org",
   "sideEffects": false,
@@ -34,7 +38,8 @@
     "@effect/experimental": "0.57.11",
     "@effect/platform": "0.93.6",
     "@effect/sql": "0.48.6",
-    "effect": "3.19.11"
+    "effect": "3.19.11",
+    "@dxos/sql-sqlite": "0.8.4-main.59c2e9b"
   },
   "devDependencies": {
     "@effect/sql-sqlite-node": "0.49.1"

package/src/index-engine.test.ts

@@ -11,6 +11,7 @@ import * as Layer from 'effect/Layer';
 import { ATTR_TYPE } from '@dxos/echo/internal';
 import { invariant } from '@dxos/invariant';
 import { DXN, ObjectId, SpaceId } from '@dxos/keys';
+import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 
 import { type DataSourceCursor, type IndexDataSource, IndexEngine } from './index-engine';
 import { type IndexCursor, IndexTracker } from './index-tracker';
@@ -20,11 +21,13 @@ const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
 const TYPE_A = DXN.parse('dxn:type:test.com/type/TypeA:0.1.0').toString();
 const TYPE_B = DXN.parse('dxn:type:test.com/type/TypeB:0.1.0').toString();
 
-const TestLayer = Layer.merge(
-  SqliteClient.layer({
-    filename: ':memory:',
-  }),
-  Reactivity.layer,
+const TestLayer = SqlTransaction.layer.pipe(
+  Layer.provideMerge(
+    SqliteClient.layer({
+      filename: ':memory:',
+    }),
+  ),
+  Layer.provideMerge(Reactivity.layer),
 );
 
 class MockIndexDataSource implements IndexDataSource {

package/src/index-engine.ts

@@ -2,16 +2,18 @@
 // Copyright 2026 DXOS.org
 //
 
-import * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlClient from '@effect/sql/SqlClient';
 import type * as SqlError from '@effect/sql/SqlError';
 import * as Effect from 'effect/Effect';
 
 import type { SpaceId } from '@dxos/keys';
+import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 
 import { type IndexCursor, IndexTracker } from './index-tracker';
 import {
   FtsIndex,
   type FtsQuery,
+  type FtsQueryResult,
   type Index,
   type IndexerObject,
   type ObjectMeta,
@@ -76,9 +78,9 @@ export class IndexEngine {
   }
 
   /**
-   * Query text index and return full object metadata.
+   * Query text index and return full object metadata with rank.
    */
-  queryText(query: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+  queryText(query: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
     return Effect.gen(this, function* () {
       return yield* this.#ftsIndex.query(query);
     });
@@ -106,7 +108,11 @@ export class IndexEngine {
   update(
     dataSource: IndexDataSource,
     opts: { spaceId: SpaceId | null; limit?: number },
-  ): Effect.Effect<{ updated: number; done: boolean }, SqlError.SqlError, SqlClient.SqlClient> {
+  ): Effect.Effect<
+    { updated: number; done: boolean },
+    SqlError.SqlError,
+    SqlTransaction.SqlTransaction | SqlClient.SqlClient
+  > {
     return Effect.gen(this, function* () {
       let updated = 0;
 
@@ -145,10 +151,15 @@ export class IndexEngine {
     index: Index,
     source: IndexDataSource,
     opts: { indexName: string; spaceId: SpaceId | null; limit?: number },
-  ): Effect.Effect<{ updated: number; done: boolean }, SqlError.SqlError, SqlClient.SqlClient> {
+  ): Effect.Effect<
+    { updated: number; done: boolean },
+    SqlError.SqlError,
+    SqlTransaction.SqlTransaction | SqlClient.SqlClient
+  > {
     return Effect.gen(this, function* () {
-      const sql = yield* SqlClient.SqlClient;
-      return yield* sql.withTransaction(
+      const sqlTransaction = yield* SqlTransaction.SqlTransaction;
+
+      return yield* sqlTransaction.withTransaction(
         Effect.gen(this, function* () {
           const cursors = yield* this.#tracker.queryCursors({
             indexName: opts.indexName,

package/src/index.ts

@@ -5,6 +5,6 @@
 export { IndexEngine, type IndexDataSource, type DataSourceCursor, type IndexEngineParams } from './index-engine';
 export { IndexTracker, type IndexCursor } from './index-tracker';
 export { type IndexerObject, type Index } from './indexes/interface';
-export { FtsIndex } from './indexes/fts-index';
+export { FtsIndex, type FtsQuery } from './indexes/fts-index';
 export { ObjectMetaIndex, type ObjectMeta } from './indexes/object-meta-index';
-export { ReverseRefIndex, type ReverseRef } from './indexes/reverse-ref-index';
+export { ReverseRefIndex, type ReverseRef, type ReverseRefQuery } from './indexes/reverse-ref-index';

package/src/indexes/fts-index.ts

@@ -49,6 +49,18 @@ export interface FtsResult extends ObjectMeta {
   snapshot: string;
 }
 
+/**
+ * Result of FTS query with rank.
+ */
+export interface FtsQueryResult extends ObjectMeta {
+  /**
+   * Relevance rank from FTS5.
+   * Higher values indicate better matches.
+   * Uses BM25 algorithm when available, falls back to 1 for non-BM25 queries.
+   */
+  rank: number;
+}
+
 /**
  * Escapes user input for safe FTS5 queries.
  *
@@ -92,7 +104,7 @@ export class FtsIndex implements Index {
     spaceId,
     includeAllQueues,
     queueIds,
-  }: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+  }: FtsQuery): Effect.Effect<readonly FtsQueryResult[], SqlError.SqlError, SqlClient.SqlClient> {
     return Effect.gen(function* () {
       const trimmed = query.trim();
       if (trimmed.length === 0) {
@@ -106,6 +118,11 @@ export class FtsIndex implements Index {
       const terms = trimmed.split(/\s+/).filter(Boolean);
       const minTermLength = Math.min(...terms.map((t) => t.length));
 
+      // Use BM25 ranking for FTS5 MATCH queries, fall back to rank 1 for LIKE queries.
+      // BM25 returns negative values where lower (more negative) means better match,
+      // so we negate it to get higher = better.
+      const useBm25 = minTermLength >= 3;
+
       const conditions =
         minTermLength < 3
           ? // LIKE fallback - scan the entire table, AND all terms.
@@ -135,7 +152,29 @@ export class FtsIndex implements Index {
         conditions.push(sql`(${sql.or(sourceConditions)})`);
       }
 
-      return yield* sql<ObjectMeta>`SELECT m.* FROM ftsIndex AS f JOIN objectMeta AS m ON f.rowid = m.recordId WHERE ${sql.and(conditions)}`;
+      if (useBm25) {
+        // Use BM25 ranking for FTS5 MATCH queries.
+        // BM25 returns negative values, negate to get higher = better match.
+        // Order by rank descending so best matches come first.
+        // Note: bm25() requires the actual table name, not an alias.
+        const rows = yield* sql<ObjectMeta & { rank: number }>`
+          SELECT m.*, -bm25(ftsIndex) AS rank 
+          FROM ftsIndex AS f 
+          JOIN objectMeta AS m ON f.rowid = m.recordId 
+          WHERE ${sql.and(conditions)}
+          ORDER BY rank DESC
+        `;
+        return rows;
+      } else {
+        // LIKE fallback - no ranking available, default to 1.
+        const rows = yield* sql<ObjectMeta>`
+          SELECT m.* 
+          FROM ftsIndex AS f 
+          JOIN objectMeta AS m ON f.rowid = m.recordId 
+          WHERE ${sql.and(conditions)}
+        `;
+        return rows.map((row) => ({ ...row, rank: 1 }));
+      }
     });
   }