@dxos/index-core 0.0.0

package/src/indexes/fts-index.ts

@@ -0,0 +1,193 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlError from '@effect/sql/SqlError';
+import type * as Statement from '@effect/sql/Statement';
+import * as Effect from 'effect/Effect';
+
+import type { Obj } from '@dxos/echo';
+import type { ObjectId, SpaceId } from '@dxos/keys';
+
+import type { Index, IndexerObject } from './interface';
+import type { ObjectMeta } from './object-meta-index';
+
+/**
+ * The space and queue constrains are combined together using a logical OR.
+ */
+export interface FtsQuery {
+  /**
+   * Text to search.
+   */
+  query: string;
+
+  /**
+   * Space ID to search within.
+   */
+  spaceId: readonly SpaceId[] | null;
+
+  /**
+   * If true, include all queues in the spaces specified by `spaceId`.
+   */
+  includeAllQueues: boolean;
+
+  /**
+   * Queue IDs to search within.
+   */
+  queueIds: readonly ObjectId[] | null;
+}
+
+/**
+ * Result of FTS query including the indexed snapshot data.
+ */
+export interface FtsResult extends ObjectMeta {
+  /**
+   * The indexed snapshot data (JSON string).
+   * Used to load queue objects without going through document loading.
+   */
+  snapshot: string;
+}
+
+/**
+ * Escapes user input for safe FTS5 queries.
+ *
+ * FTS5 has special syntax characters that can cause errors or unexpected behavior:
+ * - `*` suffix for prefix matching (e.g., `prog*` matches "program", "programming")
+ * - `"..."` for phrase queries
+ * - `.` for column specification
+ * - `AND`, `OR`, `NOT` boolean operators
+ * - `+`, `-` for required/excluded terms
+ *
+ * This function wraps each whitespace-separated term in double quotes, treating all
+ * characters as literals. Double quotes within terms are escaped by doubling (`""`).
+ *
+ * Example: `prog* AND test.` becomes `"prog*" "AND" "test."`.
+ */
+const escapeFts5Query = (text: string): string => {
+  return text
+    .split(/\s+/)
+    .filter(Boolean)
+    .map((term) => `"${term.replace(/"/g, '""')}"`)
+    .join(' ');
+};
+
+export class FtsIndex implements Index {
+  migrate = Effect.fn('FtsIndex.migrate')(function* () {
+    const sql = yield* SqlClient.SqlClient;
+
+    // https://sqlite.org/fts5.html#the_trigram_tokenizer
+    // FTS5 tables are created as virtual tables; they implicitly have a `rowid`.
+    // Trigram tokenizer enables substring matching (e.g., "rog" matches "programming").
+    //
+    // Data structure: inverted index mapping trigrams to document IDs.
+    // "hello" → trigrams ["hel", "ell", "llo"] → B-tree entries: "hel"→[1], "ell"→[1], "llo"→[1].
+    // Query "ell" → O(log n) B-tree lookup → returns [1].
+    // Posting lists are compressed, so index size scales well with document count.
+    yield* sql`CREATE VIRTUAL TABLE IF NOT EXISTS ftsIndex USING fts5(snapshot, tokenize='trigram')`;
+  });
+
+  query({
+    query,
+    spaceId,
+    includeAllQueues,
+    queueIds,
+  }: FtsQuery): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
+    return Effect.gen(function* () {
+      const trimmed = query.trim();
+      if (trimmed.length === 0) {
+        return [];
+      }
+
+      const sql = yield* SqlClient.SqlClient;
+
+      // Trigram tokenizer requires at least 3 characters per term.
+      // Check if ALL terms are at least 3 chars; otherwise use LIKE fallback.
+      const terms = trimmed.split(/\s+/).filter(Boolean);
+      const minTermLength = Math.min(...terms.map((t) => t.length));
+
+      const conditions =
+        minTermLength < 3
+          ? // LIKE fallback - scan the entire table, AND all terms.
+            terms.map((term) => sql`f.snapshot LIKE ${'%' + term + '%'}`)
+          : // MATCH - fast index lookup.
+            [sql`f.snapshot MATCH ${escapeFts5Query(trimmed)}`];
+
+      // Space and queue constraints are combined with OR.
+      const sourceConditions: Statement.Statement<{}>[] = [];
+
+      if (spaceId && spaceId.length > 0) {
+        if (includeAllQueues) {
+          // All items from these spaces (both space objects and queue objects).
+          sourceConditions.push(sql`m.spaceId IN ${sql.in(spaceId)}`);
+        } else {
+          // Only space objects (not queue objects) from these spaces.
+          sourceConditions.push(sql`(m.spaceId IN ${sql.in(spaceId)} AND m.queueId = '')`);
+        }
+      }
+
+      if (queueIds && queueIds.length > 0) {
+        // Items from specific queues.
+        sourceConditions.push(sql`m.queueId IN ${sql.in(queueIds)}`);
+      }
+
+      if (sourceConditions.length > 0) {
+        conditions.push(sql`(${sql.or(sourceConditions)})`);
+      }
+
+      return yield* sql<ObjectMeta>`SELECT m.* FROM ftsIndex AS f JOIN objectMeta AS m ON f.rowid = m.recordId WHERE ${sql.and(conditions)}`;
+    });
+  }
+
+  /**
+   * Query snapshots by recordIds.
+   * Returns the parsed JSON snapshots for queue objects.
+   */
+  querySnapshotsJSON(
+    recordIds: number[],
+  ): Effect.Effect<readonly { recordId: number; snapshot: Obj.JSON }[], SqlError.SqlError, SqlClient.SqlClient> {
+    return Effect.gen(function* () {
+      if (recordIds.length === 0) {
+        return [];
+      }
+      const sql = yield* SqlClient.SqlClient;
+      const results = yield* sql<{
+        rowid: number;
+        snapshot: string;
+      }>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(recordIds)}`;
+      return results.map((r) => ({
+        recordId: r.rowid,
+        snapshot: JSON.parse(r.snapshot),
+      }));
+    });
+  }
+
+  update = Effect.fn('FtsIndex.update')(
+    (objects: IndexerObject[]): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+
+        yield* Effect.forEach(
+          objects,
+          (object) =>
+            Effect.gen(function* () {
+              const { recordId, data } = object;
+              if (recordId === null) {
+                return yield* Effect.die(new Error('FtsIndex.update requires recordId to be set'));
+              }
+
+              const snapshot = JSON.stringify(data);
+
+              // FTS5 doesn't support UPDATE, need DELETE + INSERT for upsert.
+              const existing = yield* sql<{ rowid: number }>`SELECT rowid FROM ftsIndex WHERE rowid = ${recordId}`;
+              if (existing.length > 0) {
+                yield* sql`DELETE FROM ftsIndex WHERE rowid = ${recordId}`;
+              }
+
+              yield* sql`INSERT INTO ftsIndex (rowid, snapshot) VALUES (${recordId}, ${snapshot})`;
+            }),
+          { discard: true },
+        );
+      }),
+  );
+}

package/src/indexes/fts5.test.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Reactivity from '@effect/experimental/Reactivity';
+import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
+import * as Effect from 'effect/Effect';
+import { describe, expect, it } from 'vitest';
+
+describe('FTS5', () => {
+  it('should create an FTS5 table and search it', () =>
+    Effect.gen(function* () {
+      const sql = yield* SqliteClient.make({
+        filename: ':memory:',
+      });
+
+      yield* sql`CREATE VIRTUAL TABLE emails USING fts5(sender, title, body);`;
+
+      yield* sql`INSERT INTO emails (sender, title, body) VALUES ('bob@example.com', 'Hello', 'This is a message about Effect');`;
+      yield* sql`INSERT INTO emails (sender, title, body) VALUES ('alice@example.com', 'Meeting', 'Let us discuss SQL');`;
+
+      const result = yield* sql`SELECT * FROM emails WHERE emails MATCH 'Effect'`;
+
+      expect(result.length).toEqual(1);
+      expect(result[0].sender).toEqual('bob@example.com');
+    }).pipe(Effect.provide(Reactivity.layer), Effect.scoped, Effect.runPromise));
+});

package/src/indexes/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './fts-index';
+export * from './object-meta-index';
+export * from './reverse-ref-index';
+export * from './interface';

package/src/indexes/interface.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import type * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlError from '@effect/sql/SqlError';
+import type * as Effect from 'effect/Effect';
+
+import type { Obj } from '@dxos/echo';
+import type { ObjectId, SpaceId } from '@dxos/keys';
+
+/**
+ * Data describing objects returned from sources to the indexer.
+ */
+export interface IndexerObject {
+  spaceId: SpaceId;
+  /**
+   * Queue id if object is from the queue.
+   * If null, `documentId` must be set.
+   */
+  queueId: ObjectId | null;
+  /**
+   * Document id if object is from the automerge document.
+   * If null, `queueId` must be set.
+   */
+  documentId: string | null;
+
+  /**
+   * Record id from the objectMeta index.
+   * `Null` before the object is stored in the ObjectMetaIndex.
+   * Enriched by the IndexEngine after the object is stored in the ObjectMetaIndex.
+   */
+  recordId: number | null;
+
+  /**
+   * JSON data of the object.
+   */
+  data: Obj.JSON;
+}
+
+/**
+ * SQLite-based index for storing and querying object data.
+ */
+export interface Index {
+  /**
+   * Runs necessary migrations to the index before it is usable.
+   * Idempotent.
+   */
+  migrate: () => Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient>;
+
+  /**
+   * Updates the index with the given objects.
+   * Idempotent.
+   */
+  update: (objects: IndexerObject[]) => Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient>;
+}

package/src/indexes/object-meta-index.test.ts

@@ -0,0 +1,119 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Reactivity from '@effect/experimental/Reactivity';
+import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
+import { describe, expect, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+
+import { ATTR_DELETED, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from '@dxos/echo/internal';
+import { DXN, ObjectId, SpaceId } from '@dxos/keys';
+
+import type { IndexerObject } from './interface';
+import { ObjectMetaIndex } from './object-meta-index';
+
+const TYPE_PERSON = DXN.parse('dxn:type:example.com/type/Person:0.1.0').toString();
+const TYPE_RELATION = DXN.parse('dxn:type:example.com/type/Relation:0.1.0').toString();
+const TYPE_RELATION_UPDATED = DXN.parse('dxn:type:example.com/type/RelationUpdated:0.1.0').toString();
+
+const TestLayer = Layer.merge(
+  SqliteClient.layer({
+    filename: ':memory:',
+  }),
+  Reactivity.layer,
+);
+
+describe('ObjectMetaIndex', () => {
+  it.effect('should store and update object metadata', () =>
+    Effect.gen(function* () {
+      const index = new ObjectMetaIndex();
+      yield* index.migrate();
+
+      const spaceId = SpaceId.random();
+      const objectId1 = ObjectId.random();
+      const objectId2 = ObjectId.random();
+
+      const item1: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: null,
+        data: {
+          id: objectId1,
+          [ATTR_TYPE]: TYPE_PERSON,
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      const item2: IndexerObject = {
+        spaceId,
+        queueId: null,
+        documentId: 'doc-123',
+        recordId: null,
+        data: {
+          id: objectId2,
+          [ATTR_TYPE]: TYPE_RELATION,
+          [ATTR_RELATION_SOURCE]: DXN.parse(`dxn:echo:${spaceId}:${ObjectId.random()}}`).toString(),
+          [ATTR_RELATION_TARGET]: DXN.parse(`dxn:echo:${spaceId}:${ObjectId.random()}}`).toString(),
+          [ATTR_DELETED]: false,
+        },
+      };
+
+      // 1. Initial Insert
+      yield* index.update([item1, item2]);
+
+      // Verify Query.
+      const results = yield* index.query({ spaceId, typeDxn: TYPE_PERSON });
+      expect(results.length).toBe(1);
+      expect(results[0].objectId).toBe(objectId1);
+      expect(results[0].version).toBe(1);
+
+      const relationResults = yield* index.query({ spaceId, typeDxn: TYPE_RELATION });
+      expect(relationResults.length).toBe(1);
+      expect(relationResults[0].objectId).toBe(objectId2);
+      expect(relationResults[0].entityKind).toBe('relation');
+      expect(relationResults[0].source).toBe(item2.data[ATTR_RELATION_SOURCE]);
+      expect(relationResults[0].target).toBe(item2.data[ATTR_RELATION_TARGET]);
+      expect(relationResults[0].version).toBe(2);
+
+      // 2. Update existing object (item1 matches by queueId)
+      const item1Update: IndexerObject = {
+        ...item1,
+        data: {
+          ...item1.data,
+          [ATTR_DELETED]: true,
+        },
+      };
+
+      yield* index.update([item1Update]);
+
+      const updatedResults = yield* index.query({ spaceId, typeDxn: TYPE_PERSON });
+      // Depending on implementation, query might filter deleted or not.
+      // Current implementation is SELECT * without deleted filter for queryType
+      expect(updatedResults.length).toBe(1);
+      expect(updatedResults[0].deleted).toBe(true);
+      expect(updatedResults[0].version).toBe(3); // Incremented globally
+
+      // 3. Update existing object by documentId (item2)
+      const item2Update: IndexerObject = {
+        ...item2,
+        data: {
+          ...item2.data,
+          [ATTR_TYPE]: TYPE_RELATION_UPDATED,
+        },
+      };
+
+      yield* index.update([item2Update]);
+
+      const newTypeResults = yield* index.query({ spaceId, typeDxn: TYPE_RELATION_UPDATED });
+      expect(newTypeResults.length).toBe(1);
+      expect(newTypeResults[0].version).toBe(4);
+      expect(newTypeResults[0].objectId).toBe(objectId2);
+
+      const oldTypeResults = yield* index.query({ spaceId, typeDxn: TYPE_RELATION });
+      expect(oldTypeResults.length).toBe(0);
+    }).pipe(Effect.provide(TestLayer)),
+  );
+});

package/src/indexes/object-meta-index.ts

@@ -0,0 +1,204 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlError from '@effect/sql/SqlError';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { ATTR_DELETED, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from '@dxos/echo/internal';
+
+import type { IndexerObject } from './interface';
+import type { Index } from './interface';
+
+export const ObjectMeta = Schema.Struct({
+  recordId: Schema.Number,
+  objectId: Schema.String,
+  queueId: Schema.String,
+  spaceId: Schema.String,
+  documentId: Schema.String,
+  entityKind: Schema.String,
+  typeDxn: Schema.String,
+  deleted: Schema.Boolean,
+  source: Schema.NullOr(Schema.String),
+  target: Schema.NullOr(Schema.String),
+  /** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
+  version: Schema.Number,
+});
+export interface ObjectMeta extends Schema.Schema.Type<typeof ObjectMeta> {}
+
+export class ObjectMetaIndex implements Index {
+  migrate = Effect.fn('ObjectMetaIndex.runMigrations')(function* () {
+    const sql = yield* SqlClient.SqlClient;
+
+    yield* sql`CREATE TABLE IF NOT EXISTS objectMeta (
+      recordId INTEGER PRIMARY KEY AUTOINCREMENT,
+      objectId TEXT NOT NULL,
+      queueId TEXT NOT NULL DEFAULT '',
+      spaceId TEXT NOT NULL,
+      documentId TEXT NOT NULL DEFAULT '',
+      entityKind TEXT NOT NULL,
+      typeDxn TEXT NOT NULL,
+      deleted INTEGER NOT NULL,
+      source TEXT,
+      target TEXT,
+      version INTEGER NOT NULL
+    )`;
+
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDxn ON objectMeta(spaceId, typeDxn)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
+  });
+
+  query = Effect.fn('ObjectMetaIndex.queryType')(
+    (
+      query: Pick<ObjectMeta, 'spaceId' | 'typeDxn'>,
+    ): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        // SQLite stores booleans as integers, so we need to specify the raw row type.
+        const rows =
+          yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDxn = ${query.typeDxn}`;
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+
+  // TODO(dmaretskyi): Update recordId on objects so that we don't need to look it up separately.
+  update = Effect.fn('ObjectMetaIndex.update')(
+    (objects: IndexerObject[]): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+
+        yield* Effect.forEach(
+          objects,
+          (object) =>
+            Effect.gen(function* () {
+              const { spaceId, queueId, documentId, data } = object;
+
+              // Extract metadata (Logic emulating Echo APIs as strict imports are unavailable).
+              // TODO(agent): Verify property access matches Obj.JSON structure.
+              const castData = data;
+              const objectId = castData.id;
+
+              // Check for existing record by (spaceId, queueId) or (spaceId, documentId).
+              let existing: readonly { recordId: number }[];
+              if (documentId) {
+                existing = yield* sql<{
+                  recordId: number;
+                }>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
+              } else if (queueId) {
+                existing = yield* sql<{
+                  recordId: number;
+                }>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
+              } else {
+                // Should not happen based on IndexerObject definition (one must be present ideally), but handle gracefully.
+                existing = [];
+              }
+
+              // Get max version + 1.
+              const result = yield* sql<{ v: number | null }>`SELECT MAX(version) as v FROM objectMeta`;
+              const [{ v }] = result;
+              const version = (v ?? 0) + 1;
+
+              // Extract metadata.
+              const entityKind = castData[ATTR_RELATION_SOURCE] ? 'relation' : 'object';
+              const typeDxn = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : 'type';
+              const deleted = castData[ATTR_DELETED] ? 1 : 0;
+              // Relations.
+              const source = entityKind === 'relation' ? (castData[ATTR_RELATION_SOURCE] ?? null) : null;
+              const target = entityKind === 'relation' ? (castData[ATTR_RELATION_TARGET] ?? null) : null;
+
+              if (existing.length > 0) {
+                yield* sql`
+                  UPDATE objectMeta SET
+                    version = ${version},
+                    entityKind = ${entityKind},
+                    typeDxn = ${typeDxn},
+                    deleted = ${deleted},
+                    source = ${source},
+                    target = ${target}
+                  WHERE recordId = ${existing[0].recordId}
+                `;
+              } else {
+                yield* sql`
+                  INSERT INTO objectMeta (
+                    objectId, queueId, spaceId, documentId, 
+                    entityKind, typeDxn, deleted, source, target, version
+                  ) VALUES (
+                    ${objectId}, ${queueId ?? ''}, ${spaceId}, ${documentId ?? ''}, 
+                    ${entityKind}, ${typeDxn}, ${deleted}, 
+                    ${source}, ${target}, ${version}
+                  )
+                `;
+              }
+            }),
+          { discard: true },
+        );
+      }),
+  );
+
+  /**
+   * Look up `recordIds` for objects that are already stored in the ObjectMetaIndex.
+   * Mutates the objects in place.
+   */
+  lookupRecordIds = Effect.fn('ObjectMetaIndex.lookupRecordIds')(
+    (objects: IndexerObject[]): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+
+        for (const object of objects) {
+          const { spaceId, queueId, documentId, data } = object;
+          const objectId = data.id;
+
+          let result: readonly { recordId: number }[];
+          if (documentId) {
+            result = yield* sql<{
+              recordId: number;
+            }>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
+          } else if (queueId) {
+            result = yield* sql<{
+              recordId: number;
+            }>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
+          } else {
+            result = [];
+          }
+
+          if (result.length === 0) {
+            // TODO(mykola): Handle this case gracefully.
+            yield* Effect.die(
+              new Error(`Object not found in ObjectMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`),
+            );
+          }
+          object.recordId = result[0].recordId;
+        }
+      }),
+  );
+
+  /**
+   * Look up object metadata by recordIds.
+   */
+  lookupByRecordIds = Effect.fn('ObjectMetaIndex.lookupByRecordIds')(
+    (recordIds: number[]): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        if (recordIds.length === 0) {
+          return [];
+        }
+
+        const sql = yield* SqlClient.SqlClient;
+        const placeholders = recordIds.map(() => '?').join(', ');
+        const rows = yield* sql.unsafe<ObjectMeta>(
+          `SELECT * FROM objectMeta WHERE recordId IN (${placeholders})`,
+          recordIds,
+        );
+
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+}