@dxos/index-core 0.0.0

package/src/indexes/reverse-ref-index.test.ts

@@ -0,0 +1,251 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Reactivity from '@effect/experimental/Reactivity';
+import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
+import { describe, expect, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+
+import { ATTR_TYPE } from '@dxos/echo/internal';
+import { DXN, ObjectId, SpaceId } from '@dxos/keys';
+
+import type { IndexerObject } from './interface';
+import { ReverseRefIndex } from './reverse-ref-index';
+
+const TYPE_PERSON = DXN.parse('dxn:type:example.com/type/Person:0.1.0').toString();
+const TYPE_EXAMPLE = DXN.parse('dxn:type:example.com/type/Example:0.1.0').toString();
+
+const TestLayer = Layer.merge(
+  SqliteClient.layer({
+    filename: ':memory:',
+  }),
+  Reactivity.layer,
+);
+
+describe('ReverseRefIndex', () => {
+  it.effect('should store and query reverse references', () =>
+    Effect.gen(function* () {
+      const reverseRefIndex = new ReverseRefIndex();
+      yield* reverseRefIndex.migrate();
+
+      const spaceId = SpaceId.random();
+      const sourceObjectId = ObjectId.random();
+      const targetObjectId = ObjectId.random();
+      const targetDxn = `dxn:echo:@:${targetObjectId}`;
+
+      const sourceObject: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: 1,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_PERSON,
+          contact: { '/': targetDxn },
+        },
+      };
+
+      yield* reverseRefIndex.update([sourceObject]);
+
+      const results = yield* reverseRefIndex.query({ targetDxn });
+      expect(results.length).toBe(1);
+      expect(results[0].targetDxn).toBe(targetDxn);
+      expect(results[0].propPath).toBe('contact');
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
+  it.effect('should handle nested references', () =>
+    Effect.gen(function* () {
+      const reverseRefIndex = new ReverseRefIndex();
+      yield* reverseRefIndex.migrate();
+
+      const spaceId = SpaceId.random();
+      const sourceObjectId = ObjectId.random();
+      const targetObjectId1 = ObjectId.random();
+      const targetObjectId2 = ObjectId.random();
+      const targetDxn1 = `dxn:echo:@:${targetObjectId1}`;
+      const targetDxn2 = `dxn:echo:@:${targetObjectId2}`;
+
+      const sourceObject: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: 1,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_EXAMPLE,
+          nested: {
+            deep: {
+              ref: { '/': targetDxn1 },
+            },
+          },
+          simple: { '/': targetDxn2 },
+        },
+      };
+
+      yield* reverseRefIndex.update([sourceObject]);
+
+      const results1 = yield* reverseRefIndex.query({ targetDxn: targetDxn1 });
+      expect(results1.length).toBe(1);
+      expect(results1[0].propPath).toBe('nested.deep.ref');
+
+      const results2 = yield* reverseRefIndex.query({ targetDxn: targetDxn2 });
+      expect(results2.length).toBe(1);
+      expect(results2[0].propPath).toBe('simple');
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
+  it.effect('should handle array references', () =>
+    Effect.gen(function* () {
+      const reverseRefIndex = new ReverseRefIndex();
+      yield* reverseRefIndex.migrate();
+
+      const spaceId = SpaceId.random();
+      const sourceObjectId = ObjectId.random();
+      const targetObjectId1 = ObjectId.random();
+      const targetObjectId2 = ObjectId.random();
+      const targetDxn1 = `dxn:echo:@:${targetObjectId1}`;
+      const targetDxn2 = `dxn:echo:@:${targetObjectId2}`;
+
+      const sourceObject: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: 1,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_EXAMPLE,
+          items: [{ '/': targetDxn1 }, { '/': targetDxn2 }],
+        },
+      };
+
+      yield* reverseRefIndex.update([sourceObject]);
+
+      const results1 = yield* reverseRefIndex.query({ targetDxn: targetDxn1 });
+      expect(results1.length).toBe(1);
+      expect(results1[0].propPath).toBe('items.0');
+
+      const results2 = yield* reverseRefIndex.query({ targetDxn: targetDxn2 });
+      expect(results2.length).toBe(1);
+      expect(results2[0].propPath).toBe('items.1');
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
+  it.effect('should update references on object change', () =>
+    Effect.gen(function* () {
+      const reverseRefIndex = new ReverseRefIndex();
+      yield* reverseRefIndex.migrate();
+
+      const spaceId = SpaceId.random();
+      const queueId = ObjectId.random();
+      const sourceObjectId = ObjectId.random();
+      const targetObjectId1 = ObjectId.random();
+      const targetObjectId2 = ObjectId.random();
+      const targetDxn1 = `dxn:echo:@:${targetObjectId1}`;
+      const targetDxn2 = `dxn:echo:@:${targetObjectId2}`;
+      const recordId = 1;
+
+      // Initial object with reference to target1.
+      const sourceObject: IndexerObject = {
+        spaceId,
+        queueId,
+        documentId: null,
+        recordId,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_EXAMPLE,
+          contact: { '/': targetDxn1 },
+        },
+      };
+
+      yield* reverseRefIndex.update([sourceObject]);
+
+      let results1 = yield* reverseRefIndex.query({ targetDxn: targetDxn1 });
+      expect(results1.length).toBe(1);
+
+      // Update object to reference target2 instead (same recordId).
+      const updatedObject: IndexerObject = {
+        spaceId,
+        queueId,
+        documentId: null,
+        recordId,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_EXAMPLE,
+          contact: { '/': targetDxn2 },
+        },
+      };
+
+      yield* reverseRefIndex.update([updatedObject]);
+
+      // Old reference should be gone.
+      results1 = yield* reverseRefIndex.query({ targetDxn: targetDxn1 });
+      expect(results1.length).toBe(0);
+
+      // New reference should exist.
+      const results2 = yield* reverseRefIndex.query({ targetDxn: targetDxn2 });
+      expect(results2.length).toBe(1);
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
+  it.effect('should handle objects without references', () =>
+    Effect.gen(function* () {
+      const reverseRefIndex = new ReverseRefIndex();
+      yield* reverseRefIndex.migrate();
+
+      const spaceId = SpaceId.random();
+      const sourceObjectId = ObjectId.random();
+
+      const sourceObject: IndexerObject = {
+        spaceId,
+        queueId: ObjectId.random(),
+        documentId: null,
+        recordId: 1,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_EXAMPLE,
+          name: 'Test Object',
+          count: 42,
+        },
+      };
+
+      yield* reverseRefIndex.update([sourceObject]);
+
+      // Should not throw and no results for random DXN.
+      const results = yield* reverseRefIndex.query({ targetDxn: 'dxn:echo:@:nonexistent' });
+      expect(results.length).toBe(0);
+    }).pipe(Effect.provide(TestLayer)),
+  );
+
+  it.effect('should work with documentId instead of queueId', () =>
+    Effect.gen(function* () {
+      const reverseRefIndex = new ReverseRefIndex();
+      yield* reverseRefIndex.migrate();
+
+      const spaceId = SpaceId.random();
+      const sourceObjectId = ObjectId.random();
+      const targetObjectId = ObjectId.random();
+      const targetDxn = `dxn:echo:@:${targetObjectId}`;
+
+      const sourceObject: IndexerObject = {
+        spaceId,
+        queueId: null,
+        documentId: 'doc-123',
+        recordId: 1,
+        data: {
+          id: sourceObjectId,
+          [ATTR_TYPE]: TYPE_EXAMPLE,
+          ref: { '/': targetDxn },
+        },
+      };
+
+      yield* reverseRefIndex.update([sourceObject]);
+
+      const results = yield* reverseRefIndex.query({ targetDxn });
+      expect(results.length).toBe(1);
+      expect(results[0].propPath).toBe('ref');
+    }).pipe(Effect.provide(TestLayer)),
+  );
+});

package/src/indexes/reverse-ref-index.ts

@@ -0,0 +1,127 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlError from '@effect/sql/SqlError';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { EncodedReference, isEncodedReference } from '@dxos/echo-protocol';
+
+import { EscapedPropPath } from '../utils';
+
+import type { Index, IndexerObject } from './interface';
+
+/**
+ * Extracts all outgoing references from an object's data.
+ */
+const extractReferences = (data: Record<string, unknown>): { path: string[]; targetDxn: string }[] => {
+  const refs: { path: string[]; targetDxn: string }[] = [];
+  const visit = (path: string[], value: unknown) => {
+    if (isEncodedReference(value)) {
+      const dxn = EncodedReference.toDXN(value);
+      const echoId = dxn.asEchoDXN()?.echoId;
+      if (!echoId) {
+        return; // Skip non-echo references.
+      }
+      refs.push({ path, targetDxn: dxn.toString() });
+    } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      for (const [key, v] of Object.entries(value)) {
+        visit([...path, key], v);
+      }
+    } else if (Array.isArray(value)) {
+      for (let i = 0; i < value.length; i++) {
+        visit([...path, String(i)], value[i]);
+      }
+    }
+  };
+  visit([], data);
+  return refs;
+};
+
+export const ReverseRef = Schema.Struct({
+  recordId: Schema.Number,
+  targetDxn: Schema.String,
+  /**
+   * Escaped property path within an object.
+   *
+   * Escaping rules:
+   *
+   * - '.' -> '\.'
+   * - '\' -> '\\'
+   * - contact with .
+   */
+  propPath: Schema.String,
+});
+export interface ReverseRef extends Schema.Schema.Type<typeof ReverseRef> {}
+
+export interface ReverseRefQuery {
+  targetDxn: string;
+  // TODO: Add prop filter
+}
+
+/**
+ * Indexes reverse references - tracks which objects reference which targets.
+ * Only indexes references, not relations.
+ */
+export class ReverseRefIndex implements Index {
+  migrate = Effect.fn('ReverseRefIndex.migrate')(function* () {
+    const sql = yield* SqlClient.SqlClient;
+
+    yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
+      recordId INTEGER NOT NULL,
+      targetDxn TEXT NOT NULL,
+      propPath TEXT NOT NULL,
+      PRIMARY KEY (recordId, targetDxn, propPath)
+    )`;
+
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDxn)`;
+  });
+
+  /**
+   * Query all references pointing to a target DXN.
+   */
+  query = Effect.fn('ReverseRefIndex.query')(
+    ({ targetDxn }: ReverseRefQuery): Effect.Effect<readonly ReverseRef[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        // TODO(mykola): Join objectMeta table here.
+        const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDxn = ${targetDxn}`;
+        return rows as ReverseRef[];
+      }),
+  );
+
+  update = Effect.fn('ReverseRefIndex.update')(
+    (objects: IndexerObject[]): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+
+        yield* Effect.forEach(
+          objects,
+          (object) =>
+            Effect.gen(function* () {
+              const { recordId, data } = object;
+              if (recordId === null) {
+                yield* Effect.die(new Error('ReverseRefIndex.update requires recordId to be set'));
+              }
+
+              // Delete existing references for this record.
+              yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
+
+              // Extract references from data.
+              const refs = extractReferences(data as unknown as Record<string, unknown>);
+
+              // Insert new references.
+              yield* Effect.forEach(
+                refs,
+                (ref) =>
+                  sql`INSERT INTO reverseRef (recordId, targetDxn, propPath) VALUES (${recordId}, ${ref.targetDxn}, ${EscapedPropPath.escape(ref.path)})`,
+                { discard: true },
+              );
+            }),
+          { discard: true },
+        );
+      }),
+  );
+}

package/src/utils.ts

@@ -0,0 +1,49 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { invariant } from '@dxos/invariant';
+
+export type ObjectPropPath = (string | number)[];
+
+/**
+ * Escaped property path within an object.
+ *
+ * Escaping rules:
+ *
+ * - '.' -> '\.'
+ * - '\' -> '\\'
+ * - contact with .
+ */
+export const EscapedPropPath: Schema.SchemaClass<string, string> & {
+  escape: (path: ObjectPropPath) => EscapedPropPath;
+  unescape: (path: EscapedPropPath) => ObjectPropPath;
+} = class extends Schema.String.annotations({ title: 'EscapedPropPath' }) {
+  static escape(path: ObjectPropPath): EscapedPropPath {
+    return path.map((p) => p.toString().replaceAll('\\', '\\\\').replaceAll('.', '\\.')).join('.');
+  }
+
+  static unescape(path: EscapedPropPath): ObjectPropPath {
+    const parts: string[] = [];
+    let current = '';
+
+    for (let i = 0; i < path.length; i++) {
+      if (path[i] === '\\') {
+        invariant(i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\'), 'Malformed escaping.');
+        current = current + path[i + 1];
+        i++;
+      } else if (path[i] === '.') {
+        parts.push(current);
+        current = '';
+      } else {
+        current += path[i];
+      }
+    }
+    parts.push(current);
+
+    return parts;
+  }
+};
+export type EscapedPropPath = Schema.Schema.Type<typeof EscapedPropPath>;