@dxos/index-core 0.0.0 → 0.8.4-main.03d5cd7b56

package/src/indexes/object-meta-index.ts

@@ -4,30 +4,80 @@
 
 import * as SqlClient from '@effect/sql/SqlClient';
 import type * as SqlError from '@effect/sql/SqlError';
+import type * as Statement from '@effect/sql/Statement';
 import * as Effect from 'effect/Effect';
 import * as Schema from 'effect/Schema';
 
-import { ATTR_DELETED, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from '@dxos/echo/internal';
+import { ATTR_DELETED, ATTR_PARENT, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from '@dxos/echo/internal';
+import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
 
 import type { IndexerObject } from './interface';
 import type { Index } from './interface';
 
+const _escapeLikePrefix = (prefix: string) => {
+  // Escape LIKE metacharacters in the *literal* prefix (we still append a wildcard for the version suffix).
+  // Backslash is used as the ESCAPE character.
+  // See: https://www.sqlite.org/lang_expr.html#like
+  const escaped = prefix.replaceAll('\\', '\\\\').replaceAll('%', '\\%').replaceAll('_', '\\_');
+  return `${escaped}:%`;
+};
+
 export const ObjectMeta = Schema.Struct({
   recordId: Schema.Number,
   objectId: Schema.String,
   queueId: Schema.String,
+  /** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
+  queueNamespace: Schema.String,
   spaceId: Schema.String,
   documentId: Schema.String,
   entityKind: Schema.String,
+  /** The versioned DXN of the type of the object. */
   typeDxn: Schema.String,
   deleted: Schema.Boolean,
   source: Schema.NullOr(Schema.String),
   target: Schema.NullOr(Schema.String),
+  /** Parent object id (nullable). */
+  parent: Schema.NullOr(Schema.String),
   /** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
   version: Schema.Number,
+  /** Unix ms timestamp when the object was first indexed. */
+  createdAt: Schema.NullOr(Schema.Number),
+  /** Unix ms timestamp when the object was last re-indexed. */
+  updatedAt: Schema.NullOr(Schema.Number),
 });
 export interface ObjectMeta extends Schema.Schema.Type<typeof ObjectMeta> {}
 
+/**
+ * Builds a SQL condition for filtering by space and queue source.
+ * When `includeAllQueues` is false and no `queueIds`, only non-queue objects are returned.
+ */
+const buildSourceCondition = (
+  sql: SqlClient.SqlClient,
+  spaceIds: readonly string[],
+  includeAllQueues: boolean,
+  queueIds: readonly string[] | null,
+): Statement.Fragment => {
+  const conditions: Statement.Fragment[] = [];
+
+  if (spaceIds.length > 0) {
+    if (includeAllQueues) {
+      conditions.push(sql`${sql.in('spaceId', spaceIds)}`);
+    } else {
+      conditions.push(sql`(${sql.in('spaceId', spaceIds)} AND queueId = '')`);
+    }
+  }
+
+  if (queueIds && queueIds.length > 0) {
+    conditions.push(sql`${sql.in('queueId', queueIds)}`);
+  }
+
+  if (conditions.length === 0) {
+    return sql`1 = 0`;
+  }
+
+  return sql.or(conditions);
+};
+
 export class ObjectMetaIndex implements Index {
   migrate = Effect.fn('ObjectMetaIndex.runMigrations')(function* () {
     const sql = yield* SqlClient.SqlClient;
@@ -36,6 +86,7 @@ export class ObjectMetaIndex implements Index {
       recordId INTEGER PRIMARY KEY AUTOINCREMENT,
       objectId TEXT NOT NULL,
       queueId TEXT NOT NULL DEFAULT '',
+      queueNamespace TEXT NOT NULL DEFAULT '',
       spaceId TEXT NOT NULL,
       documentId TEXT NOT NULL DEFAULT '',
       entityKind TEXT NOT NULL,
@@ -43,23 +94,149 @@ export class ObjectMetaIndex implements Index {
       deleted INTEGER NOT NULL,
       source TEXT,
       target TEXT,
-      version INTEGER NOT NULL
+      parent TEXT,
+      version INTEGER NOT NULL,
+      createdAt INTEGER,
+      updatedAt INTEGER
     )`;
 
+    // Add `parent` column for tables created before it was introduced.
+    yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect.void);
+    // Add timestamp columns for tables created before they were introduced.
+    yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect.void);
+    yield* Effect.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect.void);
+    // Add queueNamespace column for tables created before it was introduced.
+    yield* Effect.catchAll(
+      sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`,
+      () => Effect.void,
+    );
+
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDxn ON objectMeta(spaceId, typeDxn)`;
     yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
+    yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
   });
 
-  query = Effect.fn('ObjectMetaIndex.queryType')(
+  query = Effect.fn('ObjectMetaIndex.query')(
     (
       query: Pick<ObjectMeta, 'spaceId' | 'typeDxn'>,
     ): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
       Effect.gen(function* () {
         const sql = yield* SqlClient.SqlClient;
+        const parsedType = DXN.tryParse(query.typeDxn)?.asTypeDXN();
+
         // SQLite stores booleans as integers, so we need to specify the raw row type.
         const rows =
-          yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDxn = ${query.typeDxn}`;
+          parsedType && parsedType.version === undefined
+            ? yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDxn = ${
+                query.typeDxn
+              } OR typeDxn LIKE ${_escapeLikePrefix(query.typeDxn)} ESCAPE '\\')`
+            : yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDxn = ${query.typeDxn}`;
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+
+  queryAll = Effect.fn('ObjectMetaIndex.queryAll')(
+    (query: {
+      spaceIds: readonly ObjectMeta['spaceId'][];
+      includeAllQueues?: boolean;
+      queueIds?: readonly string[] | null;
+    }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
+          return [];
+        }
+
+        const sql = yield* SqlClient.SqlClient;
+        const sourceCondition = buildSourceCondition(
+          sql,
+          query.spaceIds,
+          query.includeAllQueues ?? false,
+          query.queueIds ?? null,
+        );
+        const rows = yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+
+  queryTypes = Effect.fn('ObjectMetaIndex.queryTypes')(
+    ({
+      spaceIds,
+      typeDxns,
+      inverted = false,
+      includeAllQueues = false,
+      queueIds = null,
+    }: {
+      spaceIds: readonly ObjectMeta['spaceId'][];
+      typeDxns: readonly ObjectMeta['typeDxn'][];
+      inverted?: boolean;
+      includeAllQueues?: boolean;
+      queueIds?: readonly string[] | null;
+    }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) {
+          return [];
+        }
+
+        if (typeDxns.length === 0) {
+          if (!inverted) {
+            return [];
+          }
+
+          const sql = yield* SqlClient.SqlClient;
+          const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
+          const rows = yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
+          return rows.map((row) => ({
+            ...row,
+            deleted: !!row.deleted,
+          }));
+        }
+        const sql = yield* SqlClient.SqlClient;
+        const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
+        const typeWhere = sql.or(
+          typeDxns.map((typeDxn) => {
+            const parsedType = DXN.tryParse(typeDxn)?.asTypeDXN();
+            return parsedType && parsedType.version === undefined
+              ? sql.or([sql`typeDxn = ${typeDxn}`, sql`typeDxn LIKE ${_escapeLikePrefix(typeDxn)} ESCAPE '\\'`])
+              : sql`typeDxn = ${typeDxn}`;
+          }),
+        );
+        const rows = inverted
+          ? yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}`
+          : yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+
+  queryRelations = Effect.fn('ObjectMetaIndex.queryRelations')(
+    ({
+      endpoint,
+      anchorDxns,
+    }: {
+      endpoint: 'source' | 'target';
+      anchorDxns: readonly string[];
+    }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        if (anchorDxns.length === 0) {
+          return [];
+        }
+        const sql = yield* SqlClient.SqlClient;
+        const column = endpoint === 'source' ? 'source' : 'target';
+        const rows = yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE entityKind = 'relation' AND ${sql.in(
+          column,
+          anchorDxns,
+        )}`;
         return rows.map((row) => ({
           ...row,
           deleted: !!row.deleted,
@@ -77,10 +254,9 @@ export class ObjectMetaIndex implements Index {
           objects,
           (object) =>
             Effect.gen(function* () {
-              const { spaceId, queueId, documentId, data } = object;
+              const { spaceId, queueId, queueNamespace, documentId, data } = object;
 
               // Extract metadata (Logic emulating Echo APIs as strict imports are unavailable).
-              // TODO(agent): Verify property access matches Obj.JSON structure.
               const castData = data;
               const objectId = castData.id;
 
@@ -111,27 +287,36 @@ export class ObjectMetaIndex implements Index {
               // Relations.
               const source = entityKind === 'relation' ? (castData[ATTR_RELATION_SOURCE] ?? null) : null;
               const target = entityKind === 'relation' ? (castData[ATTR_RELATION_TARGET] ?? null) : null;
+              // Parent (nullable).
+              const parent = castData[ATTR_PARENT] ?? null;
+
+              const sourceTimestamp = object.updatedAt;
 
               if (existing.length > 0) {
                 yield* sql`
                   UPDATE objectMeta SET
                     version = ${version},
+                    queueNamespace = ${queueNamespace ?? ''},
                     entityKind = ${entityKind},
                     typeDxn = ${typeDxn},
                     deleted = ${deleted},
                     source = ${source},
-                    target = ${target}
+                    target = ${target},
+                    parent = ${parent},
+                    updatedAt = ${sourceTimestamp}
                   WHERE recordId = ${existing[0].recordId}
                 `;
               } else {
                 yield* sql`
                   INSERT INTO objectMeta (
-                    objectId, queueId, spaceId, documentId, 
-                    entityKind, typeDxn, deleted, source, target, version
+                    objectId, queueId, queueNamespace, spaceId, documentId,
+                    entityKind, typeDxn, deleted, source, target, parent, version,
+                    createdAt, updatedAt
                   ) VALUES (
-                    ${objectId}, ${queueId ?? ''}, ${spaceId}, ${documentId ?? ''}, 
-                    ${entityKind}, ${typeDxn}, ${deleted}, 
-                    ${source}, ${target}, ${version}
+                    ${objectId}, ${queueId ?? ''}, ${queueNamespace ?? ''}, ${spaceId}, ${documentId ?? ''},
+                    ${entityKind}, ${typeDxn}, ${deleted},
+                    ${source}, ${target}, ${parent}, ${version},
+                    ${sourceTimestamp}, ${sourceTimestamp}
                   )
                 `;
               }
@@ -169,7 +354,7 @@ export class ObjectMetaIndex implements Index {
 
           if (result.length === 0) {
             // TODO(mykola): Handle this case gracefully.
-            yield* Effect.die(
+            return yield* Effect.die(
               new Error(`Object not found in ObjectMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`),
             );
           }
@@ -189,12 +374,114 @@ export class ObjectMetaIndex implements Index {
         }
 
         const sql = yield* SqlClient.SqlClient;
-        const placeholders = recordIds.map(() => '?').join(', ');
-        const rows = yield* sql.unsafe<ObjectMeta>(
-          `SELECT * FROM objectMeta WHERE recordId IN (${placeholders})`,
-          recordIds,
+        const rows = yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sql.in('recordId', recordIds)}`;
+
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+
+  /**
+   * Look up object metadata by objectId, spaceId, and queueId.
+   */
+  lookupByObjectId = Effect.fn('ObjectMetaIndex.lookupByObjectId')(
+    (query: {
+      objectId: string;
+      spaceId: string;
+      queueId: string;
+    }): Effect.Effect<ObjectMeta | null, SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        const rows =
+          yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
+
+        if (rows.length === 0) {
+          return null;
+        }
+
+        return {
+          ...rows[0],
+          deleted: !!rows[0].deleted,
+        };
+      }),
+  );
+
+  /**
+   * Query objects by timestamp range.
+   */
+  queryByTimeRange = Effect.fn('ObjectMetaIndex.queryByTimeRange')(
+    (query: {
+      spaceIds: readonly string[];
+      updatedAfter?: number;
+      updatedBefore?: number;
+      createdAfter?: number;
+      createdBefore?: number;
+      includeAllQueues?: boolean;
+      queueIds?: readonly string[] | null;
+    }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
+          return [];
+        }
+
+        const sql = yield* SqlClient.SqlClient;
+        const sourceCondition = buildSourceCondition(
+          sql,
+          query.spaceIds,
+          query.includeAllQueues ?? false,
+          query.queueIds ?? null,
         );
 
+        const timeConditions: Statement.Fragment[] = [];
+        if (query.updatedAfter != null) {
+          timeConditions.push(sql`updatedAt >= ${query.updatedAfter}`);
+        }
+        if (query.updatedBefore != null) {
+          timeConditions.push(sql`updatedAt <= ${query.updatedBefore}`);
+        }
+        if (query.createdAfter != null) {
+          timeConditions.push(sql`createdAt >= ${query.createdAfter}`);
+        }
+        if (query.createdBefore != null) {
+          timeConditions.push(sql`createdAt <= ${query.createdBefore}`);
+        }
+
+        const rows =
+          timeConditions.length > 0
+            ? yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${sql.and(timeConditions)}`
+            : yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
+
+        return rows.map((row) => ({
+          ...row,
+          deleted: !!row.deleted,
+        }));
+      }),
+  );
+
+  /**
+   * Query children by parent object ids.
+   * Matches both:
+   * - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
+   * - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
+   *   DXN uses the feed's object id as its queue id — see `Feed.getQueueDxn`).
+   */
+  queryChildren = Effect.fn('ObjectMetaIndex.queryChildren')(
+    (query: {
+      spaceId: SpaceId[];
+      parentIds: ObjectId[];
+    }): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        if (query.parentIds.length === 0) {
+          return [];
+        }
+
+        const sql = yield* SqlClient.SqlClient;
+        const parentDxns = query.parentIds.map((id) => DXN.fromLocalObjectId(id).toString());
+        const rows =
+          yield* sql<ObjectMeta>`SELECT * FROM objectMeta WHERE ${sql.in('spaceId', query.spaceId)} AND (${sql.in('parent', parentDxns)} OR ${sql.in('queueId', query.parentIds)})`;
+
         return rows.map((row) => ({
           ...row,
           deleted: !!row.deleted,

package/src/indexes/reverse-ref-index.test.ts

@@ -14,8 +14,8 @@ import { DXN, ObjectId, SpaceId } from '@dxos/keys';
 import type { IndexerObject } from './interface';
 import { ReverseRefIndex } from './reverse-ref-index';
 
-const TYPE_PERSON = DXN.parse('dxn:type:example.com/type/Person:0.1.0').toString();
-const TYPE_EXAMPLE = DXN.parse('dxn:type:example.com/type/Example:0.1.0').toString();
+const TYPE_PERSON = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
+const TYPE_EXAMPLE = DXN.parse('dxn:type:com.example.type.example:0.1.0').toString();
 
 const TestLayer = Layer.merge(
   SqliteClient.layer({
@@ -38,8 +38,10 @@ describe('ReverseRefIndex', () => {
       const sourceObject: IndexerObject = {
         spaceId,
         queueId: ObjectId.random(),
+        queueNamespace: 'data',
         documentId: null,
         recordId: 1,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_PERSON,
@@ -71,8 +73,10 @@ describe('ReverseRefIndex', () => {
       const sourceObject: IndexerObject = {
         spaceId,
         queueId: ObjectId.random(),
+        queueNamespace: 'data',
         documentId: null,
         recordId: 1,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_EXAMPLE,
@@ -112,8 +116,10 @@ describe('ReverseRefIndex', () => {
       const sourceObject: IndexerObject = {
         spaceId,
         queueId: ObjectId.random(),
+        queueNamespace: 'data',
         documentId: null,
         recordId: 1,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_EXAMPLE,
@@ -151,8 +157,10 @@ describe('ReverseRefIndex', () => {
       const sourceObject: IndexerObject = {
         spaceId,
         queueId,
+        queueNamespace: 'data',
         documentId: null,
         recordId,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_EXAMPLE,
@@ -169,8 +177,10 @@ describe('ReverseRefIndex', () => {
       const updatedObject: IndexerObject = {
         spaceId,
         queueId,
+        queueNamespace: 'data',
         documentId: null,
         recordId,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_EXAMPLE,
@@ -201,8 +211,10 @@ describe('ReverseRefIndex', () => {
       const sourceObject: IndexerObject = {
         spaceId,
         queueId: ObjectId.random(),
+        queueNamespace: 'data',
         documentId: null,
         recordId: 1,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_EXAMPLE,
@@ -232,8 +244,10 @@ describe('ReverseRefIndex', () => {
       const sourceObject: IndexerObject = {
         spaceId,
         queueId: null,
+        queueNamespace: null,
         documentId: 'doc-123',
         recordId: 1,
+        updatedAt: Date.now(),
         data: {
           id: sourceObjectId,
           [ATTR_TYPE]: TYPE_EXAMPLE,

package/src/indexes/reverse-ref-index.ts

@@ -10,7 +10,6 @@ import * as Schema from 'effect/Schema';
 import { EncodedReference, isEncodedReference } from '@dxos/echo-protocol';
 
 import { EscapedPropPath } from '../utils';
-
 import type { Index, IndexerObject } from './interface';
 
 /**

package/src/utils.ts

@@ -6,7 +6,7 @@ import * as Schema from 'effect/Schema';
 
 import { invariant } from '@dxos/invariant';
 
-export type ObjectPropPath = (string | number)[];
+export type ObjectPropPath = string[];
 
 /**
  * Escaped property path within an object.