@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc674ce

package/src/query/query-executor.ts

@@ -3,11 +3,23 @@
 //
 
 import type { AutomergeUrl, DocumentId } from '@automerge/automerge-repo';
+import type * as SqlClient from '@effect/sql/SqlClient';
+import type * as Effect from 'effect/Effect';
 import * as Match from 'effect/Match';
-import * as Predicate from 'effect/Predicate';
+import * as Runtime from 'effect/Runtime';
 
 import { Context, ContextDisposedError, LifecycleState, Resource } from '@dxos/context';
-import { DatabaseDirectory, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
+import type { Obj } from '@dxos/echo';
+import { ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET } from '@dxos/echo/internal';
+import {
+  DatabaseDirectory,
+  type ObjectPropPath,
+  ObjectStructure,
+  type QueryAST,
+  isEncodedReference,
+} from '@dxos/echo-protocol';
+import { type RuntimeProvider, runAndForwardErrors, unwrapExit } from '@dxos/effect';
+import { type IndexEngine, type ObjectMeta, type ReverseRef } from '@dxos/index-core';
 import { EscapedPropPath, type FindResult, type Indexer } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { DXN, type ObjectId, PublicKey, type SpaceId } from '@dxos/keys';
@@ -19,15 +31,15 @@ import { getDeep, isNonNullable } from '@dxos/util';
 import type { AutomergeHost } from '../automerge';
 import { createIdFromSpaceKey } from '../common';
 import type { SpaceStateManager } from '../db-host';
-import { filterMatchObject } from '../filter';
+import { filterMatchObject, filterMatchObjectJSON } from '../filter';
 
 import type { QueryPlan } from './plan';
 import { QueryPlanner } from './query-planner';
 
-const isNullable: Predicate.Refinement<unknown, null | undefined> = Predicate.isNullable;
-
 type QueryExecutorOptions = {
   indexer: Indexer;
+  indexer2?: IndexEngine;
+  runtime?: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient>;
   automergeHost: AutomergeHost;
   spaceStateManager: SpaceStateManager;
 
@@ -48,11 +60,70 @@ type QueryExecutionResult = {
  */
 type QueryItem = {
   objectId: ObjectId;
-  documentId: DocumentId;
+
   spaceId: SpaceId;
-  doc: ObjectStructure;
+  queueId: ObjectId | null;
+  queueNamespace: string | null;
+
+  // For objects from automerge documents.
+  documentId: DocumentId | null;
+
+  // For objects from automerge documents.
+  doc: ObjectStructure | null;
+
+  // For objects from queues.
+  data: Obj.JSON | null;
+
+  /**
+   * Relevance rank for this item.
+   * Higher values indicate better matches for FTS/vector searches.
+   * Defaults to 1 for non-ranked queries (predicate matches).
+   */
+  rank: number;
 };
 
+const QueryItem = Object.freeze({
+  isDeleted: (item: QueryItem) => {
+    if (item.doc) {
+      return ObjectStructure.isDeleted(item.doc);
+    } else if (item.data) {
+      return item.data['@deleted'] === true;
+    } else {
+      throw new Error('Invalid query item');
+    }
+  },
+
+  getProperty: (item: QueryItem, property: ObjectPropPath) => {
+    if (item.doc) {
+      return getDeep(item.doc.data, property);
+    } else if (item.data) {
+      return getDeep(item.data, property);
+    } else {
+      throw new Error('Invalid query item');
+    }
+  },
+
+  getRelationSource: (item: QueryItem): DXN.String => {
+    if (item.doc) {
+      return ObjectStructure.getRelationSource(item.doc)?.['/'] as DXN.String;
+    } else if (item.data) {
+      return item.data[ATTR_RELATION_SOURCE] as DXN.String;
+    } else {
+      throw new Error('Invalid query item');
+    }
+  },
+
+  getRelationTarget: (item: QueryItem): DXN.String => {
+    if (item.doc) {
+      return ObjectStructure.getRelationTarget(item.doc)?.['/'] as DXN.String;
+    } else if (item.data) {
+      return item.data[ATTR_RELATION_TARGET] as DXN.String;
+    } else {
+      throw new Error('Invalid query item');
+    }
+  },
+});
+
 /**
  * Recursive data structure that represents the execution trace of a query.
  */
@@ -149,6 +220,8 @@ const TRACE_QUERY_EXECUTION = false;
  */
 export class QueryExecutor extends Resource {
   private readonly _indexer: Indexer;
+  private readonly _indexer2?: IndexEngine;
+  private readonly _runtime?: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient>;
   private readonly _automergeHost: AutomergeHost;
   private readonly _spaceStateManager: SpaceStateManager;
   /**
@@ -167,6 +240,8 @@ export class QueryExecutor extends Resource {
     super();
 
     this._indexer = options.indexer;
+    this._indexer2 = options.indexer2;
+    this._runtime = options.runtime;
     this._automergeHost = options.automergeHost;
     this._spaceStateManager = options.spaceStateManager;
 
@@ -190,19 +265,18 @@ export class QueryExecutor extends Resource {
     return this._trace;
   }
 
-  protected override async _open(ctx: Context): Promise<void> {}
-
-  protected override async _close(ctx: Context): Promise<void> {}
-
   getResults(): QueryResult[] {
     return this._lastResultSet.map(
       (item): QueryResult => ({
         id: item.objectId,
-        documentId: item.documentId,
+        documentId: item.documentId ?? undefined,
+        queueId: item.queueId ?? undefined,
+        queueNamespace: item.queueNamespace ?? undefined,
         spaceId: item.spaceId,
 
-        // TODO(dmaretskyi): Plumb through the rank.
-        rank: 0,
+        rank: item.rank,
+
+        documentJson: item.doc ? JSON.stringify(item.doc) : item.data ? JSON.stringify(item.data) : undefined,
       }),
     );
   }
@@ -230,6 +304,7 @@ export class QueryExecutor extends Resource {
     // ExecutionTrace.putOnPerformanceTimeline(trace);
 
     if (TRACE_QUERY_EXECUTION) {
+      // eslint-disable-next-line no-console
       console.log(ExecutionTrace.format(trace));
     }
 
@@ -240,7 +315,6 @@ export class QueryExecutor extends Resource {
 
   private async _execPlan(plan: QueryPlan.Plan, workingSet: QueryItem[]): Promise<StepExecutionResult> {
     const trace = ExecutionTrace.makeEmpty();
-    const begin = performance.now();
     for (const step of plan.steps) {
       if (this._ctx.disposed) {
         throw new ContextDisposedError();
@@ -288,6 +362,9 @@ export class QueryExecutor extends Resource {
       case 'OrderStep':
         ({ workingSet: newWorkingSet, trace } = await this._execOrderStep(step, workingSet));
         break;
+      case 'LimitStep':
+        ({ workingSet: newWorkingSet, trace } = await this._execLimitStep(step, workingSet));
+        break;
       default:
         throw new Error(`Unknown step type: ${(step as any)._tag}`);
     }
@@ -309,24 +386,42 @@ export class QueryExecutor extends Resource {
 
     switch (step.selector._tag) {
       case 'WildcardSelector': {
-        const beginIndexQuery = performance.now();
-        const indexHits = await this._indexer.execQuery({
-          typenames: [],
-          inverted: false,
-        });
-        trace.indexHits = +indexHits.length;
-        trace.indexQueryTime += performance.now() - beginIndexQuery;
+        if (this._supportsSqlIndexing()) {
+          const beginIndexQuery = performance.now();
+          const metas = await this._queryAllFromSqlIndex(step.spaces);
+          trace.indexHits = metas.length;
+          trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+          if (this._ctx.disposed) {
+            return { workingSet, trace };
+          }
 
-        if (this._ctx.disposed) {
-          return { workingSet, trace };
-        }
+          const documentLoadStart = performance.now();
+          const results = await this._loadDocumentsAfterSqlQuery(metas);
+          trace.documentsLoaded += results.length;
+          trace.documentLoadTime += performance.now() - documentLoadStart;
+
+          workingSet.push(...results.filter(isNonNullable));
+        } else {
+          const beginIndexQuery = performance.now();
+          const indexHits = await this._indexer.execQuery({
+            typenames: [],
+            inverted: false,
+          });
+          trace.indexHits = +indexHits.length;
+          trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+          if (this._ctx.disposed) {
+            return { workingSet, trace };
+          }
 
-        const documentLoadStart = performance.now();
-        const results = await this._loadDocumentsAfterIndexQuery(indexHits);
-        trace.documentsLoaded += results.length;
-        trace.documentLoadTime += performance.now() - documentLoadStart;
+          const documentLoadStart = performance.now();
+          const results = await this._loadDocumentsAfterIndexQuery(indexHits);
+          trace.documentsLoaded += results.length;
+          trace.documentLoadTime += performance.now() - documentLoadStart;
 
-        workingSet.push(...results.filter(isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
+          workingSet.push(...results.filter(isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
+        }
         trace.objectCount = workingSet.length;
 
         break;
@@ -347,57 +442,161 @@ export class QueryExecutor extends Resource {
       }
 
       case 'TypeSelector': {
-        const beginIndexQuery = performance.now();
-        const indexHits = await this._indexer.execQuery({
-          typenames: step.selector.typename,
-          inverted: step.selector.inverted,
-        });
-        trace.indexHits = +indexHits.length;
-        trace.indexQueryTime += performance.now() - beginIndexQuery;
+        if (this._supportsSqlIndexing()) {
+          const beginIndexQuery = performance.now();
+          const metas = await this._queryTypesFromSqlIndex(step.spaces, step.selector.typename, step.selector.inverted);
+          trace.indexHits = metas.length;
+          trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+          if (this._ctx.disposed) {
+            return { workingSet, trace };
+          }
 
-        if (this._ctx.disposed) {
-          return { workingSet, trace };
-        }
+          const documentLoadStart = performance.now();
+          const results = await this._loadDocumentsAfterSqlQuery(metas);
+          trace.documentsLoaded += results.length;
+          trace.documentLoadTime += performance.now() - documentLoadStart;
+
+          workingSet.push(...results.filter(isNonNullable));
+        } else {
+          const beginIndexQuery = performance.now();
+          const indexHits = await this._indexer.execQuery({
+            typenames: step.selector.typename,
+            inverted: step.selector.inverted,
+          });
+          trace.indexHits = +indexHits.length;
+          trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+          if (this._ctx.disposed) {
+            return { workingSet, trace };
+          }
 
-        const documentLoadStart = performance.now();
-        const results = await this._loadDocumentsAfterIndexQuery(indexHits);
-        trace.documentsLoaded += results.length;
-        trace.documentLoadTime += performance.now() - documentLoadStart;
+          const documentLoadStart = performance.now();
+          const results = await this._loadDocumentsAfterIndexQuery(indexHits);
+          trace.documentsLoaded += results.length;
+          trace.documentLoadTime += performance.now() - documentLoadStart;
 
-        workingSet.push(...results.filter(isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
+          workingSet.push(...results.filter(isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
+        }
         trace.objectCount = workingSet.length;
 
         break;
       }
 
       case 'TextSelector': {
-        const beginIndexQuery = performance.now();
-        const indexHits = await this._indexer.execQuery({
-          typenames: [],
-          text: {
-            query: step.selector.text,
-            kind: Match.type<QueryPlan.TextSearchKind>().pipe(
-              Match.withReturnType<'text' | 'vector'>(),
-              Match.when('full-text', () => 'text'),
-              Match.when('vector', () => 'vector'),
-              Match.orElseAbsurd,
-            )(step.selector.searchKind),
-          },
-        });
-        trace.indexHits = +indexHits.length;
-        trace.indexQueryTime += performance.now() - beginIndexQuery;
+        // TODO(dmaretskyi): type + FTS queries would be very common so we should support those, maybe chunk the fts index.
+        // TODO(dmaretskyi): nice to have matched text snippets/highlighting.
+        if (step.selector.searchKind === 'full-text' && this._supportsSqlIndexing()) {
+          // Use indexer2 for full-text search.
+          const beginIndexQuery = performance.now();
+          invariant(this._indexer2, 'SQL indexer is required.');
+          invariant(step.spaces.length <= 1, 'Multiple spaces are not supported for full-text search');
+          // Extract queue IDs from DXN strings.
+          const queueIds =
+            step.queues.length > 0
+              ? (step.queues.map((dxnStr) => DXN.parse(dxnStr).asQueueDXN()?.queueId).filter(Boolean) as ObjectId[])
+              : null;
+          const textResults = await this._runInRuntime(
+            this._indexer2.queryText({
+              query: step.selector.text,
+              spaceId: step.spaces,
+              includeAllQueues: step.allQueuesFromSpaces,
+              queueIds,
+            }),
+          );
+          trace.indexHits = textResults.length;
+          trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+          if (this._ctx.disposed) {
+            return { workingSet, trace };
+          }
 
-        if (this._ctx.disposed) {
-          return { workingSet, trace };
-        }
+          // Load documents from the results.
+          const documentLoadStart = performance.now();
 
-        const documentLoadStart = performance.now();
-        const results = await this._loadDocumentsAfterIndexQuery(indexHits);
-        trace.documentsLoaded += results.length;
-        trace.documentLoadTime += performance.now() - documentLoadStart;
+          // Separate queue items from space items.
+          const queueResults = textResults.filter((r) => r.queueId);
+          const spaceResults = textResults.filter((r) => !r.queueId);
 
-        workingSet.push(...results.filter(isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
-        trace.objectCount = workingSet.length;
+          // Build a map from recordId to rank for all FTS results.
+          const rankMap = new Map(textResults.map((r) => [r.recordId, r.rank]));
+
+          // Load queue items from indexed snapshots.
+          let queueItems: QueryItem[] = [];
+          if (queueResults.length > 0) {
+            const snapshots = await this._runInRuntime(
+              this._indexer2.querySnapshotsJSON(queueResults.map((r) => r.recordId)),
+            );
+            const snapshotMap = new Map(snapshots.map((s) => [s.recordId, s.snapshot]));
+            queueItems = queueResults
+              .map((result): QueryItem | null => {
+                const snapshot = snapshotMap.get(result.recordId);
+                if (!snapshot || typeof snapshot !== 'object') {
+                  return null;
+                }
+                return {
+                  objectId: result.objectId as ObjectId,
+                  spaceId: result.spaceId as SpaceId,
+                  queueId: result.queueId as ObjectId,
+                  queueNamespace: 'data',
+                  documentId: null,
+                  doc: null,
+                  data: snapshot as Obj.JSON,
+                  rank: rankMap.get(result.recordId) ?? 1,
+                };
+              })
+              .filter(isNonNullable);
+          }
+
+          // Load space items from documents.
+          const spaceItems = await Promise.all(
+            spaceResults.map(async (result): Promise<QueryItem | null> => {
+              const dxn = DXN.fromLocalObjectId(result.objectId);
+              const item = await this._loadFromDXN(dxn, { sourceSpaceId: result.spaceId as SpaceId });
+              if (item) {
+                // Override the default rank with the FTS rank.
+                item.rank = rankMap.get(result.recordId) ?? 1;
+              }
+              return item;
+            }),
+          );
+
+          const items = [...queueItems, ...spaceItems.filter(isNonNullable)];
+          trace.documentsLoaded += items.length;
+          trace.documentLoadTime += performance.now() - documentLoadStart;
+
+          workingSet.push(...items.filter((item) => step.spaces.includes(item.spaceId)));
+          trace.objectCount = workingSet.length;
+        } else {
+          // Fall back to old indexer for vector search.
+          const beginIndexQuery = performance.now();
+          const indexHits = await this._indexer.execQuery({
+            typenames: [],
+            text: {
+              query: step.selector.text,
+              kind: Match.type<QueryPlan.TextSearchKind>().pipe(
+                Match.withReturnType<'text' | 'vector'>(),
+                Match.when('full-text', () => 'text'),
+                Match.when('vector', () => 'vector'),
+                Match.orElseAbsurd,
+              )(step.selector.searchKind),
+            },
+          });
+          trace.indexHits = +indexHits.length;
+          trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+          if (this._ctx.disposed) {
+            return { workingSet, trace };
+          }
+
+          const documentLoadStart = performance.now();
+          const results = await this._loadDocumentsAfterIndexQuery(indexHits);
+          trace.documentsLoaded += results.length;
+          trace.documentLoadTime += performance.now() - documentLoadStart;
+
+          workingSet.push(...results.filter(isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
+          trace.objectCount = workingSet.length;
+        }
         break;
       }
 
@@ -405,17 +604,29 @@ export class QueryExecutor extends Resource {
         throw new Error(`Unknown selector type: ${(step.selector as any)._tag}`);
     }
 
+    // Apply limit if specified on the select step.
+    if (step.limit !== undefined && workingSet.length > step.limit) {
+      workingSet = workingSet.slice(0, step.limit);
+      trace.objectCount = workingSet.length;
+    }
+
     return { workingSet, trace };
   }
 
   private async _execFilterStep(step: QueryPlan.FilterStep, workingSet: QueryItem[]): Promise<StepExecutionResult> {
-    const result = workingSet.filter((item) =>
-      filterMatchObject(step.filter, {
-        id: item.objectId,
-        spaceId: item.spaceId,
-        doc: item.doc,
-      }),
-    );
+    const result = workingSet.filter((item) => {
+      if (item.doc) {
+        return filterMatchObject(step.filter, {
+          id: item.objectId,
+          spaceId: item.spaceId,
+          doc: item.doc,
+        });
+      } else if (item.data) {
+        return filterMatchObjectJSON(step.filter, item.data);
+      } else {
+        return false;
+      }
+    });
 
     return {
       workingSet: result,
@@ -433,7 +644,8 @@ export class QueryExecutor extends Resource {
     workingSet: QueryItem[],
   ): Promise<StepExecutionResult> {
     const expected = step.mode === 'only-deleted';
-    const result = workingSet.filter((item) => ObjectStructure.isDeleted(item.doc) === expected);
+    // TODO(dmaretskyi): How do we handle items with parents and cascade deletions? -- perhaps we forbid queue items from having parents -- i.e. queue is their parent.
+    const result = workingSet.filter((item) => QueryItem.isDeleted(item) === expected);
     return {
       workingSet: result,
       trace: {
@@ -459,11 +671,12 @@ export class QueryExecutor extends Resource {
       case 'ReferenceTraversal': {
         switch (step.traversal.direction) {
           case 'outgoing': {
+            invariant(step.traversal.property !== null, 'Outgoing reference traversal requires a property');
             const property = EscapedPropPath.unescape(step.traversal.property);
 
             const refs = workingSet
               .flatMap((item) => {
-                const ref = getDeep(item.doc.data, property);
+                const ref = QueryItem.getProperty(item, property);
                 const refs = Array.isArray(ref) ? ref : [ref];
                 return refs.map((ref) => {
                   try {
@@ -493,24 +706,39 @@ export class QueryExecutor extends Resource {
             break;
           }
           case 'incoming': {
-            const indexHits = await this._indexer.execQuery({
-              typenames: [],
-              inverted: false,
-              graph: {
-                kind: 'inbound-reference',
-                property: step.traversal.property,
-                anchors: workingSet.map((item) => item.objectId),
-              },
-            });
-            trace.indexHits += indexHits.length;
-
-            const documentLoadStart = performance.now();
-            const results = await this._loadDocumentsAfterIndexQuery(indexHits);
-            trace.documentsLoaded += results.length;
-            trace.documentLoadTime += performance.now() - documentLoadStart;
-
-            newWorkingSet.push(...results.filter(isNonNullable));
-            trace.objectCount = newWorkingSet.length;
+            if (this._supportsSqlIndexing()) {
+              const beginIndexQuery = performance.now();
+              const metas = await this._queryIncomingReferencesFromSqlIndex(workingSet, step.traversal.property);
+              trace.indexHits += metas.length;
+              trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+              const documentLoadStart = performance.now();
+              const results = await this._loadDocumentsAfterSqlQuery(metas);
+              trace.documentsLoaded += results.length;
+              trace.documentLoadTime += performance.now() - documentLoadStart;
+
+              newWorkingSet.push(...results.filter(isNonNullable));
+              trace.objectCount = newWorkingSet.length;
+            } else {
+              const indexHits = await this._indexer.execQuery({
+                typenames: [],
+                inverted: false,
+                graph: {
+                  kind: 'inbound-reference',
+                  property: step.traversal.property,
+                  anchors: workingSet.map((item) => item.objectId),
+                },
+              });
+              trace.indexHits += indexHits.length;
+
+              const documentLoadStart = performance.now();
+              const results = await this._loadDocumentsAfterIndexQuery(indexHits);
+              trace.documentsLoaded += results.length;
+              trace.documentLoadTime += performance.now() - documentLoadStart;
+
+              newWorkingSet.push(...results.filter(isNonNullable));
+              trace.objectCount = newWorkingSet.length;
+            }
 
             break;
           }
@@ -523,21 +751,17 @@ export class QueryExecutor extends Resource {
           case 'relation-to-target': {
             const refs = workingSet
               .map((item) => {
-                const ref =
+                const dxn =
                   step.traversal.direction === 'relation-to-source'
-                    ? ObjectStructure.getRelationSource(item.doc)
-                    : ObjectStructure.getRelationTarget(item.doc);
-
-                if (!isEncodedReference(ref)) {
-                  return null;
-                }
+                    ? QueryItem.getRelationSource(item)
+                    : QueryItem.getRelationTarget(item);
                 try {
                   return {
-                    ref: DXN.parse(ref['/']),
+                    ref: DXN.parse(dxn),
                     spaceId: item.spaceId,
                   };
                 } catch {
-                  log.warn('invalid reference', { ref: ref['/'] });
+                  log.warn('invalid reference', { ref: dxn });
                   return null;
                 }
               })
@@ -557,25 +781,43 @@ export class QueryExecutor extends Resource {
 
           case 'source-to-relation':
           case 'target-to-relation': {
-            const indexHits = await this._indexer.execQuery({
-              typenames: [],
-              inverted: false,
-              graph: {
-                kind: step.traversal.direction === 'source-to-relation' ? 'relation-source' : 'relation-target',
-                anchors: workingSet.map((item) => item.objectId),
-                property: null,
-              },
-            });
-
-            trace.indexHits += indexHits.length;
-
-            const documentLoadStart = performance.now();
-            const results = await this._loadDocumentsAfterIndexQuery(indexHits);
-            trace.documentsLoaded += results.length;
-            trace.documentLoadTime += performance.now() - documentLoadStart;
-
-            newWorkingSet.push(...results.filter(isNonNullable));
-            trace.objectCount = newWorkingSet.length;
+            if (this._supportsSqlIndexing()) {
+              const beginIndexQuery = performance.now();
+              const metas = await this._queryRelationsFromSqlIndex(
+                workingSet,
+                step.traversal.direction === 'source-to-relation' ? 'source' : 'target',
+              );
+              trace.indexHits += metas.length;
+              trace.indexQueryTime += performance.now() - beginIndexQuery;
+
+              const documentLoadStart = performance.now();
+              const results = await this._loadDocumentsAfterSqlQuery(metas);
+              trace.documentsLoaded += results.length;
+              trace.documentLoadTime += performance.now() - documentLoadStart;
+
+              newWorkingSet.push(...results.filter(isNonNullable));
+              trace.objectCount = newWorkingSet.length;
+            } else {
+              const indexHits = await this._indexer.execQuery({
+                typenames: [],
+                inverted: false,
+                graph: {
+                  kind: step.traversal.direction === 'source-to-relation' ? 'relation-source' : 'relation-target',
+                  anchors: workingSet.map((item) => item.objectId),
+                  property: null,
+                },
+              });
+
+              trace.indexHits += indexHits.length;
+
+              const documentLoadStart = performance.now();
+              const results = await this._loadDocumentsAfterIndexQuery(indexHits);
+              trace.documentsLoaded += results.length;
+              trace.documentLoadTime += performance.now() - documentLoadStart;
+
+              newWorkingSet.push(...results.filter(isNonNullable));
+              trace.objectCount = newWorkingSet.length;
+            }
 
             break;
           }
@@ -637,19 +879,38 @@ export class QueryExecutor extends Resource {
   }
 
   private async _execOrderStep(step: QueryPlan.OrderStep, workingSet: QueryItem[]): Promise<StepExecutionResult> {
-    const sortedWorkingSet = [...workingSet].sort((a, b) => this._compareMultiOrder(a, b, step.order));
+    let sortedWorkingSet = [...workingSet].sort((a, b) => this._compareMultiOrder(a, b, step.order));
+
+    // Apply limit if specified on the order step.
+    if (step.limit !== undefined && sortedWorkingSet.length > step.limit) {
+      sortedWorkingSet = sortedWorkingSet.slice(0, step.limit);
+    }
 
     return {
       workingSet: sortedWorkingSet,
       trace: {
         ...ExecutionTrace.makeEmpty(),
         name: 'Order',
-        details: JSON.stringify(step.order),
+        details: JSON.stringify({ order: step.order, limit: step.limit }),
         objectCount: sortedWorkingSet.length,
       },
     };
   }
 
+  private async _execLimitStep(step: QueryPlan.LimitStep, workingSet: QueryItem[]): Promise<StepExecutionResult> {
+    const limitedWorkingSet = workingSet.slice(0, step.limit);
+
+    return {
+      workingSet: limitedWorkingSet,
+      trace: {
+        ...ExecutionTrace.makeEmpty(),
+        name: 'Limit',
+        details: JSON.stringify({ limit: step.limit }),
+        objectCount: limitedWorkingSet.length,
+      },
+    };
+  }
+
   private _compareMultiOrder(a: QueryItem, b: QueryItem, orders: readonly QueryAST.Order[]): number {
     // Short circuit for common cases.
     if (orders.length === 0) {
@@ -675,15 +936,20 @@ export class QueryExecutor extends Resource {
         const comparison = this._compareByProperty(a, b, order.property);
         return order.direction === 'desc' ? -comparison : comparison;
       }
+      case 'rank': {
+        // Higher rank = better match. By default, descending order (best first).
+        const comparison = a.rank - b.rank;
+        return order.direction === 'desc' ? -comparison : comparison;
+      }
       default:
-        // Should never reach here with proper TypeScript types
+        // Should never reach here with proper TypeScript types.
         return 0;
     }
   }
 
   private _compareByProperty(a: QueryItem, b: QueryItem, property: string): number {
-    const aValue = a.doc.data[property];
-    const bValue = b.doc.data[property];
+    const aValue = QueryItem.getProperty(a, [property]);
+    const bValue = QueryItem.getProperty(b, [property]);
 
     // Both null or undefined
     if (aValue == null && bValue == null) {
@@ -727,6 +993,205 @@ export class QueryExecutor extends Resource {
     );
   }
 
+  private _supportsSqlIndexing(): boolean {
+    return !!this._indexer2 && !!this._runtime;
+  }
+
+  private async _runInRuntime<T>(effect: Effect.Effect<T, unknown, SqlClient.SqlClient>): Promise<T> {
+    const runtimeProvider = this._runtime;
+    invariant(runtimeProvider, 'SQL runtime is required.');
+    const runtime = await runAndForwardErrors(runtimeProvider);
+    return await unwrapExit(await effect.pipe(Runtime.runPromiseExit(runtime)));
+  }
+
+  private async _queryAllFromSqlIndex(spaceIds: readonly SpaceId[]): Promise<readonly ObjectMeta[]> {
+    invariant(this._indexer2, 'SQL indexer is required.');
+    return await this._runInRuntime(this._indexer2.queryAll({ spaceIds }));
+  }
+
+  private async _queryTypesFromSqlIndex(
+    spaceIds: readonly SpaceId[],
+    typeDxns: readonly string[],
+    inverted: boolean,
+  ): Promise<readonly ObjectMeta[]> {
+    invariant(this._indexer2, 'SQL indexer is required.');
+    return await this._runInRuntime(this._indexer2.queryTypes({ spaceIds, typeDxns, inverted }));
+  }
+
+  private async _queryIncomingReferencesFromSqlIndex(
+    workingSet: QueryItem[],
+    property: EscapedPropPath | null,
+  ): Promise<readonly ObjectMeta[]> {
+    const anchorDxns = workingSet.map((item) => DXN.fromLocalObjectId(item.objectId).toString());
+    invariant(this._indexer2, 'SQL indexer is required.');
+    const rows: readonly ReverseRef[] = (
+      await Promise.all(
+        anchorDxns.map((targetDxn) => this._runInRuntime(this._indexer2!.queryReverseRef({ targetDxn }))),
+      )
+    ).flat();
+
+    const recordIds = rows
+      .filter((row) => {
+        if (property === null) {
+          return true;
+        }
+
+        const queryPath = EscapedPropPath.unescape(property);
+        const rowPath = EscapedPropPath.unescape(row.propPath);
+        return QueryExecutor._matchesReferencePropertyPath(rowPath, queryPath);
+      })
+      .map((row) => row.recordId);
+
+    const uniqueRecordIds = Array.from(new Set<number>(recordIds));
+    return await this._runInRuntime(this._indexer2.lookupByRecordIds(uniqueRecordIds));
+  }
+
+  /**
+   * Matches a reverse-reference row path against a query property path.
+   * Allows numeric segments in the row path (array indices) that are not present in the query.
+   *
+   * Examples:
+   * - query: ['assignee'] matches row: ['assignee'] and ['assignee', '0'].
+   * - query: ['items', 'assignee'] matches row: ['items', '0', 'assignee'].
+   * - query: ['a', 'b'] does NOT match row: ['a'].
+   * - query: ['a'] does NOT match row: ['a', 'b'].
+   */
+  private static _matchesReferencePropertyPath(rowPath: readonly string[], queryPath: readonly string[]): boolean {
+    const isNumericSegment = (segment: string) => /^[0-9]+$/.test(segment);
+
+    let i = 0; // queryPath index.
+    let j = 0; // rowPath index.
+
+    while (i < queryPath.length && j < rowPath.length) {
+      if (rowPath[j] === queryPath[i]) {
+        i++;
+        j++;
+        continue;
+      }
+
+      // Row may contain array indices that aren't present in the query path.
+      if (isNumericSegment(rowPath[j])) {
+        j++;
+        continue;
+      }
+
+      return false;
+    }
+
+    // Must consume full query path.
+    if (i !== queryPath.length) {
+      return false;
+    }
+
+    // Any remaining row segments must be numeric (array indices).
+    for (; j < rowPath.length; j++) {
+      if (!isNumericSegment(rowPath[j])) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  private async _queryRelationsFromSqlIndex(
+    workingSet: QueryItem[],
+    endpoint: 'source' | 'target',
+  ): Promise<readonly ObjectMeta[]> {
+    invariant(this._indexer2, 'SQL indexer is required.');
+    const anchorDxns = workingSet.map((item) => DXN.fromLocalObjectId(item.objectId).toString());
+    return await this._runInRuntime(this._indexer2.queryRelations({ endpoint, anchorDxns }));
+  }
+
+  private async _loadDocumentsAfterSqlQuery(metas: readonly ObjectMeta[]): Promise<(QueryItem | null)[]> {
+    const snapshotMap = await this._loadQueueSnapshotMap(metas);
+    return await Promise.all(
+      metas.map(async (meta) => {
+        // Branch 1: Document-backed object.
+        if (meta.documentId) {
+          return this._loadFromAutomerge(meta);
+        }
+
+        // Branch 2: Queue-backed object.
+        if (meta.queueId) {
+          return this._loadFromQueue(meta, snapshotMap);
+        }
+
+        return null;
+      }),
+    );
+  }
+
+  private async _loadQueueSnapshotMap(metas: readonly ObjectMeta[]): Promise<Map<number, unknown>> {
+    const queueMetas = metas.filter((meta) => !meta.documentId && !!meta.queueId);
+    if (queueMetas.length === 0) {
+      return new Map();
+    }
+
+    invariant(this._indexer2, 'SQL indexer is required.');
+    const snapshots = await this._runInRuntime(
+      this._indexer2.querySnapshotsJSON(queueMetas.map((meta) => meta.recordId)),
+    );
+    return new Map(snapshots.map((s) => [s.recordId, s.snapshot]));
+  }
+
+  /**
+   * Loads a queue-backed object from an indexed snapshot (by `recordId`).
+   * Returns `null` when the snapshot is missing or is not a JSON object.
+   */
+  private _loadFromQueue(meta: ObjectMeta, snapshotMap: Map<number, unknown>): QueryItem | null {
+    const snapshot = snapshotMap.get(meta.recordId);
+    if (!snapshot || typeof snapshot !== 'object') {
+      return null;
+    }
+
+    return {
+      objectId: meta.objectId as ObjectId,
+      spaceId: meta.spaceId as SpaceId,
+      queueId: meta.queueId as ObjectId,
+      queueNamespace: 'data',
+      documentId: null,
+      doc: null,
+      data: snapshot as Obj.JSON,
+      rank: 1,
+    };
+  }
+
+  /**
+   * Loads a document-backed object from an Automerge `DatabaseDirectory`.
+   * Returns `null` if the document can't be loaded, the inline object isn't present, or if the meta does not have a
+   * document id (e.g. queue-backed objects).
+   */
+  private async _loadFromAutomerge(meta: ObjectMeta): Promise<QueryItem | null> {
+    if (!meta.documentId) {
+      return null;
+    }
+    const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(
+      Context.default(),
+      meta.documentId as DocumentId,
+      {
+        fetchFromNetwork: true,
+      },
+    );
+    const doc = handle.doc();
+    if (!doc) {
+      return null;
+    }
+    const object = DatabaseDirectory.getInlineObject(doc, meta.objectId);
+    if (!object) {
+      return null;
+    }
+    return {
+      objectId: meta.objectId,
+      documentId: meta.documentId as DocumentId,
+      spaceId: meta.spaceId as SpaceId,
+      queueId: null,
+      queueNamespace: null,
+      doc: object,
+      data: null,
+      rank: 1,
+    };
+  }
+
   /**
    * Space key hex -> SpaceId.
    */
@@ -735,7 +1200,9 @@ export class QueryExecutor extends Resource {
   private async _loadFromIndexHit(hit: FindResult): Promise<QueryItem | null> {
     const { objectId, documentId, spaceKey: spaceKeyInIndex } = objectPointerCodec.decode(hit.id);
 
-    const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(Context.default(), documentId as DocumentId);
+    const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(Context.default(), documentId as DocumentId, {
+      fetchFromNetwork: true,
+    });
     const doc = handle.doc();
     if (!doc) {
       return null;
@@ -761,7 +1228,11 @@ export class QueryExecutor extends Resource {
       objectId,
       documentId: documentId as DocumentId,
       spaceId,
+      queueId: null,
+      queueNamespace: null,
+      data: null,
       doc: object,
+      rank: hit.rank,
     };
   }
 
@@ -791,7 +1262,11 @@ export class QueryExecutor extends Resource {
         objectId: echoDxn.echoId,
         documentId: spaceRoot.documentId,
         spaceId,
+        queueId: null,
+        queueNamespace: null,
+        data: null,
         doc: inlineObject,
+        rank: 1,
       };
     }
 
@@ -800,7 +1275,9 @@ export class QueryExecutor extends Resource {
       return null;
     }
 
-    const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(Context.default(), link as AutomergeUrl);
+    const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(Context.default(), link as AutomergeUrl, {
+      fetchFromNetwork: true,
+    });
     const doc = handle.doc();
     if (!doc) {
       return null;
@@ -815,7 +1292,11 @@ export class QueryExecutor extends Resource {
       objectId: echoDxn.echoId,
       documentId: handle.documentId,
       spaceId,
+      queueId: null,
+      queueNamespace: null,
+      data: null,
       doc: object,
+      rank: 1,
     };
   }
 }