@dxos/echo-pipeline 0.8.2-staging.7ac8446 → 0.8.2

package/src/query/query-planner.ts

@@ -0,0 +1,470 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type QueryAST } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import type { DXN, SpaceId } from '@dxos/keys';
+
+import { QueryError } from './errors';
+import { QueryPlan } from './plan';
+
+export type QueryPlannerOptions = {
+  defaultTextSearchKind: QueryPlan.TextSearchKind;
+};
+
+const DEFAULT_OPTIONS: QueryPlannerOptions = {
+  defaultTextSearchKind: 'full-text',
+};
+
+/**
+ * Constructs an optimized query plan.
+ */
+// TODO(dmaretskyi): Implement inefficient versions of complex queries.
+export class QueryPlanner {
+  private readonly _options: QueryPlannerOptions;
+
+  constructor(options?: Partial<QueryPlannerOptions>) {
+    this._options = {
+      ...DEFAULT_OPTIONS,
+      ...options,
+    };
+  }
+
+  createPlan(query: QueryAST.Query): QueryPlan.Plan {
+    let plan = this._generate(query, { ...DEFAULT_CONTEXT, originalQuery: query });
+    plan = this._optimizeEmptyFilters(plan);
+    plan = this._optimizeSoloUnions(plan);
+    return plan;
+  }
+
+  private _generate(query: QueryAST.Query, context: GenerationContext): QueryPlan.Plan {
+    switch (query.type) {
+      case 'options':
+        return this._generateOptionsClause(query, context);
+      case 'select':
+        return this._generateSelectClause(query, context);
+      case 'filter':
+        return this._generateFilterClause(query, context);
+      case 'incoming-references':
+        return this._generateIncomingReferencesClause(query, context);
+      case 'relation':
+        return this._generateRelationClause(query, context);
+      case 'relation-traversal':
+        return this._generateRelationTraversalClause(query, context);
+      case 'reference-traversal':
+        return this._generateReferenceTraversalClause(query, context);
+      case 'union':
+        return this._generateUnionClause(query, context);
+      default:
+        throw new QueryError(`Unsupported query type: ${(query as any).type}`, {
+          context: { query: context.originalQuery },
+        });
+    }
+  }
+
+  private _generateOptionsClause(query: QueryAST.QueryOptionsClause, context: GenerationContext): QueryPlan.Plan {
+    const newContext = {
+      ...context,
+    };
+    if (query.options.spaceIds) {
+      newContext.selectionSpaces = query.options.spaceIds as readonly SpaceId[];
+    }
+    if (query.options.deleted) {
+      newContext.deletedHandling = query.options.deleted;
+    }
+    return this._generate(query.query, newContext);
+  }
+
+  private _generateSelectClause(query: QueryAST.QuerySelectClause, context: GenerationContext): QueryPlan.Plan {
+    return this._generateSelectionFromFilter(query.filter, context);
+  }
+
+  // TODO(dmaretskyi): This can be rewritten as a function of (filter[]) -> (selection ? undefined, rest: filter[]) that recurses onto itself.
+  // TODO(dmaretskyi): If the tip of the query ast is a [select, ...filter] shape we can reorder the filters so the query is most efficient.
+  private _generateSelectionFromFilter(filter: QueryAST.Filter, context: GenerationContext): QueryPlan.Plan {
+    switch (filter.type) {
+      case 'object': {
+        if (
+          context.selectionInverted &&
+          filter.id === undefined &&
+          filter.typename === null &&
+          Object.keys(filter.props).length === 0
+        ) {
+          // filter of nothing -> clear working set.
+          return QueryPlan.Plan.make([
+            {
+              _tag: 'ClearWorkingSetStep',
+            },
+            ...this._generateDeletedHandlingSteps(context),
+          ]);
+        }
+        if (context.selectionInverted) {
+          throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        }
+
+        // Try to utilize indexes during selection, prioritizing selecting by id, then by typename.
+        // After selection, filter out using the remaining predicates.
+        if (filter.id && filter.id?.length > 0) {
+          return QueryPlan.Plan.make([
+            {
+              _tag: 'SelectStep',
+              spaces: context.selectionSpaces,
+              selector: {
+                _tag: 'IdSelector',
+                objectIds: filter.id,
+              },
+            },
+            ...this._generateDeletedHandlingSteps(context),
+            {
+              _tag: 'FilterStep',
+              filter: { ...filter, id: undefined },
+            },
+          ]);
+        } else if (filter.typename) {
+          return QueryPlan.Plan.make([
+            {
+              _tag: 'SelectStep',
+              spaces: context.selectionSpaces,
+              selector: {
+                _tag: 'TypeSelector',
+                typename: [filter.typename as DXN.String],
+                inverted: false,
+              },
+            },
+            ...this._generateDeletedHandlingSteps(context),
+            {
+              _tag: 'FilterStep',
+              filter: { ...filter, typename: null },
+            },
+          ]);
+        } else {
+          return QueryPlan.Plan.make([
+            {
+              _tag: 'SelectStep',
+              spaces: context.selectionSpaces,
+              selector: {
+                _tag: 'WildcardSelector',
+              },
+            },
+            ...this._generateDeletedHandlingSteps(context),
+            {
+              _tag: 'FilterStep',
+              filter: { ...filter },
+            },
+          ]);
+        }
+      }
+      case 'text-search': {
+        return QueryPlan.Plan.make([
+          {
+            _tag: 'SelectStep',
+            spaces: context.selectionSpaces,
+            selector: {
+              _tag: 'TextSelector',
+              text: filter.text,
+              searchKind: filter.searchKind ?? this._options.defaultTextSearchKind,
+            },
+          },
+          ...this._generateDeletedHandlingSteps(context),
+        ]);
+      }
+      case 'compare':
+        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+      case 'in':
+        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+      case 'range':
+        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+      case 'not':
+        return this._generateSelectionFromFilter(filter.filter, {
+          ...context,
+          selectionInverted: !context.selectionInverted,
+        });
+      case 'and':
+        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+      case 'or':
+        // Optimized case
+        if (filter.filters.every(isTrivialTypenameFilter)) {
+          const typenames = filter.filters.map((f) => {
+            invariant(f.type === 'object' && f.typename !== null);
+            return f.typename;
+          });
+          return QueryPlan.Plan.make([
+            {
+              _tag: 'SelectStep',
+              spaces: context.selectionSpaces,
+              selector: {
+                _tag: 'TypeSelector',
+                typename: typenames as DXN.String[],
+                inverted: context.selectionInverted,
+              },
+            },
+            ...this._generateDeletedHandlingSteps(context),
+          ]);
+        } else {
+          throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        }
+
+      default:
+        throw new QueryError(`Unsupported filter type: ${(filter as any).type}`, {
+          context: { query: context.originalQuery },
+        });
+    }
+  }
+
+  private _generateDeletedHandlingSteps(context: GenerationContext): QueryPlan.Step[] {
+    switch (context.deletedHandling) {
+      case 'include':
+        return [];
+      case 'exclude':
+        return [
+          {
+            _tag: 'FilterDeletedStep',
+            mode: 'only-non-deleted',
+          },
+        ];
+      case 'only':
+        return [
+          {
+            _tag: 'FilterDeletedStep',
+            mode: 'only-deleted',
+          },
+        ];
+    }
+  }
+
+  private _generateUnionClause(query: QueryAST.QueryUnionClause, context: GenerationContext): QueryPlan.Plan {
+    return QueryPlan.Plan.make([
+      {
+        _tag: 'UnionStep',
+        plans: query.queries.map((query) => this._generate(query, context)),
+      },
+    ]);
+  }
+
+  private _generateReferenceTraversalClause(
+    query: QueryAST.QueryReferenceTraversalClause,
+    context: GenerationContext,
+  ): QueryPlan.Plan {
+    return QueryPlan.Plan.make([
+      ...this._generate(query.anchor, context).steps,
+      {
+        _tag: 'TraverseStep',
+        traversal: {
+          _tag: 'ReferenceTraversal',
+          direction: 'outgoing',
+          property: query.property,
+        },
+      },
+      ...this._generateDeletedHandlingSteps(context),
+    ]);
+  }
+
+  private _generateIncomingReferencesClause(
+    query: QueryAST.QueryIncomingReferencesClause,
+    context: GenerationContext,
+  ): QueryPlan.Plan {
+    return QueryPlan.Plan.make([
+      ...this._generate(query.anchor, context).steps,
+      {
+        _tag: 'TraverseStep',
+        traversal: {
+          _tag: 'ReferenceTraversal',
+          direction: 'incoming',
+          property: query.property,
+        },
+      },
+      ...this._generateDeletedHandlingSteps(context),
+      {
+        _tag: 'FilterStep',
+        filter: {
+          type: 'object',
+          typename: query.typename,
+          props: {},
+        },
+      },
+    ]);
+  }
+
+  private _generateRelationTraversalClause(
+    query: QueryAST.QueryRelationTraversalClause,
+    context: GenerationContext,
+  ): QueryPlan.Plan {
+    switch (query.direction) {
+      case 'source': {
+        return QueryPlan.Plan.make([
+          ...this._generate(query.anchor, context).steps,
+          createRelationTraversalStep('relation-to-source'),
+          ...this._generateDeletedHandlingSteps(context),
+        ]);
+      }
+      case 'target': {
+        return QueryPlan.Plan.make([
+          ...this._generate(query.anchor, context).steps,
+          createRelationTraversalStep('relation-to-target'),
+          ...this._generateDeletedHandlingSteps(context),
+        ]);
+      }
+      case 'both': {
+        const anchorPlan = this._generate(query.anchor, context);
+        return QueryPlan.Plan.make([
+          ...anchorPlan.steps,
+          {
+            _tag: 'UnionStep',
+            plans: [
+              QueryPlan.Plan.make([createRelationTraversalStep('relation-to-source')]),
+              QueryPlan.Plan.make([createRelationTraversalStep('relation-to-target')]),
+            ],
+          },
+          ...this._generateDeletedHandlingSteps(context),
+        ]);
+      }
+    }
+  }
+
+  private _generateRelationClause(query: QueryAST.QueryRelationClause, context: GenerationContext): QueryPlan.Plan {
+    switch (query.direction) {
+      case 'outgoing': {
+        return QueryPlan.Plan.make([
+          ...this._generate(query.anchor, context).steps,
+          createRelationTraversalStep('source-to-relation'),
+          ...this._generateDeletedHandlingSteps(context),
+          {
+            _tag: 'FilterStep',
+            filter: query.filter ?? NOOP_FILTER,
+          },
+        ]);
+      }
+      case 'incoming': {
+        return QueryPlan.Plan.make([
+          ...this._generate(query.anchor, context).steps,
+          createRelationTraversalStep('target-to-relation'),
+          ...this._generateDeletedHandlingSteps(context),
+          {
+            _tag: 'FilterStep',
+            filter: query.filter ?? NOOP_FILTER,
+          },
+        ]);
+      }
+      case 'both': {
+        const anchorPlan = this._generate(query.anchor, context);
+        return QueryPlan.Plan.make([
+          ...anchorPlan.steps,
+          {
+            _tag: 'UnionStep',
+            plans: [
+              QueryPlan.Plan.make([createRelationTraversalStep('source-to-relation')]),
+              QueryPlan.Plan.make([createRelationTraversalStep('target-to-relation')]),
+            ],
+          },
+          ...this._generateDeletedHandlingSteps(context),
+          {
+            _tag: 'FilterStep',
+            filter: query.filter ?? NOOP_FILTER,
+          },
+        ]);
+      }
+    }
+  }
+
+  private _generateFilterClause(query: QueryAST.QueryFilterClause, context: GenerationContext): QueryPlan.Plan {
+    return QueryPlan.Plan.make([
+      ...this._generate(query.selection, context).steps,
+      {
+        _tag: 'FilterStep',
+        filter: query.filter,
+      },
+    ]);
+  }
+
+  /**
+   * Removes filter steps that have no predicates.
+   */
+  private _optimizeEmptyFilters(plan: QueryPlan.Plan): QueryPlan.Plan {
+    return QueryPlan.Plan.make(
+      plan.steps
+        .filter((step) => {
+          if (step._tag === 'FilterStep') {
+            return !QueryPlan.FilterStep.isNoop(step);
+          } else {
+            return true;
+          }
+        })
+        .map((step) => {
+          if (step._tag === 'UnionStep') {
+            return {
+              _tag: 'UnionStep',
+              plans: step.plans.map((plan) => this._optimizeEmptyFilters(plan)),
+            };
+          } else {
+            return step;
+          }
+        }),
+    );
+  }
+
+  /**
+   * Removes union steps that have only one child.
+   */
+  private _optimizeSoloUnions(plan: QueryPlan.Plan): QueryPlan.Plan {
+    // TODO(dmaretskyi): Implement this.
+    return plan;
+  }
+}
+
+/**
+ * Context for query planning.
+ */
+type GenerationContext = {
+  /**
+   * The original query.
+   */
+  originalQuery: QueryAST.Query | null;
+
+  /**
+   * Which spaces to select from.
+   */
+  selectionSpaces: readonly SpaceId[];
+
+  /**
+   * How to handle deleted objects.
+   */
+  deletedHandling: 'include' | 'exclude' | 'only';
+
+  /**
+   * When generating a selection clause, whether to invert the filter.
+   */
+  selectionInverted: boolean;
+};
+
+const DEFAULT_CONTEXT: GenerationContext = {
+  originalQuery: null,
+  selectionSpaces: [],
+  deletedHandling: 'exclude',
+  selectionInverted: false,
+};
+
+const NOOP_FILTER: QueryAST.Filter = {
+  type: 'object',
+  typename: null,
+  id: [],
+  props: {},
+};
+
+const createRelationTraversalStep = (direction: QueryPlan.RelationTraversal['direction']): QueryPlan.Step => ({
+  _tag: 'TraverseStep',
+  traversal: {
+    _tag: 'RelationTraversal',
+    direction,
+  },
+});
+
+const isTrivialTypenameFilter = (filter: QueryAST.Filter): boolean => {
+  return (
+    filter.type === 'object' &&
+    filter.typename !== null &&
+    Object.keys(filter.props).length === 0 &&
+    (filter.id === undefined || filter.id.length === 0) &&
+    (filter.foreignKeys === undefined || filter.foreignKeys.length === 0)
+  );
+};

package/src/space/admission-discovery-extension.ts

@@ -40,7 +40,7 @@ export class CredentialRetrieverExtension extends RpcExtension<
     return {};
   }
 
-  override async onOpen(context: ExtensionContext) {
+  override async onOpen(context: ExtensionContext): Promise<void> {
     await super.onOpen(context);
     scheduleTask(this._ctx, async () => {
       try {
@@ -52,7 +52,7 @@ export class CredentialRetrieverExtension extends RpcExtension<
     });
   }
 
-  override async onClose() {
+  override async onClose(): Promise<void> {
     await this._ctx.dispose();
   }
 

package/src/space/control-pipeline.ts

@@ -112,13 +112,13 @@ export class ControlPipeline {
     return this._pipeline;
   }
 
-  async setWriteFeed(feed: FeedWrapper<FeedMessage>) {
+  async setWriteFeed(feed: FeedWrapper<FeedMessage>): Promise<void> {
     await this._pipeline.addFeed(feed);
     this._pipeline.setWriteFeed(feed);
   }
 
   @trace.span({ showInBrowserTimeline: true })
-  async start() {
+  async start(): Promise<void> {
     const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
     log('load snapshot', { key: this._spaceKey, present: !!snapshot, tf: snapshot?.timeframe });
     if (USE_SNAPSHOTS && snapshot) {
@@ -134,7 +134,7 @@ export class ControlPipeline {
     log('started');
   }
 
-  private async _processSnapshot(snapshot: ControlPipelineSnapshot) {
+  private async _processSnapshot(snapshot: ControlPipelineSnapshot): Promise<void> {
     await this._pipeline.setCursor(snapshot.timeframe);
 
     for (const message of snapshot.messages ?? []) {
@@ -149,7 +149,7 @@ export class ControlPipeline {
     }
   }
 
-  private async _saveSnapshot() {
+  private async _saveSnapshot(): Promise<void> {
     await this._pipeline.pause();
     const snapshot: ControlPipelineSnapshot = {
       timeframe: this._pipeline.state.timeframe,
@@ -165,7 +165,7 @@ export class ControlPipeline {
   }
 
   @trace.span()
-  private async _consumePipeline(ctx: Context) {
+  private async _consumePipeline(ctx: Context): Promise<void> {
     for await (const msg of this._pipeline.consume()) {
       const span = this._usage.beginRecording();
       this._mutations.inc();
@@ -200,7 +200,7 @@ export class ControlPipeline {
     }
   }
 
-  private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
+  private async _noteTargetStateIfNeeded(timeframe: Timeframe): Promise<void> {
     // TODO(dmaretskyi): Replace this with a proper debounce/throttle.
 
     if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
@@ -210,7 +210,7 @@ export class ControlPipeline {
     }
   }
 
-  async stop() {
+  async stop(): Promise<void> {
     log('stopping...');
     await this._ctx.dispose();
     await this._pipeline.stop();
@@ -218,7 +218,7 @@ export class ControlPipeline {
     log('stopped');
   }
 
-  private async _saveTargetTimeframe(timeframe: Timeframe) {
+  private async _saveTargetTimeframe(timeframe: Timeframe): Promise<void> {
     try {
       const newTimeframe = Timeframe.merge(this._targetTimeframe ?? new Timeframe(), timeframe);
       await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);

package/src/space/space-manager.ts

@@ -2,8 +2,9 @@
 // Copyright 2022 DXOS.org
 //
 
+import { type AutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
+
 import { synchronized, trackLeaks, Trigger } from '@dxos/async';
-import { type AutomergeUrl, parseAutomergeUrl } from '@dxos/automerge/automerge-repo';
 import { getCredentialAssertion, type DelegateInvitationCredential, type MemberInfo } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { type FeedStore } from '@dxos/feed-store';
@@ -82,10 +83,10 @@ export class SpaceManager {
   }
 
   @synchronized
-  async open() {}
+  async open(): Promise<void> {}
 
   @synchronized
-  async close() {
+  async close(): Promise<void> {
     await Promise.all([...this._spaces.values()].map((space) => space.close()));
   }
 
@@ -97,7 +98,7 @@ export class SpaceManager {
     onDelegatedInvitationStatusChange,
     onMemberRolesChanged,
     memberKey,
-  }: ConstructSpaceParams) {
+  }: ConstructSpaceParams): Promise<Space> {
     log.trace('dxos.echo.space-manager.construct-space', trace.begin({ id: this._instanceId }));
     log('constructing space...', { spaceKey: metadata.genesisFeedKey });
 

package/src/space/space-protocol.browser.test.ts

@@ -17,6 +17,7 @@ const port = process.env.SIGNAL_PORT ?? 4000;
 const SIGNAL_URL = `ws://localhost:${port}/.well-known/dx/signal`;
 
 describe('space/space-protocol', () => {
+  // TODO(dmaretskyi): Fails with the vscode test-runner for some reason.
   test('two peers discover each other', async () => {
     const builder = new TestAgentBuilder();
     onTestFinished(async () => {

package/src/space/space-protocol.test.ts

@@ -15,6 +15,7 @@ import { AuthStatus, MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, SpaceProtocol } fro
 import { TestAgentBuilder, TestFeedBuilder } from '../testing';
 
 describe('space/space-protocol', () => {
+  // Flaky.
   test('two peers discover each other via presence', async () => {
     const builder = new TestAgentBuilder();
     onTestFinished(async () => {

package/src/space/space-protocol.ts

@@ -124,7 +124,7 @@ export class SpaceProtocol {
   }
 
   // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
-  async addFeed(feed: FeedWrapper<FeedMessage>) {
+  async addFeed(feed: FeedWrapper<FeedMessage>): Promise<void> {
     log('addFeed', { key: feed.key });
 
     this._feeds.add(feed);
@@ -136,7 +136,7 @@ export class SpaceProtocol {
   }
 
   // TODO(burdon): Rename open? Common open/close interfaces for all services?
-  async start() {
+  async start(): Promise<void> {
     if (this._connection) {
       return;
     }
@@ -158,11 +158,11 @@ export class SpaceProtocol {
     log('started');
   }
 
-  public updateTopology() {
+  public updateTopology(): void {
     this._topology.forceUpdate();
   }
 
-  async stop() {
+  async stop(): Promise<void> {
     await this.blobSync.close();
 
     if (this._connection) {

package/src/space/space.ts

@@ -151,14 +151,14 @@ export class Space extends Resource {
     return this._controlPipeline.pipeline;
   }
 
-  async setControlFeed(feed: FeedWrapper<FeedMessage>) {
+  async setControlFeed(feed: FeedWrapper<FeedMessage>): Promise<this> {
     invariant(!this._controlFeed, 'Control feed already set.');
     this._controlFeed = feed;
     await this._controlPipeline.setWriteFeed(feed);
     return this;
   }
 
-  async setDataFeed(feed: FeedWrapper<FeedMessage>) {
+  async setDataFeed(feed: FeedWrapper<FeedMessage>): Promise<this> {
     invariant(!this._dataFeed, 'Data feed already set.');
     this._dataFeed = feed;
     return this;
@@ -172,7 +172,7 @@ export class Space extends Resource {
   }
 
   @trace.span()
-  protected override async _open(ctx: Context) {
+  protected override async _open(ctx: Context): Promise<void> {
     log('opening...');
 
     // Order is important.
@@ -182,14 +182,14 @@ export class Space extends Resource {
   }
 
   @synchronized
-  public async startProtocol() {
+  public async startProtocol(): Promise<void> {
     invariant(this.isOpen);
     await this.protocol.start();
     await this.protocol.addFeed(await this._feedProvider(this._genesisFeedKey));
   }
 
   @synchronized
-  protected override async _close() {
+  protected override async _close(): Promise<void> {
     log('closing...', { key: this._key });
 
     // Closes in reverse order to open.

package/src/testing/index.ts

@@ -7,3 +7,5 @@ export * from './test-agent-builder';
 export * from './test-feed-builder';
 export * from './test-network-adapter';
 export * from './test-replicator';
+export * as TestSchema from './test-schema';
+export * as TestData from './test-data';