@dxos/echo-pipeline 0.8.4-main.84f28bd → 0.8.4-main.a4bbb77

package/src/query/query-executor.ts

@@ -3,10 +3,10 @@
 //
 
 import type { AutomergeUrl, DocumentId } from '@automerge/automerge-repo';
-import { Match } from 'effect';
+import { Match, Predicate } from 'effect';
 
 import { Context, ContextDisposedError, LifecycleState, Resource } from '@dxos/context';
-import { DatabaseDirectory, isEncodedReference, ObjectStructure, type QueryAST } from '@dxos/echo-protocol';
+import { DatabaseDirectory, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
 import { EscapedPropPath, type FindResult, type Indexer } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { DXN, type ObjectId, PublicKey, type SpaceId } from '@dxos/keys';
@@ -15,13 +15,16 @@ import { objectPointerCodec } from '@dxos/protocols';
 import { type QueryReactivity, type QueryResult } from '@dxos/protocols/proto/dxos/echo/query';
 import { getDeep, isNonNullable } from '@dxos/util';
 
-import type { QueryPlan } from './plan';
-import { QueryPlanner } from './query-planner';
 import type { AutomergeHost } from '../automerge';
 import { createIdFromSpaceKey } from '../common';
 import type { SpaceStateManager } from '../db-host';
 import { filterMatchObject } from '../filter';
 
+import type { QueryPlan } from './plan';
+import { QueryPlanner } from './query-planner';
+
+const isNullable: Predicate.Refinement<unknown, null | undefined> = Predicate.isNullable;
+
 type QueryExecutorOptions = {
   indexer: Indexer;
   automergeHost: AutomergeHost;
@@ -189,7 +192,6 @@ export class QueryExecutor extends Resource {
       );
 
     if (TRACE_QUERY_EXECUTION) {
-      // eslint-disable-next-line no-console
       console.log(ExecutionTrace.format(trace));
     }
 
@@ -245,6 +247,9 @@ export class QueryExecutor extends Resource {
       case 'TraverseStep':
         ({ workingSet: newWorkingSet, trace } = await this._execTraverseStep(step, workingSet));
         break;
+      case 'OrderStep':
+        ({ workingSet: newWorkingSet, trace } = await this._execOrderStep(step, workingSet));
+        break;
       default:
         throw new Error(`Unknown step type: ${(step as any)._tag}`);
     }
@@ -286,6 +291,7 @@ export class QueryExecutor extends Resource {
 
         break;
       }
+
       case 'IdSelector': {
         const beginLoad = performance.now();
         const items = await Promise.all(
@@ -299,6 +305,7 @@ export class QueryExecutor extends Resource {
         trace.objectCount = workingSet.length;
         break;
       }
+
       case 'TypeSelector': {
         const beginIndexQuery = performance.now();
         const indexHits = await this._indexer.execQuery({
@@ -322,6 +329,7 @@ export class QueryExecutor extends Resource {
 
         break;
       }
+
       case 'TextSelector': {
         const beginIndexQuery = performance.now();
         const indexHits = await this._indexer.execQuery({
@@ -352,6 +360,7 @@ export class QueryExecutor extends Resource {
         trace.objectCount = workingSet.length;
         break;
       }
+
       default:
         throw new Error(`Unknown selector type: ${(step.selector as any)._tag}`);
     }
@@ -367,6 +376,7 @@ export class QueryExecutor extends Resource {
         doc: item.doc,
       }),
     );
+
     return {
       workingSet: result,
       trace: {
@@ -382,10 +392,6 @@ export class QueryExecutor extends Resource {
     step: QueryPlan.FilterDeletedStep,
     workingSet: QueryItem[],
   ): Promise<StepExecutionResult> {
-    if (workingSet.length === 6) {
-      log.info('FilterDeletedStep', { step, workingSet });
-    }
-
     const expected = step.mode === 'only-deleted';
     const result = workingSet.filter((item) => ObjectStructure.isDeleted(item.doc) === expected);
     return {
@@ -428,7 +434,7 @@ export class QueryExecutor extends Resource {
                         }
                       : null;
                   } catch {
-                    log.warn('Invalid reference', { ref: ref['/'] });
+                    log.warn('invalid reference', { ref: ref['/'] });
                     return null;
                   }
                 });
@@ -491,7 +497,7 @@ export class QueryExecutor extends Resource {
                     spaceId: item.spaceId,
                   };
                 } catch {
-                  log.warn('Invalid reference', { ref: ref['/'] });
+                  log.warn('invalid reference', { ref: ref['/'] });
                   return null;
                 }
               })
@@ -590,6 +596,60 @@ export class QueryExecutor extends Resource {
     };
   }
 
+  private async _execOrderStep(step: QueryPlan.OrderStep, workingSet: QueryItem[]): Promise<StepExecutionResult> {
+    const compareItems = (a: QueryItem, b: QueryItem): number =>
+      step.order.reduce((comparison, order) => {
+        // If we already have a definitive result, return it
+        if (comparison !== 0) {
+          return comparison;
+        }
+
+        return this._compareByOrder(a, b, order);
+      }, 0);
+
+    const sortedWorkingSet = [...workingSet].sort(compareItems);
+
+    return {
+      workingSet: sortedWorkingSet,
+      trace: {
+        ...ExecutionTrace.makeEmpty(),
+        name: 'Order',
+        details: JSON.stringify(step.order),
+        objectCount: sortedWorkingSet.length,
+      },
+    };
+  }
+
+  private _compareByOrder(a: QueryItem, b: QueryItem, order: QueryAST.Order): number {
+    return Match.type<QueryAST.Order>().pipe(
+      Match.withReturnType<number>(),
+      Match.when({ kind: 'natural' }, () => a.objectId.localeCompare(b.objectId)),
+      Match.when({ kind: 'property' }, ({ property, direction }) => {
+        const comparison = this._compareByProperty(a, b, property);
+        return direction === 'desc' ? -comparison : comparison;
+      }),
+      Match.exhaustive,
+    )(order);
+  }
+
+  private _compareByProperty(a: QueryItem, b: QueryItem, property: string): number {
+    const aValue = a.doc.data[property];
+    const bValue = b.doc.data[property];
+
+    return Match.type<{ a: unknown; b: unknown }>().pipe(
+      Match.withReturnType<number>(),
+      Match.when({ a: isNullable, b: isNullable }, () => 0),
+      Match.when({ a: isNullable }, () => 1),
+      Match.when({ b: isNullable }, () => -1),
+      Match.when({ a: Match.string, b: Match.string }, ({ a, b }) => a.localeCompare(b)),
+      Match.when({ a: Match.number, b: Match.number }, ({ a, b }) => a - b),
+      Match.when({ a: Match.boolean, b: Match.boolean }, ({ a, b }) => (a === b ? 0 : a ? 1 : -1)),
+      Match.when({ a: Match.defined, b: Match.defined }, ({ a, b }) => String(a).localeCompare(String(b))),
+      // TODO(wittjosiah): Why does Match.exhaustive fail here?
+      Match.orElse(() => 0),
+    )({ a: aValue, b: bValue });
+  }
+
   private async _loadDocumentsAfterIndexQuery(indexHits: FindResult[]): Promise<(QueryItem | null)[]> {
     return Promise.all(
       indexHits.map(async (hit): Promise<QueryItem | null> => {

package/src/query/query-planner.test.ts

@@ -4,13 +4,14 @@
 
 import { describe, expect, test } from 'vitest';
 
-import { Filter, Query } from '@dxos/echo';
+import { Filter, Order, Query } from '@dxos/echo';
 import { type QueryAST } from '@dxos/echo-protocol';
 import { SpaceId } from '@dxos/keys';
 
-import { QueryPlanner } from './query-planner';
 import { TestSchema } from '../testing';
 
+import { QueryPlanner } from './query-planner';
+
 describe('QueryPlanner', () => {
   const planner = new QueryPlanner();
 
@@ -38,6 +39,53 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
+        ],
+      }
+    `);
+  });
+
+  test('get all people ordered by name', () => {
+    const query = Query.select(Filter.type(TestSchema.Person)).orderBy(Order.property('name', 'asc'));
+
+    const plan = planner.createPlan(withSpaceIdOptions(query.ast));
+    expect(plan).toMatchInlineSnapshot(`
+      {
+        "steps": [
+          {
+            "_tag": "SelectStep",
+            "selector": {
+              "_tag": "TypeSelector",
+              "inverted": false,
+              "typename": [
+                "dxn:type:dxos.org/type/Person:0.1.0",
+              ],
+            },
+            "spaces": [
+              "B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO",
+            ],
+          },
+          {
+            "_tag": "FilterDeletedStep",
+            "mode": "only-non-deleted",
+          },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "direction": "asc",
+                "kind": "property",
+                "property": "name",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -82,6 +130,14 @@ describe('QueryPlanner', () => {
               "typename": null,
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -158,6 +214,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -218,6 +282,14 @@ describe('QueryPlanner', () => {
               "typename": "dxn:type:dxos.org/type/Task:0.1.0",
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -316,6 +388,14 @@ describe('QueryPlanner', () => {
               "typename": "dxn:type:dxos.org/type/Task:0.1.0",
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -378,6 +458,14 @@ describe('QueryPlanner', () => {
               },
             ],
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -487,6 +575,14 @@ describe('QueryPlanner', () => {
               ],
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -543,6 +639,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -579,6 +683,14 @@ describe('QueryPlanner', () => {
               "typename": "dxn:type:dxos.org/type/Person:0.1.0",
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -606,6 +718,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -636,6 +756,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -676,6 +804,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -705,6 +841,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);

package/src/query/query-planner.ts

@@ -2,6 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
+import { Order } from '@dxos/echo';
 import { type QueryAST } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import type { DXN, SpaceId } from '@dxos/keys';
@@ -35,6 +36,7 @@ export class QueryPlanner {
     let plan = this._generate(query, { ...DEFAULT_CONTEXT, originalQuery: query });
     plan = this._optimizeEmptyFilters(plan);
     plan = this._optimizeSoloUnions(plan);
+    plan = this._ensureOrderStep(plan);
     return plan;
   }
 
@@ -58,8 +60,11 @@ export class QueryPlanner {
         return this._generateUnionClause(query, context);
       case 'set-difference':
         return this._generateSetDifferenceClause(query, context);
+      case 'order':
+        return this._generateOrderClause(query, context);
       default:
-        throw new QueryError(`Unsupported query type: ${(query as any).type}`, {
+        throw new QueryError({
+          message: `Unsupported query type: ${(query as any).type}`,
           context: { query: context.originalQuery },
         });
     }
@@ -86,6 +91,7 @@ export class QueryPlanner {
   // TODO(dmaretskyi): If the tip of the query ast is a [select, ...filter] shape we can reorder the filters so the query is most efficient.
   private _generateSelectionFromFilter(filter: QueryAST.Filter, context: GenerationContext): QueryPlan.Plan {
     switch (filter.type) {
+      // Props
       case 'object': {
         if (
           context.selectionInverted &&
@@ -102,7 +108,10 @@ export class QueryPlanner {
           ]);
         }
         if (context.selectionInverted) {
-          throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+          throw new QueryError({
+            message: 'Query too complex',
+            context: { query: context.originalQuery },
+          });
         }
 
         // Try to utilize indexes during selection, prioritizing selecting by id, then by typename.
@@ -157,6 +166,26 @@ export class QueryPlanner {
           ]);
         }
       }
+
+      // Tag
+      case 'tag': {
+        return QueryPlan.Plan.make([
+          {
+            _tag: 'SelectStep',
+            spaces: context.selectionSpaces,
+            selector: {
+              _tag: 'WildcardSelector',
+            },
+          },
+          ...this._generateDeletedHandlingSteps(context),
+          {
+            _tag: 'FilterStep',
+            filter: { ...filter },
+          },
+        ]);
+      }
+
+      // Text
       case 'text-search': {
         return QueryPlan.Plan.make([
           {
@@ -171,26 +200,31 @@ export class QueryPlanner {
           ...this._generateDeletedHandlingSteps(context),
         ]);
       }
+
+      // Compare
       case 'compare':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'in':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'range':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
+
+      // Boolean
       case 'not':
         return this._generateSelectionFromFilter(filter.filter, {
           ...context,
           selectionInverted: !context.selectionInverted,
         });
       case 'and':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'or':
         // Optimized case
         if (filter.filters.every(isTrivialTypenameFilter)) {
-          const typenames = filter.filters.map((f) => {
-            invariant(f.type === 'object' && f.typename !== null);
-            return f.typename;
+          const typenames = filter.filters.map((filter) => {
+            invariant(filter.type === 'object' && filter.typename !== null);
+            return filter.typename;
           });
+
           return QueryPlan.Plan.make([
             {
               _tag: 'SelectStep',
@@ -204,11 +238,12 @@ export class QueryPlanner {
             ...this._generateDeletedHandlingSteps(context),
           ]);
         } else {
-          throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+          throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
         }
 
       default:
-        throw new QueryError(`Unsupported filter type: ${(filter as any).type}`, {
+        throw new QueryError({
+          message: `Unsupported filter type: ${(filter as any).type}`,
           context: { query: context.originalQuery },
         });
     }
@@ -425,6 +460,41 @@ export class QueryPlanner {
     // TODO(dmaretskyi): Implement this.
     return plan;
   }
+
+  private _generateOrderClause(query: QueryAST.QueryOrderClause, context: GenerationContext): QueryPlan.Plan {
+    return QueryPlan.Plan.make([
+      ...this._generate(query.query, context).steps,
+      {
+        _tag: 'OrderStep',
+        order: query.order,
+      },
+    ]);
+  }
+
+  // After complete plan is built, inspect it from the end:
+  //   - Walk backwards until hitting an object set changer.
+  //   - If an order step is found, skip.
+  //   - Otherwise append natural order to the end.
+  private _ensureOrderStep(plan: QueryPlan.Plan): QueryPlan.Plan {
+    const OBJECT_SET_CHANGERS = new Set(['SelectStep', 'TraverseStep', 'UnionStep', 'SetDifferenceStep']);
+    for (let i = plan.steps.length - 1; i >= 0; i--) {
+      const step = plan.steps[i];
+      if (step._tag === 'OrderStep') {
+        return plan;
+      }
+      if (OBJECT_SET_CHANGERS.has(step._tag)) {
+        break;
+      }
+    }
+
+    return QueryPlan.Plan.make([
+      ...plan.steps,
+      {
+        _tag: 'OrderStep',
+        order: [Order.natural.ast],
+      },
+    ]);
+  }
 }
 
 /**

package/src/space/admission-discovery-extension.ts

@@ -2,15 +2,15 @@
 // Copyright 2024 DXOS.org
 //
 
-import { scheduleTask, type Trigger } from '@dxos/async';
+import { type Trigger, scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { ProtocolError } from '@dxos/protocols';
 import { schema } from '@dxos/protocols/proto';
 import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import {
   type AdmissionDiscoveryService,
-  type GetAdmissionCredentialResponse,
   type GetAdmissionCredentialRequest,
+  type GetAdmissionCredentialResponse,
 } from '@dxos/protocols/proto/dxos/mesh/teleport';
 import { type ExtensionContext, RpcExtension } from '@dxos/teleport';
 

package/src/space/control-pipeline.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { CredentialGenerator, createCredential } from '@dxos/credentials';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
@@ -11,13 +11,14 @@ import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { createStorage, StorageType } from '@dxos/random-access-storage';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { Timeframe } from '@dxos/timeframe';
 
-import { ControlPipeline } from './control-pipeline';
 import { valueEncoding } from '../common';
 import { MetadataStore } from '../metadata';
 
+import { ControlPipeline } from './control-pipeline';
+
 describe('space/control-pipeline', () => {
   test('admits feeds', async () => {
     const keyring = new Keyring();

package/src/space/control-pipeline.ts

@@ -2,14 +2,14 @@
 // Copyright 2022 DXOS.org
 //
 
-import { DeferredTask, sleepWithContext, trackLeaks } from '@dxos/async';
+import { DeferredTask, scheduleMicroTask, sleepWithContext, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
 import {
-  SpaceStateMachine,
-  type SpaceState,
-  type MemberInfo,
-  type FeedInfo,
   type DelegateInvitationCredential,
+  type FeedInfo,
+  type MemberInfo,
+  type SpaceState,
+  SpaceStateMachine,
 } from '@dxos/credentials';
 import { type FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -83,9 +83,12 @@ export class ControlPipeline {
 
       // TODO(burdon): Check not stopping.
       if (info.assertion.designation === AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
-        queueMicrotask(async () => {
+        scheduleMicroTask(this._ctx, async () => {
           try {
             const feed = await feedProvider(info.key);
+            if (this._ctx.disposed) {
+              return;
+            }
             if (!this._pipeline.hasFeed(feed.key)) {
               await this._pipeline.addFeed(feed);
             }

package/src/space/space-manager.browser.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, test, onTestFinished } from 'vitest';
+import { describe, onTestFinished, test } from 'vitest';
 
 import { createStorage } from '@dxos/random-access-storage';