@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1da679c

package/src/query/query-executor.ts

@@ -3,10 +3,10 @@
 //
 
 import type { AutomergeUrl, DocumentId } from '@automerge/automerge-repo';
-import { Match } from 'effect';
+import { Match, Predicate } from 'effect';
 
 import { Context, ContextDisposedError, LifecycleState, Resource } from '@dxos/context';
-import { DatabaseDirectory, isEncodedReference, ObjectStructure, type QueryAST } from '@dxos/echo-protocol';
+import { DatabaseDirectory, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
 import { EscapedPropPath, type FindResult, type Indexer } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { DXN, type ObjectId, PublicKey, type SpaceId } from '@dxos/keys';
@@ -15,13 +15,16 @@ import { objectPointerCodec } from '@dxos/protocols';
 import { type QueryReactivity, type QueryResult } from '@dxos/protocols/proto/dxos/echo/query';
 import { getDeep, isNonNullable } from '@dxos/util';
 
-import type { QueryPlan } from './plan';
-import { QueryPlanner } from './query-planner';
 import type { AutomergeHost } from '../automerge';
 import { createIdFromSpaceKey } from '../common';
 import type { SpaceStateManager } from '../db-host';
 import { filterMatchObject } from '../filter';
 
+import type { QueryPlan } from './plan';
+import { QueryPlanner } from './query-planner';
+
+const isNullable: Predicate.Refinement<unknown, null | undefined> = Predicate.isNullable;
+
 type QueryExecutorOptions = {
   indexer: Indexer;
   automergeHost: AutomergeHost;
@@ -189,7 +192,6 @@ export class QueryExecutor extends Resource {
       );
 
     if (TRACE_QUERY_EXECUTION) {
-      // eslint-disable-next-line no-console
       console.log(ExecutionTrace.format(trace));
     }
 
@@ -245,6 +247,9 @@ export class QueryExecutor extends Resource {
       case 'TraverseStep':
         ({ workingSet: newWorkingSet, trace } = await this._execTraverseStep(step, workingSet));
         break;
+      case 'OrderStep':
+        ({ workingSet: newWorkingSet, trace } = await this._execOrderStep(step, workingSet));
+        break;
       default:
         throw new Error(`Unknown step type: ${(step as any)._tag}`);
     }
@@ -382,10 +387,6 @@ export class QueryExecutor extends Resource {
     step: QueryPlan.FilterDeletedStep,
     workingSet: QueryItem[],
   ): Promise<StepExecutionResult> {
-    if (workingSet.length === 6) {
-      log.info('FilterDeletedStep', { step, workingSet });
-    }
-
     const expected = step.mode === 'only-deleted';
     const result = workingSet.filter((item) => ObjectStructure.isDeleted(item.doc) === expected);
     return {
@@ -428,7 +429,7 @@ export class QueryExecutor extends Resource {
                         }
                       : null;
                   } catch {
-                    log.warn('Invalid reference', { ref: ref['/'] });
+                    log.warn('invalid reference', { ref: ref['/'] });
                     return null;
                   }
                 });
@@ -491,7 +492,7 @@ export class QueryExecutor extends Resource {
                     spaceId: item.spaceId,
                   };
                 } catch {
-                  log.warn('Invalid reference', { ref: ref['/'] });
+                  log.warn('invalid reference', { ref: ref['/'] });
                   return null;
                 }
               })
@@ -590,6 +591,60 @@ export class QueryExecutor extends Resource {
     };
   }
 
+  private async _execOrderStep(step: QueryPlan.OrderStep, workingSet: QueryItem[]): Promise<StepExecutionResult> {
+    const compareItems = (a: QueryItem, b: QueryItem): number =>
+      step.order.reduce((comparison, order) => {
+        // If we already have a definitive result, return it
+        if (comparison !== 0) {
+          return comparison;
+        }
+
+        return this._compareByOrder(a, b, order);
+      }, 0);
+
+    const sortedWorkingSet = [...workingSet].sort(compareItems);
+
+    return {
+      workingSet: sortedWorkingSet,
+      trace: {
+        ...ExecutionTrace.makeEmpty(),
+        name: 'Order',
+        details: JSON.stringify(step.order),
+        objectCount: sortedWorkingSet.length,
+      },
+    };
+  }
+
+  private _compareByOrder(a: QueryItem, b: QueryItem, order: QueryAST.Order): number {
+    return Match.type<QueryAST.Order>().pipe(
+      Match.withReturnType<number>(),
+      Match.when({ kind: 'natural' }, () => a.objectId.localeCompare(b.objectId)),
+      Match.when({ kind: 'property' }, ({ property, direction }) => {
+        const comparison = this._compareByProperty(a, b, property);
+        return direction === 'desc' ? -comparison : comparison;
+      }),
+      Match.exhaustive,
+    )(order);
+  }
+
+  private _compareByProperty(a: QueryItem, b: QueryItem, property: string): number {
+    const aValue = a.doc.data[property];
+    const bValue = b.doc.data[property];
+
+    return Match.type<{ a: unknown; b: unknown }>().pipe(
+      Match.withReturnType<number>(),
+      Match.when({ a: isNullable, b: isNullable }, () => 0),
+      Match.when({ a: isNullable }, () => 1),
+      Match.when({ b: isNullable }, () => -1),
+      Match.when({ a: Match.string, b: Match.string }, ({ a, b }) => a.localeCompare(b)),
+      Match.when({ a: Match.number, b: Match.number }, ({ a, b }) => a - b),
+      Match.when({ a: Match.boolean, b: Match.boolean }, ({ a, b }) => (a === b ? 0 : a ? 1 : -1)),
+      Match.when({ a: Match.defined, b: Match.defined }, ({ a, b }) => String(a).localeCompare(String(b))),
+      // TODO(wittjosiah): Why does Match.exhaustive fail here?
+      Match.orElse(() => 0),
+    )({ a: aValue, b: bValue });
+  }
+
   private async _loadDocumentsAfterIndexQuery(indexHits: FindResult[]): Promise<(QueryItem | null)[]> {
     return Promise.all(
       indexHits.map(async (hit): Promise<QueryItem | null> => {

package/src/query/query-planner.test.ts

@@ -4,13 +4,14 @@
 
 import { describe, expect, test } from 'vitest';
 
-import { Filter, Query } from '@dxos/echo';
+import { Filter, Order, Query } from '@dxos/echo';
 import { type QueryAST } from '@dxos/echo-protocol';
 import { SpaceId } from '@dxos/keys';
 
-import { QueryPlanner } from './query-planner';
 import { TestSchema } from '../testing';
 
+import { QueryPlanner } from './query-planner';
+
 describe('QueryPlanner', () => {
   const planner = new QueryPlanner();
 
@@ -38,6 +39,53 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
+        ],
+      }
+    `);
+  });
+
+  test('get all people ordered by name', () => {
+    const query = Query.select(Filter.type(TestSchema.Person)).orderBy(Order.property('name', 'asc'));
+
+    const plan = planner.createPlan(withSpaceIdOptions(query.ast));
+    expect(plan).toMatchInlineSnapshot(`
+      {
+        "steps": [
+          {
+            "_tag": "SelectStep",
+            "selector": {
+              "_tag": "TypeSelector",
+              "inverted": false,
+              "typename": [
+                "dxn:type:dxos.org/type/Person:0.1.0",
+              ],
+            },
+            "spaces": [
+              "B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO",
+            ],
+          },
+          {
+            "_tag": "FilterDeletedStep",
+            "mode": "only-non-deleted",
+          },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "direction": "asc",
+                "kind": "property",
+                "property": "name",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -82,6 +130,14 @@ describe('QueryPlanner', () => {
               "typename": null,
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -158,6 +214,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -218,6 +282,14 @@ describe('QueryPlanner', () => {
               "typename": "dxn:type:dxos.org/type/Task:0.1.0",
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -316,6 +388,14 @@ describe('QueryPlanner', () => {
               "typename": "dxn:type:dxos.org/type/Task:0.1.0",
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -378,6 +458,14 @@ describe('QueryPlanner', () => {
               },
             ],
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -487,6 +575,14 @@ describe('QueryPlanner', () => {
               ],
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -543,6 +639,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -579,6 +683,14 @@ describe('QueryPlanner', () => {
               "typename": "dxn:type:dxos.org/type/Person:0.1.0",
             },
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -606,6 +718,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -636,6 +756,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -676,6 +804,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-non-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);
@@ -705,6 +841,14 @@ describe('QueryPlanner', () => {
             "_tag": "FilterDeletedStep",
             "mode": "only-deleted",
           },
+          {
+            "_tag": "OrderStep",
+            "order": [
+              {
+                "kind": "natural",
+              },
+            ],
+          },
         ],
       }
     `);

package/src/query/query-planner.ts

@@ -2,6 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
+import { Order } from '@dxos/echo';
 import { type QueryAST } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import type { DXN, SpaceId } from '@dxos/keys';
@@ -35,6 +36,7 @@ export class QueryPlanner {
     let plan = this._generate(query, { ...DEFAULT_CONTEXT, originalQuery: query });
     plan = this._optimizeEmptyFilters(plan);
     plan = this._optimizeSoloUnions(plan);
+    plan = this._ensureOrderStep(plan);
     return plan;
   }
 
@@ -58,8 +60,11 @@ export class QueryPlanner {
         return this._generateUnionClause(query, context);
       case 'set-difference':
         return this._generateSetDifferenceClause(query, context);
+      case 'order':
+        return this._generateOrderClause(query, context);
       default:
-        throw new QueryError(`Unsupported query type: ${(query as any).type}`, {
+        throw new QueryError({
+          message: `Unsupported query type: ${(query as any).type}`,
           context: { query: context.originalQuery },
         });
     }
@@ -102,7 +107,10 @@ export class QueryPlanner {
           ]);
         }
         if (context.selectionInverted) {
-          throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+          throw new QueryError({
+            message: 'Query too complex',
+            context: { query: context.originalQuery },
+          });
         }
 
         // Try to utilize indexes during selection, prioritizing selecting by id, then by typename.
@@ -172,18 +180,18 @@ export class QueryPlanner {
         ]);
       }
       case 'compare':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'in':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'range':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'not':
         return this._generateSelectionFromFilter(filter.filter, {
           ...context,
           selectionInverted: !context.selectionInverted,
         });
       case 'and':
-        throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+        throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
       case 'or':
         // Optimized case
         if (filter.filters.every(isTrivialTypenameFilter)) {
@@ -204,11 +212,12 @@ export class QueryPlanner {
             ...this._generateDeletedHandlingSteps(context),
           ]);
         } else {
-          throw new QueryError('Query too complex', { context: { query: context.originalQuery } });
+          throw new QueryError({ message: 'Query too complex', context: { query: context.originalQuery } });
         }
 
       default:
-        throw new QueryError(`Unsupported filter type: ${(filter as any).type}`, {
+        throw new QueryError({
+          message: `Unsupported filter type: ${(filter as any).type}`,
           context: { query: context.originalQuery },
         });
     }
@@ -425,6 +434,41 @@ export class QueryPlanner {
     // TODO(dmaretskyi): Implement this.
     return plan;
   }
+
+  private _generateOrderClause(query: QueryAST.QueryOrderClause, context: GenerationContext): QueryPlan.Plan {
+    return QueryPlan.Plan.make([
+      ...this._generate(query.query, context).steps,
+      {
+        _tag: 'OrderStep',
+        order: query.order,
+      },
+    ]);
+  }
+
+  // After complete plan is built, inspect it from the end:
+  //   - Walk backwards until hitting an object set changer.
+  //   - If an order step is found, skip.
+  //   - Otherwise append natural order to the end.
+  private _ensureOrderStep(plan: QueryPlan.Plan): QueryPlan.Plan {
+    const OBJECT_SET_CHANGERS = new Set(['SelectStep', 'TraverseStep', 'UnionStep', 'SetDifferenceStep']);
+    for (let i = plan.steps.length - 1; i >= 0; i--) {
+      const step = plan.steps[i];
+      if (step._tag === 'OrderStep') {
+        return plan;
+      }
+      if (OBJECT_SET_CHANGERS.has(step._tag)) {
+        break;
+      }
+    }
+
+    return QueryPlan.Plan.make([
+      ...plan.steps,
+      {
+        _tag: 'OrderStep',
+        order: [Order.natural.ast],
+      },
+    ]);
+  }
 }
 
 /**

package/src/space/admission-discovery-extension.ts

@@ -2,15 +2,15 @@
 // Copyright 2024 DXOS.org
 //
 
-import { scheduleTask, type Trigger } from '@dxos/async';
+import { type Trigger, scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { ProtocolError } from '@dxos/protocols';
 import { schema } from '@dxos/protocols/proto';
 import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import {
   type AdmissionDiscoveryService,
-  type GetAdmissionCredentialResponse,
   type GetAdmissionCredentialRequest,
+  type GetAdmissionCredentialResponse,
 } from '@dxos/protocols/proto/dxos/mesh/teleport';
 import { type ExtensionContext, RpcExtension } from '@dxos/teleport';
 

package/src/space/control-pipeline.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { CredentialGenerator, createCredential } from '@dxos/credentials';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
@@ -11,13 +11,14 @@ import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { createStorage, StorageType } from '@dxos/random-access-storage';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { Timeframe } from '@dxos/timeframe';
 
-import { ControlPipeline } from './control-pipeline';
 import { valueEncoding } from '../common';
 import { MetadataStore } from '../metadata';
 
+import { ControlPipeline } from './control-pipeline';
+
 describe('space/control-pipeline', () => {
   test('admits feeds', async () => {
     const keyring = new Keyring();

package/src/space/control-pipeline.ts

@@ -2,14 +2,14 @@
 // Copyright 2022 DXOS.org
 //
 
-import { DeferredTask, sleepWithContext, trackLeaks } from '@dxos/async';
+import { DeferredTask, scheduleMicroTask, sleepWithContext, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
 import {
-  SpaceStateMachine,
-  type SpaceState,
-  type MemberInfo,
-  type FeedInfo,
   type DelegateInvitationCredential,
+  type FeedInfo,
+  type MemberInfo,
+  type SpaceState,
+  SpaceStateMachine,
 } from '@dxos/credentials';
 import { type FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -83,9 +83,12 @@ export class ControlPipeline {
 
       // TODO(burdon): Check not stopping.
       if (info.assertion.designation === AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
-        queueMicrotask(async () => {
+        scheduleMicroTask(this._ctx, async () => {
           try {
             const feed = await feedProvider(info.key);
+            if (this._ctx.disposed) {
+              return;
+            }
             if (!this._pipeline.hasFeed(feed.key)) {
               await this._pipeline.addFeed(feed);
             }

package/src/space/space-manager.browser.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, test, onTestFinished } from 'vitest';
+import { describe, onTestFinished, test } from 'vitest';
 
 import { createStorage } from '@dxos/random-access-storage';
 

package/src/space/space-manager.ts

@@ -4,8 +4,8 @@
 
 import { type AutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
 
-import { synchronized, trackLeaks, Trigger } from '@dxos/async';
-import { getCredentialAssertion, type DelegateInvitationCredential, type MemberInfo } from '@dxos/credentials';
+import { Trigger, synchronized, trackLeaks } from '@dxos/async';
+import { type DelegateInvitationCredential, type MemberInfo, getCredentialAssertion } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { type FeedStore } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -19,11 +19,12 @@ import { type Teleport } from '@dxos/teleport';
 import { type BlobStore } from '@dxos/teleport-extension-object-sync';
 import { ComplexMap } from '@dxos/util';
 
+import { createIdFromSpaceKey } from '../common/space-id';
+import { type MetadataStore } from '../metadata';
+
 import { CredentialRetrieverExtension } from './admission-discovery-extension';
 import { Space } from './space';
 import { SpaceProtocol, type SwarmIdentity } from './space-protocol';
-import { createIdFromSpaceKey } from '../common/space-id';
-import { type MetadataStore } from '../metadata';
 
 export type SpaceManagerParams = {
   feedStore: FeedStore<FeedMessage>;

package/src/space/space-protocol.browser.test.ts

@@ -2,14 +2,14 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { createStorage } from '@dxos/random-access-storage';
 import { Timeframe } from '@dxos/timeframe';
 
-import { TestFeedBuilder, TestAgentBuilder, WebsocketNetworkManagerProvider } from '../testing';
+import { TestAgentBuilder, TestFeedBuilder, WebsocketNetworkManagerProvider } from '../testing';
 
 // TODO(burdon): Config.
 // Signal server will be started by the setup script.

package/src/space/space-protocol.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
@@ -11,9 +11,10 @@ import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
 import { Timeframe } from '@dxos/timeframe';
 
-import { AuthStatus, MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, SpaceProtocol } from './space-protocol';
 import { TestAgentBuilder, TestFeedBuilder } from '../testing';
 
+import { AuthStatus, MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, SpaceProtocol } from './space-protocol';
+
 describe('space/space-protocol', () => {
   // Flaky.
   test('two peers discover each other via presence', async () => {

package/src/space/space-protocol.ts

@@ -9,8 +9,8 @@ import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import {
   MMSTTopology,
-  type SwarmNetworkManager,
   type SwarmConnection,
+  type SwarmNetworkManager,
   type WireProtocol,
   type WireProtocolParams,
   type WireProtocolProvider,
@@ -20,7 +20,7 @@ import { type MuxerStats, Teleport } from '@dxos/teleport';
 import { type BlobStore, BlobSync } from '@dxos/teleport-extension-object-sync';
 import { ReplicatorExtension } from '@dxos/teleport-extension-replicator';
 import { trace } from '@dxos/tracing';
-import { CallbackCollection, ComplexMap, type AsyncCallback } from '@dxos/util';
+import { type AsyncCallback, CallbackCollection, ComplexMap } from '@dxos/util';
 
 import { AuthExtension, type AuthProvider, type AuthVerifier } from './auth';
 
@@ -118,7 +118,10 @@ export class SpaceProtocol {
     this.blobSync = new BlobSync({ blobStore });
 
     // TODO(burdon): Async race condition? Move to start?
-    this._topic = subtleCrypto.digest('SHA-256', topic.asBuffer()).then(discoveryKey).then(PublicKey.from);
+    this._topic = subtleCrypto
+      .digest('SHA-256', topic.asBuffer() as ArrayBufferView<ArrayBuffer>)
+      .then(discoveryKey)
+      .then(PublicKey.from);
 
     this._disableP2pReplication = disableP2pReplication ?? false;
   }