@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1068cf700f

package/src/filter/filter-match.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { decodeReference, isEncodedReference, type QueryAST, type ObjectStructure } from '@dxos/echo-protocol';
-import { EXPANDO_TYPENAME } from '@dxos/echo-schema';
+import { ATTR_META, type ObjectJSON } from '@dxos/echo/internal';
+import { EncodedReference, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
 import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
 
 export type MatchedObject = {
@@ -14,35 +14,32 @@ export type MatchedObject = {
 
 /**
  * Matches an object against a filter AST.
+ * @param obj object structure as stored in automerge.
  */
 export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject): boolean => {
   switch (filter.type) {
     case 'object': {
-      // Check typename if specified
+      // Check typename if specified.
       if (filter.typename !== null) {
         // TODO(dmaretskyi): `system` is missing in some cases.
         if (!obj.doc.system?.type?.['/']) {
-          // Objects with no type are considered to be expando objects
-          const expectedDXN = DXN.parse(filter.typename).asTypeDXN();
-          if (expectedDXN?.type !== EXPANDO_TYPENAME) {
-            return false;
-          }
+          // Objects with no type are deprecated.
+          return false;
         } else {
           const actualDXN = DXN.parse(obj.doc.system.type['/']);
           const expectedDXN = DXN.parse(filter.typename);
-
           if (!compareTypename(expectedDXN, actualDXN)) {
             return false;
           }
         }
       }
 
-      // Check IDs if specified
+      // Check IDs if specified.
       if (filter.id && filter.id.length > 0 && !filter.id.includes(obj.id)) {
         return false;
       }
 
-      // Check properties
+      // Check properties.
       if (filter.props) {
         for (const [key, valueFilter] of Object.entries(filter.props)) {
           const value = obj.doc.data[key];
@@ -52,7 +49,7 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
         }
       }
 
-      // Check foreign keys if specified
+      // Check foreign keys if specified.
       if (filter.foreignKeys && filter.foreignKeys.length > 0) {
         const hasMatchingKey = filter.foreignKeys.some((filterKey) =>
           obj.doc.meta.keys.some((objKey) => objKey.source === filterKey.source && objKey.id === filterKey.id),
@@ -65,8 +62,13 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
       return true;
     }
 
+    case 'tag': {
+      const tags = ObjectStructure.getTags(obj.doc);
+      return tags.some((tag) => tag === filter.tag);
+    }
+
     case 'text-search': {
-      // TODO: Implement text search
+      // TODO(???): Implement text search.
       return false;
     }
 
@@ -87,6 +89,120 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
   }
 };
 
+// TODO(burdon): Reconcile with filterMatchObject.
+export const filterMatchObjectJSON = (filter: QueryAST.Filter, obj: ObjectJSON): boolean => {
+  switch (filter.type) {
+    case 'object': {
+      // Check typename if specified
+      if (filter.typename !== null) {
+        // TODO(dmaretskyi): `system` is missing in some cases.
+        if (!obj['@type']) {
+          // Objects with no type are deprecated.
+          return false;
+        } else {
+          const actualDXN = DXN.parse(obj['@type']);
+          const expectedDXN = DXN.parse(filter.typename);
+          if (!compareTypename(expectedDXN, actualDXN)) {
+            return false;
+          }
+        }
+      }
+
+      // Check IDs if specified
+      if (filter.id && filter.id.length > 0 && !filter.id.includes(obj.id)) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          if (key.startsWith('@')) {
+            // ignore meta properties
+            continue;
+          }
+          const value = (obj as any)[key];
+          if (!filterMatchValue(valueFilter, value)) {
+            return false;
+          }
+        }
+      }
+
+      // Check foreign keys if specified
+      if (filter.foreignKeys && filter.foreignKeys.length > 0) {
+        const hasMatchingKey = filter.foreignKeys.some((filterKey) =>
+          obj['@meta']?.keys?.some((objKey) => objKey.source === filterKey.source && objKey.id === filterKey.id),
+        );
+        if (!hasMatchingKey) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    case 'tag': {
+      const tags = obj[ATTR_META]?.tags ?? [];
+      return tags.some((tag) => tag === filter.tag);
+    }
+
+    // TODO: Implement text search.
+    case 'text-search': {
+      return false;
+    }
+
+    case 'not': {
+      return !filterMatchObjectJSON(filter.filter, obj);
+    }
+
+    case 'and': {
+      return filter.filters.every((f) => filterMatchObjectJSON(f, obj));
+    }
+
+    case 'or': {
+      return filter.filters.some((f) => filterMatchObjectJSON(f, obj));
+    }
+
+    default:
+      return false;
+  }
+};
+
+/**
+ * Performs structural matching between a filter object and a target object.
+ * This handles nested object comparison for array matching scenarios.
+ */
+// TODO(wittjosiah): Add ast support for non-strict matching.
+const structuralMatch = (filterObj: any, targetObj: any, strict = true): boolean => {
+  if (typeof filterObj !== 'object' || filterObj === null) {
+    return filterObj === targetObj;
+  }
+
+  if (typeof targetObj !== 'object' || targetObj === null) {
+    return false;
+  }
+
+  // Prohibit extra keys in targetObj.
+  const filterKeys = Object.keys(filterObj);
+  const targetKeys = Object.keys(targetObj);
+  if (strict && filterKeys.length !== targetKeys.length) {
+    return false;
+  }
+
+  return filterKeys.every((key) => {
+    if (!(key in targetObj)) {
+      return false;
+    }
+    const filterValue = filterObj[key];
+    const targetValue = targetObj[key];
+
+    if (typeof filterValue === 'object' && filterValue !== null) {
+      return structuralMatch(filterValue, targetValue);
+    }
+
+    return filterValue === targetValue;
+  });
+};
+
 export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boolean => {
   switch (filter.type) {
     case 'compare': {
@@ -97,7 +213,7 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
             if (!isEncodedReference(value)) {
               return false;
             }
-            return DXN.equals(decodeReference(value).toDXN(), decodeReference(compareValue).toDXN());
+            return DXN.equals(EncodedReference.toDXN(value), EncodedReference.toDXN(compareValue));
           }
           return value === compareValue;
         case 'neq':
@@ -110,12 +226,44 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
           return (value as any) < compareValue;
         case 'lte':
           return (value as any) <= compareValue;
+        default:
+          return false;
+      }
+    }
+    case 'object': {
+      // Handle nested object filters for property matching
+      if (typeof value !== 'object' || value === null) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          const nestedValue = (value as any)[key];
+          if (!filterMatchValue(valueFilter, nestedValue)) {
+            return false;
+          }
+        }
       }
-      break;
+
+      return true;
     }
     case 'in': {
       return filter.values.includes(value);
     }
+    case 'contains': {
+      if (!Array.isArray(value)) {
+        return false;
+      }
+
+      return value.some((element) => {
+        if (typeof filter.value === 'object' && filter.value !== null && !Array.isArray(filter.value)) {
+          return structuralMatch(filter.value, element);
+        }
+
+        return element === filter.value;
+      });
+    }
     case 'range': {
       return (value as any) >= filter.from && (value as any) <= filter.to;
     }

package/src/metadata/metadata-store.ts

@@ -16,11 +16,11 @@ import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/servic
 import {
   type ControlPipelineSnapshot,
   type EchoMetadata,
-  type SpaceMetadata,
+  type EdgeReplicationSetting,
   type IdentityRecord,
-  type SpaceCache,
   type LargeSpaceMetadata,
-  type EdgeReplicationSetting,
+  type SpaceCache,
+  type SpaceMetadata,
 } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { type Directory, type File } from '@dxos/random-access-storage';
 import { type Timeframe } from '@dxos/timeframe';
@@ -91,14 +91,17 @@ export class MetadataStore {
       log('loaded', { size: dataSize, checksum, name: file.filename });
 
       if (fileLength < dataSize + 8) {
-        throw new DataCorruptionError('Metadata size is smaller than expected.', { fileLength, dataSize });
+        throw new DataCorruptionError({
+          message: 'Metadata size is smaller than expected.',
+          context: { fileLength, dataSize },
+        });
       }
 
       const data = await file.read(8, dataSize);
 
       const calculatedChecksum = CRC32.buf(data);
       if (calculatedChecksum !== checksum) {
-        throw new DataCorruptionError('Metadata checksum is invalid.');
+        throw new DataCorruptionError({ message: 'Metadata checksum is invalid.' });
       }
 
       return codec.decode(data);

package/src/pipeline/pipeline-stress.test.ts

@@ -2,8 +2,9 @@
 // Copyright 2022 DXOS.org
 //
 
-import * as fc from 'fast-check';
 import { inspect } from 'node:util';
+
+import * as fc from 'fast-check';
 import { describe, expect, test } from 'vitest';
 
 import { asyncTimeout } from '@dxos/async';
@@ -15,9 +16,10 @@ import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
-import { Pipeline } from './pipeline';
 import { TestFeedBuilder } from '../testing';
 
+import { Pipeline } from './pipeline';
+
 const NUM_AGENTS = 2;
 const NUM_MESSAGES = 10;
 

package/src/pipeline/pipeline.test.ts

@@ -2,16 +2,17 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { Event, sleep } from '@dxos/async';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
-import { Pipeline } from './pipeline';
 import { TestFeedBuilder } from '../testing';
 
+import { Pipeline } from './pipeline';
+
 const TEST_MESSAGE: FeedMessage = {
   timeframe: new Timeframe(),
   payload: {},

package/src/pipeline/pipeline.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { Event, sleepWithContext, synchronized, Trigger } from '@dxos/async';
+import { Event, Trigger, sleepWithContext, synchronized } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
 import { failUndefined } from '@dxos/debug';
 import { FeedSetIterator, type FeedWrapper, type FeedWriter } from '@dxos/feed-store';
@@ -14,11 +14,12 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { ComplexMap } from '@dxos/util';
 
-import { createMessageSelector } from './message-selector';
-import { mapFeedIndexesToTimeframe, startAfter, TimeframeClock } from './timeframe-clock';
 import { createMappedFeedWriter } from '../common';
 
-export type WaitUntilReachedTargetParams = {
+import { createMessageSelector } from './message-selector';
+import { TimeframeClock, mapFeedIndexesToTimeframe, startAfter } from './timeframe-clock';
+
+export type WaitUntilReachedTargetProps = {
   /**
    * For cancellation.
    */
@@ -40,8 +41,10 @@ export class PipelineState {
    */
   _ctx = new Context();
 
-  // TODO(dmaretskyi): Remove?.
-  public readonly timeframeUpdate = this._timeframeClock.update;
+  // TODO(dmaretskyi): Remove?. Avoid accessing `_timeframeClock` before constructor initialization.
+  public get timeframeUpdate() {
+    return this._timeframeClock.update;
+  }
 
   public readonly stalled = new Event();
 
@@ -129,7 +132,7 @@ export class PipelineState {
     ctx = new Context(),
     timeout,
     breakOnStall = true,
-  }: WaitUntilReachedTargetParams = {}): Promise<void> {
+  }: WaitUntilReachedTargetProps = {}): Promise<void> {
     log('waitUntilReachedTargetTimeframe', {
       timeout,
       current: this.timeframe,

package/src/query/errors.ts

@@ -4,4 +4,6 @@
 
 import { BaseError } from '@dxos/errors';
 
-export class QueryError extends BaseError.extend('QUERY_ERROR') {}
+export class QueryError extends BaseError.extend('QueryError') {}
+
+export class InvalidQueryError extends QueryError.extend('InvalidQueryError') {}

package/src/query/plan.ts

@@ -3,7 +3,7 @@
 //
 
 import type { QueryAST } from '@dxos/echo-protocol';
-import type { EscapedPropPath } from '@dxos/indexing';
+import type { EscapedPropPath } from '@dxos/index-core';
 import type { DXN, ObjectId, SpaceId } from '@dxos/keys';
 
 export namespace QueryPlan {
@@ -32,7 +32,9 @@ export namespace QueryPlan {
     | FilterDeletedStep
     | TraverseStep
     | UnionStep
-    | SetDifferenceStep;
+    | SetDifferenceStep
+    | OrderStep
+    | LimitStep;
 
   /**
    * Clear the current working set.
@@ -43,13 +45,36 @@ export namespace QueryPlan {
 
   /**
    * Select objects based on id, typename, or other predicates.
-   * Specifies the spaces to select from.
+   * Specifies the spaces and/or queues to select from.
+   * The results from spaces and queues are unioned together.
    */
   export type SelectStep = {
     _tag: 'SelectStep';
 
+    // TODO(dmaretskyi): Extract the object container spec into a separate type in ECHO-protocol that we can share with the indexer.
+
+    /**
+     * Select from specific spaces.
+     */
     spaces: readonly SpaceId[];
+
+    /**
+     * If true, select from all queues in the spaces specified by `spaces`.
+     */
+    allQueuesFromSpaces: boolean;
+
+    /**
+     * Select from specific queues.
+     */
+    queues: readonly DXN.String[];
+
     selector: Selector;
+
+    /**
+     * Optional limit on the number of results to select.
+     * When set, the index scan can be optimized to stop early.
+     */
+    limit?: number;
   };
 
   /**
@@ -90,6 +115,11 @@ export namespace QueryPlan {
 
     text: string;
     searchKind: TextSearchKind;
+
+    /**
+     * Optionally select only objects of the given typenames.
+     */
+    typename: DXN.String[] | null;
   };
 
   /**
@@ -140,7 +170,7 @@ export namespace QueryPlan {
   /**
    * Describes a traversal of the object graph.
    */
-  export type Traversal = ReferenceTraversal | RelationTraversal;
+  export type Traversal = ReferenceTraversal | RelationTraversal | HierarchyTraversal;
 
   /**
    * Traverse a reference connection.
@@ -150,8 +180,9 @@ export namespace QueryPlan {
 
     /**
      * Property path where the reference is located.
+     * If null, matches references from any property (only valid for incoming direction).
      */
-    property: EscapedPropPath;
+    property: EscapedPropPath | null;
 
     /**
      * outgoing: the reference points from the anchor object to the target.
@@ -173,6 +204,19 @@ export namespace QueryPlan {
     direction: 'source-to-relation' | 'relation-to-source' | 'target-to-relation' | 'relation-to-target';
   };
 
+  /**
+   * Traverse parent-child hierarchy.
+   */
+  export type HierarchyTraversal = {
+    _tag: 'HierarchyTraversal';
+
+    /**
+     * to-parent: traverse from child to parent.
+     * to-children: traverse from parent to children.
+     */
+    direction: 'to-parent' | 'to-children';
+  };
+
   /**
    * Combine results from multiple plans.
    * Each of the plans starts with a copy of the current working set.
@@ -193,4 +237,30 @@ export namespace QueryPlan {
     source: Plan;
     exclude: Plan;
   };
+
+  /**
+   * Order the results of the plan.
+   */
+  export type OrderStep = {
+    _tag: 'OrderStep';
+
+    // Defaults to natural order if empty.
+    order: readonly QueryAST.Order[];
+
+    /**
+     * Optional limit on the number of results to return after ordering.
+     * When set, the sorting can be optimized to only track the top N elements.
+     */
+    // TODO(dmaretskyi): Remove. This doesn't belong here.
+    limit?: number;
+  };
+
+  /**
+   * Limit the number of results.
+   */
+  export type LimitStep = {
+    _tag: 'LimitStep';
+
+    limit: number;
+  };
 }