@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1c7ec43d41

package/src/filter/filter-match.test.ts

@@ -4,81 +4,117 @@
 
 import { describe, expect, test } from 'vitest';
 
-import { Filter } from '@dxos/echo';
+import { Filter, Ref } from '@dxos/echo';
 import { ObjectStructure } from '@dxos/echo-protocol';
-import { Expando, EXPANDO_TYPENAME, Ref } from '@dxos/echo-schema';
+import { TestSchema } from '@dxos/echo/testing';
 import { DXN, ObjectId, SpaceId } from '@dxos/keys';
 
-import { filterMatchObject, type MatchedObject } from './filter-match';
+import { type MatchedObject, filterMatchObject } from './filter-match';
 
 describe('filterMatch', () => {
+  test('everything', () => {
+    expect(filterMatchObject(Filter.everything().ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.everything().ast, OBJECT_2)).to.be.true;
+  });
+
+  test('nothing', () => {
+    expect(filterMatchObject(Filter.nothing().ast, OBJECT_1)).to.be.false;
+    expect(filterMatchObject(Filter.nothing().ast, OBJECT_2)).to.be.false;
+  });
+
   test('properties', () => {
-    expect(filterMatchObject(Filter.type(Expando, { title: 'test' }).ast, OBJECT_1)).to.be.true;
-    expect(filterMatchObject(Filter.type(Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
-    expect(filterMatchObject(Filter.type(Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
-    expect(filterMatchObject(Filter.type(Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.type(TestSchema.Expando, { title: 'test' }).ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.type(TestSchema.Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.type(TestSchema.Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
+    expect(filterMatchObject(Filter.type(TestSchema.Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.type(TestSchema.Expando, { properties: { subject: 'test' } }).ast, OBJECT_1)).to.be
+      .true;
+    expect(filterMatchObject(Filter.type(TestSchema.Expando, { array: Filter.contains('two') }).ast, OBJECT_1)).to.be
+      .true;
   });
 
   test('and', () => {
-    const filter1 = Filter.type(Expando, { title: 'test' });
-    const filter2 = Filter.type(Expando, { value: 100 });
-    const filter3 = Filter.type(Expando, { complete: true });
+    const filter1 = Filter.type(TestSchema.Expando, { title: 'test' });
+    const filter2 = Filter.type(TestSchema.Expando, { value: 100 });
+    const filter3 = Filter.type(TestSchema.Expando, { complete: true });
 
     expect(filterMatchObject(Filter.and(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
   });
 
   test('or', () => {
-    const filter1 = Filter.type(Expando, { value: 200 });
-    const filter2 = Filter.type(Expando, { title: 'test' });
-    const filter3 = Filter.type(Expando, { complete: false });
+    const filter1 = Filter.type(TestSchema.Expando, { value: 200 });
+    const filter2 = Filter.type(TestSchema.Expando, { title: 'test' });
+    const filter3 = Filter.type(TestSchema.Expando, { complete: false });
 
     expect(filterMatchObject(Filter.or(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
   });
 
   test('not', () => {
-    const filter1 = Filter.type(Expando, { title: 'test' });
+    const filter1 = Filter.type(TestSchema.Expando, { title: 'test' });
     const filter2 = Filter.not(filter1);
 
     expect(filterMatchObject(filter1.ast, OBJECT_1)).to.be.true;
     expect(filterMatchObject(filter2.ast, OBJECT_1)).to.be.false;
   });
 
+  test('contains', () => {
+    expect(
+      filterMatchObject(
+        Filter.type(TestSchema.Expando, { properties: { label: Filter.contains('test') } }).ast,
+        OBJECT_1,
+      ),
+    ).to.be.true;
+    expect(
+      filterMatchObject(
+        Filter.type(TestSchema.Expando, { fields: Filter.contains({ label: 'label', value: 'test' }) }).ast,
+        OBJECT_1,
+      ),
+    ).to.be.true;
+  });
+
   test('complex', () => {
-    const filter1 = Filter.type(Expando, { title: 'bad' });
-    const filter2 = Filter.type(Expando, { value: 0 });
+    const filter1 = Filter.type(TestSchema.Expando, { title: 'bad' });
+    const filter2 = Filter.type(TestSchema.Expando, { value: 0 });
     expect(filterMatchObject(Filter.not(Filter.or(filter1, filter2)).ast, OBJECT_1)).to.be.true;
   });
 
   test('ids', () => {
-    const filter = Filter.ids(OBJECT_1.id);
+    const filter = Filter.id(OBJECT_1.id);
     expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.true;
     expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
   });
 
-  test('everything', () => {
-    expect(filterMatchObject(Filter.everything().ast, OBJECT_1)).to.be.true;
-    expect(filterMatchObject(Filter.everything().ast, OBJECT_2)).to.be.true;
-  });
-
-  test('nothing', () => {
-    expect(filterMatchObject(Filter.nothing().ast, OBJECT_1)).to.be.false;
-    expect(filterMatchObject(Filter.nothing().ast, OBJECT_2)).to.be.false;
-  });
-
   test('refs', () => {
-    const filter = Filter.type(Expando, { parent: Ref.fromDXN(DXN.fromLocalObjectId(OBJECT_1.id)) });
+    const filter = Filter.type(TestSchema.Expando, { parent: Ref.fromDXN(DXN.fromLocalObjectId(OBJECT_1.id)) });
     expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.false;
     expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
     expect(filterMatchObject(filter.ast, OBJECT_3)).to.be.true;
   });
+
+  test('standalone timestamp filter throws invariant (must be handled at index level)', () => {
+    expect(() => filterMatchObject(Filter.updated({ after: Date.now() - 1000 }).ast, OBJECT_1)).to.throw();
+    expect(() => filterMatchObject(Filter.created({ before: Date.now() + 1000 }).ast, OBJECT_1)).to.throw();
+  });
+
+  test('and(type, timestamp) throws on timestamp part', () => {
+    const filter = Filter.and(Filter.type(TestSchema.Expando), Filter.updated({ after: Date.now() - 1000 }));
+    expect(() => filterMatchObject(filter.ast, OBJECT_1)).to.throw();
+  });
 });
 
 const OBJECT_1: MatchedObject = {
   id: ObjectId.make('01JVS9YYT5VMVJW0GGTM1YHCCH'),
   spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
   doc: ObjectStructure.makeObject({
-    type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
-    data: { title: 'test', value: 100, complete: true },
+    type: DXN.fromTypenameAndVersion(TestSchema.Expando.typename, '0.1.0').toString(),
+    data: {
+      title: 'test',
+      value: 100,
+      complete: true,
+      array: ['one', 'two', 'three'],
+      properties: { label: ['test'], subject: 'test' },
+      fields: [{ label: 'label', value: 'test' }],
+    },
   }),
 };
 
@@ -86,7 +122,7 @@ const OBJECT_2: MatchedObject = {
   id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
   spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
   doc: ObjectStructure.makeObject({
-    type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
+    type: DXN.fromTypenameAndVersion(TestSchema.Expando.typename, '0.1.0').toString(),
     data: { title: 'test', value: 100, complete: true },
     deleted: true,
   }),
@@ -96,7 +132,7 @@ const OBJECT_3: MatchedObject = {
   id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
   spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
   doc: ObjectStructure.makeObject({
-    type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
+    type: DXN.fromTypenameAndVersion(TestSchema.Expando.typename, '0.1.0').toString(),
     data: { title: 'test', value: 100, complete: true, parent: { '/': DXN.fromLocalObjectId(OBJECT_1.id).toString() } },
   }),
 };

package/src/filter/filter-match.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { decodeReference, isEncodedReference, type QueryAST, type ObjectStructure } from '@dxos/echo-protocol';
-import { EXPANDO_TYPENAME } from '@dxos/echo-schema';
+import { EncodedReference, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
+import { ATTR_META, type ObjectJSON } from '@dxos/echo/internal';
 import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
 
 export type MatchedObject = {
@@ -14,35 +14,32 @@ export type MatchedObject = {
 
 /**
  * Matches an object against a filter AST.
+ * @param obj object structure as stored in automerge.
  */
 export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject): boolean => {
   switch (filter.type) {
     case 'object': {
-      // Check typename if specified
+      // Check typename if specified.
       if (filter.typename !== null) {
         // TODO(dmaretskyi): `system` is missing in some cases.
         if (!obj.doc.system?.type?.['/']) {
-          // Objects with no type are considered to be expando objects
-          const expectedDXN = DXN.parse(filter.typename).asTypeDXN();
-          if (expectedDXN?.type !== EXPANDO_TYPENAME) {
-            return false;
-          }
+          // Objects with no type are deprecated.
+          return false;
         } else {
           const actualDXN = DXN.parse(obj.doc.system.type['/']);
           const expectedDXN = DXN.parse(filter.typename);
-
           if (!compareTypename(expectedDXN, actualDXN)) {
             return false;
           }
         }
       }
 
-      // Check IDs if specified
+      // Check IDs if specified.
       if (filter.id && filter.id.length > 0 && !filter.id.includes(obj.id)) {
         return false;
       }
 
-      // Check properties
+      // Check properties.
       if (filter.props) {
         for (const [key, valueFilter] of Object.entries(filter.props)) {
           const value = obj.doc.data[key];
@@ -52,7 +49,7 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
         }
       }
 
-      // Check foreign keys if specified
+      // Check foreign keys if specified.
       if (filter.foreignKeys && filter.foreignKeys.length > 0) {
         const hasMatchingKey = filter.foreignKeys.some((filterKey) =>
           obj.doc.meta.keys.some((objKey) => objKey.source === filterKey.source && objKey.id === filterKey.id),
@@ -65,11 +62,24 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
       return true;
     }
 
+    case 'tag': {
+      const tags = ObjectStructure.getTags(obj.doc);
+      return tags.some((tag) => tag === filter.tag);
+    }
+
     case 'text-search': {
-      // TODO: Implement text search
+      // TODO(???): Implement text search.
       return false;
     }
 
+    case 'timestamp': {
+      throw new Error('Timestamp filters must be handled at the index level, not in-memory matching.');
+    }
+
+    case 'child-of': {
+      throw new Error('child-of filters must be handled at the executor level, not in-memory matching.');
+    }
+
     case 'not': {
       return !filterMatchObject(filter.filter, obj);
     }
@@ -87,6 +97,128 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
   }
 };
 
+// TODO(burdon): Reconcile with filterMatchObject.
+export const filterMatchObjectJSON = (filter: QueryAST.Filter, obj: ObjectJSON): boolean => {
+  switch (filter.type) {
+    case 'object': {
+      // Check typename if specified
+      if (filter.typename !== null) {
+        // TODO(dmaretskyi): `system` is missing in some cases.
+        if (!obj['@type']) {
+          // Objects with no type are deprecated.
+          return false;
+        } else {
+          const actualDXN = DXN.parse(obj['@type']);
+          const expectedDXN = DXN.parse(filter.typename);
+          if (!compareTypename(expectedDXN, actualDXN)) {
+            return false;
+          }
+        }
+      }
+
+      // Check IDs if specified
+      if (filter.id && filter.id.length > 0 && !filter.id.includes(obj.id)) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          if (key.startsWith('@')) {
+            // ignore meta properties
+            continue;
+          }
+          const value = (obj as any)[key];
+          if (!filterMatchValue(valueFilter, value)) {
+            return false;
+          }
+        }
+      }
+
+      // Check foreign keys if specified
+      if (filter.foreignKeys && filter.foreignKeys.length > 0) {
+        const hasMatchingKey = filter.foreignKeys.some((filterKey) =>
+          obj['@meta']?.keys?.some((objKey) => objKey.source === filterKey.source && objKey.id === filterKey.id),
+        );
+        if (!hasMatchingKey) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    case 'tag': {
+      const tags = obj[ATTR_META]?.tags ?? [];
+      return tags.some((tag) => tag === filter.tag);
+    }
+
+    // TODO: Implement text search.
+    case 'text-search': {
+      return false;
+    }
+
+    case 'timestamp': {
+      throw new Error('Timestamp filters must be handled at the index level, not in-memory matching.');
+    }
+
+    case 'child-of': {
+      throw new Error('child-of filters must be handled at the executor level, not in-memory matching.');
+    }
+
+    case 'not': {
+      return !filterMatchObjectJSON(filter.filter, obj);
+    }
+
+    case 'and': {
+      return filter.filters.every((f) => filterMatchObjectJSON(f, obj));
+    }
+
+    case 'or': {
+      return filter.filters.some((f) => filterMatchObjectJSON(f, obj));
+    }
+
+    default:
+      return false;
+  }
+};
+
+/**
+ * Performs structural matching between a filter object and a target object.
+ * This handles nested object comparison for array matching scenarios.
+ */
+// TODO(wittjosiah): Add ast support for non-strict matching.
+const structuralMatch = (filterObj: any, targetObj: any, strict = true): boolean => {
+  if (typeof filterObj !== 'object' || filterObj === null) {
+    return filterObj === targetObj;
+  }
+
+  if (typeof targetObj !== 'object' || targetObj === null) {
+    return false;
+  }
+
+  // Prohibit extra keys in targetObj.
+  const filterKeys = Object.keys(filterObj);
+  const targetKeys = Object.keys(targetObj);
+  if (strict && filterKeys.length !== targetKeys.length) {
+    return false;
+  }
+
+  return filterKeys.every((key) => {
+    if (!(key in targetObj)) {
+      return false;
+    }
+    const filterValue = filterObj[key];
+    const targetValue = targetObj[key];
+
+    if (typeof filterValue === 'object' && filterValue !== null) {
+      return structuralMatch(filterValue, targetValue);
+    }
+
+    return filterValue === targetValue;
+  });
+};
+
 export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boolean => {
   switch (filter.type) {
     case 'compare': {
@@ -97,7 +229,7 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
             if (!isEncodedReference(value)) {
               return false;
             }
-            return DXN.equals(decodeReference(value).toDXN(), decodeReference(compareValue).toDXN());
+            return DXN.equals(EncodedReference.toDXN(value), EncodedReference.toDXN(compareValue));
           }
           return value === compareValue;
         case 'neq':
@@ -110,12 +242,44 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
           return (value as any) < compareValue;
         case 'lte':
           return (value as any) <= compareValue;
+        default:
+          return false;
+      }
+    }
+    case 'object': {
+      // Handle nested object filters for property matching
+      if (typeof value !== 'object' || value === null) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          const nestedValue = (value as any)[key];
+          if (!filterMatchValue(valueFilter, nestedValue)) {
+            return false;
+          }
+        }
       }
-      break;
+
+      return true;
     }
     case 'in': {
       return filter.values.includes(value);
     }
+    case 'contains': {
+      if (!Array.isArray(value)) {
+        return false;
+      }
+
+      return value.some((element) => {
+        if (typeof filter.value === 'object' && filter.value !== null && !Array.isArray(filter.value)) {
+          return structuralMatch(filter.value, element);
+        }
+
+        return element === filter.value;
+      });
+    }
     case 'range': {
       return (value as any) >= filter.from && (value as any) <= filter.to;
     }
@@ -143,11 +307,11 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
  *
  * Examples: (expected) (actual)
  *
- * dxn:type:example.org/type/Task       !== dxn:type:example.org/type/Contact
- * dxn:type:example.org/type/Task       === dxn:type:example.org/type/Task
- * dxn:type:example.org/type/Task:0.1.0 !== dxn:type:example.org/type/Task:0.2.0
- * dxn:type:example.org/type/Task       === dxn:type:example.org/type/Task:0.1.0
- * dxn:type:example.org/type/Task:0.1.0 === dxn:type:example.org/type/Task
+ * dxn:type:com.example.type.task       !== dxn:type:com.example.type.contact
+ * dxn:type:com.example.type.task       === dxn:type:com.example.type.task
+ * dxn:type:com.example.type.task:0.1.0 !== dxn:type:com.example.type.task:0.2.0
+ * dxn:type:com.example.type.task       === dxn:type:com.example.type.task:0.1.0
+ * dxn:type:com.example.type.task:0.1.0 === dxn:type:com.example.type.task
  *
  */
 const compareTypename = (expectedDXN: DXN, actualDXN: DXN): boolean => {

package/src/metadata/metadata-store.ts

@@ -16,11 +16,11 @@ import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/servic
 import {
   type ControlPipelineSnapshot,
   type EchoMetadata,
-  type SpaceMetadata,
+  type EdgeReplicationSetting,
   type IdentityRecord,
-  type SpaceCache,
   type LargeSpaceMetadata,
-  type EdgeReplicationSetting,
+  type SpaceCache,
+  type SpaceMetadata,
 } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { type Directory, type File } from '@dxos/random-access-storage';
 import { type Timeframe } from '@dxos/timeframe';
@@ -91,14 +91,17 @@ export class MetadataStore {
       log('loaded', { size: dataSize, checksum, name: file.filename });
 
       if (fileLength < dataSize + 8) {
-        throw new DataCorruptionError('Metadata size is smaller than expected.', { fileLength, dataSize });
+        throw new DataCorruptionError({
+          message: 'Metadata size is smaller than expected.',
+          context: { fileLength, dataSize },
+        });
       }
 
       const data = await file.read(8, dataSize);
 
       const calculatedChecksum = CRC32.buf(data);
       if (calculatedChecksum !== checksum) {
-        throw new DataCorruptionError('Metadata checksum is invalid.');
+        throw new DataCorruptionError({ message: 'Metadata checksum is invalid.' });
       }
 
       return codec.decode(data);
@@ -363,9 +366,9 @@ const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);
 export const hasInvitationExpired = (invitation: Invitation): boolean => {
   return Boolean(
     invitation.created &&
-      invitation.lifetime &&
-      invitation.lifetime !== 0 &&
-      invitation.created.getTime() + invitation.lifetime * 1000 < Date.now(),
+    invitation.lifetime &&
+    invitation.lifetime !== 0 &&
+    invitation.created.getTime() + invitation.lifetime * 1000 < Date.now(),
   );
 };
 

package/src/pipeline/pipeline-stress.test.ts

@@ -15,8 +15,8 @@ import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
-import { Pipeline } from './pipeline';
 import { TestFeedBuilder } from '../testing';
+import { Pipeline } from './pipeline';
 
 const NUM_AGENTS = 2;
 const NUM_MESSAGES = 10;

package/src/pipeline/pipeline.test.ts

@@ -2,15 +2,15 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { Event, sleep } from '@dxos/async';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
-import { Pipeline } from './pipeline';
 import { TestFeedBuilder } from '../testing';
+import { Pipeline } from './pipeline';
 
 const TEST_MESSAGE: FeedMessage = {
   timeframe: new Timeframe(),

package/src/pipeline/pipeline.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { Event, sleepWithContext, synchronized, Trigger } from '@dxos/async';
+import { Event, Trigger, sleepWithContext, synchronized } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
 import { failUndefined } from '@dxos/debug';
 import { FeedSetIterator, type FeedWrapper, type FeedWriter } from '@dxos/feed-store';
@@ -14,11 +14,11 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { ComplexMap } from '@dxos/util';
 
-import { createMessageSelector } from './message-selector';
-import { mapFeedIndexesToTimeframe, startAfter, TimeframeClock } from './timeframe-clock';
 import { createMappedFeedWriter } from '../common';
+import { createMessageSelector } from './message-selector';
+import { TimeframeClock, mapFeedIndexesToTimeframe, startAfter } from './timeframe-clock';
 
-export type WaitUntilReachedTargetParams = {
+export type WaitUntilReachedTargetProps = {
   /**
    * For cancellation.
    */
@@ -40,8 +40,10 @@ export class PipelineState {
    */
   _ctx = new Context();
 
-  // TODO(dmaretskyi): Remove?.
-  public readonly timeframeUpdate = this._timeframeClock.update;
+  // TODO(dmaretskyi): Remove?. Avoid accessing `_timeframeClock` before constructor initialization.
+  public get timeframeUpdate() {
+    return this._timeframeClock.update;
+  }
 
   public readonly stalled = new Event();
 
@@ -129,7 +131,7 @@ export class PipelineState {
     ctx = new Context(),
     timeout,
     breakOnStall = true,
-  }: WaitUntilReachedTargetParams = {}): Promise<void> {
+  }: WaitUntilReachedTargetProps = {}): Promise<void> {
     log('waitUntilReachedTargetTimeframe', {
       timeout,
       current: this.timeframe,

package/src/query/errors.ts

@@ -4,4 +4,6 @@
 
 import { BaseError } from '@dxos/errors';
 
-export class QueryError extends BaseError.extend('QUERY_ERROR') {}
+export class QueryError extends BaseError.extend('QueryError') {}
+
+export class InvalidQueryError extends QueryError.extend('InvalidQueryError') {}