@dxos/echo-pipeline 0.8.2-staging.7ac8446 → 0.8.2

package/src/filter/filter-match.ts

@@ -0,0 +1,174 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { decodeReference, isEncodedReference, type QueryAST, type ObjectStructure } from '@dxos/echo-protocol';
+import { EXPANDO_TYPENAME } from '@dxos/echo-schema';
+import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
+
+export type MatchedObject = {
+  id: ObjectId;
+  spaceId: SpaceId;
+  doc: ObjectStructure;
+};
+
+/**
+ * Matches an object against a filter AST.
+ */
+export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject): boolean => {
+  switch (filter.type) {
+    case 'object': {
+      // Check typename if specified
+      if (filter.typename !== null) {
+        // TODO(dmaretskyi): `system` is missing in some cases.
+        if (!obj.doc.system?.type?.['/']) {
+          // Objects with no type are considered to be expando objects
+          const expectedDXN = DXN.parse(filter.typename).asTypeDXN();
+          if (expectedDXN?.type !== EXPANDO_TYPENAME) {
+            return false;
+          }
+        } else {
+          const actualDXN = DXN.parse(obj.doc.system.type['/']);
+          const expectedDXN = DXN.parse(filter.typename);
+
+          if (!compareTypename(expectedDXN, actualDXN)) {
+            return false;
+          }
+        }
+      }
+
+      // Check IDs if specified
+      if (filter.id && filter.id.length > 0 && !filter.id.includes(obj.id)) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          const value = obj.doc.data[key];
+          if (!filterMatchValue(valueFilter, value)) {
+            return false;
+          }
+        }
+      }
+
+      // Check foreign keys if specified
+      if (filter.foreignKeys && filter.foreignKeys.length > 0) {
+        const hasMatchingKey = filter.foreignKeys.some((filterKey) =>
+          obj.doc.meta.keys.some((objKey) => objKey.source === filterKey.source && objKey.id === filterKey.id),
+        );
+        if (!hasMatchingKey) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    case 'text-search': {
+      // TODO: Implement text search
+      return false;
+    }
+
+    case 'not': {
+      return !filterMatchObject(filter.filter, obj);
+    }
+
+    case 'and': {
+      return filter.filters.every((f) => filterMatchObject(f, obj));
+    }
+
+    case 'or': {
+      return filter.filters.some((f) => filterMatchObject(f, obj));
+    }
+
+    default:
+      return false;
+  }
+};
+
+export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boolean => {
+  switch (filter.type) {
+    case 'compare': {
+      const compareValue = filter.value as any;
+      switch (filter.operator) {
+        case 'eq':
+          if (isEncodedReference(compareValue)) {
+            if (!isEncodedReference(value)) {
+              return false;
+            }
+            return DXN.equals(decodeReference(value).toDXN(), decodeReference(compareValue).toDXN());
+          }
+          return value === compareValue;
+        case 'neq':
+          return value !== compareValue;
+        case 'gt':
+          return (value as any) > compareValue;
+        case 'gte':
+          return (value as any) >= compareValue;
+        case 'lt':
+          return (value as any) < compareValue;
+        case 'lte':
+          return (value as any) <= compareValue;
+      }
+      break;
+    }
+    case 'in': {
+      return filter.values.includes(value);
+    }
+    case 'range': {
+      return (value as any) >= filter.from && (value as any) <= filter.to;
+    }
+    case 'not': {
+      return !filterMatchValue(filter.filter, value);
+    }
+    case 'and': {
+      return filter.filters.every((f) => filterMatchValue(f, value));
+    }
+    case 'or': {
+      return filter.filters.some((f) => filterMatchValue(f, value));
+    }
+    default:
+      return false;
+  }
+};
+
+/**
+ * Compares typename DXNs.
+ * @returns true if they match
+ *
+ * Compares typename string.
+ * Missing version (on either actual or expected) matches any version.
+ * non `type` DXNs are compared exactly.
+ *
+ * Examples: (expected) (actual)
+ *
+ * dxn:type:example.org/type/Task       !== dxn:type:example.org/type/Contact
+ * dxn:type:example.org/type/Task       === dxn:type:example.org/type/Task
+ * dxn:type:example.org/type/Task:0.1.0 !== dxn:type:example.org/type/Task:0.2.0
+ * dxn:type:example.org/type/Task       === dxn:type:example.org/type/Task:0.1.0
+ * dxn:type:example.org/type/Task:0.1.0 === dxn:type:example.org/type/Task
+ *
+ */
+const compareTypename = (expectedDXN: DXN, actualDXN: DXN): boolean => {
+  const expectedTypeDXN = expectedDXN.asTypeDXN();
+  if (expectedTypeDXN) {
+    const actualTypeDXN = actualDXN.asTypeDXN();
+    if (!actualTypeDXN) {
+      return false;
+    }
+    if (
+      actualTypeDXN.type !== expectedTypeDXN.type ||
+      (expectedTypeDXN.version !== undefined &&
+        actualTypeDXN.version !== undefined &&
+        actualTypeDXN.version !== expectedTypeDXN.version)
+    ) {
+      return false;
+    }
+  } else {
+    if (!DXN.equals(actualDXN, expectedDXN)) {
+      return false;
+    }
+  }
+  return true;
+};

package/src/filter/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './filter-match';

package/src/index.ts

@@ -10,3 +10,4 @@ export * from './space';
 export * from './automerge';
 export * from './edge';
 export * from './util';
+export * from './query';

package/src/metadata/metadata-store.ts

@@ -126,7 +126,7 @@ export class MetadataStore {
     log('saved', { size: encoded.length, checksum });
   }
 
-  async close() {
+  async close(): Promise<void> {
     await this._invitationCleanupCtx.dispose();
     await this.flush();
     await this._metadataFile?.close();
@@ -220,7 +220,7 @@ export class MetadataStore {
     await this._writeFile(file, LargeSpaceMetadata, data);
   }
 
-  async flush() {
+  async flush(): Promise<void> {
     await this._directory.flush();
   }
 
@@ -268,7 +268,7 @@ export class MetadataStore {
     return this._metadata.identity;
   }
 
-  async setIdentityRecord(record: IdentityRecord) {
+  async setIdentityRecord(record: IdentityRecord): Promise<void> {
     invariant(!this._metadata.identity, 'Cannot overwrite existing identity in metadata');
 
     this._metadata.identity = record;
@@ -280,7 +280,7 @@ export class MetadataStore {
     return this._metadata.invitations ?? [];
   }
 
-  async addInvitation(invitation: Invitation) {
+  async addInvitation(invitation: Invitation): Promise<void> {
     if (this._metadata.invitations?.find((i) => i.invitationId === invitation.invitationId)) {
       return;
     }
@@ -290,13 +290,13 @@ export class MetadataStore {
     await this.flush();
   }
 
-  async removeInvitation(invitationId: string) {
+  async removeInvitation(invitationId: string): Promise<void> {
     this._metadata.invitations = (this._metadata.invitations ?? []).filter((i) => i.invitationId !== invitationId);
     await this._save();
     await this.flush();
   }
 
-  async addSpace(record: SpaceMetadata) {
+  async addSpace(record: SpaceMetadata): Promise<void> {
     invariant(
       !(this._metadata.spaces ?? []).find((space) => space.key.equals(record.key)),
       'Cannot overwrite existing space in metadata',
@@ -307,23 +307,23 @@ export class MetadataStore {
     await this.flush();
   }
 
-  async setSpaceDataLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe) {
+  async setSpaceDataLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe): Promise<void> {
     this._getSpace(spaceKey).dataTimeframe = timeframe;
     await this._save();
   }
 
-  async setSpaceControlLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe) {
+  async setSpaceControlLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe): Promise<void> {
     this._getSpace(spaceKey).controlTimeframe = timeframe;
     await this._save();
     await this.flush();
   }
 
-  async setCache(spaceKey: PublicKey, cache: SpaceCache) {
+  async setCache(spaceKey: PublicKey, cache: SpaceCache): Promise<void> {
     this._getSpace(spaceKey).cache = cache;
     await this._save();
   }
 
-  async setWritableFeedKeys(spaceKey: PublicKey, controlFeedKey: PublicKey, dataFeedKey: PublicKey) {
+  async setWritableFeedKeys(spaceKey: PublicKey, controlFeedKey: PublicKey, dataFeedKey: PublicKey): Promise<void> {
     const space = this._getSpace(spaceKey);
     space.controlFeedKey = controlFeedKey;
     space.dataFeedKey = dataFeedKey;
@@ -331,7 +331,7 @@ export class MetadataStore {
     await this.flush();
   }
 
-  async setSpaceState(spaceKey: PublicKey, state: SpaceState) {
+  async setSpaceState(spaceKey: PublicKey, state: SpaceState): Promise<void> {
     this._getSpace(spaceKey).state = state;
     await this._save();
     await this.flush();
@@ -341,7 +341,7 @@ export class MetadataStore {
     return this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot;
   }
 
-  async setSpaceControlPipelineSnapshot(spaceKey: PublicKey, snapshot: ControlPipelineSnapshot) {
+  async setSpaceControlPipelineSnapshot(spaceKey: PublicKey, snapshot: ControlPipelineSnapshot): Promise<void> {
     this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot = snapshot;
     await this._saveSpaceLargeMetadata(spaceKey);
     await this.flush();
@@ -351,7 +351,7 @@ export class MetadataStore {
     return this.hasSpace(spaceKey) ? this._getSpace(spaceKey).edgeReplication : undefined;
   }
 
-  async setSpaceEdgeReplicationSetting(spaceKey: PublicKey, setting: EdgeReplicationSetting) {
+  async setSpaceEdgeReplicationSetting(spaceKey: PublicKey, setting: EdgeReplicationSetting): Promise<void> {
     this._getSpace(spaceKey).edgeReplication = setting;
     await this._save();
     await this.flush();

package/src/pipeline/pipeline-stress.test.ts

@@ -83,12 +83,12 @@ class Agent {
     public id: string,
   ) {}
 
-  async open() {
+  async open(): Promise<void> {
     const key = await this.builder.keyring.createKey();
     this.feed = await this.feedStore.openFeed(key, { writable: true });
   }
 
-  async start() {
+  async start(): Promise<void> {
     this.pipeline = new Pipeline();
     await this.pipeline.setCursor(this.startingTimeframe);
     await this.pipeline.start();
@@ -101,22 +101,22 @@ class Agent {
     void this.consume();
   }
 
-  async stop() {
+  async stop(): Promise<void> {
     await this.pipeline.stop();
     this.startingTimeframe = this.pipeline.state.timeframe;
   }
 
-  async close() {
+  async close(): Promise<void> {
     await this.feed.close();
   }
 
-  write(message: FeedMessage.Payload) {
+  write(message: FeedMessage.Payload): void {
     const prev = this.writePromise;
     const promise = this.pipeline.writer!.write(message);
     this.writePromise = Promise.all([prev, promise]);
   }
 
-  async consume() {
+  async consume(): Promise<void> {
     for await (const msg of this.pipeline.consume()) {
       this.messages.push(msg);
     }
@@ -139,7 +139,7 @@ class WriteCommand implements fc.AsyncCommand<Model, Real> {
 
   check = () => true;
 
-  async run(model: Model, real: Real) {
+  async run(model: Model, real: Real): Promise<void> {
     // console.log(`WriteCommand(${this.agent}, ${this.count})`);
     const agent = real.agents.get(this.agent)!;
     const toWrite = Math.min(this.count, NUM_MESSAGES - agent.feed.length);
@@ -156,7 +156,7 @@ class WriteCommand implements fc.AsyncCommand<Model, Real> {
 class SyncCommand implements fc.AsyncCommand<Model, Real> {
   check = () => true;
 
-  async run(model: Model, real: Real) {
+  async run(model: Model, real: Real): Promise<void> {
     // console.log('SyncCommand()');
     const targets: any = {};
 
@@ -217,7 +217,7 @@ class RestartCommand implements fc.AsyncCommand<Model, Real> {
 
   check = () => true;
 
-  async run(model: Model, real: Real) {
+  async run(model: Model, real: Real): Promise<void> {
     log(`RestartCommand(${this.agent})`);
 
     const agent = real.agents.get(this.agent)!;

package/src/pipeline/pipeline.ts

@@ -110,11 +110,11 @@ export class PipelineState {
     return Array.from(this._feeds.values());
   }
 
-  async waitUntilTimeframe(target: Timeframe) {
+  async waitUntilTimeframe(target: Timeframe): Promise<void> {
     await this._timeframeClock.waitUntilReached(target);
   }
 
-  setTargetTimeframe(target: Timeframe) {
+  setTargetTimeframe(target: Timeframe): void {
     this._targetTimeframe = target;
   }
 
@@ -129,7 +129,7 @@ export class PipelineState {
     ctx = new Context(),
     timeout,
     breakOnStall = true,
-  }: WaitUntilReachedTargetParams = {}) {
+  }: WaitUntilReachedTargetParams = {}): Promise<void> {
     log('waitUntilReachedTargetTimeframe', {
       timeout,
       current: this.timeframe,
@@ -244,7 +244,7 @@ export class Pipeline implements PipelineAccessor {
     return this._writer;
   }
 
-  hasFeed(feedKey: PublicKey) {
+  hasFeed(feedKey: PublicKey): boolean {
     return this._feeds.has(feedKey);
   }
 
@@ -254,7 +254,7 @@ export class Pipeline implements PipelineAccessor {
 
   // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
   // which might be opening feeds during the mutation processing, which w
-  async addFeed(feed: FeedWrapper<FeedMessage>) {
+  async addFeed(feed: FeedWrapper<FeedMessage>): Promise<void> {
     this._feeds.set(feed.key, feed);
 
     if (this._feedSetIterator) {
@@ -266,7 +266,7 @@ export class Pipeline implements PipelineAccessor {
     }
   }
 
-  setWriteFeed(feed: FeedWrapper<FeedMessage>) {
+  setWriteFeed(feed: FeedWrapper<FeedMessage>): void {
     invariant(!this._writer, 'Writer already set.');
     invariant(feed.properties.writable, 'Feed must be writable.');
 
@@ -280,7 +280,7 @@ export class Pipeline implements PipelineAccessor {
   }
 
   @synchronized
-  async start() {
+  async start(): Promise<void> {
     invariant(!this._isStarted, 'Pipeline is already started.');
     log('starting...');
     await this._initIterator();
@@ -296,7 +296,7 @@ export class Pipeline implements PipelineAccessor {
   }
 
   @synchronized
-  async stop() {
+  async stop(): Promise<void> {
     log('stopping...');
     this._isStopping = true;
     for (const [feed, handle] of this._downloads.entries()) {
@@ -318,7 +318,7 @@ export class Pipeline implements PipelineAccessor {
    *  The pipeline will start processing messages AFTER this timeframe.
    */
   @synchronized
-  async setCursor(timeframe: Timeframe) {
+  async setCursor(timeframe: Timeframe): Promise<void> {
     invariant(!this._isStarted || this._isPaused, 'Invalid state.');
 
     this._state._startTimeframe = timeframe;
@@ -336,7 +336,7 @@ export class Pipeline implements PipelineAccessor {
    * Calling pause while processing will cause a deadlock.
    */
   @synchronized
-  async pause() {
+  async pause(): Promise<void> {
     if (this._isPaused) {
       return;
     }
@@ -347,7 +347,7 @@ export class Pipeline implements PipelineAccessor {
   }
 
   @synchronized
-  async unpause() {
+  async unpause(): Promise<void> {
     invariant(this._isPaused, 'Pipeline is not paused.');
 
     this._pauseTrigger.wake();
@@ -396,7 +396,7 @@ export class Pipeline implements PipelineAccessor {
     this._isBeingConsumed = false;
   }
 
-  private _setFeedDownloadState(feed: FeedWrapper<FeedMessage>) {
+  private _setFeedDownloadState(feed: FeedWrapper<FeedMessage>): void {
     let handle = this._downloads.get(feed); // TODO(burdon): Always undefined?
     if (handle) {
       feed.undownload(handle);
@@ -416,7 +416,7 @@ export class Pipeline implements PipelineAccessor {
     this._downloads.set(feed, handle);
   }
 
-  private async _initIterator() {
+  private async _initIterator(): Promise<void> {
     this._feedSetIterator = new FeedSetIterator<FeedMessage>(createMessageSelector(this._timeframeClock), {
       start: startAfter(this._timeframeClock.timeframe),
       stallTimeout: 1000,

package/src/pipeline/timeframe-clock.ts

@@ -45,28 +45,28 @@ export class TimeframeClock {
     return this._pendingTimeframe;
   }
 
-  setTimeframe(timeframe: Timeframe) {
+  setTimeframe(timeframe: Timeframe): void {
     this._timeframe = timeframe;
     this._pendingTimeframe = timeframe;
     this.update.emit(this._timeframe);
   }
 
-  updatePendingTimeframe(key: PublicKey, seq: number) {
+  updatePendingTimeframe(key: PublicKey, seq: number): void {
     this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([[key, seq]]));
   }
 
-  updateTimeframe() {
+  updateTimeframe(): void {
     this._timeframe = this._pendingTimeframe;
     this.update.emit(this._timeframe);
   }
 
-  hasGaps(timeframe: Timeframe) {
+  hasGaps(timeframe: Timeframe): boolean {
     const gaps = Timeframe.dependencies(timeframe, this._timeframe);
     return !gaps.isEmpty();
   }
 
   @timed(5_000)
-  async waitUntilReached(target: Timeframe) {
+  async waitUntilReached(target: Timeframe): Promise<void> {
     log('waitUntilReached', { target, current: this._timeframe });
     await this.update.waitForCondition(() => {
       log('check if reached', {

package/src/query/errors.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { BaseError } from '@dxos/errors';
+
+export class QueryError extends BaseError.extend('QUERY_ERROR') {}

package/src/query/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './query-executor';
+export * from './query-planner';
+export * from './plan';
+export * from '../filter/filter-match';