@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1da679c

package/src/edge/echo-edge-replicator.ts

@@ -2,17 +2,24 @@
 // Copyright 2024 DXOS.org
 //
 
-import { cbor } from '@automerge/automerge-repo';
+import { type DocumentId, type Heads, cbor } from '@automerge/automerge-repo';
 
-import { Mutex, scheduleTask, scheduleMicroTask } from '@dxos/async';
+import { Mutex, scheduleMicroTask, scheduleTask } from '@dxos/async';
 import { Context, Resource } from '@dxos/context';
 import { randomUUID } from '@dxos/crypto';
 import type { CollectionId } from '@dxos/echo-protocol';
-import { type EdgeConnection } from '@dxos/edge-client';
+import { type EdgeConnection, type EdgeHttpClient } from '@dxos/edge-client';
 import { invariant } from '@dxos/invariant';
 import type { SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { EdgeService, type AutomergeProtocolMessage, type PeerId } from '@dxos/protocols';
+import {
+  type AutomergeProtocolMessage,
+  DocumentCodec,
+  EdgeService,
+  type ExportBundleRequest,
+  type ImportBundleRequest,
+  type PeerId,
+} from '@dxos/protocols';
 import { buf } from '@dxos/protocols/buf';
 import {
   type Message as RouterMessage,
@@ -20,16 +27,17 @@ import {
 } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
 import { bufferToArray } from '@dxos/util';
 
-import { InflightRequestLimiter } from './inflight-request-limiter';
 import {
-  getSpaceIdFromCollectionId,
   type EchoReplicator,
   type EchoReplicatorContext,
   type ReplicatorConnection,
   type ShouldAdvertiseParams,
   type ShouldSyncCollectionParams,
+  getSpaceIdFromCollectionId,
 } from '../automerge';
 
+import { InflightRequestLimiter } from './inflight-request-limiter';
+
 /**
  * Delay before restarting the connection after receiving a forbidden error.
  */
@@ -39,11 +47,13 @@ const MAX_RESTART_DELAY = 5000;
 
 export type EchoEdgeReplicatorParams = {
   edgeConnection: EdgeConnection;
+  edgeHttpClient: EdgeHttpClient;
   disableSharePolicy?: boolean;
 };
 
 export class EchoEdgeReplicator implements EchoReplicator {
   private readonly _edgeConnection: EdgeConnection;
+  private readonly _edgeHttpClient: EdgeHttpClient;
   private readonly _mutex = new Mutex();
 
   private _ctx?: Context = undefined;
@@ -52,8 +62,9 @@ export class EchoEdgeReplicator implements EchoReplicator {
   private _connections = new Map<SpaceId, EdgeReplicatorConnection>();
   private _sharePolicyEnabled = true;
 
-  constructor({ edgeConnection, disableSharePolicy }: EchoEdgeReplicatorParams) {
+  constructor({ edgeConnection, edgeHttpClient, disableSharePolicy }: EchoEdgeReplicatorParams) {
     this._edgeConnection = edgeConnection;
+    this._edgeHttpClient = edgeHttpClient;
     this._sharePolicyEnabled = !disableSharePolicy;
   }
 
@@ -127,14 +138,17 @@ export class EchoEdgeReplicator implements EchoReplicator {
     let restartScheduled = false;
 
     const connection = new EdgeReplicatorConnection({
+      edgeHttpClient: this._edgeHttpClient,
       edgeConnection: this._edgeConnection,
       spaceId,
       context: this._context,
       sharedPolicyEnabled: this._sharePolicyEnabled,
       onRemoteConnected: async () => {
+        log.trace('dxos.echo.edge.replicator.onRemoteConnected', { spaceId });
         this._context?.onConnectionOpen(connection);
       },
       onRemoteDisconnected: async () => {
+        log.trace('dxos.echo.edge.replicator.onRemoteDisconnected', { spaceId });
         this._context?.onConnectionClosed(connection);
       },
       onRestartRequested: async () => {
@@ -162,6 +176,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
             if (ctx?.disposed) {
               return;
             }
+            log.trace('dxos.echo.edge.replicator.restart', { spaceId, reconnects, restartDelay });
             await this._openConnection(spaceId, reconnects + 1);
           },
           restartDelay,
@@ -176,6 +191,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
 
 type EdgeReplicatorConnectionsParams = {
   edgeConnection: EdgeConnection;
+  edgeHttpClient: EdgeHttpClient;
   spaceId: SpaceId;
   context: EchoReplicatorContext;
   sharedPolicyEnabled: boolean;
@@ -189,6 +205,7 @@ const MAX_RATE_LIMIT_WAIT_TIME_MS = 3000;
 
 class EdgeReplicatorConnection extends Resource implements ReplicatorConnection {
   private readonly _edgeConnection: EdgeConnection;
+  private readonly _edgeHttpClient: EdgeHttpClient;
   private readonly _remotePeerId: string | null = null;
   private readonly _targetServiceId: string;
   private readonly _spaceId: SpaceId;
@@ -210,6 +227,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
 
   constructor({
     edgeConnection,
+    edgeHttpClient,
     spaceId,
     context,
     sharedPolicyEnabled,
@@ -219,6 +237,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
   }: EdgeReplicatorConnectionsParams) {
     super();
     this._edgeConnection = edgeConnection;
+    this._edgeHttpClient = edgeHttpClient;
     this._spaceId = spaceId;
     this._context = context;
     // Generate a unique peer id for every connection.
@@ -252,13 +271,26 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
 
     await this._requestLimiter.open();
 
-    // TODO: handle reconnects
     this._ctx.onDispose(
       this._edgeConnection.onMessage((msg: RouterMessage) => {
         this._onMessage(msg);
       }),
     );
 
+    let firstReconnect = true;
+    this._ctx.onDispose(
+      // NOTE: This will fire immediately if the connection is already open.
+      this._edgeConnection.onReconnected(async () => {
+        if (firstReconnect) {
+          log.verbose('first reconnect skipped');
+          firstReconnect = false;
+          return;
+        }
+
+        this._onRestartRequested();
+      }),
+    );
+
     await this._onRemoteConnected();
   }
 
@@ -327,6 +359,27 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
     this._processMessage(payload);
   }
 
+  get bundleSyncEnabled(): boolean {
+    return true;
+  }
+
+  async pushBundle(bundle: { documentId: DocumentId; data: Uint8Array; heads: Heads }[]) {
+    const request: ImportBundleRequest = {
+      bundle: bundle.map(({ documentId, data, heads }) => ({
+        documentId,
+        mutation: DocumentCodec.encode(data),
+        heads,
+      })),
+    };
+    await this._edgeHttpClient.importBundle(this._spaceId, request);
+  }
+
+  async pullBundle(docHeads: Record<DocumentId, Heads>): Promise<Record<DocumentId, Uint8Array>> {
+    const request: ExportBundleRequest = { docHeads };
+    const response = await this._edgeHttpClient.exportBundle(this._spaceId, request);
+    return Object.fromEntries(response.bundle.map((doc) => [doc.documentId, DocumentCodec.decode(doc.mutation)]));
+  }
+
   private _processMessage(message: AutomergeProtocolMessage): void {
     // There's a race between the credentials being replicated that are needed for access control and the data replication.
     // AutomergeReplicator might return a Forbidden error if the credentials are not yet replicated.
@@ -353,16 +406,20 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
 
     const encoded = cbor.encode(message);
 
-    await this._edgeConnection.send(
-      buf.create(RouterMessageSchema, {
-        serviceId: this._targetServiceId,
-        source: {
-          identityKey: this._edgeConnection.identityKey,
-          peerKey: this._edgeConnection.peerKey,
-        },
-        payload: { value: bufferToArray(encoded) },
-      }),
-    );
+    try {
+      await this._edgeConnection.send(
+        buf.create(RouterMessageSchema, {
+          serviceId: this._targetServiceId,
+          source: {
+            identityKey: this._edgeConnection.identityKey,
+            peerKey: this._edgeConnection.peerKey,
+          },
+          payload: { value: bufferToArray(encoded) },
+        }),
+      );
+    } catch (err) {
+      log.error('failed to send message', { err });
+    }
   }
 }
 

package/src/filter/filter-match.test.ts

@@ -6,10 +6,10 @@ import { describe, expect, test } from 'vitest';
 
 import { Filter } from '@dxos/echo';
 import { ObjectStructure } from '@dxos/echo-protocol';
-import { Expando, EXPANDO_TYPENAME, Ref } from '@dxos/echo-schema';
+import { EXPANDO_TYPENAME, Expando, Ref } from '@dxos/echo-schema';
 import { DXN, ObjectId, SpaceId } from '@dxos/keys';
 
-import { filterMatchObject, type MatchedObject } from './filter-match';
+import { type MatchedObject, filterMatchObject } from './filter-match';
 
 describe('filterMatch', () => {
   test('properties', () => {
@@ -17,6 +17,19 @@ describe('filterMatch', () => {
     expect(filterMatchObject(Filter.type(Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
     expect(filterMatchObject(Filter.type(Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
     expect(filterMatchObject(Filter.type(Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.type(Expando, { properties: { subject: 'test' } }).ast, OBJECT_1)).to.be.true;
+    expect(filterMatchObject(Filter.type(Expando, { array: Filter.contains('two') }).ast, OBJECT_1)).to.be.true;
+  });
+
+  test('contains', () => {
+    expect(filterMatchObject(Filter.type(Expando, { properties: { label: Filter.contains('test') } }).ast, OBJECT_1)).to
+      .be.true;
+    expect(
+      filterMatchObject(
+        Filter.type(Expando, { fields: Filter.contains({ label: 'label', value: 'test' }) }).ast,
+        OBJECT_1,
+      ),
+    ).to.be.true;
   });
 
   test('and', () => {
@@ -78,7 +91,14 @@ const OBJECT_1: MatchedObject = {
   spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
   doc: ObjectStructure.makeObject({
     type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
-    data: { title: 'test', value: 100, complete: true },
+    data: {
+      title: 'test',
+      value: 100,
+      complete: true,
+      array: ['one', 'two', 'three'],
+      properties: { label: ['test'], subject: 'test' },
+      fields: [{ label: 'label', value: 'test' }],
+    },
   }),
 };
 

package/src/filter/filter-match.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { decodeReference, isEncodedReference, type QueryAST, type ObjectStructure } from '@dxos/echo-protocol';
-import { EXPANDO_TYPENAME } from '@dxos/echo-schema';
+import { type ObjectStructure, type QueryAST, decodeReference, isEncodedReference } from '@dxos/echo-protocol';
+import { EXPANDO_TYPENAME, type ObjectJSON } from '@dxos/echo-schema';
 import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
 
 export type MatchedObject = {
@@ -14,6 +14,7 @@ export type MatchedObject = {
 
 /**
  * Matches an object against a filter AST.
+ * @param obj object structure as stored in automerge.
  */
 export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject): boolean => {
   switch (filter.type) {
@@ -87,6 +88,118 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
   }
 };
 
+export const filterMatchObjectJSON = (filter: QueryAST.Filter, obj: ObjectJSON): boolean => {
+  switch (filter.type) {
+    case 'object': {
+      // Check typename if specified
+      if (filter.typename !== null) {
+        // TODO(dmaretskyi): `system` is missing in some cases.
+        if (!obj['@type']) {
+          // Objects with no type are considered to be expando objects
+          const expectedDXN = DXN.parse(filter.typename).asTypeDXN();
+          if (expectedDXN?.type !== EXPANDO_TYPENAME) {
+            return false;
+          }
+        } else {
+          const actualDXN = DXN.parse(obj['@type']);
+          const expectedDXN = DXN.parse(filter.typename);
+
+          if (!compareTypename(expectedDXN, actualDXN)) {
+            return false;
+          }
+        }
+      }
+
+      // Check IDs if specified
+      if (filter.id && filter.id.length > 0 && !filter.id.includes(obj.id)) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          if (key.startsWith('@')) {
+            // ignore meta properties
+            continue;
+          }
+          const value = (obj as any)[key];
+          if (!filterMatchValue(valueFilter, value)) {
+            return false;
+          }
+        }
+      }
+
+      // Check foreign keys if specified
+      if (filter.foreignKeys && filter.foreignKeys.length > 0) {
+        const hasMatchingKey = filter.foreignKeys.some((filterKey) =>
+          obj['@meta']?.keys?.some((objKey) => objKey.source === filterKey.source && objKey.id === filterKey.id),
+        );
+        if (!hasMatchingKey) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    case 'text-search': {
+      // TODO: Implement text search
+      return false;
+    }
+
+    case 'not': {
+      return !filterMatchObjectJSON(filter.filter, obj);
+    }
+
+    case 'and': {
+      return filter.filters.every((f) => filterMatchObjectJSON(f, obj));
+    }
+
+    case 'or': {
+      return filter.filters.some((f) => filterMatchObjectJSON(f, obj));
+    }
+
+    default:
+      return false;
+  }
+};
+
+/**
+ * Performs structural matching between a filter object and a target object.
+ * This handles nested object comparison for array matching scenarios.
+ */
+// TODO(wittjosiah): Add ast support for non-strict matching.
+const structuralMatch = (filterObj: any, targetObj: any, strict = true): boolean => {
+  if (typeof filterObj !== 'object' || filterObj === null) {
+    return filterObj === targetObj;
+  }
+
+  if (typeof targetObj !== 'object' || targetObj === null) {
+    return false;
+  }
+
+  // Prohibit extra keys in targetObj.
+  const filterKeys = Object.keys(filterObj);
+  const targetKeys = Object.keys(targetObj);
+  if (strict && filterKeys.length !== targetKeys.length) {
+    return false;
+  }
+
+  return filterKeys.every((key) => {
+    if (!(key in targetObj)) {
+      return false;
+    }
+    const filterValue = filterObj[key];
+    const targetValue = targetObj[key];
+
+    if (typeof filterValue === 'object' && filterValue !== null) {
+      return structuralMatch(filterValue, targetValue);
+    }
+
+    return filterValue === targetValue;
+  });
+};
+
 export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boolean => {
   switch (filter.type) {
     case 'compare': {
@@ -110,12 +223,44 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
           return (value as any) < compareValue;
         case 'lte':
           return (value as any) <= compareValue;
+        default:
+          return false;
       }
-      break;
+    }
+    case 'object': {
+      // Handle nested object filters for property matching
+      if (typeof value !== 'object' || value === null) {
+        return false;
+      }
+
+      // Check properties
+      if (filter.props) {
+        for (const [key, valueFilter] of Object.entries(filter.props)) {
+          const nestedValue = (value as any)[key];
+          if (!filterMatchValue(valueFilter, nestedValue)) {
+            return false;
+          }
+        }
+      }
+
+      return true;
     }
     case 'in': {
       return filter.values.includes(value);
     }
+    case 'contains': {
+      if (!Array.isArray(value)) {
+        return false;
+      }
+
+      return value.some((element) => {
+        if (typeof filter.value === 'object' && filter.value !== null && !Array.isArray(filter.value)) {
+          return structuralMatch(filter.value, element);
+        }
+
+        return element === filter.value;
+      });
+    }
     case 'range': {
       return (value as any) >= filter.from && (value as any) <= filter.to;
     }

package/src/metadata/metadata-store.ts

@@ -16,11 +16,11 @@ import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/servic
 import {
   type ControlPipelineSnapshot,
   type EchoMetadata,
-  type SpaceMetadata,
+  type EdgeReplicationSetting,
   type IdentityRecord,
-  type SpaceCache,
   type LargeSpaceMetadata,
-  type EdgeReplicationSetting,
+  type SpaceCache,
+  type SpaceMetadata,
 } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { type Directory, type File } from '@dxos/random-access-storage';
 import { type Timeframe } from '@dxos/timeframe';

package/src/pipeline/pipeline-stress.test.ts

@@ -2,8 +2,9 @@
 // Copyright 2022 DXOS.org
 //
 
-import * as fc from 'fast-check';
 import { inspect } from 'node:util';
+
+import * as fc from 'fast-check';
 import { describe, expect, test } from 'vitest';
 
 import { asyncTimeout } from '@dxos/async';
@@ -15,9 +16,10 @@ import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
-import { Pipeline } from './pipeline';
 import { TestFeedBuilder } from '../testing';
 
+import { Pipeline } from './pipeline';
+
 const NUM_AGENTS = 2;
 const NUM_MESSAGES = 10;
 

package/src/pipeline/pipeline.test.ts

@@ -2,16 +2,17 @@
 // Copyright 2022 DXOS.org
 //
 
-import { describe, expect, test, onTestFinished } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { Event, sleep } from '@dxos/async';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
-import { Pipeline } from './pipeline';
 import { TestFeedBuilder } from '../testing';
 
+import { Pipeline } from './pipeline';
+
 const TEST_MESSAGE: FeedMessage = {
   timeframe: new Timeframe(),
   payload: {},

package/src/pipeline/pipeline.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { Event, sleepWithContext, synchronized, Trigger } from '@dxos/async';
+import { Event, Trigger, sleepWithContext, synchronized } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
 import { failUndefined } from '@dxos/debug';
 import { FeedSetIterator, type FeedWrapper, type FeedWriter } from '@dxos/feed-store';
@@ -14,10 +14,11 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 import { ComplexMap } from '@dxos/util';
 
-import { createMessageSelector } from './message-selector';
-import { mapFeedIndexesToTimeframe, startAfter, TimeframeClock } from './timeframe-clock';
 import { createMappedFeedWriter } from '../common';
 
+import { createMessageSelector } from './message-selector';
+import { TimeframeClock, mapFeedIndexesToTimeframe, startAfter } from './timeframe-clock';
+
 export type WaitUntilReachedTargetParams = {
   /**
    * For cancellation.
@@ -40,8 +41,10 @@ export class PipelineState {
    */
   _ctx = new Context();
 
-  // TODO(dmaretskyi): Remove?.
-  public readonly timeframeUpdate = this._timeframeClock.update;
+  // TODO(dmaretskyi): Remove?. Avoid accessing `_timeframeClock` before constructor initialization.
+  public get timeframeUpdate() {
+    return this._timeframeClock.update;
+  }
 
   public readonly stalled = new Event();
 

package/src/query/errors.ts

@@ -5,3 +5,5 @@
 import { BaseError } from '@dxos/errors';
 
 export class QueryError extends BaseError.extend('QUERY_ERROR') {}
+
+export class InvalidQueryError extends QueryError.extend('INVALID_QUERY') {}

package/src/query/plan.ts

@@ -32,7 +32,8 @@ export namespace QueryPlan {
     | FilterDeletedStep
     | TraverseStep
     | UnionStep
-    | SetDifferenceStep;
+    | SetDifferenceStep
+    | OrderStep;
 
   /**
    * Clear the current working set.
@@ -193,4 +194,14 @@ export namespace QueryPlan {
     source: Plan;
     exclude: Plan;
   };
+
+  /**
+   * Order the results of the plan.
+   */
+  export type OrderStep = {
+    _tag: 'OrderStep';
+
+    // Defaults to natural order if empty.
+    order: readonly QueryAST.Order[];
+  };
 }