@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/metadata/metadata-store.ts

@@ -91,14 +91,17 @@ export class MetadataStore {
       log('loaded', { size: dataSize, checksum, name: file.filename });
 
       if (fileLength < dataSize + 8) {
-        throw new DataCorruptionError('Metadata size is smaller than expected.', { fileLength, dataSize });
+        throw new DataCorruptionError({
+          message: 'Metadata size is smaller than expected.',
+          context: { fileLength, dataSize },
+        });
       }
 
       const data = await file.read(8, dataSize);
 
       const calculatedChecksum = CRC32.buf(data);
       if (calculatedChecksum !== checksum) {
-        throw new DataCorruptionError('Metadata checksum is invalid.');
+        throw new DataCorruptionError({ message: 'Metadata checksum is invalid.' });
       }
 
       return codec.decode(data);
@@ -363,9 +366,9 @@ const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);
 export const hasInvitationExpired = (invitation: Invitation): boolean => {
   return Boolean(
     invitation.created &&
-      invitation.lifetime &&
-      invitation.lifetime !== 0 &&
-      invitation.created.getTime() + invitation.lifetime * 1000 < Date.now(),
+    invitation.lifetime &&
+    invitation.lifetime !== 0 &&
+    invitation.created.getTime() + invitation.lifetime * 1000 < Date.now(),
   );
 };
 

package/src/pipeline/pipeline-stress.test.ts

@@ -2,9 +2,8 @@
 // Copyright 2022 DXOS.org
 //
 
-import { inspect } from 'node:util';
-
 import * as fc from 'fast-check';
+import { inspect } from 'node:util';
 import { describe, expect, test } from 'vitest';
 
 import { asyncTimeout } from '@dxos/async';
@@ -17,7 +16,6 @@ import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
 import { TestFeedBuilder } from '../testing';
-
 import { Pipeline } from './pipeline';
 
 const NUM_AGENTS = 2;

package/src/pipeline/pipeline.test.ts

@@ -10,7 +10,6 @@ import { Timeframe } from '@dxos/timeframe';
 import { range } from '@dxos/util';
 
 import { TestFeedBuilder } from '../testing';
-
 import { Pipeline } from './pipeline';
 
 const TEST_MESSAGE: FeedMessage = {

package/src/pipeline/pipeline.ts

@@ -15,11 +15,10 @@ import { Timeframe } from '@dxos/timeframe';
 import { ComplexMap } from '@dxos/util';
 
 import { createMappedFeedWriter } from '../common';
-
 import { createMessageSelector } from './message-selector';
 import { TimeframeClock, mapFeedIndexesToTimeframe, startAfter } from './timeframe-clock';
 
-export type WaitUntilReachedTargetParams = {
+export type WaitUntilReachedTargetProps = {
   /**
    * For cancellation.
    */
@@ -132,7 +131,7 @@ export class PipelineState {
     ctx = new Context(),
     timeout,
     breakOnStall = true,
-  }: WaitUntilReachedTargetParams = {}): Promise<void> {
+  }: WaitUntilReachedTargetProps = {}): Promise<void> {
     log('waitUntilReachedTargetTimeframe', {
       timeout,
       current: this.timeframe,

package/src/query/errors.ts

@@ -4,6 +4,6 @@
 
 import { BaseError } from '@dxos/errors';
 
-export class QueryError extends BaseError.extend('QUERY_ERROR') {}
+export class QueryError extends BaseError.extend('QueryError') {}
 
-export class InvalidQueryError extends QueryError.extend('INVALID_QUERY') {}
+export class InvalidQueryError extends QueryError.extend('InvalidQueryError') {}

package/src/query/plan.ts

@@ -3,8 +3,8 @@
 //
 
 import type { QueryAST } from '@dxos/echo-protocol';
-import type { EscapedPropPath } from '@dxos/indexing';
-import type { DXN, ObjectId, SpaceId } from '@dxos/keys';
+import type { EscapedPropPath } from '@dxos/index-core';
+import type { DXN, ObjectId } from '@dxos/keys';
 
 export namespace QueryPlan {
   export type TextSearchKind = 'full-text' | 'vector' | 'hybrid';
@@ -33,7 +33,8 @@ export namespace QueryPlan {
     | TraverseStep
     | UnionStep
     | SetDifferenceStep
-    | OrderStep;
+    | OrderStep
+    | LimitStep;
 
   /**
    * Clear the current working set.
@@ -44,20 +45,34 @@ export namespace QueryPlan {
 
   /**
    * Select objects based on id, typename, or other predicates.
-   * Specifies the spaces to select from.
+   * Specifies the spaces and/or queues to select from.
+   * The results from spaces and queues are unioned together.
    */
   export type SelectStep = {
     _tag: 'SelectStep';
 
-    spaces: readonly SpaceId[];
+    /**
+     * What to select from the scope.
+     */
     selector: Selector;
+
+    /**
+     * Where to select from.
+     */
+    scope: QueryAST.Scope;
+
+    /**
+     * Optional limit on the number of results to select.
+     * When set, the index scan can be optimized to stop early.
+     */
+    limit?: number;
   };
 
   /**
    * Specifier to scan the database for objects.
    * Optimized to utilize database indexes.
    */
-  export type Selector = WildcardSelector | IdSelector | TypeSelector | TextSelector;
+  export type Selector = WildcardSelector | IdSelector | TypeSelector | TextSelector | TimestampSelector;
 
   export type WildcardSelector = {
     _tag: 'WildcardSelector';
@@ -91,6 +106,24 @@ export namespace QueryPlan {
 
     text: string;
     searchKind: TextSearchKind;
+
+    /**
+     * Optionally select only objects of the given typenames.
+     */
+    typename: DXN.String[] | null;
+  };
+
+  /**
+   * Select objects by timestamp range (createdAt / updatedAt).
+   * Uses the ObjectMetaIndex timestamp columns.
+   */
+  export type TimestampSelector = {
+    _tag: 'TimestampSelector';
+
+    updatedAfter?: number;
+    updatedBefore?: number;
+    createdAfter?: number;
+    createdBefore?: number;
   };
 
   /**
@@ -111,7 +144,8 @@ export namespace QueryPlan {
             step.filter.typename === null &&
             (step.filter.id === undefined || step.filter.id.length === 0) &&
             (step.filter.props === undefined || Object.keys(step.filter.props).length === 0) &&
-            (step.filter.foreignKeys === undefined || step.filter.foreignKeys.length === 0)
+            (step.filter.foreignKeys === undefined || step.filter.foreignKeys.length === 0) &&
+            step.filter.metaKey === undefined
           );
         }
         default:
@@ -141,7 +175,7 @@ export namespace QueryPlan {
   /**
    * Describes a traversal of the object graph.
    */
-  export type Traversal = ReferenceTraversal | RelationTraversal;
+  export type Traversal = ReferenceTraversal | RelationTraversal | HierarchyTraversal;
 
   /**
    * Traverse a reference connection.
@@ -151,8 +185,9 @@ export namespace QueryPlan {
 
     /**
      * Property path where the reference is located.
+     * If null, matches references from any property (only valid for incoming direction).
      */
-    property: EscapedPropPath;
+    property: EscapedPropPath | null;
 
     /**
      * outgoing: the reference points from the anchor object to the target.
@@ -174,6 +209,19 @@ export namespace QueryPlan {
     direction: 'source-to-relation' | 'relation-to-source' | 'target-to-relation' | 'relation-to-target';
   };
 
+  /**
+   * Traverse parent-child hierarchy.
+   */
+  export type HierarchyTraversal = {
+    _tag: 'HierarchyTraversal';
+
+    /**
+     * to-parent: traverse from child to parent.
+     * to-children: traverse from parent to children.
+     */
+    direction: 'to-parent' | 'to-children';
+  };
+
   /**
    * Combine results from multiple plans.
    * Each of the plans starts with a copy of the current working set.
@@ -203,5 +251,21 @@ export namespace QueryPlan {
 
     // Defaults to natural order if empty.
     order: readonly QueryAST.Order[];
+
+    /**
+     * Optional limit on the number of results to return after ordering.
+     * When set, the sorting can be optimized to only track the top N elements.
+     */
+    // TODO(dmaretskyi): Remove. This doesn't belong here.
+    limit?: number;
+  };
+
+  /**
+   * Limit the number of results.
+   */
+  export type LimitStep = {
+    _tag: 'LimitStep';
+
+    limit: number;
   };
 }