@dxos/echo-protocol 0.8.4-main.72ec0f3 → 0.8.4-main.74a063c4e0

package/package.json

@@ -1,20 +1,22 @@
 {
   "name": "@dxos/echo-protocol",
-  "version": "0.8.4-main.72ec0f3",
+  "version": "0.8.4-main.74a063c4e0",
   "description": "Core ECHO APIs.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
-  "repository": "github:dxos/dxos",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "DXOS.org",
-  "sideEffects": true,
+  "sideEffects": false,
   "type": "module",
   "exports": {
     ".": {
       "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "default": "./dist/lib/neutral/index.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
@@ -26,12 +28,12 @@
     "src"
   ],
   "dependencies": {
-    "effect": "3.18.3",
-    "@dxos/crypto": "0.8.4-main.72ec0f3",
-    "@dxos/keys": "0.8.4-main.72ec0f3",
-    "@dxos/invariant": "0.8.4-main.72ec0f3",
-    "@dxos/util": "0.8.4-main.72ec0f3",
-    "@dxos/protocols": "0.8.4-main.72ec0f3"
+    "effect": "3.20.0",
+    "@dxos/crypto": "0.8.4-main.74a063c4e0",
+    "@dxos/keys": "0.8.4-main.74a063c4e0",
+    "@dxos/invariant": "0.8.4-main.74a063c4e0",
+    "@dxos/protocols": "0.8.4-main.74a063c4e0",
+    "@dxos/util": "0.8.4-main.74a063c4e0"
   },
   "publishConfig": {
     "access": "public"

package/src/document-structure.ts

@@ -137,6 +137,10 @@ export const ObjectStructure = Object.freeze({
     return object.system?.target;
   },
 
+  getParent: (object: ObjectStructure): EncodedReference | undefined => {
+    return object.system?.parent;
+  },
+
   /**
    * @returns All references in the data section of the object.
    */
@@ -248,13 +252,19 @@ export type ObjectSystem = {
    */
   deleted?: boolean;
 
+  /**
+   * Object parent.
+   * Objects with no parent are at the top level of the object hierarchy in the space.
+   */
+  parent?: EncodedReference;
+
   /**
    * Only for relations.
    */
   source?: EncodedReference;
 
   /**
-   * Only for relations.w
+   * Only for relations.
    */
   target?: EncodedReference;
 };

package/src/echo-feed-codec.ts

@@ -0,0 +1,67 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { FeedProtocol } from '@dxos/protocols';
+
+import type { ForeignKey } from './foreign-key';
+
+/** Property name for meta when object is serialized to JSON. Matches @dxos/echo/internal ATTR_META. */
+const ATTR_META = '@meta';
+
+/**
+ * Codec for ECHO objects in feed block payload: JSON object ↔ UTF-8 bytes.
+ * Encodes with queue position stripped; decodes with optional position injection.
+ */
+export class EchoFeedCodec {
+  static readonly #encoder = new TextEncoder();
+  static readonly #decoder = new TextDecoder();
+
+  /**
+   * Prepares a value for feed storage (strips queue position from metadata) and encodes to bytes.
+   */
+  static encode(value: Record<string, unknown>): Uint8Array {
+    const prepared = EchoFeedCodec.#stripQueuePosition(value);
+    return EchoFeedCodec.#encoder.encode(JSON.stringify(prepared));
+  }
+
+  /**
+   * Decodes feed block bytes to a JSON value.
+   * If position is provided, injects queue position into the decoded object's metadata.
+   */
+  static decode(data: Uint8Array, position?: number): Record<string, unknown> {
+    const decoded = JSON.parse(EchoFeedCodec.#decoder.decode(data));
+    if (position !== undefined && typeof decoded === 'object' && decoded !== null) {
+      EchoFeedCodec.#setQueuePosition(decoded, position);
+    }
+    return decoded;
+  }
+
+  static #stripQueuePosition(value: Record<string, unknown>): Record<string, unknown> {
+    if (typeof value !== 'object' || value === null) {
+      return value;
+    }
+    const obj = structuredClone(value);
+    const meta = obj[ATTR_META] as { keys?: ForeignKey[] } | undefined;
+    if (meta?.keys?.some((key: ForeignKey) => key.source === FeedProtocol.KEY_QUEUE_POSITION)) {
+      meta.keys = meta.keys.filter((key: ForeignKey) => key.source !== FeedProtocol.KEY_QUEUE_POSITION);
+    }
+    return obj;
+  }
+
+  static #setQueuePosition(obj: Record<string, any>, position: number): void {
+    obj[ATTR_META] ??= { keys: [] };
+    obj[ATTR_META]!.keys ??= [];
+    const keys = obj[ATTR_META]!.keys!;
+    for (let i = 0; i < keys.length; i++) {
+      if (keys[i].source === FeedProtocol.KEY_QUEUE_POSITION) {
+        keys.splice(i, 1);
+        i--;
+      }
+    }
+    keys.push({
+      source: FeedProtocol.KEY_QUEUE_POSITION,
+      id: position.toString(),
+    });
+  }
+}

package/src/index.ts

@@ -2,10 +2,11 @@
 // Copyright 2023 DXOS.org
 //
 
+export type * from './collection-sync';
 export * from './document-structure';
+export * from './echo-feed-codec';
+export * from './foreign-key';
+export * from './query';
 export * from './reference';
 export * from './space-doc-version';
-export type * from './collection-sync';
 export * from './space-id';
-export * from './foreign-key';
-export * from './query';

package/src/query/ast.ts

@@ -10,7 +10,7 @@ import { DXN, ObjectId } from '@dxos/keys';
 import { ForeignKey } from '../foreign-key';
 
 const TypenameSpecifier = Schema.Union(DXN.Schema, Schema.Null).annotations({
-  description: 'DXN or null. Null means any type will match',
+  description: 'DXN or null; null matches any type',
 });
 
 // NOTE: This pattern with 3 definitions per schema is need to make the types opaque, and circular references in AST to not cause compiler errors.
@@ -75,7 +75,9 @@ const FilterContains_ = Schema.Struct({
   type: Schema.Literal('contains'),
   value: Schema.Any,
 });
+
 export interface FilterContains extends Schema.Schema.Type<typeof FilterContains_> {}
+
 /**
  * Predicate for an array property to contain the provided value.
  * Nested objects are matched using strict structural matching.
@@ -89,6 +91,7 @@ const FilterTag_ = Schema.Struct({
   type: Schema.Literal('tag'),
   tag: Schema.String, // TODO(burdon): Make OR-collection?
 });
+
 export interface FilterTag extends Schema.Schema.Type<typeof FilterTag_> {}
 export const FilterTag: Schema.Schema<FilterTag> = FilterTag_;
 
@@ -100,9 +103,24 @@ const FilterRange_ = Schema.Struct({
   from: Schema.Any,
   to: Schema.Any,
 });
+
 export interface FilterRange extends Schema.Schema.Type<typeof FilterRange_> {}
 export const FilterRange: Schema.Schema<FilterRange> = FilterRange_;
 
+/**
+ * Filter by system timestamp (createdAt / updatedAt).
+ * Timestamps are unix milliseconds stored in the object meta index.
+ */
+const FilterTimestamp_ = Schema.Struct({
+  type: Schema.Literal('timestamp'),
+  field: Schema.Literal('createdAt', 'updatedAt'),
+  operator: Schema.Literal('gt', 'gte', 'lt', 'lte'),
+  value: Schema.Number,
+});
+
+export interface FilterTimestamp extends Schema.Schema.Type<typeof FilterTimestamp_> {}
+export const FilterTimestamp: Schema.Schema<FilterTimestamp> = FilterTimestamp_;
+
 /**
  * Text search.
  */
@@ -111,6 +129,7 @@ const FilterTextSearch_ = Schema.Struct({
   text: Schema.String,
   searchKind: Schema.optional(Schema.Literal('full-text', 'vector')),
 });
+
 export interface FilterTextSearch extends Schema.Schema.Type<typeof FilterTextSearch_> {}
 export const FilterTextSearch: Schema.Schema<FilterTextSearch> = FilterTextSearch_;
 
@@ -121,6 +140,7 @@ const FilterNot_ = Schema.Struct({
   type: Schema.Literal('not'),
   filter: Schema.suspend(() => Filter),
 });
+
 export interface FilterNot extends Schema.Schema.Type<typeof FilterNot_> {}
 export const FilterNot: Schema.Schema<FilterNot> = FilterNot_;
 
@@ -131,6 +151,7 @@ const FilterAnd_ = Schema.Struct({
   type: Schema.Literal('and'),
   filters: Schema.Array(Schema.suspend(() => Filter)),
 });
+
 export interface FilterAnd extends Schema.Schema.Type<typeof FilterAnd_> {}
 export const FilterAnd: Schema.Schema<FilterAnd> = FilterAnd_;
 
@@ -141,6 +162,7 @@ const FilterOr_ = Schema.Struct({
   type: Schema.Literal('or'),
   filters: Schema.Array(Schema.suspend(() => Filter)),
 });
+
 export interface FilterOr extends Schema.Schema.Type<typeof FilterOr_> {}
 export const FilterOr: Schema.Schema<FilterOr> = FilterOr_;
 
@@ -154,11 +176,13 @@ export const Filter = Schema.Union(
   FilterContains,
   FilterTag,
   FilterRange,
+  FilterTimestamp,
   FilterTextSearch,
   FilterNot,
   FilterAnd,
   FilterOr,
-).annotations({ identifier: 'dxos.org/schema/Filter' });
+).annotations({ identifier: 'org.dxos.schema.filter' });
+
 export type Filter = Schema.Schema.Type<typeof Filter>;
 
 /**
@@ -168,6 +192,7 @@ const QuerySelectClause_ = Schema.Struct({
   type: Schema.Literal('select'),
   filter: Schema.suspend(() => Filter),
 });
+
 export interface QuerySelectClause extends Schema.Schema.Type<typeof QuerySelectClause_> {}
 export const QuerySelectClause: Schema.Schema<QuerySelectClause> = QuerySelectClause_;
 
@@ -179,6 +204,7 @@ const QueryFilterClause_ = Schema.Struct({
   selection: Schema.suspend(() => Query),
   filter: Schema.suspend(() => Filter),
 });
+
 export interface QueryFilterClause extends Schema.Schema.Type<typeof QueryFilterClause_> {}
 export const QueryFilterClause: Schema.Schema<QueryFilterClause> = QueryFilterClause_;
 
@@ -190,6 +216,7 @@ const QueryReferenceTraversalClause_ = Schema.Struct({
   anchor: Schema.suspend(() => Query),
   property: Schema.String, // TODO(dmaretskyi): Change to EscapedPropPath.
 });
+
 export interface QueryReferenceTraversalClause extends Schema.Schema.Type<typeof QueryReferenceTraversalClause_> {}
 export const QueryReferenceTraversalClause: Schema.Schema<QueryReferenceTraversalClause> =
   QueryReferenceTraversalClause_;
@@ -200,9 +227,14 @@ export const QueryReferenceTraversalClause: Schema.Schema<QueryReferenceTraversa
 const QueryIncomingReferencesClause_ = Schema.Struct({
   type: Schema.Literal('incoming-references'),
   anchor: Schema.suspend(() => Query),
-  property: Schema.String,
+  /**
+   * Property path where the reference is located.
+   * If null, matches references from any property.
+   */
+  property: Schema.NullOr(Schema.String),
   typename: TypenameSpecifier,
 });
+
 export interface QueryIncomingReferencesClause extends Schema.Schema.Type<typeof QueryIncomingReferencesClause_> {}
 export const QueryIncomingReferencesClause: Schema.Schema<QueryIncomingReferencesClause> =
   QueryIncomingReferencesClause_;
@@ -221,6 +253,7 @@ const QueryRelationClause_ = Schema.Struct({
   direction: Schema.Literal('outgoing', 'incoming', 'both'),
   filter: Schema.optional(Schema.suspend(() => Filter)),
 });
+
 export interface QueryRelationClause extends Schema.Schema.Type<typeof QueryRelationClause_> {}
 export const QueryRelationClause: Schema.Schema<QueryRelationClause> = QueryRelationClause_;
 
@@ -232,9 +265,27 @@ const QueryRelationTraversalClause_ = Schema.Struct({
   anchor: Schema.suspend(() => Query),
   direction: Schema.Literal('source', 'target', 'both'),
 });
+
 export interface QueryRelationTraversalClause extends Schema.Schema.Type<typeof QueryRelationTraversalClause_> {}
 export const QueryRelationTraversalClause: Schema.Schema<QueryRelationTraversalClause> = QueryRelationTraversalClause_;
 
+/**
+ * Traverse parent-child hierarchy.
+ */
+const QueryHierarchyTraversalClause_ = Schema.Struct({
+  type: Schema.Literal('hierarchy-traversal'),
+  anchor: Schema.suspend(() => Query),
+  /**
+   * to-parent: traverse from child to parent.
+   * to-children: traverse from parent to children.
+   */
+  direction: Schema.Literal('to-parent', 'to-children'),
+});
+
+export interface QueryHierarchyTraversalClause extends Schema.Schema.Type<typeof QueryHierarchyTraversalClause_> {}
+export const QueryHierarchyTraversalClause: Schema.Schema<QueryHierarchyTraversalClause> =
+  QueryHierarchyTraversalClause_;
+
 /**
  * Union of multiple queries.
  */
@@ -242,6 +293,7 @@ const QueryUnionClause_ = Schema.Struct({
   type: Schema.Literal('union'),
   queries: Schema.Array(Schema.suspend(() => Query)),
 });
+
 export interface QueryUnionClause extends Schema.Schema.Type<typeof QueryUnionClause_> {}
 export const QueryUnionClause: Schema.Schema<QueryUnionClause> = QueryUnionClause_;
 
@@ -253,6 +305,7 @@ const QuerySetDifferenceClause_ = Schema.Struct({
   source: Schema.suspend(() => Query),
   exclude: Schema.suspend(() => Query),
 });
+
 export interface QuerySetDifferenceClause extends Schema.Schema.Type<typeof QuerySetDifferenceClause_> {}
 export const QuerySetDifferenceClause: Schema.Schema<QuerySetDifferenceClause> = QuerySetDifferenceClause_;
 
@@ -269,7 +322,14 @@ const Order_ = Schema.Union(
     property: Schema.String,
     direction: OrderDirection,
   }),
+  Schema.Struct({
+    // Order by relevance rank (for FTS/vector search results).
+    // Default direction is 'desc' (higher rank = better match first).
+    kind: Schema.Literal('rank'),
+    direction: OrderDirection,
+  }),
 );
+
 export type Order = Schema.Schema.Type<typeof Order_>;
 export const Order: Schema.Schema<Order> = Order_;
 
@@ -282,6 +342,7 @@ const QueryOrderClause_ = Schema.Struct({
   query: Schema.suspend(() => Query),
   order: Schema.Array(Order),
 });
+
 export interface QueryOrderClause extends Schema.Schema.Type<typeof QueryOrderClause_> {}
 export const QueryOrderClause: Schema.Schema<QueryOrderClause> = QueryOrderClause_;
 
@@ -293,9 +354,37 @@ const QueryOptionsClause_ = Schema.Struct({
   query: Schema.suspend(() => Query),
   options: Schema.suspend(() => QueryOptions),
 });
+
 export interface QueryOptionsClause extends Schema.Schema.Type<typeof QueryOptionsClause_> {}
 export const QueryOptionsClause: Schema.Schema<QueryOptionsClause> = QueryOptionsClause_;
 
+/**
+ * Limit the number of results.
+ */
+const QueryLimitClause_ = Schema.Struct({
+  type: Schema.Literal('limit'),
+  query: Schema.suspend(() => Query),
+  limit: Schema.Number,
+});
+
+export interface QueryLimitClause extends Schema.Schema.Type<typeof QueryLimitClause_> {}
+export const QueryLimitClause: Schema.Schema<QueryLimitClause> = QueryLimitClause_;
+
+export const QueryFromClause_ = Schema.Struct({
+  type: Schema.Literal('from'),
+  query: Schema.suspend(() => Query),
+  from: Schema.Union(
+    Schema.TaggedStruct('scope', {
+      scope: Schema.suspend(() => Scope),
+    }),
+    Schema.TaggedStruct('query', {
+      query: Schema.suspend(() => Query),
+    }),
+  ),
+});
+export interface QueryFromClause extends Schema.Schema.Type<typeof QueryFromClause_> {}
+export const QueryFromClause: Schema.Schema<QueryFromClause> = QueryFromClause_;
+
 const Query_ = Schema.Union(
   QuerySelectClause,
   QueryFilterClause,
@@ -303,16 +392,31 @@ const Query_ = Schema.Union(
   QueryIncomingReferencesClause,
   QueryRelationClause,
   QueryRelationTraversalClause,
+  QueryHierarchyTraversalClause,
   QueryUnionClause,
   QuerySetDifferenceClause,
   QueryOrderClause,
   QueryOptionsClause,
-).annotations({ identifier: 'dxos.org/schema/Query' });
+  QueryLimitClause,
+  QueryFromClause,
+).annotations({ identifier: 'org.dxos.schema.query' });
 
 export type Query = Schema.Schema.Type<typeof Query_>;
 export const Query: Schema.Schema<Query> = Query_;
 
 export const QueryOptions = Schema.Struct({
+  /**
+   * Nested select statements will use this option to filter deleted objects.
+   */
+  deleted: Schema.optional(Schema.Literal('include', 'exclude', 'only')),
+});
+
+export interface QueryOptions extends Schema.Schema.Type<typeof QueryOptions> {}
+
+/**
+ * Specifies the scope of the data to query from.
+ */
+export const Scope = Schema.Struct({
   /**
    * The nested select statemets will select from the given spaces.
    *
@@ -320,19 +424,19 @@ export const QueryOptions = Schema.Struct({
    */
   spaceIds: Schema.optional(Schema.Array(Schema.String)),
 
+  /**
+   * If true, the nested select statements will select from all queues in the spaces specified by `spaceIds`.
+   */
+  allQueuesFromSpaces: Schema.optional(Schema.Boolean),
+
   /**
    * The nested select statemets will select from the given queues.
    *
    * NOTE: Spaces and queues are unioned together if both are specified.
    */
   queues: Schema.optional(Schema.Array(DXN.Schema)),
-
-  /**
-   * Nested select statements will use this option to filter deleted objects.
-   */
-  deleted: Schema.optional(Schema.Literal('include', 'exclude', 'only')),
 });
-export interface QueryOptions extends Schema.Schema.Type<typeof QueryOptions> {}
+export interface Scope extends Schema.Schema.Type<typeof Scope> {}
 
 export const visit = (query: Query, visitor: (node: Query) => void) => {
   visitor(query);
@@ -344,17 +448,57 @@ export const visit = (query: Query, visitor: (node: Query) => void) => {
     Match.when({ type: 'relation' }, ({ anchor }) => visit(anchor, visitor)),
     Match.when({ type: 'options' }, ({ query }) => visit(query, visitor)),
     Match.when({ type: 'relation-traversal' }, ({ anchor }) => visit(anchor, visitor)),
+    Match.when({ type: 'hierarchy-traversal' }, ({ anchor }) => visit(anchor, visitor)),
     Match.when({ type: 'union' }, ({ queries }) => queries.forEach((q) => visit(q, visitor))),
     Match.when({ type: 'set-difference' }, ({ source, exclude }) => {
       visit(source, visitor);
       visit(exclude, visitor);
     }),
     Match.when({ type: 'order' }, ({ query }) => visit(query, visitor)),
+    Match.when({ type: 'limit' }, ({ query }) => visit(query, visitor)),
+    Match.when({ type: 'from' }, (node) => {
+      visit(node.query, visitor);
+      if (node.from._tag === 'query') {
+        visit(node.from.query, visitor);
+      }
+    }),
     Match.when({ type: 'select' }, () => {}),
     Match.exhaustive,
   );
 };
 
+/**
+ * Recursively transforms a query tree bottom-up.
+ * The mapper receives each node with its children already transformed.
+ */
+export const map = (query: Query, mapper: (node: Query) => Query): Query => {
+  const mapped: Query = Match.value(query).pipe(
+    Match.when({ type: 'filter' }, (node) => ({ ...node, selection: map(node.selection, mapper) })),
+    Match.when({ type: 'reference-traversal' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'incoming-references' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'relation' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'relation-traversal' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'hierarchy-traversal' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'options' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'order' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'limit' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'from' }, (node) => ({
+      ...node,
+      query: map(node.query, mapper),
+      ...(node.from._tag === 'query' ? { from: { _tag: 'query' as const, query: map(node.from.query, mapper) } } : {}),
+    })),
+    Match.when({ type: 'union' }, (node) => ({ ...node, queries: node.queries.map((q) => map(q, mapper)) })),
+    Match.when({ type: 'set-difference' }, (node) => ({
+      ...node,
+      source: map(node.source, mapper),
+      exclude: map(node.exclude, mapper),
+    })),
+    Match.when({ type: 'select' }, (node) => node),
+    Match.exhaustive,
+  );
+  return mapper(mapped);
+};
+
 export const fold = <T>(query: Query, reducer: (node: Query) => T): T[] => {
   return Match.value(query).pipe(
     Match.withReturnType<T[]>(),
@@ -364,11 +508,20 @@ export const fold = <T>(query: Query, reducer: (node: Query) => T): T[] => {
     Match.when({ type: 'relation' }, ({ anchor }) => fold(anchor, reducer)),
     Match.when({ type: 'options' }, ({ query }) => fold(query, reducer)),
     Match.when({ type: 'relation-traversal' }, ({ anchor }) => fold(anchor, reducer)),
+    Match.when({ type: 'hierarchy-traversal' }, ({ anchor }) => fold(anchor, reducer)),
     Match.when({ type: 'union' }, ({ queries }) => queries.flatMap((q) => fold(q, reducer))),
     Match.when({ type: 'set-difference' }, ({ source, exclude }) =>
       fold(source, reducer).concat(fold(exclude, reducer)),
     ),
     Match.when({ type: 'order' }, ({ query }) => fold(query, reducer)),
+    Match.when({ type: 'limit' }, ({ query }) => fold(query, reducer)),
+    Match.when({ type: 'from' }, (node) => {
+      const results = fold(node.query, reducer);
+      if (node.from._tag === 'query') {
+        return results.concat(fold(node.from.query, reducer));
+      }
+      return results;
+    }),
     Match.when({ type: 'select' }, () => []),
     Match.exhaustive,
   );

package/src/reference.ts

@@ -177,6 +177,9 @@ export const EncodedReference = Object.freeze({
     return DXN.parse(EncodedReference.getReferenceString(value));
   },
   fromDXN: (dxn: DXN): EncodedReference => {
-    return encodeReference(Reference.fromDXN(dxn));
+    return { '/': dxn.toString() };
+  },
+  fromLegacyTypename: (typename: string): EncodedReference => {
+    return { '/': DXN.fromTypename(typename).toString() };
   },
 });