@dxos/echo-protocol 0.8.4-main.ef1bc66f44 → 0.8.4-main.effb148878

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-protocol",
-  "version": "0.8.4-main.ef1bc66f44",
+  "version": "0.8.4-main.effb148878",
   "description": "Core ECHO APIs.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/dxos/dxos"
   },
-  "license": "MIT",
+  "license": "FSL-1.1-Apache-2.0",
   "author": "DXOS.org",
   "sideEffects": false,
   "type": "module",
@@ -20,20 +20,17 @@
     }
   },
   "types": "dist/types/src/index.d.ts",
-  "typesVersions": {
-    "*": {}
-  },
   "files": [
     "dist",
     "src"
   ],
   "dependencies": {
-    "effect": "3.19.16",
-    "@dxos/crypto": "0.8.4-main.ef1bc66f44",
-    "@dxos/invariant": "0.8.4-main.ef1bc66f44",
-    "@dxos/keys": "0.8.4-main.ef1bc66f44",
-    "@dxos/util": "0.8.4-main.ef1bc66f44",
-    "@dxos/protocols": "0.8.4-main.ef1bc66f44"
+    "effect": "3.20.0",
+    "@dxos/invariant": "0.8.4-main.effb148878",
+    "@dxos/util": "0.8.4-main.effb148878",
+    "@dxos/protocols": "0.8.4-main.effb148878",
+    "@dxos/crypto": "0.8.4-main.effb148878",
+    "@dxos/keys": "0.8.4-main.effb148878"
   },
   "publishConfig": {
     "access": "public"

package/src/document-structure.ts

@@ -3,7 +3,7 @@
 //
 
 import { invariant } from '@dxos/invariant';
-import type { DXN, ObjectId } from '@dxos/keys';
+import type { ObjectId, URI } from '@dxos/keys';
 import { visitValues } from '@dxos/util';
 
 import { type RawString } from './automerge';
@@ -41,7 +41,7 @@ export interface DatabaseDirectory {
    * Object id points to an automerge doc url where the object is embedded.
    */
   links?: {
-    [echoId: string]: string | RawString;
+    [echoUri: string]: string | RawString;
   };
 
   /**
@@ -119,9 +119,9 @@ export const ObjectStructure = Object.freeze({
   /**
    * @throws On invalid object structure.
    */
-  getEntityKind: (object: ObjectStructure): 'object' | 'relation' => {
+  getEntityKind: (object: ObjectStructure): 'object' | 'relation' | 'type' => {
     const kind = object.system?.kind ?? 'object';
-    invariant(kind === 'object' || kind === 'relation', 'Invalid kind');
+    invariant(kind === 'object' || kind === 'relation' || kind === 'type', 'Invalid kind');
     return kind;
   },
 
@@ -166,7 +166,7 @@ export const ObjectStructure = Object.freeze({
     data,
     keys,
   }: {
-    type: DXN.String;
+    type: URI.URI;
     deleted?: boolean;
     keys?: ForeignKey[];
     data?: unknown;
@@ -191,7 +191,7 @@ export const ObjectStructure = Object.freeze({
     keys,
     data,
   }: {
-    type: DXN.String;
+    type: URI.URI;
     source: EncodedReference;
     target: EncodedReference;
     deleted?: boolean;
@@ -212,6 +212,19 @@ export const ObjectStructure = Object.freeze({
       data: data ?? {},
     };
   },
+
+  makeType: ({ type, keys, data }: { type: URI.URI; keys?: ForeignKey[]; data?: unknown }): ObjectStructure => {
+    return {
+      system: {
+        kind: 'type',
+        type: { '/': type },
+      },
+      meta: {
+        keys: keys ?? [],
+      },
+      data: data ?? {},
+    };
+  },
 });
 
 /**
@@ -230,6 +243,18 @@ export type ObjectMeta = {
    * NOTE: Optional for backwards compatibilty.
    */
   tags?: string[];
+
+  /**
+   * Fully-qualified registry key for the object (FQN format, e.g. `org.example.type.foo`).
+   * Identifies the canonical registry entry the object instance was created from.
+   */
+  key?: string;
+
+  /**
+   * Semantic version of the registry entry the object was created from.
+   * Must be a valid semver string (e.g. `1.2.3`).
+   */
+  version?: string;
 };
 
 /**
@@ -238,12 +263,22 @@ export type ObjectMeta = {
  */
 export type ObjectSystem = {
   /**
-   * Entity kind.
+   * Entity kind. `'type'` covers persisted ECHO type definitions (instances of
+   * the `Type.Type` meta-schema); `'object'` / `'relation'` cover regular ECHO
+   * instances.
    */
-  kind?: 'object' | 'relation';
+  kind?: 'object' | 'relation' | 'type';
 
   /**
-   * Object reference ('protobuf' protocol) type.
+   * Object reference ('protobuf' protocol) type — DXN of the schema this
+   * entity instantiates.
+   *
+   * - For `kind === 'object'` / `'relation'` instances, this is the URI of the
+   *   user-defined schema the entity was created from (e.g. `dxn:type:org.example.Person:1.0.0`).
+   * - For `kind === 'type'` entities (persisted Type.Type meta-instances) this
+   *   is always the URI of the `TypeSchema` meta-schema itself
+   *   (`dxn:org.dxos.type.schema:0.1.0`). The kind that the meta-instance
+   *   _describes_ (object/relation/type) lives in `data.jsonSchema.entityKind`.
    */
   type?: EncodedReference;
 

package/src/echo-feed-codec.ts

@@ -0,0 +1,67 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { FeedProtocol } from '@dxos/protocols';
+
+import type { ForeignKey } from './foreign-key';
+
+/** Property name for meta when object is serialized to JSON. Matches @dxos/echo/internal ATTR_META. */
+const ATTR_META = '@meta';
+
+/**
+ * Codec for ECHO objects in feed block payload: JSON object ↔ UTF-8 bytes.
+ * Encodes with queue position stripped; decodes with optional position injection.
+ */
+export class EchoFeedCodec {
+  static readonly #encoder = new TextEncoder();
+  static readonly #decoder = new TextDecoder();
+
+  /**
+   * Prepares a value for feed storage (strips queue position from metadata) and encodes to bytes.
+   */
+  static encode(value: Record<string, unknown>): Uint8Array {
+    const prepared = EchoFeedCodec.#stripQueuePosition(value);
+    return EchoFeedCodec.#encoder.encode(JSON.stringify(prepared));
+  }
+
+  /**
+   * Decodes feed block bytes to a JSON value.
+   * If position is provided, injects queue position into the decoded object's metadata.
+   */
+  static decode(data: Uint8Array, position?: number): Record<string, unknown> {
+    const decoded = JSON.parse(EchoFeedCodec.#decoder.decode(data));
+    if (position !== undefined && typeof decoded === 'object' && decoded !== null) {
+      EchoFeedCodec.#setQueuePosition(decoded, position);
+    }
+    return decoded;
+  }
+
+  static #stripQueuePosition(value: Record<string, unknown>): Record<string, unknown> {
+    if (typeof value !== 'object' || value === null) {
+      return value;
+    }
+    const obj = structuredClone(value);
+    const meta = obj[ATTR_META] as { keys?: ForeignKey[] } | undefined;
+    if (meta?.keys?.some((key: ForeignKey) => key.source === FeedProtocol.KEY_QUEUE_POSITION)) {
+      meta.keys = meta.keys.filter((key: ForeignKey) => key.source !== FeedProtocol.KEY_QUEUE_POSITION);
+    }
+    return obj;
+  }
+
+  static #setQueuePosition(obj: Record<string, any>, position: number): void {
+    obj[ATTR_META] ??= { keys: [] };
+    obj[ATTR_META]!.keys ??= [];
+    const keys = obj[ATTR_META]!.keys!;
+    for (let i = 0; i < keys.length; i++) {
+      if (keys[i].source === FeedProtocol.KEY_QUEUE_POSITION) {
+        keys.splice(i, 1);
+        i--;
+      }
+    }
+    keys.push({
+      source: FeedProtocol.KEY_QUEUE_POSITION,
+      id: position.toString(),
+    });
+  }
+}

package/src/edge-peer.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type SpaceId } from '@dxos/keys';
+import { EdgeService } from '@dxos/protocols';
+import { compositeKey } from '@dxos/util';
+
+/**
+ * Returns true if the given peerId belongs to an EDGE replicator (Automerge or Subduction).
+ *
+ * When `spaceId` is provided, the match is scoped to that space (peerId must start with
+ * `<service>:<spaceId>`). When omitted, only the leading service segment is checked
+ * (peerId must start with `<service>:`), which is useful when the caller doesn't have a
+ * spaceId on hand or wants to match any edge replicator regardless of space.
+ */
+export const isEdgePeerId = (peerId: string, spaceId?: SpaceId): boolean => {
+  const automergePrefix =
+    spaceId !== undefined
+      ? compositeKey(EdgeService.AUTOMERGE_REPLICATOR, spaceId)
+      : `${EdgeService.AUTOMERGE_REPLICATOR}:`;
+  const subductionPrefix =
+    spaceId !== undefined
+      ? compositeKey(EdgeService.SUBDUCTION_REPLICATOR, spaceId)
+      : `${EdgeService.SUBDUCTION_REPLICATOR}:`;
+  return peerId.startsWith(automergePrefix) || peerId.startsWith(subductionPrefix);
+};

package/src/index.ts

@@ -2,10 +2,12 @@
 // Copyright 2023 DXOS.org
 //
 
+export type * from './collection-sync';
 export * from './document-structure';
+export * from './edge-peer';
+export * from './echo-feed-codec';
+export * from './foreign-key';
+export * from './query';
 export * from './reference';
 export * from './space-doc-version';
-export type * from './collection-sync';
 export * from './space-id';
-export * from './foreign-key';
-export * from './query';

package/src/query/ast.ts

@@ -5,13 +5,14 @@
 import * as Match from 'effect/Match';
 import * as Schema from 'effect/Schema';
 
-import { DXN, ObjectId } from '@dxos/keys';
+import { EchoURI, ObjectId, URI } from '@dxos/keys';
 
 import { ForeignKey } from '../foreign-key';
 
-const TypenameSpecifier = Schema.Union(DXN.Schema, Schema.Null).annotations({
-  description: 'DXN or null; null matches any type',
-});
+// Type identifier URI — either a DXN (typename) or an EchoURI (stored-schema-as-object).
+// Matches the URI written into an object's `system.type` (see `getSchemaURI`). Null
+// matches any type.
+const TypenameSpecifier = Schema.Union(URI.Schema, Schema.Null);
 
 // NOTE: This pattern with 3 definitions per schema is need to make the types opaque, and circular references in AST to not cause compiler errors.
 
@@ -42,6 +43,17 @@ const FilterObject_ = Schema.Struct({
    */
   foreignKeys: Schema.optional(Schema.Array(ForeignKey)),
 
+  /**
+   * Match objects whose meta `key` equals this fully-qualified registry key (FQN format).
+   */
+  metaKey: Schema.optional(Schema.String),
+
+  /**
+   * Semver range matched against the object's meta `version`.
+   * Only consulted when {@link metaKey} is set. Objects with no `version` do not satisfy a version-constrained filter.
+   */
+  metaVersion: Schema.optional(Schema.String),
+
   // NOTE: Make sure to update `FilterStep.isNoop` if you change this.
 });
 export interface FilterObject extends Schema.Schema.Type<typeof FilterObject_> {}
@@ -107,6 +119,20 @@ const FilterRange_ = Schema.Struct({
 export interface FilterRange extends Schema.Schema.Type<typeof FilterRange_> {}
 export const FilterRange: Schema.Schema<FilterRange> = FilterRange_;
 
+/**
+ * Filter by system timestamp (createdAt / updatedAt).
+ * Timestamps are unix milliseconds stored in the object meta index.
+ */
+const FilterTimestamp_ = Schema.Struct({
+  type: Schema.Literal('timestamp'),
+  field: Schema.Literal('createdAt', 'updatedAt'),
+  operator: Schema.Literal('gt', 'gte', 'lt', 'lte'),
+  value: Schema.Number,
+});
+
+export interface FilterTimestamp extends Schema.Schema.Type<typeof FilterTimestamp_> {}
+export const FilterTimestamp: Schema.Schema<FilterTimestamp> = FilterTimestamp_;
+
 /**
  * Text search.
  */
@@ -152,6 +178,21 @@ const FilterOr_ = Schema.Struct({
 export interface FilterOr extends Schema.Schema.Type<typeof FilterOr_> {}
 export const FilterOr: Schema.Schema<FilterOr> = FilterOr_;
 
+/**
+ * Filter objects that are children of the specified parents.
+ * With transitive=true (default), matches grandchildren and beyond.
+ */
+const FilterChildOf_ = Schema.Struct({
+  type: Schema.Literal('child-of'),
+  /** Parent DXNs to match children of. */
+  parents: Schema.Array(EchoURI.Schema),
+  /** Whether to match transitively (grandchildren, etc.). Defaults to true. */
+  transitive: Schema.Boolean,
+});
+
+export interface FilterChildOf extends Schema.Schema.Type<typeof FilterChildOf_> {}
+export const FilterChildOf: Schema.Schema<FilterChildOf> = FilterChildOf_;
+
 /**
  * Union of filters.
  */
@@ -162,11 +203,13 @@ export const Filter = Schema.Union(
   FilterContains,
   FilterTag,
   FilterRange,
+  FilterTimestamp,
   FilterTextSearch,
+  FilterChildOf,
   FilterNot,
   FilterAnd,
   FilterOr,
-).annotations({ identifier: 'dxos.org/schema/Filter' });
+).annotations({ identifier: 'org.dxos.schema.filter' });
 
 export type Filter = Schema.Schema.Type<typeof Filter>;
 
@@ -355,6 +398,21 @@ const QueryLimitClause_ = Schema.Struct({
 export interface QueryLimitClause extends Schema.Schema.Type<typeof QueryLimitClause_> {}
 export const QueryLimitClause: Schema.Schema<QueryLimitClause> = QueryLimitClause_;
 
+export const QueryFromClause_ = Schema.Struct({
+  type: Schema.Literal('from'),
+  query: Schema.suspend(() => Query),
+  from: Schema.Union(
+    Schema.TaggedStruct('scope', {
+      scope: Schema.suspend(() => Scope),
+    }),
+    Schema.TaggedStruct('query', {
+      query: Schema.suspend(() => Query),
+    }),
+  ),
+});
+export interface QueryFromClause extends Schema.Schema.Type<typeof QueryFromClause_> {}
+export const QueryFromClause: Schema.Schema<QueryFromClause> = QueryFromClause_;
+
 const Query_ = Schema.Union(
   QuerySelectClause,
   QueryFilterClause,
@@ -368,38 +426,50 @@ const Query_ = Schema.Union(
   QueryOrderClause,
   QueryOptionsClause,
   QueryLimitClause,
-).annotations({ identifier: 'dxos.org/schema/Query' });
+  QueryFromClause,
+).annotations({ identifier: 'org.dxos.schema.query' });
 
 export type Query = Schema.Schema.Type<typeof Query_>;
 export const Query: Schema.Schema<Query> = Query_;
 
 export const QueryOptions = Schema.Struct({
   /**
-   * The nested select statemets will select from the given spaces.
-   *
-   * NOTE: Spaces and queues are unioned together if both are specified.
+   * Nested select statements will use this option to filter deleted objects.
    */
-  spaceIds: Schema.optional(Schema.Array(Schema.String)),
+  deleted: Schema.optional(Schema.Literal('include', 'exclude', 'only')),
 
   /**
-   * If true, the nested select statements will select from all queues in the spaces specified by `spaceIds`.
+   * Diagnostics-only label for logs / tooling (not used by execution semantics).
    */
-  allQueuesFromSpaces: Schema.optional(Schema.Boolean),
+  debugLabel: Schema.optional(Schema.String),
+});
 
+export interface QueryOptions extends Schema.Schema.Type<typeof QueryOptions> {}
+
+/**
+ * Specifies the scope of the data to query from.
+ */
+export const Scope = Schema.Struct({
   /**
-   * The nested select statemets will select from the given queues.
+   * The nested select statemets will select from the given spaces.
    *
-   * NOTE: Spaces and queues are unioned together if both are specified.
+   * NOTE: Spaces and feeds are unioned together if both are specified.
    */
-  queues: Schema.optional(Schema.Array(DXN.Schema)),
+  spaceIds: Schema.optional(Schema.Array(Schema.String)),
 
   /**
-   * Nested select statements will use this option to filter deleted objects.
+   * If true, the nested select statements will select from all feeds in the spaces specified by `spaceIds`.
    */
-  deleted: Schema.optional(Schema.Literal('include', 'exclude', 'only')),
-});
+  allFeedsFromSpaces: Schema.optional(Schema.Boolean),
 
-export interface QueryOptions extends Schema.Schema.Type<typeof QueryOptions> {}
+  /**
+   * The nested select statemets will select from the given feeds (by EchoURI or legacy DXN).
+   *
+   * NOTE: Spaces and feeds are unioned together if both are specified.
+   */
+  feeds: Schema.optional(Schema.Array(EchoURI.Schema)),
+});
+export interface Scope extends Schema.Schema.Type<typeof Scope> {}
 
 export const visit = (query: Query, visitor: (node: Query) => void) => {
   visitor(query);
@@ -419,11 +489,49 @@ export const visit = (query: Query, visitor: (node: Query) => void) => {
     }),
     Match.when({ type: 'order' }, ({ query }) => visit(query, visitor)),
     Match.when({ type: 'limit' }, ({ query }) => visit(query, visitor)),
+    Match.when({ type: 'from' }, (node) => {
+      visit(node.query, visitor);
+      if (node.from._tag === 'query') {
+        visit(node.from.query, visitor);
+      }
+    }),
     Match.when({ type: 'select' }, () => {}),
     Match.exhaustive,
   );
 };
 
+/**
+ * Recursively transforms a query tree bottom-up.
+ * The mapper receives each node with its children already transformed.
+ */
+export const map = (query: Query, mapper: (node: Query) => Query): Query => {
+  const mapped: Query = Match.value(query).pipe(
+    Match.when({ type: 'filter' }, (node) => ({ ...node, selection: map(node.selection, mapper) })),
+    Match.when({ type: 'reference-traversal' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'incoming-references' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'relation' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'relation-traversal' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'hierarchy-traversal' }, (node) => ({ ...node, anchor: map(node.anchor, mapper) })),
+    Match.when({ type: 'options' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'order' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'limit' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'from' }, (node) => ({
+      ...node,
+      query: map(node.query, mapper),
+      ...(node.from._tag === 'query' ? { from: { _tag: 'query' as const, query: map(node.from.query, mapper) } } : {}),
+    })),
+    Match.when({ type: 'union' }, (node) => ({ ...node, queries: node.queries.map((q) => map(q, mapper)) })),
+    Match.when({ type: 'set-difference' }, (node) => ({
+      ...node,
+      source: map(node.source, mapper),
+      exclude: map(node.exclude, mapper),
+    })),
+    Match.when({ type: 'select' }, (node) => node),
+    Match.exhaustive,
+  );
+  return mapper(mapped);
+};
+
 export const fold = <T>(query: Query, reducer: (node: Query) => T): T[] => {
   return Match.value(query).pipe(
     Match.withReturnType<T[]>(),
@@ -440,6 +548,13 @@ export const fold = <T>(query: Query, reducer: (node: Query) => T): T[] => {
     ),
     Match.when({ type: 'order' }, ({ query }) => fold(query, reducer)),
     Match.when({ type: 'limit' }, ({ query }) => fold(query, reducer)),
+    Match.when({ type: 'from' }, (node) => {
+      const results = fold(node.query, reducer);
+      if (node.from._tag === 'query') {
+        return results.concat(fold(node.from.query, reducer));
+      }
+      return results;
+    }),
     Match.when({ type: 'select' }, () => []),
     Match.exhaustive,
   );