@dxos/echo-protocol 0.10.0 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-protocol",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Core ECHO APIs.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -16,7 +16,7 @@
     ".": {
       "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "default": "./dist/lib/neutral/index.mjs"
+      "import": "./dist/lib/index.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
@@ -26,11 +26,11 @@
   ],
   "dependencies": {
     "effect": "3.21.4",
-    "@dxos/crypto": "0.10.0",
-    "@dxos/keys": "0.10.0",
-    "@dxos/util": "0.10.0",
-    "@dxos/invariant": "0.10.0",
-    "@dxos/protocols": "0.10.0"
+    "@dxos/crypto": "0.11.0",
+    "@dxos/invariant": "0.11.0",
+    "@dxos/keys": "0.11.0",
+    "@dxos/protocols": "0.11.0",
+    "@dxos/util": "0.11.0"
   },
   "publishConfig": {
     "access": "public"

package/src/blob.ts

@@ -0,0 +1,36 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type SpaceId } from '@dxos/keys';
+
+export interface BlobPutRequest {
+  spaceId: SpaceId;
+  data: Uint8Array;
+  contentType?: string;
+  /** Lowercase hex SHA-256 digest of `data`, computed by the manager. The backend does not verify it. */
+  contentHash: string;
+  /** For path-addressed extension backends. */
+  name?: string;
+}
+
+export interface BlobPutResponse {
+  /** URI locating the stored bytes; must use a scheme the backend resolves. */
+  uri: string;
+}
+
+/**
+ * Implemented by pluggable blob storage backends and registered on the Hypergraph via
+ * `registerBlobBackend`.
+ */
+export interface BlobBackend {
+  /** URI schemes this backend resolves at read time. */
+  readonly schemes: readonly string[];
+  /** Largest `data.byteLength` this backend accepts, in bytes. `undefined` means unlimited. */
+  readonly maxSize?: number;
+  put(request: BlobPutRequest): Promise<BlobPutResponse>;
+  /** `undefined` means the URI was not found. Rejects on transport failure (e.g. offline). */
+  get(request: { spaceId: SpaceId; uri: string }): Promise<Uint8Array | undefined>;
+  has(request: { spaceId: SpaceId; uri: string }): Promise<boolean>;
+  getUrl?(request: { spaceId: SpaceId; uri: string; contentType?: string }): Promise<string | undefined>;
+}

package/src/document-structure.ts

@@ -3,13 +3,14 @@
 //
 
 import { invariant } from '@dxos/invariant';
-import type { EntityId, URI } from '@dxos/keys';
+import { type EntityId, PublicKey, SpaceId, type URI } from '@dxos/keys';
 import { visitValues } from '@dxos/util';
 
 import { type RawString } from './automerge';
 import type { ForeignKey } from './foreign-key';
 import { type EncodedReference, isEncodedReference } from './reference';
 import { type SpaceDocVersion } from './space-doc-version';
+import { createIdFromSpaceKey } from './space-id';
 
 export type SpaceState = {
   // Url of the root automerge document.
@@ -29,7 +30,19 @@ export interface DatabaseDirectory {
   version?: SpaceDocVersion;
 
   access?: {
-    spaceKey: string;
+    /**
+     * ID of the space that owns the document.
+     */
+    spaceId?: SpaceId;
+
+    /**
+     * @deprecated Use {@link spaceId}. Still written alongside `spaceId` so older clients
+     * (and code paths that need the space public key, which cannot be recovered from the id)
+     * keep working.
+     *
+     * Space key of the owning space in hex format without the 0x prefix.
+     */
+    spaceKey?: string;
   };
   /**
    * Objects inlined in the current document.
@@ -44,6 +57,21 @@ export interface DatabaseDirectory {
     [echoUri: string]: string | RawString;
   };
 
+  /**
+   * Per-object branch registry. Keyed by the subtree-root object id, then by branch name; each
+   * branch records the automerge doc url holding each subtree member at that branch.
+   *
+   * This is the single synced source of truth for branches: it is both the branch list/membership
+   * AND the set of branch documents the space must replicate (the host collects these urls in
+   * {@link getAllBranchDocUrls}). The client document loader does NOT treat these urls as object
+   * links, so branch docs never materialize as phantom objects. The implicit `'main'` branch is
+   * never listed here (it is the object's main doc via {@link links}).
+   *
+   * Which branch a device is currently viewing is NOT stored here — that is device-local,
+   * non-synced state.
+   */
+  branches?: SpaceBranchRegistry;
+
   /**
    * @deprecated
    * For backward compatibility.
@@ -51,8 +79,51 @@ export interface DatabaseDirectory {
   experimental_spaceKey?: string;
 }
 
+/**
+ * @see DatabaseDirectory.branches
+ */
+export type SpaceBranchRegistry = {
+  [rootObjectId: string]: {
+    [branchName: string]: BranchRecord;
+  };
+};
+
+export type BranchRecord = {
+  /** Subtree member object id -> automerge doc url holding that member on this branch. */
+  members: { [objectId: string]: string | RawString };
+  /**
+   * The root object's main-doc heads at fork time. Provenance only — currently written but never
+   * read; the merge relies on shared automerge ancestry, not this field.
+   */
+  baseHeads?: string[];
+  /** Unix ms timestamp at branch creation. */
+  createdAt?: number;
+};
+
 export const DatabaseDirectory = Object.freeze({
   /**
+   * @returns ID of the space that owns the document.
+   * Coalesces `access.spaceId` with the deprecated space key fields (`access.spaceKey`,
+   * `experimental_spaceKey`), deriving the id from the key for documents that predate `spaceId`.
+   */
+  getSpaceId: async (doc: DatabaseDirectory): Promise<SpaceId | null> => {
+    if (doc.access?.spaceId != null) {
+      invariant(SpaceId.isValid(doc.access.spaceId), 'Invalid space ID');
+      return doc.access.spaceId;
+    }
+
+    const spaceKeyHex = DatabaseDirectory.getSpaceKey(doc);
+    if (spaceKeyHex == null) {
+      return null;
+    }
+
+    return createIdFromSpaceKey(PublicKey.fromHex(spaceKeyHex));
+  },
+
+  /**
+   * @deprecated Use {@link DatabaseDirectory.getSpaceId}. Only paths that require the space
+   * public key (which cannot be derived from the space id) should read the key.
+   *
    * @returns Space key in hex of the space that owns the document. In hex format. Without 0x prefix.
    */
   getSpaceKey: (doc: DatabaseDirectory): string | null => {
@@ -75,17 +146,46 @@ export const DatabaseDirectory = Object.freeze({
     return doc.links?.[id]?.toString();
   },
 
+  /**
+   * @returns The branch registry for a subtree-root object, or undefined if it has no branches.
+   */
+  getBranches: (doc: DatabaseDirectory, rootObjectId: EntityId): Record<string, BranchRecord> | undefined => {
+    return doc.branches?.[rootObjectId];
+  },
+
+  /**
+   * @returns All branch document urls referenced anywhere in the registry. Used by the host to
+   * decide which documents to replicate (branch docs are NOT object links).
+   */
+  getAllBranchDocUrls: (doc: DatabaseDirectory): string[] => {
+    const urls: string[] = [];
+    for (const byName of Object.values(doc.branches ?? {})) {
+      for (const record of Object.values(byName)) {
+        for (const url of Object.values(record.members ?? {})) {
+          urls.push(url.toString());
+        }
+      }
+    }
+    return urls;
+  },
+
   make: ({
+    spaceId,
     spaceKey,
     objects,
     links,
   }: {
-    spaceKey: string;
+    spaceId?: SpaceId;
+    /**
+     * @deprecated Provide {@link spaceId}. The key is still stamped for older clients.
+     */
+    spaceKey?: string;
     objects?: Record<string, EntityStructure>;
     links?: Record<string, RawString>;
   }): DatabaseDirectory => ({
     access: {
-      spaceKey,
+      ...(spaceId != null ? { spaceId } : {}),
+      ...(spaceKey != null ? { spaceKey } : {}),
     },
     objects: objects ?? {},
     links: links ?? {},

package/src/echo-feed-codec.ts

@@ -18,10 +18,12 @@ export class EchoFeedCodec {
   static readonly #decoder = new TextDecoder();
 
   /**
-   * Prepares a value for feed storage (strips queue position from metadata) and encodes to bytes.
+   * Feed blocks are always whole-object snapshots; the index collapses entries by id to the latest
+   * block. TODO(wittjosiah): Follow-up — a partial-object update block format with field-level
+   * last-write-wins merge at the index (see EntityMetaIndex.update).
    */
   static encode(value: Record<string, unknown>): Uint8Array {
-    const prepared = EchoFeedCodec.#stripQueuePosition(value);
+    const prepared = EchoFeedCodec.stripQueuePosition(value);
     return EchoFeedCodec.#encoder.encode(JSON.stringify(prepared));
   }
 
@@ -37,7 +39,11 @@ export class EchoFeedCodec {
     return decoded;
   }
 
-  static #stripQueuePosition(value: Record<string, unknown>): Record<string, unknown> {
+  /**
+   * Strips the queue-position foreign key from an object's metadata, producing a canonical form
+   * comparable across a local snapshot and an inbound feed block (positions differ per-append).
+   */
+  static stripQueuePosition(value: Record<string, unknown>): Record<string, unknown> {
     if (typeof value !== 'object' || value === null) {
       return value;
     }
@@ -65,3 +71,9 @@ export class EchoFeedCodec {
     });
   }
 }
+
+/**
+ * Foreign-key source for the global position a feed block was assigned.
+ * Re-exported so `@dxos/echo` can read it without depending on `@dxos/protocols`.
+ */
+export const KEY_QUEUE_POSITION = FeedProtocol.KEY_QUEUE_POSITION;

package/src/index.ts

@@ -2,6 +2,7 @@
 // Copyright 2023 DXOS.org
 //
 
+export * from './blob';
 export type * from './collection-sync';
 export * from './document-structure';
 export * from './edge-peer';

package/src/query/ast.ts

@@ -80,6 +80,23 @@ const FilterIn_ = Schema.Struct({
 export interface FilterIn extends Schema.Schema.Type<typeof FilterIn_> {}
 export const FilterIn: Schema.Schema<FilterIn> = FilterIn_;
 
+/**
+ * In (subquery form) — membership against a value projected from a subquery's results,
+ * e.g. `threadId IN (SELECT threadId FROM feed WHERE tag = 'inbox')`.
+ *
+ * Nested-only (like {@link FilterIn}): valid inside an `object` filter's `props`, not at
+ * the query root — the planner has no selector for a standalone membership predicate.
+ * The subquery may target a different scope than the parent query; it is resolved once at
+ * execution time by projecting `property` from its results into a set.
+ */
+const FilterInQuery_ = Schema.Struct({
+  type: Schema.Literal('in-query'),
+  subquery: Schema.suspend(() => Query),
+  property: Schema.String,
+});
+export interface FilterInQuery extends Schema.Schema.Type<typeof FilterInQuery_> {}
+export const FilterInQuery: Schema.Schema<FilterInQuery> = FilterInQuery_;
+
 /**
  * Contains.
  */
@@ -200,6 +217,7 @@ export const Filter = Schema.Union(
   FilterObject,
   FilterCompare,
   FilterIn,
+  FilterInQuery,
   FilterContains,
   FilterTag,
   FilterRange,
@@ -342,8 +360,10 @@ export type OrderDirection = Schema.Schema.Type<typeof OrderDirection>;
 
 const Order_ = Schema.Union(
   Schema.Struct({
-    // How database wants to order them (in practice - by id).
+    // How the database wants to order them by default. For non-feed sources this is by id;
+    // for feed sources this is insertion order, so `desc` gives newest-first head reads.
     kind: Schema.Literal('natural'),
+    direction: OrderDirection,
   }),
   Schema.Struct({
     kind: Schema.Literal('property'),
@@ -404,6 +424,78 @@ const QueryLimitClause_ = Schema.Struct({
 export interface QueryLimitClause extends Schema.Schema.Type<typeof QueryLimitClause_> {}
 export const QueryLimitClause: Schema.Schema<QueryLimitClause> = QueryLimitClause_;
 
+/**
+ * Skip a number of results (offset). Combined with `limit` and a `natural` order, this expresses
+ * a windowed (paginated) read without any feed-specific query surface.
+ */
+const QuerySkipClause_ = Schema.Struct({
+  type: Schema.Literal('skip'),
+  query: Schema.suspend(() => Query),
+  skip: Schema.Number,
+});
+
+export interface QuerySkipClause extends Schema.Schema.Type<typeof QuerySkipClause_> {}
+export const QuerySkipClause: Schema.Schema<QuerySkipClause> = QuerySkipClause_;
+
+/**
+ * A named aggregate computed per group over its members, exposed as a top-level field on the flat
+ * result record (`row[name]`) and orderable via a following `orderBy(Order.property(name))`. A
+ * tagged union per kind — `property`/`limit`/`order` are present exactly when the kind uses them,
+ * so read sites narrow by `kind` instead of guarding an unused optional field.
+ * - `group` partitions members by a scalar `property`; its coerced key value is the field's value.
+ *   Composite keys are formed from multiple `group` entries. A query with no `group` entries
+ *   aggregates its entire input into a single row.
+ * - `max`/`min` reduce a scalar member `property`.
+ * - `items` collects the group's members, optionally ordered by `order` and capped to `limit`.
+ *   Opt-in — a row carries no members otherwise. `order` is this aggregate's own per-group
+ *   ordering, independent of any `orderBy` clause elsewhere in the query (which orders the whole
+ *   input stream / the resulting groups, not this aggregate's member selection).
+ * - `count` yields the member count. Opt-in — a row carries no count otherwise.
+ */
+const GroupAggregateGroup_ = Schema.Struct({
+  name: Schema.String,
+  kind: Schema.Literal('group'),
+  property: Schema.String,
+});
+const GroupAggregateMax_ = Schema.Struct({ name: Schema.String, kind: Schema.Literal('max'), property: Schema.String });
+const GroupAggregateMin_ = Schema.Struct({ name: Schema.String, kind: Schema.Literal('min'), property: Schema.String });
+const GroupAggregateItems_ = Schema.Struct({
+  name: Schema.String,
+  kind: Schema.Literal('items'),
+  limit: Schema.optional(Schema.Number),
+  order: Schema.optional(Schema.Array(Order)),
+});
+const GroupAggregateCount_ = Schema.Struct({ name: Schema.String, kind: Schema.Literal('count') });
+
+const GroupAggregate_ = Schema.Union(
+  GroupAggregateGroup_,
+  GroupAggregateMax_,
+  GroupAggregateMin_,
+  GroupAggregateItems_,
+  GroupAggregateCount_,
+);
+
+export type GroupAggregate = Schema.Schema.Type<typeof GroupAggregate_>;
+export const GroupAggregate: Schema.Schema<GroupAggregate> = GroupAggregate_;
+
+/**
+ * Aggregates results into flat records. `group`-kind entries partition members into contiguous
+ * groups (one row each); with no `group` entries the whole input aggregates into a single row.
+ * Groups are ordered by the first occurrence of their key in the incoming (already-ordered) result
+ * stream — this lets a preceding `orderBy` also control group order (e.g. ordering thread groups by
+ * their most recent message). A following `orderBy(Order.property(name))` referencing an aggregate
+ * or group field reorders whole groups instead. Must be the outermost data clause: only
+ * `from`/`options`/`order` may wrap it.
+ */
+const QueryAggregateClause_ = Schema.Struct({
+  type: Schema.Literal('aggregate'),
+  query: Schema.suspend(() => Query),
+  aggregates: Schema.Array(GroupAggregate),
+});
+
+export interface QueryAggregateClause extends Schema.Schema.Type<typeof QueryAggregateClause_> {}
+export const QueryAggregateClause: Schema.Schema<QueryAggregateClause> = QueryAggregateClause_;
+
 export const QueryFromClause_ = Schema.Struct({
   type: Schema.Literal('from'),
   query: Schema.suspend(() => Query),
@@ -432,6 +524,8 @@ const Query_ = Schema.Union(
   QueryOrderClause,
   QueryOptionsClause,
   QueryLimitClause,
+  QuerySkipClause,
+  QueryAggregateClause,
   QueryFromClause,
 ).annotations({ identifier: 'org.dxos.schema.query' });
 
@@ -511,6 +605,8 @@ export const visit = (query: Query, visitor: (node: Query) => void) => {
     }),
     Match.when({ type: 'order' }, ({ query }) => visit(query, visitor)),
     Match.when({ type: 'limit' }, ({ query }) => visit(query, visitor)),
+    Match.when({ type: 'skip' }, ({ query }) => visit(query, visitor)),
+    Match.when({ type: 'aggregate' }, ({ query }) => visit(query, visitor)),
     Match.when({ type: 'from' }, (node) => {
       visit(node.query, visitor);
       if (node.from._tag === 'query') {
@@ -537,6 +633,8 @@ export const map = (query: Query, mapper: (node: Query) => Query): Query => {
     Match.when({ type: 'options' }, (node) => ({ ...node, query: map(node.query, mapper) })),
     Match.when({ type: 'order' }, (node) => ({ ...node, query: map(node.query, mapper) })),
     Match.when({ type: 'limit' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'skip' }, (node) => ({ ...node, query: map(node.query, mapper) })),
+    Match.when({ type: 'aggregate' }, (node) => ({ ...node, query: map(node.query, mapper) })),
     Match.when({ type: 'from' }, (node) => ({
       ...node,
       query: map(node.query, mapper),
@@ -570,6 +668,8 @@ export const fold = <T>(query: Query, reducer: (node: Query) => T): T[] => {
     ),
     Match.when({ type: 'order' }, ({ query }) => fold(query, reducer)),
     Match.when({ type: 'limit' }, ({ query }) => fold(query, reducer)),
+    Match.when({ type: 'skip' }, ({ query }) => fold(query, reducer)),
+    Match.when({ type: 'aggregate' }, ({ query }) => fold(query, reducer)),
     Match.when({ type: 'from' }, (node) => {
       const results = fold(node.query, reducer);
       if (node.from._tag === 'query') {