@dxos/echo-pipeline 0.8.3 → 0.8.4-main.05e74ebcff

package/src/automerge/echo-replicator.ts

@@ -2,14 +2,19 @@
 // Copyright 2024 DXOS.org
 //
 
+import { type Heads } from '@automerge/automerge';
+import { type DocumentId } from '@automerge/automerge-repo';
+
+import { type Context } from '@dxos/context';
 import { type PublicKey, type SpaceId } from '@dxos/keys';
-import type { AutomergeProtocolMessage } from '@dxos/protocols';
+import { type AutomergeProtocolMessage } from '@dxos/protocols';
 
-export interface EchoReplicator {
+// TODO(burdon): Rename AutomergeReplicator?
+export interface AutomergeReplicator {
   /**
    * Called on when replicator is added to EchoHost.
    */
-  connect(context: EchoReplicatorContext): Promise<void>;
+  connect(ctx: Context, context: AutomergeReplicatorContext): Promise<void>;
 
   /**
    * Called on when replicator is removed from EchoHost.
@@ -17,7 +22,16 @@ export interface EchoReplicator {
   disconnect(): Promise<void>;
 }
 
-export interface EchoReplicatorContext {
+/**
+ * Replicator with explicit per-space wiring (used by edge replicators that maintain one logical
+ * connection per `SpaceId`). `DataSpaceManager` drives these lifecycle hooks on space open/close.
+ */
+export interface EdgeAutomergeReplicator extends AutomergeReplicator {
+  connectToSpace(ctx: Context, spaceId: SpaceId): Promise<void>;
+  disconnectFromSpace(spaceId: SpaceId): Promise<void>;
+}
+
+export interface AutomergeReplicatorContext {
   /**
    * Our own peer id.
    */
@@ -28,16 +42,20 @@ export interface EchoReplicatorContext {
    */
   getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
   getContainingSpaceIdForDocument(documentId: string): Promise<SpaceId | null>;
-  isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckParams): Promise<boolean>;
 
-  onConnectionOpen(connection: ReplicatorConnection): void;
-  onConnectionClosed(connection: ReplicatorConnection): void;
-  onConnectionAuthScopeChanged(connection: ReplicatorConnection): void;
+  /**
+   * Returns false if collection sync hasn't happened yet.
+   */
+  isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckProps): Promise<boolean>;
+
+  onConnectionOpen(connection: AutomergeReplicatorConnection): void;
+  onConnectionClosed(connection: AutomergeReplicatorConnection): void;
+  onConnectionAuthScopeChanged(connection: AutomergeReplicatorConnection): void;
 }
 
-export interface ReplicatorConnection {
+export interface AutomergeReplicatorConnection {
   /**
-   * Remove peer id.
+   * Remote peer id.
    */
   get peerId(): string;
 
@@ -55,23 +73,40 @@ export interface ReplicatorConnection {
    * @returns true if the document should be advertised to this peer.
    * The remote peer can still request the document by its id bypassing this check.
    */
-  shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean>;
+  shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean>;
 
   /**
    * @returns true if the collection should be synced to this peer.
    */
-  shouldSyncCollection(params: ShouldSyncCollectionParams): boolean;
+  shouldSyncCollection(params: ShouldSyncCollectionProps): boolean;
+
+  /**
+   * Batch syncing considered enabled if AutomergeReplicatorConnection implements `pushBatch` and `pullBatch` methods.
+   * @returns true if the batch syncing is enabled.
+   */
+  get bundleSyncEnabled(): boolean;
+
+  /**
+   * Pushes the batch of documents to the remote peer.
+   */
+  pushBundle?(ctx: Context, bundle: { documentId: DocumentId; data: Uint8Array; heads: Heads }[]): Promise<void>;
+
+  /**
+   * Pulls the batch of documents from the remote peer.
+   */
+  // TODO(mykola): Use automerge-repo-bundles Bundle type here.
+  pullBundle?(ctx: Context, docHeads: Record<DocumentId, Heads>): Promise<Record<DocumentId, Uint8Array>>;
 }
 
-export type ShouldAdvertiseParams = {
+export type ShouldAdvertiseProps = {
   documentId: string;
 };
 
-export type ShouldSyncCollectionParams = {
+export type ShouldSyncCollectionProps = {
   collectionId: string;
 };
 
-export type RemoteDocumentExistenceCheckParams = {
+export type RemoteDocumentExistenceCheckProps = {
   peerId: string;
   documentId: string;
 };

package/src/automerge/errors.ts

@@ -0,0 +1,10 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { BaseError } from '@dxos/errors';
+
+/**
+ * Thrown when attempting to send a message to a peer that is not connected.
+ */
+export class PeerNotFoundError extends BaseError.extend('PeerNotFoundError', 'Peer not found') {}

package/src/automerge/handle-state.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type DocumentId, type QueryState, type Repo } from '@automerge/automerge-repo';
+
+/**
+ * Variants reported by {@link QueryState}: `'loading' | 'ready' | 'unavailable' | 'failed'`.
+ */
+export type HandleQueryState = QueryState<unknown>['state'];
+
+/**
+ * Returns the {@link QueryState} variant for a document already known to the
+ * repo (i.e. one for which a handle has been registered via `Repo.find` /
+ * `Repo.findWithProgress` / `Repo.create`), or `undefined` if no handle/query
+ * exists for this document yet.
+ *
+ * Use this anywhere you need to ask "is this document loaded?" / "did sync
+ * give up?" — read the result against the {@link HandleQueryState} variants
+ * (e.g. `=== 'ready'`).
+ *
+ * # Why we can't just call `DocHandle.isReady()` / read `DocHandle.state`
+ *
+ * The current subduction fork of `@automerge/automerge-repo`
+ * (`2.6.0-subduction.10`) **gutted** the per-handle state machine. Inside the
+ * fork, every `DocHandle` state predicate is a constant placeholder:
+ *
+ * ```
+ * isReady       = () => true
+ * isUnloaded    = () => false
+ * isDeleted     = () => false
+ * isUnavailable = () => false
+ * inState       = (states) => states.includes('ready')
+ * get state     = () => 'ready'
+ * whenReady     = async () => {}     // resolves immediately, ignores argument
+ * ```
+ *
+ * That means `handle.isReady()` always lies "yes", `handle.state` always reads
+ * `'ready'`, and `await handle.whenReady()` never blocks — even for handles
+ * whose document has never been loaded from storage or synced from a peer.
+ * Reading `handle.doc()` on such a "ready" handle returns the empty initial
+ * doc (`A.init()` or `A.emptyChange(A.init())`) rather than throwing or
+ * returning undefined, which can silently corrupt downstream logic that
+ * trusted the handle's `isReady()` signal.
+ *
+ * Real liveness moved to the {@link DocumentQuery} returned by
+ * `Repo.findWithProgress`, whose state goes through
+ * `'loading' → 'ready' | 'unavailable' | 'failed'`. Production code that
+ * needs to know whether a document is actually loaded must consult the query
+ * state through this helper instead of any `DocHandle.*` predicate.
+ *
+ * TODO(mykola): Remove this helper once the fork either (a) restores the
+ * `DocHandle` state machine, or (b) every caller is migrated to query
+ * `DocumentProgress` directly with a richer API surface (await on state
+ * transitions, etc.).
+ */
+export const getHandleState = (repo: Repo, documentId: DocumentId): HandleQueryState | undefined => {
+  if (!repo.handles[documentId]) {
+    return undefined;
+  }
+  return repo.findWithProgress(documentId).peek().state;
+};

package/src/automerge/heads-store.ts

@@ -4,18 +4,45 @@
 
 import type { Heads } from '@automerge/automerge';
 import type { DocumentId } from '@automerge/automerge-repo';
+import { type MixedEncoding } from 'level-transcoder';
 
-import { headsEncoding } from '@dxos/indexing';
+import type { ProtoCodec } from '@dxos/codec-protobuf';
 import type { BatchLevel, SublevelDB } from '@dxos/kv-store';
+import { log } from '@dxos/log';
+import { schema } from '@dxos/protocols/proto';
+import type { Heads as HeadsProto } from '@dxos/protocols/proto/dxos/echo/query';
 
-export type HeadsStoreParams = {
+// NOTE: Lazy so that code that doesn't use indexing doesn't need to load the codec (breaks in workerd).
+let headsCodec: ProtoCodec<HeadsProto>;
+const getHeadsCodec = () => (headsCodec ??= schema.getCodecForType('dxos.echo.query.Heads'));
+
+const headsEncoding: MixedEncoding<Heads, Uint8Array, Heads> = {
+  encode: (value: Heads): Uint8Array => getHeadsCodec().encode({ hashes: value }),
+  decode: (encodedValue: Uint8Array): Heads => {
+    try {
+      return getHeadsCodec().decode(encodedValue).hashes!;
+    } catch {
+      // Legacy encoding migration path.
+      log.warn('Detected legacy encoding of heads in storage.');
+      const concatenatedHeads = Buffer.from(encodedValue).toString('utf8').replace(/"/g, '');
+      const heads = [];
+      for (let i = 0; i < concatenatedHeads.length; i += 64) {
+        heads.push(concatenatedHeads.slice(i, i + 64));
+      }
+      return heads;
+    }
+  },
+  format: 'buffer',
+};
+
+export type HeadsStoreProps = {
   db: SublevelDB;
 };
 
 export class HeadsStore {
   private readonly _db: SublevelDB;
 
-  constructor({ db }: HeadsStoreParams) {
+  constructor({ db }: HeadsStoreProps) {
     this._db = db;
   }
 
@@ -34,4 +61,16 @@ export class HeadsStore {
       valueEncoding: headsEncoding,
     });
   }
+
+  /**
+   * Iterate over all document IDs and their heads.
+   */
+  async *iterateAll(): AsyncGenerator<{ documentId: DocumentId; heads: Heads }> {
+    for await (const [documentId, heads] of this._db.iterator<DocumentId, Heads>({
+      keyEncoding: 'utf8',
+      valueEncoding: headsEncoding,
+    })) {
+      yield { documentId, heads };
+    }
+  }
 }

package/src/automerge/index.ts

@@ -3,9 +3,11 @@
 //
 
 export * from './automerge-host';
+export * from './handle-state';
 export * from './leveldb-storage-adapter';
 export * from './mesh-echo-replicator';
-export * from './echo-replicator';
+export type * from './echo-replicator';
 export { diffCollectionState } from './collection-synchronizer';
 export * from './space-collection';
 export * from './echo-data-monitor';
+export * from './errors';

package/src/automerge/leveldb-storage-adapter.ts

@@ -2,10 +2,10 @@
 // Copyright 2024 DXOS.org
 // s
 
-import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@automerge/automerge-repo';
+import { type Chunk, type StorageAdapterInterface, type StorageKey } from '@automerge/automerge-repo';
 import { type MixedEncoding } from 'level-transcoder';
 
-import { LifecycleState, Resource } from '@dxos/context';
+import { Resource } from '@dxos/context';
 import { type BatchLevel, type SublevelDB } from '@dxos/kv-store';
 import { type MaybePromise } from '@dxos/util';
 
@@ -16,27 +16,56 @@ export interface StorageAdapterDataMonitor {
   recordStoreDuration(durationMs: number): void;
 }
 
-export type LevelDBStorageAdapterParams = {
+export type LevelDBStorageAdapterProps = {
   db: SublevelDB;
   callbacks?: StorageCallbacks;
   monitor?: StorageAdapterDataMonitor;
 };
 
-export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
+export type BeforeSaveProps = { path: StorageKey; batch: BatchLevel };
 
 export interface StorageCallbacks {
-  beforeSave(params: BeforeSaveParams): MaybePromise<void>;
+  beforeSave(params: BeforeSaveProps): MaybePromise<void>;
   afterSave(path: StorageKey): MaybePromise<void>;
 }
 
 export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
-  constructor(private readonly _params: LevelDBStorageAdapterParams) {
+  /**
+   * In-flight `loadRange` / `removeRange` iterations. Awaited in `_close` so
+   * the adapter doesn't return from close while a `for await` on the sublevel
+   * is still pending — otherwise the kv owner upstream would close the
+   * sublevel and the iterator's next `.next()` would reject with
+   * `LEVEL_ITERATOR_NOT_OPEN`.
+   *
+   * Promises stored here resolve regardless of outcome (the wrapper
+   * `.catch`es) so `Promise.all` won't reject.
+   */
+  readonly #inFlightIterations = new Set<Promise<unknown>>();
+
+  constructor(private readonly _params: LevelDBStorageAdapterProps) {
     super();
   }
 
+  protected override async _close(): Promise<void> {
+    // New `loadRange`/`removeRange` calls already short-circuit on `!isOpen`
+    // (the Resource base flips state before invoking `_close`). Wait for any
+    // iterations that started while we were still open to finish before
+    // returning, so the upstream `kv` owner can safely close the sublevel.
+    if (this.#inFlightIterations.size > 0) {
+      await Promise.all(this.#inFlightIterations);
+    }
+  }
+
+  #trackIteration<T>(work: Promise<T>): Promise<T> {
+    const settled = work.catch(() => undefined);
+    this.#inFlightIterations.add(settled);
+    void settled.finally(() => this.#inFlightIterations.delete(settled));
+    return work;
+  }
+
   async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
     try {
-      if (this._lifecycleState !== LifecycleState.OPEN) {
+      if (!this.isOpen) {
         // TODO(mykola): this should be an error.
         return undefined;
       }
@@ -54,7 +83,7 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
   }
 
   async save(keyArray: StorageKey, binary: Uint8Array): Promise<void> {
-    if (this._lifecycleState !== LifecycleState.OPEN) {
+    if (!this.isOpen) {
       return undefined;
     }
     const startMs = Date.now();
@@ -71,17 +100,50 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
     this._params.monitor?.recordStoreDuration(Date.now() - startMs);
   }
 
+  /**
+   * Atomically persist multiple key/value entries in a single LevelDB batch write.
+   */
+  async saveBatch(entries: Array<[StorageKey, Uint8Array]>): Promise<void> {
+    if (!this.isOpen || entries.length === 0) {
+      return undefined;
+    }
+    const startMs = Date.now();
+    const batch = this._params.db.batch();
+
+    for (const [keyArray] of entries) {
+      await this._params.callbacks?.beforeSave?.({ path: keyArray, batch });
+    }
+    for (const [keyArray, binary] of entries) {
+      batch.put<StorageKey, Uint8Array>(keyArray, Buffer.from(binary), {
+        ...encodingOptions,
+      });
+    }
+    await batch.write();
+
+    let bytesStored = 0;
+    for (const [keyArray, binary] of entries) {
+      bytesStored += binary.byteLength;
+      await this._params.callbacks?.afterSave?.(keyArray);
+    }
+    this._params.monitor?.recordBytesStored(bytesStored);
+    this._params.monitor?.recordStoreDuration(Date.now() - startMs);
+  }
+
   async remove(keyArray: StorageKey): Promise<void> {
-    if (this._lifecycleState !== LifecycleState.OPEN) {
+    if (!this.isOpen) {
       return undefined;
     }
     await this._params.db.del<StorageKey>(keyArray, { ...encodingOptions });
   }
 
   async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
-    if (this._lifecycleState !== LifecycleState.OPEN) {
+    if (!this.isOpen) {
       return [];
     }
+    return this.#trackIteration(this.#doLoadRange(keyPrefix));
+  }
+
+  async #doLoadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
     const startMs = Date.now();
     const result: Chunk[] = [];
     for await (const [key, value] of this._params.db.iterator<StorageKey, Uint8Array>({
@@ -100,9 +162,13 @@ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInt
   }
 
   async removeRange(keyPrefix: StorageKey): Promise<void> {
-    if (this._lifecycleState !== LifecycleState.OPEN) {
+    if (!this.isOpen) {
       return undefined;
     }
+    await this.#trackIteration(this.#doRemoveRange(keyPrefix));
+  }
+
+  async #doRemoveRange(keyPrefix: StorageKey): Promise<void> {
     const batch = this._params.db.batch();
 
     for await (const [key] of this._params.db.iterator<StorageKey, Uint8Array>({

package/src/automerge/mesh-echo-replicator-connection.ts

@@ -12,20 +12,20 @@ import { log } from '@dxos/log';
 import type { AutomergeProtocolMessage } from '@dxos/protocols';
 import { AutomergeReplicator, type AutomergeReplicatorFactory } from '@dxos/teleport-extension-automerge-replicator';
 
-import type { ReplicatorConnection, ShouldAdvertiseParams, ShouldSyncCollectionParams } from './echo-replicator';
+import type { AutomergeReplicatorConnection, ShouldAdvertiseProps, ShouldSyncCollectionProps } from './echo-replicator';
 
 const DEFAULT_FACTORY: AutomergeReplicatorFactory = (params) => new AutomergeReplicator(...params);
 
-export type MeshReplicatorConnectionParams = {
+export type MeshReplicatorConnectionProps = {
   ownPeerId: string;
   onRemoteConnected: () => void;
   onRemoteDisconnected: () => void;
-  shouldAdvertise: (params: ShouldAdvertiseParams) => Promise<boolean>;
-  shouldSyncCollection: (params: ShouldSyncCollectionParams) => boolean;
+  shouldAdvertise: (params: ShouldAdvertiseProps) => Promise<boolean>;
+  shouldSyncCollection: (params: ShouldSyncCollectionProps) => boolean;
   replicatorFactory?: AutomergeReplicatorFactory;
 };
 
-export class MeshReplicatorConnection extends Resource implements ReplicatorConnection {
+export class MeshReplicatorConnection extends Resource implements AutomergeReplicatorConnection {
   public readable: ReadableStream<AutomergeProtocolMessage>;
   public writable: WritableStream<AutomergeProtocolMessage>;
   public remoteDeviceKey: PublicKey | null = null;
@@ -35,7 +35,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
   private _remotePeerId: string | null = null;
   private _isEnabled = false;
 
-  constructor(private readonly _params: MeshReplicatorConnectionParams) {
+  constructor(private readonly _params: MeshReplicatorConnectionProps) {
     super();
 
     let readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
@@ -111,11 +111,15 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
     return this._isEnabled;
   }
 
-  async shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean> {
+  get bundleSyncEnabled(): boolean {
+    return false;
+  }
+
+  async shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean> {
     return this._params.shouldAdvertise(params);
   }
 
-  shouldSyncCollection(params: ShouldSyncCollectionParams): boolean {
+  shouldSyncCollection(params: ShouldSyncCollectionProps): boolean {
     return this._params.shouldSyncCollection(params);
   }
 

package/src/automerge/mesh-echo-replicator.ts

@@ -2,27 +2,29 @@
 // Copyright 2024 DXOS.org
 //
 
+import { type Context } from '@dxos/context';
 import type { CollectionId } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import { PublicKey, type SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
-import {
-  type AutomergeReplicator,
-  type AutomergeReplicatorFactory,
-} from '@dxos/teleport-extension-automerge-replicator';
+import type * as TeleportAutomergeReplicator from '@dxos/teleport-extension-automerge-replicator';
 import { ComplexSet, defaultMap } from '@dxos/util';
 
-import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseParams } from './echo-replicator';
+import { createIdFromSpaceKey } from '../common/space-id';
+import {
+  type AutomergeReplicator,
+  type AutomergeReplicatorContext,
+  type ShouldAdvertiseProps,
+} from './echo-replicator';
 import { MeshReplicatorConnection } from './mesh-echo-replicator-connection';
 import { getSpaceIdFromCollectionId } from './space-collection';
-import { createIdFromSpaceKey } from '../common/space-id';
 
 // TODO(dmaretskyi): Move out of @dxos/echo-pipeline.
 
 /**
  * Used to replicate with other peers over the network.
  */
-export class MeshEchoReplicator implements EchoReplicator {
+export class MeshEchoReplicator implements AutomergeReplicator {
   /**
    * We might have multiple connections open with a peer (one per space), but there'll be only one enabled
    * connection at any given moment, because there's a single repo for all the spaces.
@@ -40,9 +42,9 @@ export class MeshEchoReplicator implements EchoReplicator {
    */
   private readonly _authorizedDevices = new Map<SpaceId, ComplexSet<PublicKey>>();
 
-  private _context: EchoReplicatorContext | null = null;
+  private _context: AutomergeReplicatorContext | null = null;
 
-  async connect(context: EchoReplicatorContext): Promise<void> {
+  async connect(_ctx: Context, context: AutomergeReplicatorContext): Promise<void> {
     this._context = context;
   }
 
@@ -63,7 +65,9 @@ export class MeshEchoReplicator implements EchoReplicator {
     this._context = null;
   }
 
-  createExtension(extensionFactory?: AutomergeReplicatorFactory): AutomergeReplicator {
+  createExtension(
+    extensionFactory?: TeleportAutomergeReplicator.AutomergeReplicatorFactory,
+  ): TeleportAutomergeReplicator.AutomergeReplicator {
     invariant(this._context);
 
     const connection: MeshReplicatorConnection = new MeshReplicatorConnection({
@@ -110,7 +114,7 @@ export class MeshEchoReplicator implements EchoReplicator {
           }
         }
       },
-      shouldAdvertise: async (params: ShouldAdvertiseParams) => {
+      shouldAdvertise: async (params: ShouldAdvertiseProps) => {
         log('shouldAdvertise', { peerId: connection.peerId, documentId: params.documentId });
         invariant(this._context);
         try {

package/src/automerge/storage-adapter.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { onTestFinished, describe, expect, test } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { randomBytes } from '@dxos/crypto';
 import { PublicKey } from '@dxos/keys';

package/src/common/space-id.ts

@@ -18,7 +18,7 @@ export const createIdFromSpaceKey = async (spaceKey: PublicKey): Promise<SpaceId
     return cachedValue;
   }
 
-  const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array());
+  const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array() as Uint8Array<ArrayBuffer>);
 
   const bytes = new Uint8Array(digest).slice(0, SpaceId.byteLength);
   const spaceId = SpaceId.encode(bytes);