@automerge/automerge-repo 1.1.0-alpha.1 → 1.1.0-alpha.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@automerge/automerge-repo",
-  "version": "1.1.0-alpha.1",
+  "version": "1.1.0-alpha.13",
   "description": "A repository object to manage a collection of automerge documents",
   "repository": "https://github.com/automerge/automerge-repo/tree/master/packages/automerge-repo",
   "author": "Peter van Hardenberg <pvh@pvh.ca>",
@@ -10,20 +10,20 @@
   "scripts": {
     "build": "tsc",
     "watch": "npm-watch build",
-    "test:coverage": "c8 --reporter=lcov --reporter=html --reporter=text yarn test",
+    "test:coverage": "c8 --reporter=lcov --reporter=html --reporter=text pnpm test",
     "test": "vitest",
     "test:watch": "npm-watch test",
-    "test:log": "cross-env DEBUG='automerge-repo:*' yarn test",
     "fuzz": "ts-node --esm --experimentalSpecifierResolution=node fuzz/fuzz.ts"
   },
   "browser": {
     "crypto": false
   },
   "devDependencies": {
-    "http-server": "^14.1.0"
+    "http-server": "^14.1.0",
+    "vite": "^5.0.8"
   },
   "dependencies": {
-    "@automerge/automerge": "^2.1.8-alpha.1",
+    "@automerge/automerge": "^2.1.9",
     "bs58check": "^3.0.1",
     "cbor-x": "^1.3.0",
     "debug": "^4.3.4",
@@ -55,5 +55,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "11805d698f860bd6ffb3ca028d3b57e718690b5a"
+  "gitHead": "f4ce1376d900ad98f00a638626be9611077460b5"
 }

package/src/AutomergeUrl.ts

@@ -13,7 +13,7 @@ export const urlPrefix = "automerge:"
 /** Given an Automerge URL, returns the DocumentId in both base58check-encoded form and binary form */
 export const parseAutomergeUrl = (url: AutomergeUrl) => {
   const regex = new RegExp(`^${urlPrefix}(\\w+)$`)
-  const [_, docMatch] = url.match(regex) || []
+  const [, docMatch] = url.match(regex) || []
   const documentId = docMatch as DocumentId
   const binaryDocumentId = documentIdToBinary(documentId)
 
@@ -33,7 +33,7 @@ export const parseAutomergeUrl = (url: AutomergeUrl) => {
 export const stringifyAutomergeUrl = (
   arg: UrlOptions | DocumentId | BinaryDocumentId
 ) => {
-  let documentId =
+  const documentId =
     arg instanceof Uint8Array || typeof arg === "string"
       ? arg
       : "documentId" in arg

package/src/RemoteHeadsSubscriptions.ts

@@ -44,6 +44,9 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
   #theirSubscriptions: Map<StorageId, Set<PeerId>> = new Map()
   // Peers we will always share remote heads with even if they are not subscribed
   #generousPeers: Set<PeerId> = new Set()
+  // Documents each peer has open, we need this information so we only send remote heads of documents that the peer knows
+  #subscribedDocsByPeer: Map<PeerId, Set<DocumentId>> = new Map()
+
   #log = debug("automerge-repo:remote-heads-subscriptions")
 
   subscribeToRemotes(remotes: StorageId[]) {
@@ -89,11 +92,17 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
   handleControlMessage(control: RemoteSubscriptionControlMessage) {
     const remotesToAdd: StorageId[] = []
     const remotesToRemove: StorageId[] = []
+    const addedRemotesWeKnow: StorageId[] = []
 
     this.#log("handleControlMessage", control)
     if (control.add) {
       for (const remote of control.add) {
         let theirSubs = this.#theirSubscriptions.get(remote)
+
+        if (this.#ourSubscriptions.has(remote) || theirSubs) {
+          addedRemotesWeKnow.push(remote)
+        }
+
         if (!theirSubs) {
           theirSubs = new Set()
           this.#theirSubscriptions.set(remote, theirSubs)
@@ -128,6 +137,30 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
         remove: remotesToRemove,
       })
     }
+
+    // send all our stored heads of documents the peer knows for the remotes they've added
+    for (const remote of addedRemotesWeKnow) {
+      const subscribedDocs = this.#subscribedDocsByPeer.get(control.senderId)
+      if (subscribedDocs) {
+        for (const documentId of subscribedDocs) {
+          const knownHeads = this.#knownHeads.get(documentId)
+          if (!knownHeads) {
+            continue
+          }
+
+          const lastHeads = knownHeads.get(remote)
+          if (lastHeads) {
+            this.emit("notify-remote-heads", {
+              targetId: control.senderId,
+              documentId,
+              heads: lastHeads.heads,
+              timestamp: lastHeads.timestamp,
+              storageId: remote,
+            })
+          }
+        }
+      }
+    }
   }
 
   /** A peer we are not directly connected to has changed their heads */
@@ -165,13 +198,15 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
       const theirSubs = this.#theirSubscriptions.get(event.storageId)
       if (theirSubs) {
         for (const peerId of theirSubs) {
-          this.emit("notify-remote-heads", {
-            targetId: peerId,
-            documentId: event.documentId,
-            heads: event.remoteHeads,
-            timestamp: event.timestamp,
-            storageId: event.storageId,
-          })
+          if (this.#isPeerSubscribedToDoc(peerId, event.documentId)) {
+            this.emit("notify-remote-heads", {
+              targetId: peerId,
+              documentId: event.documentId,
+              heads: event.remoteHeads,
+              timestamp: event.timestamp,
+              storageId: event.storageId,
+            })
+          }
         }
       }
     }
@@ -200,13 +235,15 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
     const theirSubs = this.#theirSubscriptions.get(storageId)
     if (theirSubs) {
       for (const peerId of theirSubs) {
-        this.emit("notify-remote-heads", {
-          targetId: peerId,
-          documentId: documentId,
-          heads: heads,
-          timestamp: timestamp,
-          storageId: storageId,
-        })
+        if (this.#isPeerSubscribedToDoc(peerId, documentId)) {
+          this.emit("notify-remote-heads", {
+            targetId: peerId,
+            documentId: documentId,
+            heads: heads,
+            timestamp: timestamp,
+            storageId: storageId,
+          })
+        }
       }
     }
   }
@@ -241,6 +278,7 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
     const remotesToRemove = []
 
     this.#generousPeers.delete(peerId)
+    this.#subscribedDocsByPeer.delete(peerId)
 
     for (const [storageId, peerIds] of this.#theirSubscriptions) {
       if (peerIds.has(peerId)) {
@@ -261,6 +299,37 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
     }
   }
 
+  subscribePeerToDoc(peerId: PeerId, documentId: DocumentId) {
+    let subscribedDocs = this.#subscribedDocsByPeer.get(peerId)
+    if (!subscribedDocs) {
+      subscribedDocs = new Set()
+      this.#subscribedDocsByPeer.set(peerId, subscribedDocs)
+    }
+
+    subscribedDocs.add(documentId)
+
+    const remoteHeads = this.#knownHeads.get(documentId)
+    if (remoteHeads) {
+      for (const [storageId, lastHeads] of remoteHeads) {
+        const subscribedPeers = this.#theirSubscriptions.get(storageId)
+        if (subscribedPeers && subscribedPeers.has(peerId)) {
+          this.emit("notify-remote-heads", {
+            targetId: peerId,
+            documentId,
+            heads: lastHeads.heads,
+            timestamp: lastHeads.timestamp,
+            storageId,
+          })
+        }
+      }
+    }
+  }
+
+  #isPeerSubscribedToDoc(peerId: PeerId, documentId: DocumentId) {
+    const subscribedDocs = this.#subscribedDocsByPeer.get(peerId)
+    return subscribedDocs && subscribedDocs.has(documentId)
+  }
+
   /** Returns the (document, storageId) pairs which have changed after processing msg */
   #changedHeads(msg: RemoteHeadsChanged): {
     documentId: DocumentId
@@ -279,12 +348,12 @@ export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscripti
       }
       let remote = this.#knownHeads.get(documentId)
       if (!remote) {
-        remote = new Map([[storageId as StorageId, { heads, timestamp }]])
+        remote = new Map()
         this.#knownHeads.set(documentId, remote)
       }
 
       const docRemote = remote.get(storageId as StorageId)
-      if (docRemote && docRemote.timestamp > timestamp) {
+      if (docRemote && docRemote.timestamp >= timestamp) {
         continue
       } else {
         remote.set(storageId as StorageId, { timestamp, heads })

package/src/Repo.ts

@@ -7,17 +7,18 @@ import {
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
 import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
+import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
+import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
-import { NetworkAdapter } from "./network/NetworkAdapter.js"
+import { NetworkAdapter, type PeerMetadata } from "./network/NetworkAdapter.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
+import { RepoMessage } from "./network/messages.js"
 import { StorageAdapter } from "./storage/StorageAdapter.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
+import { StorageId } from "./storage/types.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
+import { SyncStatePayload } from "./synchronizer/Synchronizer.js"
 import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
-import { RepoMessage, SyncStateMessage } from "./network/messages.js"
-import { StorageId } from "./storage/types.js"
-import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
-import { headsAreSame } from "./helpers/headsAreSame.js"
 
 /** A Repo is a collection of documents with networking, syncing, and storage capabilities. */
 /** The `Repo` is the main entry point of this library
@@ -47,11 +48,12 @@ export class Repo extends EventEmitter<RepoEvents> {
   /** @hidden */
   sharePolicy: SharePolicy = async () => true
 
-  /** maps peer id to to persistance information (storageId, isEphemeral), access by collection synchronizer  */
+  /** maps peer id to to persistence information (storageId, isEphemeral), access by collection synchronizer  */
   /** @hidden */
-  persistanceInfoByPeerId: Record<PeerId, PersistanceInfo> = {}
+  peerMetadataByPeerId: Record<PeerId, PeerMetadata> = {}
 
   #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+  #remoteHeadsGossipingEnabled = false
 
   constructor({
     storage,
@@ -59,8 +61,10 @@ export class Repo extends EventEmitter<RepoEvents> {
     peerId,
     sharePolicy,
     isEphemeral = storage === undefined,
+    enableRemoteHeadsGossiping = false,
   }: RepoConfig) {
     super()
+    this.#remoteHeadsGossipingEnabled = enableRemoteHeadsGossiping
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
 
@@ -77,10 +81,7 @@ export class Repo extends EventEmitter<RepoEvents> {
         }: DocHandleEncodedChangePayload<any>) => {
           void storageSubsystem.saveDoc(handle.documentId, doc)
         }
-        const debouncedSaveFn = handle.on(
-          "heads-changed",
-          throttle(saveFn, this.saveDebounceRate)
-        )
+        handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
 
         if (isNew) {
           // this is a new document, immediately save it
@@ -140,6 +141,12 @@ export class Repo extends EventEmitter<RepoEvents> {
       networkSubsystem.send(message)
     })
 
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#synchronizer.on("open-doc", ({ peerId, documentId }) => {
+        this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
+      })
+    }
+
     // STORAGE
     // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
     const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
@@ -147,28 +154,34 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
+
+    const myPeerMetadata: Promise<PeerMetadata> = new Promise(
+      // eslint-disable-next-line no-async-promise-executor -- TODO: fix
+      async resolve =>
+        resolve({
+          storageId: await storageSubsystem?.id(),
+          isEphemeral,
+        } as PeerMetadata)
+    )
+
     const networkSubsystem = new NetworkSubsystem(
       network,
       peerId,
-      storageSubsystem?.id() ?? Promise.resolve(undefined),
-      isEphemeral
+      myPeerMetadata
     )
     this.networkSubsystem = networkSubsystem
 
     // When we get a new peer, register it with the synchronizer
-    networkSubsystem.on("peer", async ({ peerId, storageId, isEphemeral }) => {
+    networkSubsystem.on("peer", async ({ peerId, peerMetadata }) => {
       this.#log("peer connected", { peerId })
 
-      if (storageId) {
-        this.persistanceInfoByPeerId[peerId] = {
-          storageId,
-          isEphemeral,
-        }
+      if (peerMetadata) {
+        this.peerMetadataByPeerId[peerId] = { ...peerMetadata }
       }
 
       this.sharePolicy(peerId)
         .then(shouldShare => {
-          if (shouldShare) {
+          if (shouldShare && this.#remoteHeadsGossipingEnabled) {
             this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
           }
         })
@@ -195,12 +208,11 @@ export class Repo extends EventEmitter<RepoEvents> {
 
       const handle = this.#handleCache[message.documentId]
 
-      const info = this.persistanceInfoByPeerId[message.peerId]
-      if (!info) {
+      const { storageId } = this.peerMetadataByPeerId[message.peerId] || {}
+      if (!storageId) {
         return
       }
 
-      const { storageId } = info
       const heads = handle.getRemoteHeads(storageId)
       const haveHeadsChanged =
         message.syncState.theirHeads &&
@@ -209,7 +221,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       if (haveHeadsChanged) {
         handle.setRemoteHeads(storageId, message.syncState.theirHeads)
 
-        if (storageId) {
+        if (storageId && this.#remoteHeadsGossipingEnabled) {
           this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
             message.documentId,
             storageId,
@@ -219,45 +231,51 @@ export class Repo extends EventEmitter<RepoEvents> {
       }
     })
 
-    this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
-      this.networkSubsystem.send({
-        type: "remote-heads-changed",
-        targetId: message.targetId,
-        documentId: message.documentId,
-        newHeads: {
-          [message.storageId]: {
-            heads: message.heads,
-            timestamp: message.timestamp,
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
+        this.networkSubsystem.send({
+          type: "remote-heads-changed",
+          targetId: message.targetId,
+          documentId: message.documentId,
+          newHeads: {
+            [message.storageId]: {
+              heads: message.heads,
+              timestamp: message.timestamp,
+            },
           },
-        },
+        })
       })
-    })
 
-    this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
-      this.#log("change-remote-subs", message)
-      for (const peer of message.peers) {
-        this.networkSubsystem.send({
-          type: "remote-subscription-change",
-          targetId: peer,
-          add: message.add,
-          remove: message.remove,
-        })
-      }
-    })
+      this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
+        this.#log("change-remote-subs", message)
+        for (const peer of message.peers) {
+          this.networkSubsystem.send({
+            type: "remote-subscription-change",
+            targetId: peer,
+            add: message.add,
+            remove: message.remove,
+          })
+        }
+      })
 
-    this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
-      const handle = this.#handleCache[message.documentId]
-      handle.setRemoteHeads(message.storageId, message.remoteHeads)
-    })
+      this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
+        const handle = this.#handleCache[message.documentId]
+        handle.setRemoteHeads(message.storageId, message.remoteHeads)
+      })
+    }
   }
 
   #receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "remote-subscription-change":
-        this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        }
         break
       case "remote-heads-changed":
-        this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        }
         break
       case "sync":
       case "request":
@@ -271,34 +289,37 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   #throttledSaveSyncStateHandlers: Record<
     StorageId,
-    (message: SyncStateMessage) => void
+    (payload: SyncStatePayload) => void
   > = {}
 
   /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
-  #saveSyncState(message: SyncStateMessage) {
+  #saveSyncState(payload: SyncStatePayload) {
     if (!this.storageSubsystem) {
       return
     }
 
-    const persistanceInfo = this.persistanceInfoByPeerId[message.peerId]
+    const { storageId, isEphemeral } =
+      this.peerMetadataByPeerId[payload.peerId] || {}
 
-    if (!persistanceInfo || persistanceInfo.isEphemeral) {
+    if (!storageId || isEphemeral) {
       return
     }
 
-    const { storageId } = persistanceInfo
-
     let handler = this.#throttledSaveSyncStateHandlers[storageId]
     if (!handler) {
       handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
-        ({ documentId, syncState }: SyncStateMessage) => {
-          this.storageSubsystem!.saveSyncState(documentId, storageId, syncState)
+        ({ documentId, syncState }: SyncStatePayload) => {
+          void this.storageSubsystem!.saveSyncState(
+            documentId,
+            storageId,
+            syncState
+          )
         },
         this.saveDebounceRate
       )
     }
 
-    handler(message)
+    handler(payload)
   }
 
   /** Returns an existing handle if we have it; creates one otherwise. */
@@ -329,6 +350,10 @@ export class Repo extends EventEmitter<RepoEvents> {
     return this.#synchronizer.peers
   }
 
+  getStorageIdOfPeer(peerId: PeerId): StorageId | undefined {
+    return this.peerMetadataByPeerId[peerId]?.storageId
+  }
+
   /**
    * Creates a new document and returns a handle to it. The initial value of the document is
    * an empty object `{}`. Its documentId is generated by the system. we emit a `document` event
@@ -439,9 +464,47 @@ export class Repo extends EventEmitter<RepoEvents> {
     this.emit("delete-document", { documentId })
   }
 
+  /**
+   * Exports a document to a binary format.
+   * @param id - The url or documentId of the handle to export
+   *
+   * @returns Promise<Uint8Array | undefined> - A Promise containing the binary document,
+   * or undefined if the document is unavailable.
+   */
+  async export(id: AnyDocumentId): Promise<Uint8Array | undefined> {
+    const documentId = interpretAsDocumentId(id)
+
+    const handle = this.#getHandle(documentId, false)
+    const doc = await handle.doc()
+    if (!doc) return undefined
+    return Automerge.save(doc)
+  }
+
+  /**
+   * Imports document binary into the repo.
+   * @param binary - The binary to import
+   */
+  import<T>(binary: Uint8Array) {
+    const doc = Automerge.load<T>(binary)
+
+    const handle = this.create<T>()
+
+    handle.update(() => {
+      return Automerge.clone(doc)
+    })
+
+    return handle
+  }
+
   subscribeToRemotes = (remotes: StorageId[]) => {
-    this.#log("subscribeToRemotes", { remotes })
-    this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#log("subscribeToRemotes", { remotes })
+      this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+    } else {
+      this.#log(
+        "WARN: subscribeToRemotes called but remote heads gossiping is not enabled"
+      )
+    }
   }
 
   storageId = async (): Promise<StorageId | undefined> => {
@@ -453,11 +516,6 @@ export class Repo extends EventEmitter<RepoEvents> {
   }
 }
 
-interface PersistanceInfo {
-  storageId: StorageId
-  isEphemeral: boolean
-}
-
 export interface RepoConfig {
   /** Our unique identifier */
   peerId?: PeerId
@@ -477,6 +535,11 @@ export interface RepoConfig {
    * all peers). A server only syncs documents that a peer explicitly requests by ID.
    */
   sharePolicy?: SharePolicy
+
+  /**
+   * Whether to enable the experimental remote heads gossiping feature
+   */
+  enableRemoteHeadsGossiping?: boolean
 }
 
 /** A function that determines whether we should share a document with a peer

package/src/helpers/debounce.ts

@@ -19,7 +19,7 @@ export const throttle = <F extends (...args: Parameters<F>) => ReturnType<F>>(
   return function (...args: Parameters<F>) {
     clearTimeout(timeout)
     timeout = setTimeout(() => {
-      fn.apply(null, args)
+      fn(...args)
     }, rate)
   }
 }

package/src/helpers/pause.ts

@@ -1,3 +1,5 @@
+/* c8 ignore start */
+
 export const pause = (t = 0) =>
   new Promise<void>(resolve => setTimeout(() => resolve(), t))
 
@@ -12,3 +14,5 @@ export function rejectOnTimeout<T>(
     }),
   ])
 }
+
+/* c8 ignore end */

package/src/helpers/throttle.ts

@@ -36,7 +36,7 @@ export const throttle = <F extends (...args: Parameters<F>) => ReturnType<F>>(
     wait = lastCall + delay - Date.now()
     clearTimeout(timeout)
     timeout = setTimeout(() => {
-      fn.apply(null, args)
+      fn(...args)
       lastCall = Date.now()
     }, wait)
   }

package/src/helpers/withTimeout.ts

@@ -1,3 +1,4 @@
+/* c8 ignore start */
 /**
  * If `promise` is resolved before `t` ms elapse, the timeout is cleared and the result of the
  * promise is returned. If the timeout ends first, a `TimeoutError` is thrown.
@@ -26,3 +27,4 @@ export class TimeoutError extends Error {
     this.name = "TimeoutError"
   }
 }
+/* c8 ignore end */

package/src/index.ts

@@ -34,7 +34,7 @@ export {
 } from "./AutomergeUrl.js"
 export { Repo } from "./Repo.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
-export { isValidRepoMessage } from "./network/messages.js"
+export { isRepoMessage } from "./network/messages.js"
 export { StorageAdapter } from "./storage/StorageAdapter.js"
 
 /** @hidden **/
@@ -67,6 +67,7 @@ export type {
   OpenPayload,
   PeerCandidatePayload,
   PeerDisconnectedPayload,
+  PeerMetadata,
 } from "./network/NetworkAdapter.js"
 
 export type {