@automerge/automerge-repo 1.0.19 → 1.1.0-alpha.13

package/src/RemoteHeadsSubscriptions.ts

@@ -0,0 +1,375 @@
+import { next as A } from "@automerge/automerge"
+import { EventEmitter } from "eventemitter3"
+import { DocumentId, PeerId } from "./types.js"
+import {
+  RemoteHeadsChanged,
+  RemoteSubscriptionControlMessage,
+} from "./network/messages.js"
+import { StorageId } from "./index.js"
+import debug from "debug"
+
+// Notify a DocHandle that remote heads have changed
+export type RemoteHeadsSubscriptionEventPayload = {
+  documentId: DocumentId
+  storageId: StorageId
+  remoteHeads: A.Heads
+  timestamp: number
+}
+
+// Send a message to the given peer notifying them of new heads
+export type NotifyRemoteHeadsPayload = {
+  targetId: PeerId
+  documentId: DocumentId
+  storageId: StorageId
+  heads: A.Heads
+  timestamp: number
+}
+
+type RemoteHeadsSubscriptionEvents = {
+  "remote-heads-changed": (payload: RemoteHeadsSubscriptionEventPayload) => void
+  "change-remote-subs": (payload: {
+    peers: PeerId[]
+    add?: StorageId[]
+    remove?: StorageId[]
+  }) => void
+  "notify-remote-heads": (payload: NotifyRemoteHeadsPayload) => void
+}
+
+export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscriptionEvents> {
+  // Storage IDs we have received remote heads from
+  #knownHeads: Map<DocumentId, Map<StorageId, LastHeads>> = new Map()
+  // Storage IDs we have subscribed to via Repo.subscribeToRemoteHeads
+  #ourSubscriptions: Set<StorageId> = new Set()
+  // Storage IDs other peers have subscribed to by sending us a control message
+  #theirSubscriptions: Map<StorageId, Set<PeerId>> = new Map()
+  // Peers we will always share remote heads with even if they are not subscribed
+  #generousPeers: Set<PeerId> = new Set()
+  // Documents each peer has open, we need this information so we only send remote heads of documents that the peer knows
+  #subscribedDocsByPeer: Map<PeerId, Set<DocumentId>> = new Map()
+
+  #log = debug("automerge-repo:remote-heads-subscriptions")
+
+  subscribeToRemotes(remotes: StorageId[]) {
+    this.#log("subscribeToRemotes", remotes)
+    const remotesToAdd = []
+    for (const remote of remotes) {
+      if (!this.#ourSubscriptions.has(remote)) {
+        this.#ourSubscriptions.add(remote)
+        remotesToAdd.push(remote)
+      }
+    }
+
+    if (remotesToAdd.length > 0) {
+      this.emit("change-remote-subs", {
+        add: remotesToAdd,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  unsubscribeFromRemotes(remotes: StorageId[]) {
+    this.#log("subscribeToRemotes", remotes)
+    const remotesToRemove = []
+
+    for (const remote of remotes) {
+      if (this.#ourSubscriptions.has(remote)) {
+        this.#ourSubscriptions.delete(remote)
+
+        if (!this.#theirSubscriptions.has(remote)) {
+          remotesToRemove.push(remote)
+        }
+      }
+    }
+
+    if (remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        remove: remotesToRemove,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  handleControlMessage(control: RemoteSubscriptionControlMessage) {
+    const remotesToAdd: StorageId[] = []
+    const remotesToRemove: StorageId[] = []
+    const addedRemotesWeKnow: StorageId[] = []
+
+    this.#log("handleControlMessage", control)
+    if (control.add) {
+      for (const remote of control.add) {
+        let theirSubs = this.#theirSubscriptions.get(remote)
+
+        if (this.#ourSubscriptions.has(remote) || theirSubs) {
+          addedRemotesWeKnow.push(remote)
+        }
+
+        if (!theirSubs) {
+          theirSubs = new Set()
+          this.#theirSubscriptions.set(remote, theirSubs)
+
+          if (!this.#ourSubscriptions.has(remote)) {
+            remotesToAdd.push(remote)
+          }
+        }
+
+        theirSubs.add(control.senderId)
+      }
+    }
+
+    if (control.remove) {
+      for (const remote of control.remove) {
+        const theirSubs = this.#theirSubscriptions.get(remote)
+        if (theirSubs) {
+          theirSubs.delete(control.senderId)
+
+          // if no one is subscribed anymore remove remote
+          if (theirSubs.size == 0 && !this.#ourSubscriptions.has(remote)) {
+            remotesToRemove.push(remote)
+          }
+        }
+      }
+    }
+
+    if (remotesToAdd.length > 0 || remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        peers: Array.from(this.#generousPeers),
+        add: remotesToAdd,
+        remove: remotesToRemove,
+      })
+    }
+
+    // send all our stored heads of documents the peer knows for the remotes they've added
+    for (const remote of addedRemotesWeKnow) {
+      const subscribedDocs = this.#subscribedDocsByPeer.get(control.senderId)
+      if (subscribedDocs) {
+        for (const documentId of subscribedDocs) {
+          const knownHeads = this.#knownHeads.get(documentId)
+          if (!knownHeads) {
+            continue
+          }
+
+          const lastHeads = knownHeads.get(remote)
+          if (lastHeads) {
+            this.emit("notify-remote-heads", {
+              targetId: control.senderId,
+              documentId,
+              heads: lastHeads.heads,
+              timestamp: lastHeads.timestamp,
+              storageId: remote,
+            })
+          }
+        }
+      }
+    }
+  }
+
+  /** A peer we are not directly connected to has changed their heads */
+  handleRemoteHeads(msg: RemoteHeadsChanged) {
+    this.#log("handleRemoteHeads", msg)
+    const changedHeads = this.#changedHeads(msg)
+
+    // Emit a remote-heads-changed event to update local dochandles
+    for (const event of changedHeads) {
+      if (this.#ourSubscriptions.has(event.storageId)) {
+        this.emit("remote-heads-changed", event)
+      }
+    }
+
+    // Notify generous peers of these changes regardless of if they are subscribed to us
+    for (const event of changedHeads) {
+      for (const peer of this.#generousPeers) {
+        // don't emit event to sender if sender is a generous peer
+        if (peer === msg.senderId) {
+          continue
+        }
+
+        this.emit("notify-remote-heads", {
+          targetId: peer,
+          documentId: event.documentId,
+          heads: event.remoteHeads,
+          timestamp: event.timestamp,
+          storageId: event.storageId,
+        })
+      }
+    }
+
+    // Notify subscribers of these changes
+    for (const event of changedHeads) {
+      const theirSubs = this.#theirSubscriptions.get(event.storageId)
+      if (theirSubs) {
+        for (const peerId of theirSubs) {
+          if (this.#isPeerSubscribedToDoc(peerId, event.documentId)) {
+            this.emit("notify-remote-heads", {
+              targetId: peerId,
+              documentId: event.documentId,
+              heads: event.remoteHeads,
+              timestamp: event.timestamp,
+              storageId: event.storageId,
+            })
+          }
+        }
+      }
+    }
+  }
+
+  /** A peer we are directly connected to has updated their heads */
+  handleImmediateRemoteHeadsChanged(
+    documentId: DocumentId,
+    storageId: StorageId,
+    heads: A.Heads
+  ) {
+    this.#log("handleLocalHeadsChanged", documentId, storageId, heads)
+    const remote = this.#knownHeads.get(documentId)
+    const timestamp = Date.now()
+    if (!remote) {
+      this.#knownHeads.set(
+        documentId,
+        new Map([[storageId, { heads, timestamp }]])
+      )
+    } else {
+      const docRemote = remote.get(storageId)
+      if (!docRemote || docRemote.timestamp < Date.now()) {
+        remote.set(storageId, { heads, timestamp: Date.now() })
+      }
+    }
+    const theirSubs = this.#theirSubscriptions.get(storageId)
+    if (theirSubs) {
+      for (const peerId of theirSubs) {
+        if (this.#isPeerSubscribedToDoc(peerId, documentId)) {
+          this.emit("notify-remote-heads", {
+            targetId: peerId,
+            documentId: documentId,
+            heads: heads,
+            timestamp: timestamp,
+            storageId: storageId,
+          })
+        }
+      }
+    }
+  }
+
+  addGenerousPeer(peerId: PeerId) {
+    this.#log("addGenerousPeer", peerId)
+    this.#generousPeers.add(peerId)
+
+    if (this.#ourSubscriptions.size > 0) {
+      this.emit("change-remote-subs", {
+        add: Array.from(this.#ourSubscriptions),
+        peers: [peerId],
+      })
+    }
+
+    for (const [documentId, remote] of this.#knownHeads) {
+      for (const [storageId, { heads, timestamp }] of remote) {
+        this.emit("notify-remote-heads", {
+          targetId: peerId,
+          documentId: documentId,
+          heads: heads,
+          timestamp: timestamp,
+          storageId: storageId,
+        })
+      }
+    }
+  }
+
+  removePeer(peerId: PeerId) {
+    this.#log("removePeer", peerId)
+
+    const remotesToRemove = []
+
+    this.#generousPeers.delete(peerId)
+    this.#subscribedDocsByPeer.delete(peerId)
+
+    for (const [storageId, peerIds] of this.#theirSubscriptions) {
+      if (peerIds.has(peerId)) {
+        peerIds.delete(peerId)
+
+        if (peerIds.size == 0) {
+          remotesToRemove.push(storageId)
+          this.#theirSubscriptions.delete(storageId)
+        }
+      }
+    }
+
+    if (remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        remove: remotesToRemove,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  subscribePeerToDoc(peerId: PeerId, documentId: DocumentId) {
+    let subscribedDocs = this.#subscribedDocsByPeer.get(peerId)
+    if (!subscribedDocs) {
+      subscribedDocs = new Set()
+      this.#subscribedDocsByPeer.set(peerId, subscribedDocs)
+    }
+
+    subscribedDocs.add(documentId)
+
+    const remoteHeads = this.#knownHeads.get(documentId)
+    if (remoteHeads) {
+      for (const [storageId, lastHeads] of remoteHeads) {
+        const subscribedPeers = this.#theirSubscriptions.get(storageId)
+        if (subscribedPeers && subscribedPeers.has(peerId)) {
+          this.emit("notify-remote-heads", {
+            targetId: peerId,
+            documentId,
+            heads: lastHeads.heads,
+            timestamp: lastHeads.timestamp,
+            storageId,
+          })
+        }
+      }
+    }
+  }
+
+  #isPeerSubscribedToDoc(peerId: PeerId, documentId: DocumentId) {
+    const subscribedDocs = this.#subscribedDocsByPeer.get(peerId)
+    return subscribedDocs && subscribedDocs.has(documentId)
+  }
+
+  /** Returns the (document, storageId) pairs which have changed after processing msg */
+  #changedHeads(msg: RemoteHeadsChanged): {
+    documentId: DocumentId
+    storageId: StorageId
+    remoteHeads: A.Heads
+    timestamp: number
+  }[] {
+    const changedHeads = []
+    const { documentId, newHeads } = msg
+    for (const [storageId, { heads, timestamp }] of Object.entries(newHeads)) {
+      if (
+        !this.#ourSubscriptions.has(storageId as StorageId) &&
+        !this.#theirSubscriptions.has(storageId as StorageId)
+      ) {
+        continue
+      }
+      let remote = this.#knownHeads.get(documentId)
+      if (!remote) {
+        remote = new Map()
+        this.#knownHeads.set(documentId, remote)
+      }
+
+      const docRemote = remote.get(storageId as StorageId)
+      if (docRemote && docRemote.timestamp >= timestamp) {
+        continue
+      } else {
+        remote.set(storageId as StorageId, { timestamp, heads })
+        changedHeads.push({
+          documentId,
+          storageId: storageId as StorageId,
+          remoteHeads: heads,
+          timestamp,
+        })
+      }
+    }
+    return changedHeads
+  }
+}
+
+type LastHeads = {
+  timestamp: number
+  heads: A.Heads
+}

package/src/Repo.ts

@@ -7,14 +7,18 @@ import {
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
 import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
+import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
+import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
-import { NetworkAdapter } from "./network/NetworkAdapter.js"
+import { NetworkAdapter, type PeerMetadata } from "./network/NetworkAdapter.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
+import { RepoMessage } from "./network/messages.js"
 import { StorageAdapter } from "./storage/StorageAdapter.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
+import { StorageId } from "./storage/types.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
+import { SyncStatePayload } from "./synchronizer/Synchronizer.js"
 import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
-import { SyncStateMessage } from "./network/messages.js"
 
 /** A Repo is a collection of documents with networking, syncing, and storage capabilities. */
 /** The `Repo` is the main entry point of this library
@@ -44,8 +48,23 @@ export class Repo extends EventEmitter<RepoEvents> {
   /** @hidden */
   sharePolicy: SharePolicy = async () => true
 
-  constructor({ storage, network, peerId, sharePolicy }: RepoConfig) {
+  /** maps peer id to to persistence information (storageId, isEphemeral), access by collection synchronizer  */
+  /** @hidden */
+  peerMetadataByPeerId: Record<PeerId, PeerMetadata> = {}
+
+  #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+  #remoteHeadsGossipingEnabled = false
+
+  constructor({
+    storage,
+    network,
+    peerId,
+    sharePolicy,
+    isEphemeral = storage === undefined,
+    enableRemoteHeadsGossiping = false,
+  }: RepoConfig) {
     super()
+    this.#remoteHeadsGossipingEnabled = enableRemoteHeadsGossiping
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
 
@@ -62,10 +81,7 @@ export class Repo extends EventEmitter<RepoEvents> {
         }: DocHandleEncodedChangePayload<any>) => {
           void storageSubsystem.saveDoc(handle.documentId, doc)
         }
-        const debouncedSaveFn = handle.on(
-          "heads-changed",
-          throttle(saveFn, this.saveDebounceRate)
-        )
+        handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
 
         if (isNew) {
           // this is a new document, immediately save it
@@ -125,6 +141,12 @@ export class Repo extends EventEmitter<RepoEvents> {
       networkSubsystem.send(message)
     })
 
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#synchronizer.on("open-doc", ({ peerId, documentId }) => {
+        this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
+      })
+    }
+
     // STORAGE
     // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
     const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
@@ -132,33 +154,172 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
-    const networkSubsystem = new NetworkSubsystem(network, peerId)
+
+    const myPeerMetadata: Promise<PeerMetadata> = new Promise(
+      // eslint-disable-next-line no-async-promise-executor -- TODO: fix
+      async resolve =>
+        resolve({
+          storageId: await storageSubsystem?.id(),
+          isEphemeral,
+        } as PeerMetadata)
+    )
+
+    const networkSubsystem = new NetworkSubsystem(
+      network,
+      peerId,
+      myPeerMetadata
+    )
     this.networkSubsystem = networkSubsystem
 
     // When we get a new peer, register it with the synchronizer
-    networkSubsystem.on("peer", async ({ peerId }) => {
+    networkSubsystem.on("peer", async ({ peerId, peerMetadata }) => {
       this.#log("peer connected", { peerId })
+
+      if (peerMetadata) {
+        this.peerMetadataByPeerId[peerId] = { ...peerMetadata }
+      }
+
+      this.sharePolicy(peerId)
+        .then(shouldShare => {
+          if (shouldShare && this.#remoteHeadsGossipingEnabled) {
+            this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
+          }
+        })
+        .catch(err => {
+          console.log("error in share policy", { err })
+        })
+
       this.#synchronizer.addPeer(peerId)
     })
 
     // When a peer disconnects, remove it from the synchronizer
     networkSubsystem.on("peer-disconnected", ({ peerId }) => {
       this.#synchronizer.removePeer(peerId)
+      this.#remoteHeadsSubscriptions.removePeer(peerId)
     })
 
     // Handle incoming messages
     networkSubsystem.on("message", async msg => {
-      await this.#synchronizer.receiveMessage(msg)
+      this.#receiveMessage(msg)
     })
 
-    if (storageSubsystem) {
-      const debouncedSaveSyncState: (syncState: SyncStateMessage) => void =
-        throttle(({ documentId, peerId, syncState }: SyncStateMessage) => {
-          storageSubsystem.saveSyncState(documentId, peerId, syncState)
-        }, this.saveDebounceRate)
+    this.#synchronizer.on("sync-state", message => {
+      this.#saveSyncState(message)
+
+      const handle = this.#handleCache[message.documentId]
+
+      const { storageId } = this.peerMetadataByPeerId[message.peerId] || {}
+      if (!storageId) {
+        return
+      }
+
+      const heads = handle.getRemoteHeads(storageId)
+      const haveHeadsChanged =
+        message.syncState.theirHeads &&
+        (!heads || !headsAreSame(heads, message.syncState.theirHeads))
+
+      if (haveHeadsChanged) {
+        handle.setRemoteHeads(storageId, message.syncState.theirHeads)
+
+        if (storageId && this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
+            message.documentId,
+            storageId,
+            message.syncState.theirHeads
+          )
+        }
+      }
+    })
+
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
+        this.networkSubsystem.send({
+          type: "remote-heads-changed",
+          targetId: message.targetId,
+          documentId: message.documentId,
+          newHeads: {
+            [message.storageId]: {
+              heads: message.heads,
+              timestamp: message.timestamp,
+            },
+          },
+        })
+      })
+
+      this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
+        this.#log("change-remote-subs", message)
+        for (const peer of message.peers) {
+          this.networkSubsystem.send({
+            type: "remote-subscription-change",
+            targetId: peer,
+            add: message.add,
+            remove: message.remove,
+          })
+        }
+      })
+
+      this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
+        const handle = this.#handleCache[message.documentId]
+        handle.setRemoteHeads(message.storageId, message.remoteHeads)
+      })
+    }
+  }
+
+  #receiveMessage(message: RepoMessage) {
+    switch (message.type) {
+      case "remote-subscription-change":
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        }
+        break
+      case "remote-heads-changed":
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        }
+        break
+      case "sync":
+      case "request":
+      case "ephemeral":
+      case "doc-unavailable":
+        this.#synchronizer.receiveMessage(message).catch(err => {
+          console.log("error receiving message", { err })
+        })
+    }
+  }
+
+  #throttledSaveSyncStateHandlers: Record<
+    StorageId,
+    (payload: SyncStatePayload) => void
+  > = {}
 
-      this.#synchronizer.on("sync-state", debouncedSaveSyncState)
+  /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
+  #saveSyncState(payload: SyncStatePayload) {
+    if (!this.storageSubsystem) {
+      return
     }
+
+    const { storageId, isEphemeral } =
+      this.peerMetadataByPeerId[payload.peerId] || {}
+
+    if (!storageId || isEphemeral) {
+      return
+    }
+
+    let handler = this.#throttledSaveSyncStateHandlers[storageId]
+    if (!handler) {
+      handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
+        ({ documentId, syncState }: SyncStatePayload) => {
+          void this.storageSubsystem!.saveSyncState(
+            documentId,
+            storageId,
+            syncState
+          )
+        },
+        this.saveDebounceRate
+      )
+    }
+
+    handler(payload)
   }
 
   /** Returns an existing handle if we have it; creates one otherwise. */
@@ -189,6 +350,10 @@ export class Repo extends EventEmitter<RepoEvents> {
     return this.#synchronizer.peers
   }
 
+  getStorageIdOfPeer(peerId: PeerId): StorageId | undefined {
+    return this.peerMetadataByPeerId[peerId]?.storageId
+  }
+
   /**
    * Creates a new document and returns a handle to it. The initial value of the document is
    * an empty object `{}`. Its documentId is generated by the system. we emit a `document` event
@@ -298,12 +463,67 @@ export class Repo extends EventEmitter<RepoEvents> {
     delete this.#handleCache[documentId]
     this.emit("delete-document", { documentId })
   }
+
+  /**
+   * Exports a document to a binary format.
+   * @param id - The url or documentId of the handle to export
+   *
+   * @returns Promise<Uint8Array | undefined> - A Promise containing the binary document,
+   * or undefined if the document is unavailable.
+   */
+  async export(id: AnyDocumentId): Promise<Uint8Array | undefined> {
+    const documentId = interpretAsDocumentId(id)
+
+    const handle = this.#getHandle(documentId, false)
+    const doc = await handle.doc()
+    if (!doc) return undefined
+    return Automerge.save(doc)
+  }
+
+  /**
+   * Imports document binary into the repo.
+   * @param binary - The binary to import
+   */
+  import<T>(binary: Uint8Array) {
+    const doc = Automerge.load<T>(binary)
+
+    const handle = this.create<T>()
+
+    handle.update(() => {
+      return Automerge.clone(doc)
+    })
+
+    return handle
+  }
+
+  subscribeToRemotes = (remotes: StorageId[]) => {
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#log("subscribeToRemotes", { remotes })
+      this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+    } else {
+      this.#log(
+        "WARN: subscribeToRemotes called but remote heads gossiping is not enabled"
+      )
+    }
+  }
+
+  storageId = async (): Promise<StorageId | undefined> => {
+    if (!this.storageSubsystem) {
+      return undefined
+    } else {
+      return this.storageSubsystem.id()
+    }
+  }
 }
 
 export interface RepoConfig {
   /** Our unique identifier */
   peerId?: PeerId
 
+  /** Indicates whether other peers should persist the sync state of this peer.
+   * Sync state is only persisted for non-ephemeral peers */
+  isEphemeral?: boolean
+
   /** A storage adapter can be provided, or not */
   storage?: StorageAdapter
 
@@ -315,6 +535,11 @@ export interface RepoConfig {
    * all peers). A server only syncs documents that a peer explicitly requests by ID.
    */
   sharePolicy?: SharePolicy
+
+  /**
+   * Whether to enable the experimental remote heads gossiping feature
+   */
+  enableRemoteHeadsGossiping?: boolean
 }
 
 /** A function that determines whether we should share a document with a peer