@automerge/automerge-repo 1.0.19 → 1.1.0-alpha.2

package/src/RemoteHeadsSubscriptions.ts

@@ -0,0 +1,306 @@
+import { next as A } from "@automerge/automerge"
+import { EventEmitter } from "eventemitter3"
+import { DocumentId, PeerId } from "./types.js"
+import {
+  RemoteHeadsChanged,
+  RemoteSubscriptionControlMessage,
+} from "./network/messages.js"
+import { StorageId } from "./index.js"
+import debug from "debug"
+
+// Notify a DocHandle that remote heads have changed
+export type RemoteHeadsSubscriptionEventPayload = {
+  documentId: DocumentId
+  storageId: StorageId
+  remoteHeads: A.Heads
+  timestamp: number
+}
+
+// Send a message to the given peer notifying them of new heads
+export type NotifyRemoteHeadsPayload = {
+  targetId: PeerId
+  documentId: DocumentId
+  storageId: StorageId
+  heads: A.Heads
+  timestamp: number
+}
+
+type RemoteHeadsSubscriptionEvents = {
+  "remote-heads-changed": (payload: RemoteHeadsSubscriptionEventPayload) => void
+  "change-remote-subs": (payload: {
+    peers: PeerId[]
+    add?: StorageId[]
+    remove?: StorageId[]
+  }) => void
+  "notify-remote-heads": (payload: NotifyRemoteHeadsPayload) => void
+}
+
+export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscriptionEvents> {
+  // Storage IDs we have received remote heads from
+  #knownHeads: Map<DocumentId, Map<StorageId, LastHeads>> = new Map()
+  // Storage IDs we have subscribed to via Repo.subscribeToRemoteHeads
+  #ourSubscriptions: Set<StorageId> = new Set()
+  // Storage IDs other peers have subscribed to by sending us a control message
+  #theirSubscriptions: Map<StorageId, Set<PeerId>> = new Map()
+  // Peers we will always share remote heads with even if they are not subscribed
+  #generousPeers: Set<PeerId> = new Set()
+  #log = debug("automerge-repo:remote-heads-subscriptions")
+
+  subscribeToRemotes(remotes: StorageId[]) {
+    this.#log("subscribeToRemotes", remotes)
+    const remotesToAdd = []
+    for (const remote of remotes) {
+      if (!this.#ourSubscriptions.has(remote)) {
+        this.#ourSubscriptions.add(remote)
+        remotesToAdd.push(remote)
+      }
+    }
+
+    if (remotesToAdd.length > 0) {
+      this.emit("change-remote-subs", {
+        add: remotesToAdd,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  unsubscribeFromRemotes(remotes: StorageId[]) {
+    this.#log("subscribeToRemotes", remotes)
+    const remotesToRemove = []
+
+    for (const remote of remotes) {
+      if (this.#ourSubscriptions.has(remote)) {
+        this.#ourSubscriptions.delete(remote)
+
+        if (!this.#theirSubscriptions.has(remote)) {
+          remotesToRemove.push(remote)
+        }
+      }
+    }
+
+    if (remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        remove: remotesToRemove,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  handleControlMessage(control: RemoteSubscriptionControlMessage) {
+    const remotesToAdd: StorageId[] = []
+    const remotesToRemove: StorageId[] = []
+
+    this.#log("handleControlMessage", control)
+    if (control.add) {
+      for (const remote of control.add) {
+        let theirSubs = this.#theirSubscriptions.get(remote)
+        if (!theirSubs) {
+          theirSubs = new Set()
+          this.#theirSubscriptions.set(remote, theirSubs)
+
+          if (!this.#ourSubscriptions.has(remote)) {
+            remotesToAdd.push(remote)
+          }
+        }
+
+        theirSubs.add(control.senderId)
+      }
+    }
+
+    if (control.remove) {
+      for (const remote of control.remove) {
+        const theirSubs = this.#theirSubscriptions.get(remote)
+        if (theirSubs) {
+          theirSubs.delete(control.senderId)
+
+          // if no one is subscribed anymore remove remote
+          if (theirSubs.size == 0 && !this.#ourSubscriptions.has(remote)) {
+            remotesToRemove.push(remote)
+          }
+        }
+      }
+    }
+
+    if (remotesToAdd.length > 0 || remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        peers: Array.from(this.#generousPeers),
+        add: remotesToAdd,
+        remove: remotesToRemove,
+      })
+    }
+  }
+
+  /** A peer we are not directly connected to has changed their heads */
+  handleRemoteHeads(msg: RemoteHeadsChanged) {
+    this.#log("handleRemoteHeads", msg)
+    const changedHeads = this.#changedHeads(msg)
+
+    // Emit a remote-heads-changed event to update local dochandles
+    for (const event of changedHeads) {
+      if (this.#ourSubscriptions.has(event.storageId)) {
+        this.emit("remote-heads-changed", event)
+      }
+    }
+
+    // Notify generous peers of these changes regardless of if they are subscribed to us
+    for (const event of changedHeads) {
+      for (const peer of this.#generousPeers) {
+        // don't emit event to sender if sender is a generous peer
+        if (peer === msg.senderId) {
+          continue
+        }
+
+        this.emit("notify-remote-heads", {
+          targetId: peer,
+          documentId: event.documentId,
+          heads: event.remoteHeads,
+          timestamp: event.timestamp,
+          storageId: event.storageId,
+        })
+      }
+    }
+
+    // Notify subscribers of these changes
+    for (const event of changedHeads) {
+      const theirSubs = this.#theirSubscriptions.get(event.storageId)
+      if (theirSubs) {
+        for (const peerId of theirSubs) {
+          this.emit("notify-remote-heads", {
+            targetId: peerId,
+            documentId: event.documentId,
+            heads: event.remoteHeads,
+            timestamp: event.timestamp,
+            storageId: event.storageId,
+          })
+        }
+      }
+    }
+  }
+
+  /** A peer we are directly connected to has updated their heads */
+  handleImmediateRemoteHeadsChanged(
+    documentId: DocumentId,
+    storageId: StorageId,
+    heads: A.Heads
+  ) {
+    this.#log("handleLocalHeadsChanged", documentId, storageId, heads)
+    const remote = this.#knownHeads.get(documentId)
+    const timestamp = Date.now()
+    if (!remote) {
+      this.#knownHeads.set(
+        documentId,
+        new Map([[storageId, { heads, timestamp }]])
+      )
+    } else {
+      const docRemote = remote.get(storageId)
+      if (!docRemote || docRemote.timestamp < Date.now()) {
+        remote.set(storageId, { heads, timestamp: Date.now() })
+      }
+    }
+    const theirSubs = this.#theirSubscriptions.get(storageId)
+    if (theirSubs) {
+      for (const peerId of theirSubs) {
+        this.emit("notify-remote-heads", {
+          targetId: peerId,
+          documentId: documentId,
+          heads: heads,
+          timestamp: timestamp,
+          storageId: storageId,
+        })
+      }
+    }
+  }
+
+  addGenerousPeer(peerId: PeerId) {
+    this.#log("addGenerousPeer", peerId)
+    this.#generousPeers.add(peerId)
+
+    if (this.#ourSubscriptions.size > 0) {
+      this.emit("change-remote-subs", {
+        add: Array.from(this.#ourSubscriptions),
+        peers: [peerId],
+      })
+    }
+
+    for (const [documentId, remote] of this.#knownHeads) {
+      for (const [storageId, { heads, timestamp }] of remote) {
+        this.emit("notify-remote-heads", {
+          targetId: peerId,
+          documentId: documentId,
+          heads: heads,
+          timestamp: timestamp,
+          storageId: storageId,
+        })
+      }
+    }
+  }
+
+  removePeer(peerId: PeerId) {
+    this.#log("removePeer", peerId)
+
+    const remotesToRemove = []
+
+    this.#generousPeers.delete(peerId)
+
+    for (const [storageId, peerIds] of this.#theirSubscriptions) {
+      if (peerIds.has(peerId)) {
+        peerIds.delete(peerId)
+
+        if (peerIds.size == 0) {
+          remotesToRemove.push(storageId)
+          this.#theirSubscriptions.delete(storageId)
+        }
+      }
+    }
+
+    if (remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        remove: remotesToRemove,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  /** Returns the (document, storageId) pairs which have changed after processing msg */
+  #changedHeads(msg: RemoteHeadsChanged): {
+    documentId: DocumentId
+    storageId: StorageId
+    remoteHeads: A.Heads
+    timestamp: number
+  }[] {
+    const changedHeads = []
+    const { documentId, newHeads } = msg
+    for (const [storageId, { heads, timestamp }] of Object.entries(newHeads)) {
+      if (
+        !this.#ourSubscriptions.has(storageId as StorageId) &&
+        !this.#theirSubscriptions.has(storageId as StorageId)
+      ) {
+        continue
+      }
+      let remote = this.#knownHeads.get(documentId)
+      if (!remote) {
+        remote = new Map([[storageId as StorageId, { heads, timestamp }]])
+        this.#knownHeads.set(documentId, remote)
+      }
+
+      const docRemote = remote.get(storageId as StorageId)
+      if (docRemote && docRemote.timestamp > timestamp) {
+        continue
+      } else {
+        remote.set(storageId as StorageId, { timestamp, heads })
+        changedHeads.push({
+          documentId,
+          storageId: storageId as StorageId,
+          remoteHeads: heads,
+          timestamp,
+        })
+      }
+    }
+    return changedHeads
+  }
+}
+
+type LastHeads = {
+  timestamp: number
+  heads: A.Heads
+}

package/src/Repo.ts

@@ -8,13 +8,16 @@ import {
 } from "./AutomergeUrl.js"
 import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
 import { throttle } from "./helpers/throttle.js"
-import { NetworkAdapter } from "./network/NetworkAdapter.js"
+import { NetworkAdapter, type PeerMetadata } from "./network/NetworkAdapter.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
 import { StorageAdapter } from "./storage/StorageAdapter.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
 import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
-import { SyncStateMessage } from "./network/messages.js"
+import { RepoMessage, SyncStateMessage } from "./network/messages.js"
+import { StorageId } from "./storage/types.js"
+import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
+import { headsAreSame } from "./helpers/headsAreSame.js"
 
 /** A Repo is a collection of documents with networking, syncing, and storage capabilities. */
 /** The `Repo` is the main entry point of this library
@@ -44,7 +47,19 @@ export class Repo extends EventEmitter<RepoEvents> {
   /** @hidden */
   sharePolicy: SharePolicy = async () => true
 
-  constructor({ storage, network, peerId, sharePolicy }: RepoConfig) {
+  /** maps peer id to to persistence information (storageId, isEphemeral), access by collection synchronizer  */
+  /** @hidden */
+  peerMetadataByPeerId: Record<PeerId, PeerMetadata> = {}
+
+  #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+
+  constructor({
+    storage,
+    network,
+    peerId,
+    sharePolicy,
+    isEphemeral = storage === undefined,
+  }: RepoConfig) {
     super()
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
@@ -132,33 +147,161 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
-    const networkSubsystem = new NetworkSubsystem(network, peerId)
+
+    const myPeerMetadata: Promise<PeerMetadata> = new Promise(
+      async (resolve, reject) =>
+        resolve({
+          storageId: await storageSubsystem?.id(),
+          isEphemeral,
+        } as PeerMetadata)
+    )
+
+    const networkSubsystem = new NetworkSubsystem(
+      network,
+      peerId,
+      myPeerMetadata
+    )
     this.networkSubsystem = networkSubsystem
 
     // When we get a new peer, register it with the synchronizer
-    networkSubsystem.on("peer", async ({ peerId }) => {
+    networkSubsystem.on("peer", async ({ peerId, peerMetadata }) => {
       this.#log("peer connected", { peerId })
+
+      if (peerMetadata) {
+        this.peerMetadataByPeerId[peerId] = { ...peerMetadata }
+      }
+
+      this.sharePolicy(peerId)
+        .then(shouldShare => {
+          if (shouldShare) {
+            this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
+          }
+        })
+        .catch(err => {
+          console.log("error in share policy", { err })
+        })
+
       this.#synchronizer.addPeer(peerId)
     })
 
     // When a peer disconnects, remove it from the synchronizer
     networkSubsystem.on("peer-disconnected", ({ peerId }) => {
       this.#synchronizer.removePeer(peerId)
+      this.#remoteHeadsSubscriptions.removePeer(peerId)
     })
 
     // Handle incoming messages
     networkSubsystem.on("message", async msg => {
-      await this.#synchronizer.receiveMessage(msg)
+      this.#receiveMessage(msg)
+    })
+
+    this.#synchronizer.on("sync-state", message => {
+      this.#saveSyncState(message)
+
+      const handle = this.#handleCache[message.documentId]
+
+      const { storageId } = this.peerMetadataByPeerId[message.peerId] || {}
+      if (!storageId) {
+        return
+      }
+
+      const heads = handle.getRemoteHeads(storageId)
+      const haveHeadsChanged =
+        message.syncState.theirHeads &&
+        (!heads || !headsAreSame(heads, message.syncState.theirHeads))
+
+      if (haveHeadsChanged) {
+        handle.setRemoteHeads(storageId, message.syncState.theirHeads)
+
+        if (storageId) {
+          this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
+            message.documentId,
+            storageId,
+            message.syncState.theirHeads
+          )
+        }
+      }
+    })
+
+    this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
+      this.networkSubsystem.send({
+        type: "remote-heads-changed",
+        targetId: message.targetId,
+        documentId: message.documentId,
+        newHeads: {
+          [message.storageId]: {
+            heads: message.heads,
+            timestamp: message.timestamp,
+          },
+        },
+      })
     })
 
-    if (storageSubsystem) {
-      const debouncedSaveSyncState: (syncState: SyncStateMessage) => void =
-        throttle(({ documentId, peerId, syncState }: SyncStateMessage) => {
-          storageSubsystem.saveSyncState(documentId, peerId, syncState)
-        }, this.saveDebounceRate)
+    this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
+      this.#log("change-remote-subs", message)
+      for (const peer of message.peers) {
+        this.networkSubsystem.send({
+          type: "remote-subscription-change",
+          targetId: peer,
+          add: message.add,
+          remove: message.remove,
+        })
+      }
+    })
+
+    this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
+      const handle = this.#handleCache[message.documentId]
+      handle.setRemoteHeads(message.storageId, message.remoteHeads)
+    })
+  }
+
+  #receiveMessage(message: RepoMessage) {
+    switch (message.type) {
+      case "remote-subscription-change":
+        this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        break
+      case "remote-heads-changed":
+        this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        break
+      case "sync":
+      case "request":
+      case "ephemeral":
+      case "doc-unavailable":
+        this.#synchronizer.receiveMessage(message).catch(err => {
+          console.log("error receiving message", { err })
+        })
+    }
+  }
+
+  #throttledSaveSyncStateHandlers: Record<
+    StorageId,
+    (message: SyncStateMessage) => void
+  > = {}
+
+  /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
+  #saveSyncState(message: SyncStateMessage) {
+    if (!this.storageSubsystem) {
+      return
+    }
+
+    const { storageId, isEphemeral } =
+      this.peerMetadataByPeerId[message.peerId] || {}
+
+    if (!storageId || isEphemeral) {
+      return
+    }
 
-      this.#synchronizer.on("sync-state", debouncedSaveSyncState)
+    let handler = this.#throttledSaveSyncStateHandlers[storageId]
+    if (!handler) {
+      handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
+        ({ documentId, syncState }: SyncStateMessage) => {
+          this.storageSubsystem!.saveSyncState(documentId, storageId, syncState)
+        },
+        this.saveDebounceRate
+      )
     }
+
+    handler(message)
   }
 
   /** Returns an existing handle if we have it; creates one otherwise. */
@@ -298,12 +441,29 @@ export class Repo extends EventEmitter<RepoEvents> {
     delete this.#handleCache[documentId]
     this.emit("delete-document", { documentId })
   }
+
+  subscribeToRemotes = (remotes: StorageId[]) => {
+    this.#log("subscribeToRemotes", { remotes })
+    this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+  }
+
+  storageId = async (): Promise<StorageId | undefined> => {
+    if (!this.storageSubsystem) {
+      return undefined
+    } else {
+      return this.storageSubsystem.id()
+    }
+  }
 }
 
 export interface RepoConfig {
   /** Our unique identifier */
   peerId?: PeerId
 
+  /** Indicates whether other peers should persist the sync state of this peer.
+   * Sync state is only persisted for non-ephemeral peers */
+  isEphemeral?: boolean
+
   /** A storage adapter can be provided, or not */
   storage?: StorageAdapter
 

package/src/index.ts

@@ -67,6 +67,7 @@ export type {
   OpenPayload,
   PeerCandidatePayload,
   PeerDisconnectedPayload,
+  PeerMetadata,
 } from "./network/NetworkAdapter.js"
 
 export type {
@@ -83,6 +84,7 @@ export type {
   ChunkInfo,
   ChunkType,
   StorageKey,
+  StorageId,
 } from "./storage/types.js"
 
 export * from "./types.js"

package/src/network/NetworkAdapter.ts

@@ -1,6 +1,18 @@
 import { EventEmitter } from "eventemitter3"
 import { PeerId } from "../types.js"
 import { Message } from "./messages.js"
+import { StorageId } from "../storage/types.js"
+
+/** 
+ * Describes a peer intent to the system
+ * storageId: the key for syncState to decide what the other peer already has
+ * isEphemeral: to decide if we bother recording this peer's sync state
+ * 
+ */
+export interface PeerMetadata {
+  storageId?: StorageId
+  isEphemeral?: boolean
+}
 
 /** An interface representing some way to connect to other peers
  *
@@ -11,12 +23,17 @@ import { Message } from "./messages.js"
  */
 export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
   peerId?: PeerId // hmmm, maybe not
+  peerMetadata?: PeerMetadata
 
   /** Called by the {@link Repo} to start the connection process
    *
    * @argument peerId - the peerId of this repo
+   * @argument peerMetadata - how this adapter should present itself to other peers
    */
-  abstract connect(peerId: PeerId): void
+  abstract connect(
+    peerId: PeerId,
+    peerMetadata?: PeerMetadata
+  ): void
 
   /** Called by the {@link Repo} to send a message to a peer
    *
@@ -53,6 +70,7 @@ export interface OpenPayload {
 
 export interface PeerCandidatePayload {
   peerId: PeerId
+  peerMetadata: PeerMetadata
 }
 
 export interface PeerDisconnectedPayload {

package/src/network/NetworkSubsystem.ts

@@ -1,7 +1,11 @@
 import debug from "debug"
 import { EventEmitter } from "eventemitter3"
 import { PeerId, SessionId } from "../types.js"
-import { NetworkAdapter, PeerDisconnectedPayload } from "./NetworkAdapter.js"
+import type {
+  NetworkAdapter,
+  PeerDisconnectedPayload,
+  PeerMetadata,
+} from "./NetworkAdapter.js"
 import {
   EphemeralMessage,
   MessageContents,
@@ -9,6 +13,7 @@ import {
   isEphemeralMessage,
   isValidRepoMessage,
 } from "./messages.js"
+import { StorageId } from "../storage/types.js"
 
 type EphemeralMessageSource = `${PeerId}:${SessionId}`
 
@@ -25,7 +30,11 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #readyAdapterCount = 0
   #adapters: NetworkAdapter[] = []
 
-  constructor(adapters: NetworkAdapter[], public peerId = randomPeerId()) {
+  constructor(
+    adapters: NetworkAdapter[],
+    public peerId = randomPeerId(),
+    private peerMetadata: Promise<PeerMetadata>
+  ) {
     super()
     this.#log = debug(`automerge-repo:network:${this.peerId}`)
     adapters.forEach(a => this.addNetworkAdapter(a))
@@ -46,9 +55,8 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       }
     })
 
-    networkAdapter.on("peer-candidate", ({ peerId }) => {
+    networkAdapter.on("peer-candidate", ({ peerId, peerMetadata }) => {
       this.#log(`peer candidate: ${peerId} `)
-
       // TODO: This is where authentication would happen
 
       if (!this.#adaptersByPeer[peerId]) {
@@ -56,7 +64,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
         this.#adaptersByPeer[peerId] = networkAdapter
       }
 
-      this.emit("peer", { peerId })
+      this.emit("peer", { peerId, peerMetadata })
     })
 
     networkAdapter.on("peer-disconnected", ({ peerId }) => {
@@ -98,7 +106,9 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       })
     })
 
-    networkAdapter.connect(this.peerId)
+    this.peerMetadata.then(peerMetadata => {
+      networkAdapter.connect(this.peerId, peerMetadata)
+    })
   }
 
   send(message: MessageContents) {
@@ -171,4 +181,5 @@ export interface NetworkSubsystemEvents {
 
 export interface PeerPayload {
   peerId: PeerId
+  peerMetadata: PeerMetadata
 }