@automerge/automerge-repo 1.0.18 → 1.1.0-alpha.1

package/src/RemoteHeadsSubscriptions.ts

@@ -0,0 +1,306 @@
+import { next as A } from "@automerge/automerge"
+import { EventEmitter } from "eventemitter3"
+import { DocumentId, PeerId } from "./types.js"
+import {
+  RemoteHeadsChanged,
+  RemoteSubscriptionControlMessage,
+} from "./network/messages.js"
+import { StorageId } from "./index.js"
+import debug from "debug"
+
+// Notify a DocHandle that remote heads have changed
+export type RemoteHeadsSubscriptionEventPayload = {
+  documentId: DocumentId
+  storageId: StorageId
+  remoteHeads: A.Heads
+  timestamp: number
+}
+
+// Send a message to the given peer notifying them of new heads
+export type NotifyRemoteHeadsPayload = {
+  targetId: PeerId
+  documentId: DocumentId
+  storageId: StorageId
+  heads: A.Heads
+  timestamp: number
+}
+
+type RemoteHeadsSubscriptionEvents = {
+  "remote-heads-changed": (payload: RemoteHeadsSubscriptionEventPayload) => void
+  "change-remote-subs": (payload: {
+    peers: PeerId[]
+    add?: StorageId[]
+    remove?: StorageId[]
+  }) => void
+  "notify-remote-heads": (payload: NotifyRemoteHeadsPayload) => void
+}
+
+export class RemoteHeadsSubscriptions extends EventEmitter<RemoteHeadsSubscriptionEvents> {
+  // Storage IDs we have received remote heads from
+  #knownHeads: Map<DocumentId, Map<StorageId, LastHeads>> = new Map()
+  // Storage IDs we have subscribed to via Repo.subscribeToRemoteHeads
+  #ourSubscriptions: Set<StorageId> = new Set()
+  // Storage IDs other peers have subscribed to by sending us a control message
+  #theirSubscriptions: Map<StorageId, Set<PeerId>> = new Map()
+  // Peers we will always share remote heads with even if they are not subscribed
+  #generousPeers: Set<PeerId> = new Set()
+  #log = debug("automerge-repo:remote-heads-subscriptions")
+
+  subscribeToRemotes(remotes: StorageId[]) {
+    this.#log("subscribeToRemotes", remotes)
+    const remotesToAdd = []
+    for (const remote of remotes) {
+      if (!this.#ourSubscriptions.has(remote)) {
+        this.#ourSubscriptions.add(remote)
+        remotesToAdd.push(remote)
+      }
+    }
+
+    if (remotesToAdd.length > 0) {
+      this.emit("change-remote-subs", {
+        add: remotesToAdd,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  unsubscribeFromRemotes(remotes: StorageId[]) {
+    this.#log("subscribeToRemotes", remotes)
+    const remotesToRemove = []
+
+    for (const remote of remotes) {
+      if (this.#ourSubscriptions.has(remote)) {
+        this.#ourSubscriptions.delete(remote)
+
+        if (!this.#theirSubscriptions.has(remote)) {
+          remotesToRemove.push(remote)
+        }
+      }
+    }
+
+    if (remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        remove: remotesToRemove,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  handleControlMessage(control: RemoteSubscriptionControlMessage) {
+    const remotesToAdd: StorageId[] = []
+    const remotesToRemove: StorageId[] = []
+
+    this.#log("handleControlMessage", control)
+    if (control.add) {
+      for (const remote of control.add) {
+        let theirSubs = this.#theirSubscriptions.get(remote)
+        if (!theirSubs) {
+          theirSubs = new Set()
+          this.#theirSubscriptions.set(remote, theirSubs)
+
+          if (!this.#ourSubscriptions.has(remote)) {
+            remotesToAdd.push(remote)
+          }
+        }
+
+        theirSubs.add(control.senderId)
+      }
+    }
+
+    if (control.remove) {
+      for (const remote of control.remove) {
+        const theirSubs = this.#theirSubscriptions.get(remote)
+        if (theirSubs) {
+          theirSubs.delete(control.senderId)
+
+          // if no one is subscribed anymore remove remote
+          if (theirSubs.size == 0 && !this.#ourSubscriptions.has(remote)) {
+            remotesToRemove.push(remote)
+          }
+        }
+      }
+    }
+
+    if (remotesToAdd.length > 0 || remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        peers: Array.from(this.#generousPeers),
+        add: remotesToAdd,
+        remove: remotesToRemove,
+      })
+    }
+  }
+
+  /** A peer we are not directly connected to has changed their heads */
+  handleRemoteHeads(msg: RemoteHeadsChanged) {
+    this.#log("handleRemoteHeads", msg)
+    const changedHeads = this.#changedHeads(msg)
+
+    // Emit a remote-heads-changed event to update local dochandles
+    for (const event of changedHeads) {
+      if (this.#ourSubscriptions.has(event.storageId)) {
+        this.emit("remote-heads-changed", event)
+      }
+    }
+
+    // Notify generous peers of these changes regardless of if they are subscribed to us
+    for (const event of changedHeads) {
+      for (const peer of this.#generousPeers) {
+        // don't emit event to sender if sender is a generous peer
+        if (peer === msg.senderId) {
+          continue
+        }
+
+        this.emit("notify-remote-heads", {
+          targetId: peer,
+          documentId: event.documentId,
+          heads: event.remoteHeads,
+          timestamp: event.timestamp,
+          storageId: event.storageId,
+        })
+      }
+    }
+
+    // Notify subscribers of these changes
+    for (const event of changedHeads) {
+      const theirSubs = this.#theirSubscriptions.get(event.storageId)
+      if (theirSubs) {
+        for (const peerId of theirSubs) {
+          this.emit("notify-remote-heads", {
+            targetId: peerId,
+            documentId: event.documentId,
+            heads: event.remoteHeads,
+            timestamp: event.timestamp,
+            storageId: event.storageId,
+          })
+        }
+      }
+    }
+  }
+
+  /** A peer we are directly connected to has updated their heads */
+  handleImmediateRemoteHeadsChanged(
+    documentId: DocumentId,
+    storageId: StorageId,
+    heads: A.Heads
+  ) {
+    this.#log("handleLocalHeadsChanged", documentId, storageId, heads)
+    const remote = this.#knownHeads.get(documentId)
+    const timestamp = Date.now()
+    if (!remote) {
+      this.#knownHeads.set(
+        documentId,
+        new Map([[storageId, { heads, timestamp }]])
+      )
+    } else {
+      const docRemote = remote.get(storageId)
+      if (!docRemote || docRemote.timestamp < Date.now()) {
+        remote.set(storageId, { heads, timestamp: Date.now() })
+      }
+    }
+    const theirSubs = this.#theirSubscriptions.get(storageId)
+    if (theirSubs) {
+      for (const peerId of theirSubs) {
+        this.emit("notify-remote-heads", {
+          targetId: peerId,
+          documentId: documentId,
+          heads: heads,
+          timestamp: timestamp,
+          storageId: storageId,
+        })
+      }
+    }
+  }
+
+  addGenerousPeer(peerId: PeerId) {
+    this.#log("addGenerousPeer", peerId)
+    this.#generousPeers.add(peerId)
+
+    if (this.#ourSubscriptions.size > 0) {
+      this.emit("change-remote-subs", {
+        add: Array.from(this.#ourSubscriptions),
+        peers: [peerId],
+      })
+    }
+
+    for (const [documentId, remote] of this.#knownHeads) {
+      for (const [storageId, { heads, timestamp }] of remote) {
+        this.emit("notify-remote-heads", {
+          targetId: peerId,
+          documentId: documentId,
+          heads: heads,
+          timestamp: timestamp,
+          storageId: storageId,
+        })
+      }
+    }
+  }
+
+  removePeer(peerId: PeerId) {
+    this.#log("removePeer", peerId)
+
+    const remotesToRemove = []
+
+    this.#generousPeers.delete(peerId)
+
+    for (const [storageId, peerIds] of this.#theirSubscriptions) {
+      if (peerIds.has(peerId)) {
+        peerIds.delete(peerId)
+
+        if (peerIds.size == 0) {
+          remotesToRemove.push(storageId)
+          this.#theirSubscriptions.delete(storageId)
+        }
+      }
+    }
+
+    if (remotesToRemove.length > 0) {
+      this.emit("change-remote-subs", {
+        remove: remotesToRemove,
+        peers: Array.from(this.#generousPeers),
+      })
+    }
+  }
+
+  /** Returns the (document, storageId) pairs which have changed after processing msg */
+  #changedHeads(msg: RemoteHeadsChanged): {
+    documentId: DocumentId
+    storageId: StorageId
+    remoteHeads: A.Heads
+    timestamp: number
+  }[] {
+    const changedHeads = []
+    const { documentId, newHeads } = msg
+    for (const [storageId, { heads, timestamp }] of Object.entries(newHeads)) {
+      if (
+        !this.#ourSubscriptions.has(storageId as StorageId) &&
+        !this.#theirSubscriptions.has(storageId as StorageId)
+      ) {
+        continue
+      }
+      let remote = this.#knownHeads.get(documentId)
+      if (!remote) {
+        remote = new Map([[storageId as StorageId, { heads, timestamp }]])
+        this.#knownHeads.set(documentId, remote)
+      }
+
+      const docRemote = remote.get(storageId as StorageId)
+      if (docRemote && docRemote.timestamp > timestamp) {
+        continue
+      } else {
+        remote.set(storageId as StorageId, { timestamp, heads })
+        changedHeads.push({
+          documentId,
+          storageId: storageId as StorageId,
+          remoteHeads: heads,
+          timestamp,
+        })
+      }
+    }
+    return changedHeads
+  }
+}
+
+type LastHeads = {
+  timestamp: number
+  heads: A.Heads
+}

package/src/Repo.ts

@@ -14,7 +14,10 @@ import { StorageAdapter } from "./storage/StorageAdapter.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
 import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
-import { SyncStateMessage } from "./network/messages.js"
+import { RepoMessage, SyncStateMessage } from "./network/messages.js"
+import { StorageId } from "./storage/types.js"
+import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
+import { headsAreSame } from "./helpers/headsAreSame.js"
 
 /** A Repo is a collection of documents with networking, syncing, and storage capabilities. */
 /** The `Repo` is the main entry point of this library
@@ -44,7 +47,19 @@ export class Repo extends EventEmitter<RepoEvents> {
   /** @hidden */
   sharePolicy: SharePolicy = async () => true
 
-  constructor({ storage, network, peerId, sharePolicy }: RepoConfig) {
+  /** maps peer id to to persistance information (storageId, isEphemeral), access by collection synchronizer  */
+  /** @hidden */
+  persistanceInfoByPeerId: Record<PeerId, PersistanceInfo> = {}
+
+  #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+
+  constructor({
+    storage,
+    network,
+    peerId,
+    sharePolicy,
+    isEphemeral = storage === undefined,
+  }: RepoConfig) {
     super()
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
@@ -132,33 +147,158 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
-    const networkSubsystem = new NetworkSubsystem(network, peerId)
+    const networkSubsystem = new NetworkSubsystem(
+      network,
+      peerId,
+      storageSubsystem?.id() ?? Promise.resolve(undefined),
+      isEphemeral
+    )
     this.networkSubsystem = networkSubsystem
 
     // When we get a new peer, register it with the synchronizer
-    networkSubsystem.on("peer", async ({ peerId }) => {
+    networkSubsystem.on("peer", async ({ peerId, storageId, isEphemeral }) => {
       this.#log("peer connected", { peerId })
+
+      if (storageId) {
+        this.persistanceInfoByPeerId[peerId] = {
+          storageId,
+          isEphemeral,
+        }
+      }
+
+      this.sharePolicy(peerId)
+        .then(shouldShare => {
+          if (shouldShare) {
+            this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
+          }
+        })
+        .catch(err => {
+          console.log("error in share policy", { err })
+        })
+
       this.#synchronizer.addPeer(peerId)
     })
 
     // When a peer disconnects, remove it from the synchronizer
     networkSubsystem.on("peer-disconnected", ({ peerId }) => {
       this.#synchronizer.removePeer(peerId)
+      this.#remoteHeadsSubscriptions.removePeer(peerId)
     })
 
     // Handle incoming messages
     networkSubsystem.on("message", async msg => {
-      await this.#synchronizer.receiveMessage(msg)
+      this.#receiveMessage(msg)
     })
 
-    if (storageSubsystem) {
-      const debouncedSaveSyncState: (syncState: SyncStateMessage) => void =
-        throttle(({ documentId, peerId, syncState }: SyncStateMessage) => {
-          storageSubsystem.saveSyncState(documentId, peerId, syncState)
-        }, this.saveDebounceRate)
+    this.#synchronizer.on("sync-state", message => {
+      this.#saveSyncState(message)
+
+      const handle = this.#handleCache[message.documentId]
+
+      const info = this.persistanceInfoByPeerId[message.peerId]
+      if (!info) {
+        return
+      }
+
+      const { storageId } = info
+      const heads = handle.getRemoteHeads(storageId)
+      const haveHeadsChanged =
+        message.syncState.theirHeads &&
+        (!heads || !headsAreSame(heads, message.syncState.theirHeads))
+
+      if (haveHeadsChanged) {
+        handle.setRemoteHeads(storageId, message.syncState.theirHeads)
+
+        if (storageId) {
+          this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
+            message.documentId,
+            storageId,
+            message.syncState.theirHeads
+          )
+        }
+      }
+    })
+
+    this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
+      this.networkSubsystem.send({
+        type: "remote-heads-changed",
+        targetId: message.targetId,
+        documentId: message.documentId,
+        newHeads: {
+          [message.storageId]: {
+            heads: message.heads,
+            timestamp: message.timestamp,
+          },
+        },
+      })
+    })
+
+    this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
+      this.#log("change-remote-subs", message)
+      for (const peer of message.peers) {
+        this.networkSubsystem.send({
+          type: "remote-subscription-change",
+          targetId: peer,
+          add: message.add,
+          remove: message.remove,
+        })
+      }
+    })
 
-      this.#synchronizer.on("sync-state", debouncedSaveSyncState)
+    this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
+      const handle = this.#handleCache[message.documentId]
+      handle.setRemoteHeads(message.storageId, message.remoteHeads)
+    })
+  }
+
+  #receiveMessage(message: RepoMessage) {
+    switch (message.type) {
+      case "remote-subscription-change":
+        this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        break
+      case "remote-heads-changed":
+        this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        break
+      case "sync":
+      case "request":
+      case "ephemeral":
+      case "doc-unavailable":
+        this.#synchronizer.receiveMessage(message).catch(err => {
+          console.log("error receiving message", { err })
+        })
+    }
+  }
+
+  #throttledSaveSyncStateHandlers: Record<
+    StorageId,
+    (message: SyncStateMessage) => void
+  > = {}
+
+  /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
+  #saveSyncState(message: SyncStateMessage) {
+    if (!this.storageSubsystem) {
+      return
+    }
+
+    const persistanceInfo = this.persistanceInfoByPeerId[message.peerId]
+
+    if (!persistanceInfo || persistanceInfo.isEphemeral) {
+      return
+    }
+
+    const { storageId } = persistanceInfo
+
+    let handler = this.#throttledSaveSyncStateHandlers[storageId]
+    if (!handler) {
+      handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
+        ({ documentId, syncState }: SyncStateMessage) => {
+          this.storageSubsystem!.saveSyncState(documentId, storageId, syncState)
+        },
+        this.saveDebounceRate
+      )
     }
+
+    handler(message)
   }
 
   /** Returns an existing handle if we have it; creates one otherwise. */
@@ -298,12 +438,34 @@ export class Repo extends EventEmitter<RepoEvents> {
     delete this.#handleCache[documentId]
     this.emit("delete-document", { documentId })
   }
+
+  subscribeToRemotes = (remotes: StorageId[]) => {
+    this.#log("subscribeToRemotes", { remotes })
+    this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+  }
+
+  storageId = async (): Promise<StorageId | undefined> => {
+    if (!this.storageSubsystem) {
+      return undefined
+    } else {
+      return this.storageSubsystem.id()
+    }
+  }
+}
+
+interface PersistanceInfo {
+  storageId: StorageId
+  isEphemeral: boolean
 }
 
 export interface RepoConfig {
   /** Our unique identifier */
   peerId?: PeerId
 
+  /** Indicates whether other peers should persist the sync state of this peer.
+   * Sync state is only persisted for non-ephemeral peers */
+  isEphemeral?: boolean
+
   /** A storage adapter can be provided, or not */
   storage?: StorageAdapter
 

package/src/index.ts

@@ -83,6 +83,7 @@ export type {
   ChunkInfo,
   ChunkType,
   StorageKey,
+  StorageId,
 } from "./storage/types.js"
 
 export * from "./types.js"

package/src/network/NetworkAdapter.ts

@@ -1,6 +1,7 @@
 import { EventEmitter } from "eventemitter3"
 import { PeerId } from "../types.js"
 import { Message } from "./messages.js"
+import { StorageId } from "../storage/types.js"
 
 /** An interface representing some way to connect to other peers
  *
@@ -11,12 +12,20 @@ import { Message } from "./messages.js"
  */
 export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
   peerId?: PeerId // hmmm, maybe not
+  storageId?: StorageId
+  isEphemeral = true
 
   /** Called by the {@link Repo} to start the connection process
    *
    * @argument peerId - the peerId of this repo
+   * @argument storageId - the storage id of the peer
+   * @argument isEphemeral - weather or not the other end should persist our sync state
    */
-  abstract connect(peerId: PeerId): void
+  abstract connect(
+    peerId: PeerId,
+    storageId: StorageId | undefined,
+    isEphemeral: boolean
+  ): void
 
   /** Called by the {@link Repo} to send a message to a peer
    *
@@ -53,6 +62,8 @@ export interface OpenPayload {
 
 export interface PeerCandidatePayload {
   peerId: PeerId
+  storageId?: StorageId
+  isEphemeral: boolean
 }
 
 export interface PeerDisconnectedPayload {

package/src/network/NetworkSubsystem.ts

@@ -9,6 +9,7 @@ import {
   isEphemeralMessage,
   isValidRepoMessage,
 } from "./messages.js"
+import { StorageId } from "../storage/types.js"
 
 type EphemeralMessageSource = `${PeerId}:${SessionId}`
 
@@ -25,7 +26,12 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #readyAdapterCount = 0
   #adapters: NetworkAdapter[] = []
 
-  constructor(adapters: NetworkAdapter[], public peerId = randomPeerId()) {
+  constructor(
+    adapters: NetworkAdapter[],
+    public peerId = randomPeerId(),
+    private storageId: Promise<StorageId | undefined>, // todo: we shouldn't pass a promise here
+    private isEphemeral: boolean
+  ) {
     super()
     this.#log = debug(`automerge-repo:network:${this.peerId}`)
     adapters.forEach(a => this.addNetworkAdapter(a))
@@ -46,18 +52,21 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       }
     })
 
-    networkAdapter.on("peer-candidate", ({ peerId }) => {
-      this.#log(`peer candidate: ${peerId} `)
+    networkAdapter.on(
+      "peer-candidate",
+      ({ peerId, storageId, isEphemeral }) => {
+        this.#log(`peer candidate: ${peerId} `)
 
-      // TODO: This is where authentication would happen
+        // TODO: This is where authentication would happen
 
-      if (!this.#adaptersByPeer[peerId]) {
-        // TODO: handle losing a server here
-        this.#adaptersByPeer[peerId] = networkAdapter
-      }
+        if (!this.#adaptersByPeer[peerId]) {
+          // TODO: handle losing a server here
+          this.#adaptersByPeer[peerId] = networkAdapter
+        }
 
-      this.emit("peer", { peerId })
-    })
+        this.emit("peer", { peerId, storageId, isEphemeral })
+      }
+    )
 
     networkAdapter.on("peer-disconnected", ({ peerId }) => {
       this.#log(`peer disconnected: ${peerId} `)
@@ -98,7 +107,9 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       })
     })
 
-    networkAdapter.connect(this.peerId)
+    this.storageId.then(storageId => {
+      networkAdapter.connect(this.peerId, storageId, this.isEphemeral)
+    })
   }
 
   send(message: MessageContents) {
@@ -171,4 +182,6 @@ export interface NetworkSubsystemEvents {
 
 export interface PeerPayload {
   peerId: PeerId
+  storageId?: StorageId
+  isEphemeral: boolean
 }