@automerge/automerge-repo 2.0.0-alpha.6 → 2.0.0-collectionsync-alpha.1

package/src/Repo.ts

@@ -6,8 +6,14 @@ import {
   interpretAsDocumentId,
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
-import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
-import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
+import {
+  DELETED,
+  DocHandle,
+  DocHandleEncodedChangePayload,
+  READY,
+  UNAVAILABLE,
+  UNLOADED,
+} from "./DocHandle.js"
 import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
 import {
@@ -15,13 +21,25 @@ import {
   type PeerMetadata,
 } from "./network/NetworkAdapterInterface.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
-import { RepoMessage } from "./network/messages.js"
+import { MessageContents, RepoMessage } from "./network/messages.js"
 import { StorageAdapterInterface } from "./storage/StorageAdapterInterface.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { StorageId } from "./storage/types.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
-import { SyncStatePayload } from "./synchronizer/Synchronizer.js"
-import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
+import {
+  DocSyncMetrics,
+  SyncStatePayload,
+} from "./synchronizer/Synchronizer.js"
+import type {
+  AnyDocumentId,
+  AutomergeUrl,
+  DocumentId,
+  PeerId,
+} from "./types.js"
+import { Progress } from "./ferigan.js"
+import { CollectionHandle } from "./CollectionHandle.js"
+import { next as A } from "@automerge/automerge/slim"
+import { InMemoryStorageAdapter } from "./storage/StorageAdapter.js"
 
 function randomPeerId() {
   return ("peer-" + Math.random().toString(36).slice(4)) as PeerId
@@ -40,12 +58,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   /** @hidden */
   networkSubsystem: NetworkSubsystem
-  /** @hidden */
-  storageSubsystem?: StorageSubsystem
-
-  /** The debounce rate is adjustable on the repo. */
-  /** @hidden */
-  saveDebounceRate = 100
+  storageSubsystem: StorageSubsystem
 
   #handleCache: Record<DocumentId, DocHandle<any>> = {}
 
@@ -60,8 +73,7 @@ export class Repo extends EventEmitter<RepoEvents> {
   /** @hidden */
   peerMetadataByPeerId: Record<PeerId, PeerMetadata> = {}
 
-  #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
-  #remoteHeadsGossipingEnabled = false
+  #beelay: A.beelay.Beelay
 
   constructor({
     storage,
@@ -70,49 +82,78 @@ export class Repo extends EventEmitter<RepoEvents> {
     sharePolicy,
     isEphemeral = storage === undefined,
     enableRemoteHeadsGossiping = false,
+    denylist = [],
   }: RepoConfig = {}) {
     super()
-    this.#remoteHeadsGossipingEnabled = enableRemoteHeadsGossiping
+    if (storage == null) {
+      // beelayStorage = new InMemoryStorageAdapter()
+      storage = new InMemoryStorageAdapter()
+    }
+    this.#beelay = new A.beelay.Beelay({
+      storage,
+      peerId,
+      requestPolicy: async ({ docId }) => {
+        const peers = Array.from(this.networkSubsystem.peers)
+        const generousPeers: PeerId[] = []
+        for (const peerId of peers) {
+          const okToShare = await this.sharePolicy(peerId)
+          if (okToShare) generousPeers.push(peerId)
+        }
+        return generousPeers
+      },
+    })
+    this.storageSubsystem = new StorageSubsystem(this.#beelay, storage)
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
 
-    this.on("delete-document", ({ documentId }) => {
-      // TODO Pass the delete on to the network
-      // synchronizer.removeDocument(documentId)
+    this.#beelay.on("message", ({ message }) => {
+      this.#log(`sending ${message} message to ${message.recipient}`)
+      networkSubsystem.send({
+        targetId: message.recipient as PeerId,
+        type: "beelay",
+        ...message,
+      } as MessageContents)
+    })
 
-      if (storageSubsystem) {
-        storageSubsystem.removeDoc(documentId).catch(err => {
-          this.#log("error deleting document", { documentId, err })
-        })
+    this.#beelay.on("docEvent", event => {
+      this.#log(`received ${event.data.type} event for ${event.docId}`)
+      const handle = this.#handleCache[event.docId as DocumentId]
+      if (handle != null) {
+        handle.update(d => Automerge.loadIncremental(d, event.data.contents))
       }
     })
 
+    this.#beelay.on("bundleRequired", ({ start, end, checkpoints, docId }) => {
+      ;(async () => {
+        const doc = await this.storageSubsystem.loadDoc(docId as DocumentId)
+        if (doc == null) {
+          console.warn("document not found when creating bundle")
+          return
+        }
+        const bundle = A.saveBundle(doc, start, end)
+        this.#beelay.addBundle({
+          docId,
+          checkpoints,
+          start,
+          end,
+          data: bundle,
+        })
+      })()
+    })
+
     // SYNCHRONIZER
-    // The synchronizer uses the network subsystem to keep documents in sync with peers.
-    this.synchronizer = new CollectionSynchronizer(this)
+    this.synchronizer = new CollectionSynchronizer(this.#beelay, this, [])
 
-    // When the synchronizer emits messages, send them to peers
     this.synchronizer.on("message", message => {
       this.#log(`sending ${message.type} message to ${message.targetId}`)
       networkSubsystem.send(message)
     })
 
-    if (this.#remoteHeadsGossipingEnabled) {
-      this.synchronizer.on("open-doc", ({ peerId, documentId }) => {
-        this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
-      })
-    }
-
-    // STORAGE
-    // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
-    const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
-    this.storageSubsystem = storageSubsystem
-
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
 
     const myPeerMetadata: Promise<PeerMetadata> = (async () => ({
-      storageId: await storageSubsystem?.id(),
+      // storageId: await this.storageSubsystem.id(),
       isEphemeral,
     }))()
 
@@ -126,109 +167,48 @@ export class Repo extends EventEmitter<RepoEvents> {
     // When we get a new peer, register it with the synchronizer
     networkSubsystem.on("peer", async ({ peerId, peerMetadata }) => {
       this.#log("peer connected", { peerId })
-
       if (peerMetadata) {
         this.peerMetadataByPeerId[peerId] = { ...peerMetadata }
       }
-
-      this.sharePolicy(peerId)
-        .then(shouldShare => {
-          if (shouldShare && this.#remoteHeadsGossipingEnabled) {
-            this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
-          }
-        })
-        .catch(err => {
-          console.log("error in share policy", { err })
-        })
-
       this.synchronizer.addPeer(peerId)
     })
 
-    // When a peer disconnects, remove it from the synchronizer
-    networkSubsystem.on("peer-disconnected", ({ peerId }) => {
-      this.synchronizer.removePeer(peerId)
-      this.#remoteHeadsSubscriptions.removePeer(peerId)
-    })
-
     // Handle incoming messages
     networkSubsystem.on("message", async msg => {
-      this.#receiveMessage(msg)
-    })
-
-    this.synchronizer.on("sync-state", message => {
-      this.#saveSyncState(message)
-
-      const handle = this.#handleCache[message.documentId]
-
-      const { storageId } = this.peerMetadataByPeerId[message.peerId] || {}
-      if (!storageId) {
-        return
-      }
-
-      const heads = handle.getRemoteHeads(storageId)
-      const haveHeadsChanged =
-        message.syncState.theirHeads &&
-        (!heads || !headsAreSame(heads, message.syncState.theirHeads))
-
-      if (haveHeadsChanged && message.syncState.theirHeads) {
-        handle.setRemoteHeads(storageId, message.syncState.theirHeads)
-
-        if (storageId && this.#remoteHeadsGossipingEnabled) {
-          this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
-            message.documentId,
-            storageId,
-            message.syncState.theirHeads
-          )
+      //@ts-ignore
+      // const inspected = A.beelay.inspectMessage(msg.message)
+      // this.#log(`received msg: ${JSON.stringify(inspected)}`)
+      //@ts-ignore
+      if (msg.type === "beelay") {
+        if (!(msg.message instanceof Uint8Array)) {
+          // The Uint8Array instance in the vitest VM is _different_ from the
+          // Uint8Array instance which is available in this file for some reason.
+          // So, even though `msg.message` _is_ a `Uint8Array`, we have to do this
+          // absurd thing to get the tests to pass
+          msg.message = Uint8Array.from(msg.message)
         }
-      }
-    })
-
-    if (this.#remoteHeadsGossipingEnabled) {
-      this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
-        this.networkSubsystem.send({
-          type: "remote-heads-changed",
-          targetId: message.targetId,
-          documentId: message.documentId,
-          newHeads: {
-            [message.storageId]: {
-              heads: message.heads,
-              timestamp: message.timestamp,
-            },
+        this.#beelay.receiveMessage({
+          message: {
+            sender: msg.senderId,
+            recipient: msg.targetId,
+            message: msg.message,
           },
         })
-      })
-
-      this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
-        this.#log("change-remote-subs", message)
-        for (const peer of message.peers) {
-          this.networkSubsystem.send({
-            type: "remote-subscription-change",
-            targetId: peer,
-            add: message.add,
-            remove: message.remove,
-          })
-        }
-      })
-
-      this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
-        const handle = this.#handleCache[message.documentId]
-        handle.setRemoteHeads(message.storageId, message.remoteHeads)
-      })
-    }
+      } else {
+        this.#receiveMessage(msg)
+      }
+    })
   }
 
   // The `document` event is fired by the DocCollection any time we create a new document or look
   // up a document by ID. We listen for it in order to wire up storage and network synchronization.
   #registerHandleWithSubsystems(handle: DocHandle<any>) {
-    const { storageSubsystem } = this
-    if (storageSubsystem) {
-      // Save when the document changes, but no more often than saveDebounceRate.
-      const saveFn = ({ handle, doc }: DocHandleEncodedChangePayload<any>) => {
-        void storageSubsystem.saveDoc(handle.documentId, doc)
+    handle.on("heads-changed", () => {
+      const doc = handle.docSync()
+      if (doc != null) {
+        this.storageSubsystem.saveDoc(handle.documentId, doc)
       }
-      handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
-    }
-
+    })
     handle.on("unavailable", () => {
       this.#log("document unavailable", { documentId: handle.documentId })
       this.emit("unavailable-document", {
@@ -236,7 +216,6 @@ export class Repo extends EventEmitter<RepoEvents> {
       })
     })
 
-    // Register the document with the synchronizer. This advertises our interest in the document.
     this.synchronizer.addDocument(handle.documentId)
 
     // Preserve the old event in case anyone was using it.
@@ -246,60 +225,19 @@ export class Repo extends EventEmitter<RepoEvents> {
   #receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "remote-subscription-change":
-        if (this.#remoteHeadsGossipingEnabled) {
-          this.#remoteHeadsSubscriptions.handleControlMessage(message)
-        }
-        break
       case "remote-heads-changed":
-        if (this.#remoteHeadsGossipingEnabled) {
-          this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
-        }
         break
       case "sync":
       case "request":
       case "ephemeral":
       case "doc-unavailable":
         this.synchronizer.receiveMessage(message).catch(err => {
-          console.log("error receiving message", { err })
+          console.error("error receiving message", { err })
         })
+        break
     }
   }
 
-  #throttledSaveSyncStateHandlers: Record<
-    StorageId,
-    (payload: SyncStatePayload) => void
-  > = {}
-
-  /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
-  #saveSyncState(payload: SyncStatePayload) {
-    if (!this.storageSubsystem) {
-      return
-    }
-
-    const { storageId, isEphemeral } =
-      this.peerMetadataByPeerId[payload.peerId] || {}
-
-    if (!storageId || isEphemeral) {
-      return
-    }
-
-    let handler = this.#throttledSaveSyncStateHandlers[storageId]
-    if (!handler) {
-      handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
-        ({ documentId, syncState }: SyncStatePayload) => {
-          void this.storageSubsystem!.saveSyncState(
-            documentId,
-            storageId,
-            syncState
-          )
-        },
-        this.saveDebounceRate
-      )
-    }
-
-    handler(payload)
-  }
-
   /** Returns an existing handle if we have it; creates one otherwise. */
   #getHandle<T>({
     documentId,
@@ -324,7 +262,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   /** Returns a list of all connected peer ids */
   get peers(): PeerId[] {
-    return this.synchronizer.peers
+    return this.networkSubsystem.peers
   }
 
   getStorageIdOfPeer(peerId: PeerId): StorageId | undefined {
@@ -343,7 +281,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       documentId,
     }) as DocHandle<T>
 
-    this.#registerHandleWithSubsystems(handle)
+    let initialLinks: A.Link[] = []
 
     handle.update(() => {
       let nextDoc: Automerge.Doc<T>
@@ -352,9 +290,33 @@ export class Repo extends EventEmitter<RepoEvents> {
       } else {
         nextDoc = Automerge.emptyChange(Automerge.init())
       }
+      const patches = A.diff(nextDoc, [], A.getHeads(nextDoc))
+      for (const patch of patches) {
+        initialLinks = patches
+          .map(patch => {
+            if (patch.action === "put") {
+              if (patch.value instanceof A.Link) {
+                return patch.value
+              }
+            }
+            return null
+          })
+          .filter(v => v != null)
+      }
       return nextDoc
     })
 
+    for (const link of initialLinks) {
+      const { documentId: target } = parseAutomergeUrl(
+        link.target as AutomergeUrl
+      )
+      this.#beelay.addLink({ from: documentId, to: target })
+    }
+
+    this.storageSubsystem.saveDoc(handle.documentId, handle.docSync()!)
+
+    this.#registerHandleWithSubsystems(handle)
+
     handle.doneLoading()
     return handle
   }
@@ -378,7 +340,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     if (!clonedHandle.isReady()) {
       throw new Error(
         `Cloned handle is not yet in ready state.
-        (Try await handle.waitForReady() first.)`
+        (Try await handle.whenReady() first.)`
       )
     }
 
@@ -405,6 +367,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     /** The url or documentId of the handle to retrieve */
     id: AnyDocumentId
   ): DocHandle<T> {
+    this.#log("find", { id })
     const documentId = interpretAsDocumentId(id)
 
     // If we have the handle cached, return it
@@ -427,9 +390,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // Loading & network is going to be asynchronous no matter what,
     // but we want to return the handle immediately.
-    const attemptLoad = this.storageSubsystem
-      ? this.storageSubsystem.loadDoc(handle.documentId)
-      : Promise.resolve(null)
+    const attemptLoad = this.storageSubsystem.loadDoc(handle.documentId)
 
     attemptLoad
       .then(async loadedDoc => {
@@ -441,6 +402,7 @@ export class Repo extends EventEmitter<RepoEvents> {
           // we want to wait for the network subsystem to be ready before
           // we request the document. this prevents entering unavailable during initialization.
           await this.networkSubsystem.whenReady()
+          console.log("we didn't find it so we're requesting")
           handle.request()
         }
         this.#registerHandleWithSubsystems(handle)
@@ -496,16 +458,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     return handle
   }
 
-  subscribeToRemotes = (remotes: StorageId[]) => {
-    if (this.#remoteHeadsGossipingEnabled) {
-      this.#log("subscribeToRemotes", { remotes })
-      this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
-    } else {
-      this.#log(
-        "WARN: subscribeToRemotes called but remote heads gossiping is not enabled"
-      )
-    }
-  }
+  subscribeToRemotes = (remotes: StorageId[]) => {}
 
   storageId = async (): Promise<StorageId | undefined> => {
     if (!this.storageSubsystem) {
@@ -539,6 +492,39 @@ export class Repo extends EventEmitter<RepoEvents> {
     )
   }
 
+  /**
+   * Removes a DocHandle from the handleCache.
+   * @hidden this API is experimental and may change.
+   * @param documentId - documentId of the DocHandle to remove from handleCache, if present in cache.
+   * @returns Promise<void>
+   */
+  async removeFromCache(documentId: DocumentId) {
+    if (!this.#handleCache[documentId]) {
+      this.#log(
+        `WARN: removeFromCache called but handle not found in handleCache for documentId: ${documentId}`
+      )
+      return
+    }
+    const handle = this.#getHandle({ documentId })
+    const doc = await handle.doc([READY, UNLOADED, DELETED, UNAVAILABLE])
+    if (doc) {
+      if (handle.isReady()) {
+        handle.unload()
+      } else {
+        this.#log(
+          `WARN: removeFromCache called but handle for documentId: ${documentId} in unexpected state: ${handle.state}`
+        )
+      }
+      delete this.#handleCache[documentId]
+      // TODO: remove document from synchronizer when removeDocument is implemented
+      // this.synchronizer.removeDocument(documentId)
+    } else {
+      this.#log(
+        `WARN: removeFromCache called but doc undefined for documentId: ${documentId}`
+      )
+    }
+  }
+
   shutdown(): Promise<void> {
     this.networkSubsystem.adapters.forEach(adapter => {
       adapter.disconnect()
@@ -547,7 +533,8 @@ export class Repo extends EventEmitter<RepoEvents> {
   }
 
   metrics(): { documents: { [key: string]: any } } {
-    return { documents: this.synchronizer.metrics() }
+    //return { documents: this.synchronizer.metrics() }
+    return { documents: {} }
   }
 }
 
@@ -575,6 +562,13 @@ export interface RepoConfig {
    * Whether to enable the experimental remote heads gossiping feature
    */
   enableRemoteHeadsGossiping?: boolean
+
+  /**
+   * A list of automerge URLs which should never be loaded regardless of what
+   * messages are received or what the share policy is. This is useful to avoid
+   * loading documents that are known to be too resource intensive.
+   */
+  denylist?: AutomergeUrl[]
 }
 
 /** A function that determines whether we should share a document with a peer
@@ -598,6 +592,7 @@ export interface RepoEvents {
   "delete-document": (arg: DeleteDocumentPayload) => void
   /** A document was marked as unavailable (we don't have it and none of our peers have it) */
   "unavailable-document": (arg: DeleteDocumentPayload) => void
+  "doc-metrics": (arg: DocMetrics) => void
 }
 
 export interface DocumentPayload {
@@ -607,3 +602,17 @@ export interface DocumentPayload {
 export interface DeleteDocumentPayload {
   documentId: DocumentId
 }
+
+export type DocMetrics =
+  | DocSyncMetrics
+  | {
+      type: "doc-loaded"
+      documentId: DocumentId
+      durationMillis: number
+      numOps: number
+      numChanges: number
+    }
+  | {
+      type: "doc-denied"
+      documentId: DocumentId
+    }