@automerge/automerge-repo 2.0.0-collectionsync-alpha.1 → 2.0.1

package/src/Repo.ts

@@ -2,8 +2,10 @@ import { next as Automerge } from "@automerge/automerge/slim"
 import debug from "debug"
 import { EventEmitter } from "eventemitter3"
 import {
+  encodeHeads,
   generateAutomergeUrl,
   interpretAsDocumentId,
+  isValidAutomergeUrl,
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
 import {
@@ -14,6 +16,7 @@ import {
   UNAVAILABLE,
   UNLOADED,
 } from "./DocHandle.js"
+import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
 import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
 import {
@@ -21,7 +24,7 @@ import {
   type PeerMetadata,
 } from "./network/NetworkAdapterInterface.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
-import { MessageContents, RepoMessage } from "./network/messages.js"
+import { RepoMessage } from "./network/messages.js"
 import { StorageAdapterInterface } from "./storage/StorageAdapterInterface.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { StorageId } from "./storage/types.js"
@@ -36,10 +39,20 @@ import type {
   DocumentId,
   PeerId,
 } from "./types.js"
-import { Progress } from "./ferigan.js"
-import { CollectionHandle } from "./CollectionHandle.js"
-import { next as A } from "@automerge/automerge/slim"
-import { InMemoryStorageAdapter } from "./storage/StorageAdapter.js"
+import { abortable, AbortOptions } from "./helpers/abortable.js"
+import { FindProgress } from "./FindProgress.js"
+
+export type FindProgressWithMethods<T> = FindProgress<T> & {
+  untilReady: (allowableStates: string[]) => Promise<DocHandle<T>>
+  peek: () => FindProgress<T>
+  subscribe: (callback: (progress: FindProgress<T>) => void) => () => void
+}
+
+export type ProgressSignal<T> = {
+  peek: () => FindProgress<T>
+  subscribe: (callback: (progress: FindProgress<T>) => void) => () => void
+  untilReady: (allowableStates: string[]) => Promise<DocHandle<T>>
+}
 
 function randomPeerId() {
   return ("peer-" + Math.random().toString(36).slice(4)) as PeerId
@@ -58,7 +71,12 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   /** @hidden */
   networkSubsystem: NetworkSubsystem
-  storageSubsystem: StorageSubsystem
+  /** @hidden */
+  storageSubsystem?: StorageSubsystem
+
+  /** The debounce rate is adjustable on the repo. */
+  /** @hidden */
+  saveDebounceRate = 100
 
   #handleCache: Record<DocumentId, DocHandle<any>> = {}
 
@@ -73,7 +91,9 @@ export class Repo extends EventEmitter<RepoEvents> {
   /** @hidden */
   peerMetadataByPeerId: Record<PeerId, PeerMetadata> = {}
 
-  #beelay: A.beelay.Beelay
+  #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+  #remoteHeadsGossipingEnabled = false
+  #progressCache: Record<DocumentId, FindProgress<any>> = {}
 
   constructor({
     storage,
@@ -85,75 +105,56 @@ export class Repo extends EventEmitter<RepoEvents> {
     denylist = [],
   }: RepoConfig = {}) {
     super()
-    if (storage == null) {
-      // beelayStorage = new InMemoryStorageAdapter()
-      storage = new InMemoryStorageAdapter()
-    }
-    this.#beelay = new A.beelay.Beelay({
-      storage,
-      peerId,
-      requestPolicy: async ({ docId }) => {
-        const peers = Array.from(this.networkSubsystem.peers)
-        const generousPeers: PeerId[] = []
-        for (const peerId of peers) {
-          const okToShare = await this.sharePolicy(peerId)
-          if (okToShare) generousPeers.push(peerId)
-        }
-        return generousPeers
-      },
-    })
-    this.storageSubsystem = new StorageSubsystem(this.#beelay, storage)
+    this.#remoteHeadsGossipingEnabled = enableRemoteHeadsGossiping
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
 
-    this.#beelay.on("message", ({ message }) => {
-      this.#log(`sending ${message} message to ${message.recipient}`)
-      networkSubsystem.send({
-        targetId: message.recipient as PeerId,
-        type: "beelay",
-        ...message,
-      } as MessageContents)
-    })
-
-    this.#beelay.on("docEvent", event => {
-      this.#log(`received ${event.data.type} event for ${event.docId}`)
-      const handle = this.#handleCache[event.docId as DocumentId]
-      if (handle != null) {
-        handle.update(d => Automerge.loadIncremental(d, event.data.contents))
-      }
-    })
+    this.on("delete-document", ({ documentId }) => {
+      // TODO Pass the delete on to the network
+      // synchronizer.removeDocument(documentId)
 
-    this.#beelay.on("bundleRequired", ({ start, end, checkpoints, docId }) => {
-      ;(async () => {
-        const doc = await this.storageSubsystem.loadDoc(docId as DocumentId)
-        if (doc == null) {
-          console.warn("document not found when creating bundle")
-          return
-        }
-        const bundle = A.saveBundle(doc, start, end)
-        this.#beelay.addBundle({
-          docId,
-          checkpoints,
-          start,
-          end,
-          data: bundle,
+      if (storageSubsystem) {
+        storageSubsystem.removeDoc(documentId).catch(err => {
+          this.#log("error deleting document", { documentId, err })
         })
-      })()
+      }
     })
 
     // SYNCHRONIZER
-    this.synchronizer = new CollectionSynchronizer(this.#beelay, this, [])
+    // The synchronizer uses the network subsystem to keep documents in sync with peers.
+    this.synchronizer = new CollectionSynchronizer(this, denylist)
 
+    // When the synchronizer emits messages, send them to peers
     this.synchronizer.on("message", message => {
       this.#log(`sending ${message.type} message to ${message.targetId}`)
       networkSubsystem.send(message)
     })
 
+    // Forward metrics from doc synchronizers
+    this.synchronizer.on("metrics", event => this.emit("doc-metrics", event))
+
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.synchronizer.on("open-doc", ({ peerId, documentId }) => {
+        this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
+      })
+    }
+
+    // STORAGE
+    // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
+    const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
+    if (storageSubsystem) {
+      storageSubsystem.on("document-loaded", event =>
+        this.emit("doc-metrics", { type: "doc-loaded", ...event })
+      )
+    }
+
+    this.storageSubsystem = storageSubsystem
+
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
 
     const myPeerMetadata: Promise<PeerMetadata> = (async () => ({
-      // storageId: await this.storageSubsystem.id(),
+      storageId: await storageSubsystem?.id(),
       isEphemeral,
     }))()
 
@@ -167,77 +168,174 @@ export class Repo extends EventEmitter<RepoEvents> {
     // When we get a new peer, register it with the synchronizer
     networkSubsystem.on("peer", async ({ peerId, peerMetadata }) => {
       this.#log("peer connected", { peerId })
+
       if (peerMetadata) {
         this.peerMetadataByPeerId[peerId] = { ...peerMetadata }
       }
+
+      this.sharePolicy(peerId)
+        .then(shouldShare => {
+          if (shouldShare && this.#remoteHeadsGossipingEnabled) {
+            this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
+          }
+        })
+        .catch(err => {
+          console.log("error in share policy", { err })
+        })
+
       this.synchronizer.addPeer(peerId)
     })
 
+    // When a peer disconnects, remove it from the synchronizer
+    networkSubsystem.on("peer-disconnected", ({ peerId }) => {
+      this.synchronizer.removePeer(peerId)
+      this.#remoteHeadsSubscriptions.removePeer(peerId)
+    })
+
     // Handle incoming messages
     networkSubsystem.on("message", async msg => {
-      //@ts-ignore
-      // const inspected = A.beelay.inspectMessage(msg.message)
-      // this.#log(`received msg: ${JSON.stringify(inspected)}`)
-      //@ts-ignore
-      if (msg.type === "beelay") {
-        if (!(msg.message instanceof Uint8Array)) {
-          // The Uint8Array instance in the vitest VM is _different_ from the
-          // Uint8Array instance which is available in this file for some reason.
-          // So, even though `msg.message` _is_ a `Uint8Array`, we have to do this
-          // absurd thing to get the tests to pass
-          msg.message = Uint8Array.from(msg.message)
+      this.#receiveMessage(msg)
+    })
+
+    this.synchronizer.on("sync-state", message => {
+      this.#saveSyncState(message)
+
+      const handle = this.#handleCache[message.documentId]
+
+      const { storageId } = this.peerMetadataByPeerId[message.peerId] || {}
+      if (!storageId) {
+        return
+      }
+
+      const heads = handle.getRemoteHeads(storageId)
+      const haveHeadsChanged =
+        message.syncState.theirHeads &&
+        (!heads ||
+          !headsAreSame(heads, encodeHeads(message.syncState.theirHeads)))
+
+      if (haveHeadsChanged && message.syncState.theirHeads) {
+        handle.setRemoteHeads(
+          storageId,
+          encodeHeads(message.syncState.theirHeads)
+        )
+
+        if (storageId && this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
+            message.documentId,
+            storageId,
+            encodeHeads(message.syncState.theirHeads)
+          )
         }
-        this.#beelay.receiveMessage({
-          message: {
-            sender: msg.senderId,
-            recipient: msg.targetId,
-            message: msg.message,
-          },
-        })
-      } else {
-        this.#receiveMessage(msg)
       }
     })
+
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
+        this.networkSubsystem.send({
+          type: "remote-heads-changed",
+          targetId: message.targetId,
+          documentId: message.documentId,
+          newHeads: {
+            [message.storageId]: {
+              heads: message.heads,
+              timestamp: message.timestamp,
+            },
+          },
+        })
+      })
+
+      this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
+        this.#log("change-remote-subs", message)
+        for (const peer of message.peers) {
+          this.networkSubsystem.send({
+            type: "remote-subscription-change",
+            targetId: peer,
+            add: message.add,
+            remove: message.remove,
+          })
+        }
+      })
+
+      this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
+        const handle = this.#handleCache[message.documentId]
+        handle.setRemoteHeads(message.storageId, message.remoteHeads)
+      })
+    }
   }
 
   // The `document` event is fired by the DocCollection any time we create a new document or look
   // up a document by ID. We listen for it in order to wire up storage and network synchronization.
   #registerHandleWithSubsystems(handle: DocHandle<any>) {
-    handle.on("heads-changed", () => {
-      const doc = handle.docSync()
-      if (doc != null) {
-        this.storageSubsystem.saveDoc(handle.documentId, doc)
+    const { storageSubsystem } = this
+    if (storageSubsystem) {
+      // Save when the document changes, but no more often than saveDebounceRate.
+      const saveFn = ({ handle, doc }: DocHandleEncodedChangePayload<any>) => {
+        void storageSubsystem.saveDoc(handle.documentId, doc)
       }
-    })
-    handle.on("unavailable", () => {
-      this.#log("document unavailable", { documentId: handle.documentId })
-      this.emit("unavailable-document", {
-        documentId: handle.documentId,
-      })
-    })
-
-    this.synchronizer.addDocument(handle.documentId)
+      handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
+    }
 
-    // Preserve the old event in case anyone was using it.
-    this.emit("document", { handle })
+    // Register the document with the synchronizer. This advertises our interest in the document.
+    this.synchronizer.addDocument(handle)
   }
 
   #receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "remote-subscription-change":
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        }
+        break
       case "remote-heads-changed":
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        }
         break
       case "sync":
       case "request":
       case "ephemeral":
       case "doc-unavailable":
         this.synchronizer.receiveMessage(message).catch(err => {
-          console.error("error receiving message", { err })
+          console.log("error receiving message", { err })
         })
-        break
     }
   }
 
+  #throttledSaveSyncStateHandlers: Record<
+    StorageId,
+    (payload: SyncStatePayload) => void
+  > = {}
+
+  /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
+  #saveSyncState(payload: SyncStatePayload) {
+    if (!this.storageSubsystem) {
+      return
+    }
+
+    const { storageId, isEphemeral } =
+      this.peerMetadataByPeerId[payload.peerId] || {}
+
+    if (!storageId || isEphemeral) {
+      return
+    }
+
+    let handler = this.#throttledSaveSyncStateHandlers[storageId]
+    if (!handler) {
+      handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
+        ({ documentId, syncState }: SyncStatePayload) => {
+          void this.storageSubsystem!.saveSyncState(
+            documentId,
+            storageId,
+            syncState
+          )
+        },
+        this.saveDebounceRate
+      )
+    }
+
+    handler(payload)
+  }
+
   /** Returns an existing handle if we have it; creates one otherwise. */
   #getHandle<T>({
     documentId,
@@ -262,7 +360,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   /** Returns a list of all connected peer ids */
   get peers(): PeerId[] {
-    return this.networkSubsystem.peers
+    return this.synchronizer.peers
   }
 
   getStorageIdOfPeer(peerId: PeerId): StorageId | undefined {
@@ -281,7 +379,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       documentId,
     }) as DocHandle<T>
 
-    let initialLinks: A.Link[] = []
+    this.#registerHandleWithSubsystems(handle)
 
     handle.update(() => {
       let nextDoc: Automerge.Doc<T>
@@ -290,33 +388,9 @@ export class Repo extends EventEmitter<RepoEvents> {
       } else {
         nextDoc = Automerge.emptyChange(Automerge.init())
       }
-      const patches = A.diff(nextDoc, [], A.getHeads(nextDoc))
-      for (const patch of patches) {
-        initialLinks = patches
-          .map(patch => {
-            if (patch.action === "put") {
-              if (patch.value instanceof A.Link) {
-                return patch.value
-              }
-            }
-            return null
-          })
-          .filter(v => v != null)
-      }
       return nextDoc
     })
 
-    for (const link of initialLinks) {
-      const { documentId: target } = parseAutomergeUrl(
-        link.target as AutomergeUrl
-      )
-      this.#beelay.addLink({ from: documentId, to: target })
-    }
-
-    this.storageSubsystem.saveDoc(handle.documentId, handle.docSync()!)
-
-    this.#registerHandleWithSubsystems(handle)
-
     handle.doneLoading()
     return handle
   }
@@ -333,8 +407,6 @@ export class Repo extends EventEmitter<RepoEvents> {
    * Any peers this `Repo` is connected to for whom `sharePolicy` returns `true` will
    * be notified of the newly created DocHandle.
    *
-   * @throws if the cloned handle is not yet ready or if
-   * `clonedHandle.docSync()` returns `undefined` (i.e. the handle is unavailable).
    */
   clone<T>(clonedHandle: DocHandle<T>) {
     if (!clonedHandle.isReady()) {
@@ -344,11 +416,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       )
     }
 
-    const sourceDoc = clonedHandle.docSync()
-    if (!sourceDoc) {
-      throw new Error("Cloned handle doesn't have a document.")
-    }
-
+    const sourceDoc = clonedHandle.doc()
     const handle = this.create<T>()
 
     handle.update(() => {
@@ -359,60 +427,267 @@ export class Repo extends EventEmitter<RepoEvents> {
     return handle
   }
 
-  /**
-   * Retrieves a document by id. It gets data from the local system, but also emits a `document`
-   * event to advertise interest in the document.
-   */
-  find<T>(
-    /** The url or documentId of the handle to retrieve */
-    id: AnyDocumentId
-  ): DocHandle<T> {
-    this.#log("find", { id })
-    const documentId = interpretAsDocumentId(id)
+  findWithProgress<T>(
+    id: AnyDocumentId,
+    options: AbortOptions = {}
+  ): FindProgressWithMethods<T> | FindProgress<T> {
+    const { signal } = options
+    const { documentId, heads } = isValidAutomergeUrl(id)
+      ? parseAutomergeUrl(id)
+      : { documentId: interpretAsDocumentId(id), heads: undefined }
 
-    // If we have the handle cached, return it
+    // Check handle cache first - return plain FindStep for terminal states
     if (this.#handleCache[documentId]) {
-      if (this.#handleCache[documentId].isUnavailable()) {
-        // this ensures that the event fires after the handle has been returned
-        setTimeout(() => {
-          this.#handleCache[documentId].emit("unavailable", {
-            handle: this.#handleCache[documentId],
-          })
-        })
+      const handle = this.#handleCache[documentId]
+      if (handle.state === UNAVAILABLE) {
+        const result = {
+          state: "unavailable" as const,
+          error: new Error(`Document ${id} is unavailable`),
+          handle,
+        }
+        return result
+      }
+      if (handle.state === DELETED) {
+        const result = {
+          state: "failed" as const,
+          error: new Error(`Document ${id} was deleted`),
+          handle,
+        }
+        return result
+      }
+      if (handle.state === READY) {
+        const result = {
+          state: "ready" as const,
+          handle: heads ? handle.view(heads) : handle,
+        }
+        return result
       }
-      return this.#handleCache[documentId]
     }
 
-    // If we don't already have the handle, make an empty one and try loading it
-    const handle = this.#getHandle<T>({
+    // Check progress cache for any existing signal
+    const cachedProgress = this.#progressCache[documentId]
+    if (cachedProgress) {
+      const handle = this.#handleCache[documentId]
+      // Return cached progress if we have a handle and it's either in a terminal state or loading
+      if (
+        handle &&
+        (handle.state === READY ||
+          handle.state === UNAVAILABLE ||
+          handle.state === DELETED ||
+          handle.state === "loading")
+      ) {
+        return cachedProgress as FindProgressWithMethods<T>
+      }
+    }
+
+    const handle = this.#getHandle<T>({ documentId })
+    const initial = {
+      state: "loading" as const,
+      progress: 0,
+      handle,
+    }
+
+    // Create a new progress signal
+    const progressSignal = {
+      subscribers: new Set<(progress: FindProgress<T>) => void>(),
+      currentProgress: undefined as FindProgress<T> | undefined,
+      notify: (progress: FindProgress<T>) => {
+        progressSignal.currentProgress = progress
+        progressSignal.subscribers.forEach(callback => callback(progress))
+        // Cache all states, not just terminal ones
+        this.#progressCache[documentId] = progress
+      },
+      peek: () => progressSignal.currentProgress || initial,
+      subscribe: (callback: (progress: FindProgress<T>) => void) => {
+        progressSignal.subscribers.add(callback)
+        return () => progressSignal.subscribers.delete(callback)
+      },
+    }
+
+    progressSignal.notify(initial)
+
+    // Start the loading process
+    void this.#loadDocumentWithProgress(
+      id,
       documentId,
-    }) as DocHandle<T>
+      handle,
+      progressSignal,
+      signal ? abortable(new Promise(() => {}), signal) : new Promise(() => {})
+    )
+
+    const result = {
+      ...initial,
+      peek: progressSignal.peek,
+      subscribe: progressSignal.subscribe,
+    }
+    this.#progressCache[documentId] = result
+    return result
+  }
+
+  async #loadDocumentWithProgress<T>(
+    id: AnyDocumentId,
+    documentId: DocumentId,
+    handle: DocHandle<T>,
+    progressSignal: {
+      notify: (progress: FindProgress<T>) => void
+    },
+    abortPromise: Promise<never>
+  ) {
+    try {
+      progressSignal.notify({
+        state: "loading" as const,
+        progress: 25,
+        handle,
+      })
+
+      const loadingPromise = await (this.storageSubsystem
+        ? this.storageSubsystem.loadDoc(handle.documentId)
+        : Promise.resolve(null))
+
+      const loadedDoc = await Promise.race([loadingPromise, abortPromise])
+
+      if (loadedDoc) {
+        handle.update(() => loadedDoc as Automerge.Doc<T>)
+        handle.doneLoading()
+        progressSignal.notify({
+          state: "loading" as const,
+          progress: 50,
+          handle,
+        })
+      } else {
+        await Promise.race([this.networkSubsystem.whenReady(), abortPromise])
+        handle.request()
+        progressSignal.notify({
+          state: "loading" as const,
+          progress: 75,
+          handle,
+        })
+      }
+
+      this.#registerHandleWithSubsystems(handle)
+
+      await Promise.race([handle.whenReady([READY, UNAVAILABLE]), abortPromise])
 
-    // Loading & network is going to be asynchronous no matter what,
-    // but we want to return the handle immediately.
-    const attemptLoad = this.storageSubsystem.loadDoc(handle.documentId)
-
-    attemptLoad
-      .then(async loadedDoc => {
-        if (loadedDoc) {
-          // uhhhh, sorry if you're reading this because we were lying to the type system
-          handle.update(() => loadedDoc as Automerge.Doc<T>)
-          handle.doneLoading()
-        } else {
-          // we want to wait for the network subsystem to be ready before
-          // we request the document. this prevents entering unavailable during initialization.
-          await this.networkSubsystem.whenReady()
-          console.log("we didn't find it so we're requesting")
-          handle.request()
+      if (handle.state === UNAVAILABLE) {
+        const unavailableProgress = {
+          state: "unavailable" as const,
+          handle,
         }
-        this.#registerHandleWithSubsystems(handle)
+        progressSignal.notify(unavailableProgress)
+        return
+      }
+      if (handle.state === DELETED) {
+        throw new Error(`Document ${id} was deleted`)
+      }
+
+      progressSignal.notify({ state: "ready" as const, handle })
+    } catch (error) {
+      progressSignal.notify({
+        state: "failed" as const,
+        error: error instanceof Error ? error : new Error(String(error)),
+        handle: this.#getHandle<T>({ documentId }),
       })
-      .catch(err => {
-        this.#log("error waiting for network", { err })
+    }
+  }
+
+  async find<T>(
+    id: AnyDocumentId,
+    options: RepoFindOptions & AbortOptions = {}
+  ): Promise<DocHandle<T>> {
+    const { allowableStates = ["ready"], signal } = options
+
+    // Check if already aborted
+    if (signal?.aborted) {
+      throw new Error("Operation aborted")
+    }
+
+    const progress = this.findWithProgress<T>(id, { signal })
+
+    if ("subscribe" in progress) {
+      this.#registerHandleWithSubsystems(progress.handle)
+      return new Promise((resolve, reject) => {
+        const unsubscribe = progress.subscribe(state => {
+          if (allowableStates.includes(state.handle.state)) {
+            unsubscribe()
+            resolve(state.handle)
+          } else if (state.state === "unavailable") {
+            unsubscribe()
+            reject(new Error(`Document ${id} is unavailable`))
+          } else if (state.state === "failed") {
+            unsubscribe()
+            reject(state.error)
+          }
+        })
       })
+    } else {
+      if (progress.handle.state === READY) {
+        return progress.handle
+      }
+      // If the handle isn't ready, wait for it and then return it
+      await progress.handle.whenReady([READY, UNAVAILABLE])
+      return progress.handle
+    }
+  }
+
+  /**
+   * Loads a document without waiting for ready state
+   */
+  async #loadDocument<T>(documentId: DocumentId): Promise<DocHandle<T>> {
+    // If we have the handle cached, return it
+    if (this.#handleCache[documentId]) {
+      return this.#handleCache[documentId]
+    }
+
+    // If we don't already have the handle, make an empty one and try loading it
+    const handle = this.#getHandle<T>({ documentId })
+    const loadedDoc = await (this.storageSubsystem
+      ? this.storageSubsystem.loadDoc(handle.documentId)
+      : Promise.resolve(null))
+
+    if (loadedDoc) {
+      // We need to cast this to <T> because loadDoc operates in <unknowns>.
+      // This is really where we ought to be validating the input matches <T>.
+      handle.update(() => loadedDoc as Automerge.Doc<T>)
+      handle.doneLoading()
+    } else {
+      // Because the network subsystem might still be booting up, we wait
+      // here so that we don't immediately give up loading because we're still
+      // making our initial connection to a sync server.
+      await this.networkSubsystem.whenReady()
+      handle.request()
+    }
+
+    this.#registerHandleWithSubsystems(handle)
     return handle
   }
 
+  /**
+   * Retrieves a document by id. It gets data from the local system, but also emits a `document`
+   * event to advertise interest in the document.
+   */
+  async findClassic<T>(
+    /** The url or documentId of the handle to retrieve */
+    id: AnyDocumentId,
+    options: RepoFindOptions & AbortOptions = {}
+  ): Promise<DocHandle<T>> {
+    const documentId = interpretAsDocumentId(id)
+    const { allowableStates, signal } = options
+
+    return abortable(
+      (async () => {
+        const handle = await this.#loadDocument<T>(documentId)
+        if (!allowableStates) {
+          await handle.whenReady([READY, UNAVAILABLE])
+          if (handle.state === UNAVAILABLE && !signal?.aborted) {
+            throw new Error(`Document ${id} is unavailable`)
+          }
+        }
+        return handle
+      })(),
+      signal
+    )
+  }
+
   delete(
     /** The url or documentId of the handle to delete */
     id: AnyDocumentId
@@ -423,6 +698,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     handle.delete()
 
     delete this.#handleCache[documentId]
+    delete this.#progressCache[documentId]
     this.emit("delete-document", { documentId })
   }
 
@@ -437,8 +713,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     const documentId = interpretAsDocumentId(id)
 
     const handle = this.#getHandle({ documentId })
-    const doc = await handle.doc()
-    if (!doc) return undefined
+    const doc = handle.doc()
     return Automerge.save(doc)
   }
 
@@ -458,7 +733,16 @@ export class Repo extends EventEmitter<RepoEvents> {
     return handle
   }
 
-  subscribeToRemotes = (remotes: StorageId[]) => {}
+  subscribeToRemotes = (remotes: StorageId[]) => {
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#log("subscribeToRemotes", { remotes })
+      this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+    } else {
+      this.#log(
+        "WARN: subscribeToRemotes called but remote heads gossiping is not enabled"
+      )
+    }
+  }
 
   storageId = async (): Promise<StorageId | undefined> => {
     if (!this.storageSubsystem) {
@@ -483,11 +767,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       : Object.values(this.#handleCache)
     await Promise.all(
       handles.map(async handle => {
-        const doc = handle.docSync()
-        if (!doc) {
-          return
-        }
-        return this.storageSubsystem!.saveDoc(handle.documentId, doc)
+        return this.storageSubsystem!.saveDoc(handle.documentId, handle.doc())
       })
     )
   }
@@ -506,7 +786,9 @@ export class Repo extends EventEmitter<RepoEvents> {
       return
     }
     const handle = this.#getHandle({ documentId })
-    const doc = await handle.doc([READY, UNLOADED, DELETED, UNAVAILABLE])
+    await handle.whenReady([READY, UNLOADED, DELETED, UNAVAILABLE])
+    const doc = handle.doc()
+    // because this is an internal-ish function, we'll be extra careful about undefined docs here
     if (doc) {
       if (handle.isReady()) {
         handle.unload()
@@ -533,8 +815,7 @@ export class Repo extends EventEmitter<RepoEvents> {
   }
 
   metrics(): { documents: { [key: string]: any } } {
-    //return { documents: this.synchronizer.metrics() }
-    return { documents: {} }
+    return { documents: this.synchronizer.metrics() }
   }
 }
 
@@ -595,6 +876,10 @@ export interface RepoEvents {
   "doc-metrics": (arg: DocMetrics) => void
 }
 
+export interface RepoFindOptions {
+  allowableStates?: string[]
+}
+
 export interface DocumentPayload {
   handle: DocHandle<any>
 }