@automerge/automerge-repo 2.0.0-alpha.7 → 2.0.0-beta.1

package/src/Repo.ts

@@ -2,11 +2,20 @@ import { next as Automerge } from "@automerge/automerge/slim"
 import debug from "debug"
 import { EventEmitter } from "eventemitter3"
 import {
+  encodeHeads,
   generateAutomergeUrl,
   interpretAsDocumentId,
+  isValidAutomergeUrl,
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
-import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
+import {
+  DELETED,
+  DocHandle,
+  DocHandleEncodedChangePayload,
+  READY,
+  UNAVAILABLE,
+  UNLOADED,
+} from "./DocHandle.js"
 import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
 import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
@@ -20,8 +29,30 @@ import { StorageAdapterInterface } from "./storage/StorageAdapterInterface.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { StorageId } from "./storage/types.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
-import { SyncStatePayload } from "./synchronizer/Synchronizer.js"
-import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
+import {
+  DocSyncMetrics,
+  SyncStatePayload,
+} from "./synchronizer/Synchronizer.js"
+import type {
+  AnyDocumentId,
+  AutomergeUrl,
+  DocumentId,
+  PeerId,
+} from "./types.js"
+import { abortable, AbortOptions } from "./helpers/abortable.js"
+import { FindProgress } from "./FindProgress.js"
+
+export type FindProgressWithMethods<T> = FindProgress<T> & {
+  untilReady: (allowableStates: string[]) => Promise<DocHandle<T>>
+  peek: () => FindProgress<T>
+  subscribe: (callback: (progress: FindProgress<T>) => void) => () => void
+}
+
+export type ProgressSignal<T> = {
+  peek: () => FindProgress<T>
+  subscribe: (callback: (progress: FindProgress<T>) => void) => () => void
+  untilReady: (allowableStates: string[]) => Promise<DocHandle<T>>
+}
 
 function randomPeerId() {
   return ("peer-" + Math.random().toString(36).slice(4)) as PeerId
@@ -62,6 +93,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
   #remoteHeadsGossipingEnabled = false
+  #progressCache: Record<DocumentId, FindProgress<any>> = {}
 
   constructor({
     storage,
@@ -70,6 +102,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     sharePolicy,
     isEphemeral = storage === undefined,
     enableRemoteHeadsGossiping = false,
+    denylist = [],
   }: RepoConfig = {}) {
     super()
     this.#remoteHeadsGossipingEnabled = enableRemoteHeadsGossiping
@@ -89,7 +122,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // SYNCHRONIZER
     // The synchronizer uses the network subsystem to keep documents in sync with peers.
-    this.synchronizer = new CollectionSynchronizer(this)
+    this.synchronizer = new CollectionSynchronizer(this, denylist)
 
     // When the synchronizer emits messages, send them to peers
     this.synchronizer.on("message", message => {
@@ -97,6 +130,9 @@ export class Repo extends EventEmitter<RepoEvents> {
       networkSubsystem.send(message)
     })
 
+    // Forward metrics from doc synchronizers
+    this.synchronizer.on("metrics", event => this.emit("doc-metrics", event))
+
     if (this.#remoteHeadsGossipingEnabled) {
       this.synchronizer.on("open-doc", ({ peerId, documentId }) => {
         this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
@@ -106,6 +142,12 @@ export class Repo extends EventEmitter<RepoEvents> {
     // STORAGE
     // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
     const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
+    if (storageSubsystem) {
+      storageSubsystem.on("document-loaded", event =>
+        this.emit("doc-metrics", { type: "doc-loaded", ...event })
+      )
+    }
+
     this.storageSubsystem = storageSubsystem
 
     // NETWORK
@@ -168,16 +210,20 @@ export class Repo extends EventEmitter<RepoEvents> {
       const heads = handle.getRemoteHeads(storageId)
       const haveHeadsChanged =
         message.syncState.theirHeads &&
-        (!heads || !headsAreSame(heads, message.syncState.theirHeads))
+        (!heads ||
+          !headsAreSame(heads, encodeHeads(message.syncState.theirHeads)))
 
       if (haveHeadsChanged && message.syncState.theirHeads) {
-        handle.setRemoteHeads(storageId, message.syncState.theirHeads)
+        handle.setRemoteHeads(
+          storageId,
+          encodeHeads(message.syncState.theirHeads)
+        )
 
         if (storageId && this.#remoteHeadsGossipingEnabled) {
           this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
             message.documentId,
             storageId,
-            message.syncState.theirHeads
+            encodeHeads(message.syncState.theirHeads)
           )
         }
       }
@@ -229,18 +275,8 @@ export class Repo extends EventEmitter<RepoEvents> {
       handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
     }
 
-    handle.on("unavailable", () => {
-      this.#log("document unavailable", { documentId: handle.documentId })
-      this.emit("unavailable-document", {
-        documentId: handle.documentId,
-      })
-    })
-
     // Register the document with the synchronizer. This advertises our interest in the document.
-    this.synchronizer.addDocument(handle.documentId)
-
-    // Preserve the old event in case anyone was using it.
-    this.emit("document", { handle })
+    this.synchronizer.addDocument(handle)
   }
 
   #receiveMessage(message: RepoMessage) {
@@ -371,22 +407,16 @@ export class Repo extends EventEmitter<RepoEvents> {
    * Any peers this `Repo` is connected to for whom `sharePolicy` returns `true` will
    * be notified of the newly created DocHandle.
    *
-   * @throws if the cloned handle is not yet ready or if
-   * `clonedHandle.docSync()` returns `undefined` (i.e. the handle is unavailable).
    */
   clone<T>(clonedHandle: DocHandle<T>) {
     if (!clonedHandle.isReady()) {
       throw new Error(
         `Cloned handle is not yet in ready state.
-        (Try await handle.waitForReady() first.)`
+        (Try await handle.whenReady() first.)`
       )
     }
 
-    const sourceDoc = clonedHandle.docSync()
-    if (!sourceDoc) {
-      throw new Error("Cloned handle doesn't have a document.")
-    }
-
+    const sourceDoc = clonedHandle.doc()
     const handle = this.create<T>()
 
     handle.update(() => {
@@ -397,60 +427,263 @@ export class Repo extends EventEmitter<RepoEvents> {
     return handle
   }
 
-  /**
-   * Retrieves a document by id. It gets data from the local system, but also emits a `document`
-   * event to advertise interest in the document.
-   */
-  find<T>(
-    /** The url or documentId of the handle to retrieve */
-    id: AnyDocumentId
-  ): DocHandle<T> {
-    const documentId = interpretAsDocumentId(id)
+  findWithProgress<T>(
+    id: AnyDocumentId,
+    options: AbortOptions = {}
+  ): FindProgressWithMethods<T> | FindProgress<T> {
+    const { signal } = options
+    const abortPromise = abortable(signal)
 
-    // If we have the handle cached, return it
+    const { documentId, heads } = isValidAutomergeUrl(id)
+      ? parseAutomergeUrl(id)
+      : { documentId: interpretAsDocumentId(id), heads: undefined }
+
+    // Check handle cache first - return plain FindStep for terminal states
     if (this.#handleCache[documentId]) {
-      if (this.#handleCache[documentId].isUnavailable()) {
-        // this ensures that the event fires after the handle has been returned
-        setTimeout(() => {
-          this.#handleCache[documentId].emit("unavailable", {
-            handle: this.#handleCache[documentId],
-          })
-        })
+      const handle = this.#handleCache[documentId]
+      if (handle.state === UNAVAILABLE) {
+        const result = {
+          state: "unavailable" as const,
+          error: new Error(`Document ${id} is unavailable`),
+          handle,
+        }
+        return result
+      }
+      if (handle.state === DELETED) {
+        const result = {
+          state: "failed" as const,
+          error: new Error(`Document ${id} was deleted`),
+          handle,
+        }
+        return result
+      }
+      if (handle.state === READY) {
+        const result = {
+          state: "ready" as const,
+          handle: heads ? handle.view(heads) : handle,
+        }
+        return result
       }
-      return this.#handleCache[documentId]
     }
 
-    // If we don't already have the handle, make an empty one and try loading it
-    const handle = this.#getHandle<T>({
+    // Check progress cache for any existing signal
+    const cachedProgress = this.#progressCache[documentId]
+    if (cachedProgress) {
+      const handle = this.#handleCache[documentId]
+      // Return cached progress if we have a handle and it's either in a terminal state or loading
+      if (
+        handle &&
+        (handle.state === READY ||
+          handle.state === UNAVAILABLE ||
+          handle.state === DELETED ||
+          handle.state === "loading")
+      ) {
+        return cachedProgress as FindProgressWithMethods<T>
+      }
+    }
+
+    const handle = this.#getHandle<T>({ documentId })
+    const initial = {
+      state: "loading" as const,
+      progress: 0,
+      handle,
+    }
+
+    // Create a new progress signal
+    const progressSignal = {
+      subscribers: new Set<(progress: FindProgress<T>) => void>(),
+      currentProgress: undefined as FindProgress<T> | undefined,
+      notify: (progress: FindProgress<T>) => {
+        progressSignal.currentProgress = progress
+        progressSignal.subscribers.forEach(callback => callback(progress))
+        // Cache all states, not just terminal ones
+        this.#progressCache[documentId] = progress
+      },
+      peek: () => progressSignal.currentProgress || initial,
+      subscribe: (callback: (progress: FindProgress<T>) => void) => {
+        progressSignal.subscribers.add(callback)
+        return () => progressSignal.subscribers.delete(callback)
+      },
+    }
+
+    progressSignal.notify(initial)
+
+    // Start the loading process
+    void this.#loadDocumentWithProgress(
+      id,
       documentId,
-    }) as DocHandle<T>
+      handle,
+      progressSignal,
+      abortPromise
+    )
 
-    // Loading & network is going to be asynchronous no matter what,
-    // but we want to return the handle immediately.
-    const attemptLoad = this.storageSubsystem
-      ? this.storageSubsystem.loadDoc(handle.documentId)
-      : Promise.resolve(null)
-
-    attemptLoad
-      .then(async loadedDoc => {
-        if (loadedDoc) {
-          // uhhhh, sorry if you're reading this because we were lying to the type system
-          handle.update(() => loadedDoc as Automerge.Doc<T>)
-          handle.doneLoading()
-        } else {
-          // we want to wait for the network subsystem to be ready before
-          // we request the document. this prevents entering unavailable during initialization.
-          await this.networkSubsystem.whenReady()
-          handle.request()
+    const result = {
+      ...initial,
+      peek: progressSignal.peek,
+      subscribe: progressSignal.subscribe,
+    }
+    this.#progressCache[documentId] = result
+    return result
+  }
+
+  async #loadDocumentWithProgress<T>(
+    id: AnyDocumentId,
+    documentId: DocumentId,
+    handle: DocHandle<T>,
+    progressSignal: {
+      notify: (progress: FindProgress<T>) => void
+    },
+    abortPromise: Promise<never>
+  ) {
+    try {
+      progressSignal.notify({
+        state: "loading" as const,
+        progress: 25,
+        handle,
+      })
+
+      const loadingPromise = await (this.storageSubsystem
+        ? this.storageSubsystem.loadDoc(handle.documentId)
+        : Promise.resolve(null))
+
+      const loadedDoc = await Promise.race([loadingPromise, abortPromise])
+
+      if (loadedDoc) {
+        handle.update(() => loadedDoc as Automerge.Doc<T>)
+        handle.doneLoading()
+        progressSignal.notify({
+          state: "loading" as const,
+          progress: 50,
+          handle,
+        })
+      } else {
+        await Promise.race([this.networkSubsystem.whenReady(), abortPromise])
+        handle.request()
+        progressSignal.notify({
+          state: "loading" as const,
+          progress: 75,
+          handle,
+        })
+      }
+
+      this.#registerHandleWithSubsystems(handle)
+
+      await Promise.race([handle.whenReady([READY, UNAVAILABLE]), abortPromise])
+
+      if (handle.state === UNAVAILABLE) {
+        const unavailableProgress = {
+          state: "unavailable" as const,
+          handle,
         }
-        this.#registerHandleWithSubsystems(handle)
+        progressSignal.notify(unavailableProgress)
+        return
+      }
+      if (handle.state === DELETED) {
+        throw new Error(`Document ${id} was deleted`)
+      }
+
+      progressSignal.notify({ state: "ready" as const, handle })
+    } catch (error) {
+      progressSignal.notify({
+        state: "failed" as const,
+        error: error instanceof Error ? error : new Error(String(error)),
+        handle: this.#getHandle<T>({ documentId }),
       })
-      .catch(err => {
-        this.#log("error waiting for network", { err })
+    }
+  }
+
+  async find<T>(
+    id: AnyDocumentId,
+    options: RepoFindOptions & AbortOptions = {}
+  ): Promise<DocHandle<T>> {
+    const { allowableStates = ["ready"], signal } = options
+    const progress = this.findWithProgress<T>(id, { signal })
+
+    if ("subscribe" in progress) {
+      this.#registerHandleWithSubsystems(progress.handle)
+      return new Promise((resolve, reject) => {
+        const unsubscribe = progress.subscribe(state => {
+          if (allowableStates.includes(state.handle.state)) {
+            unsubscribe()
+            resolve(state.handle)
+          } else if (state.state === "unavailable") {
+            unsubscribe()
+            reject(new Error(`Document ${id} is unavailable`))
+          } else if (state.state === "failed") {
+            unsubscribe()
+            reject(state.error)
+          }
+        })
       })
+    } else {
+      if (progress.handle.state === READY) {
+        return progress.handle
+      }
+      // If the handle isn't ready, wait for it and then return it
+      await progress.handle.whenReady([READY, UNAVAILABLE])
+      return progress.handle
+    }
+  }
+
+  /**
+   * Loads a document without waiting for ready state
+   */
+  async #loadDocument<T>(documentId: DocumentId): Promise<DocHandle<T>> {
+    // If we have the handle cached, return it
+    if (this.#handleCache[documentId]) {
+      return this.#handleCache[documentId]
+    }
+
+    // If we don't already have the handle, make an empty one and try loading it
+    const handle = this.#getHandle<T>({ documentId })
+    const loadedDoc = await (this.storageSubsystem
+      ? this.storageSubsystem.loadDoc(handle.documentId)
+      : Promise.resolve(null))
+
+    if (loadedDoc) {
+      // We need to cast this to <T> because loadDoc operates in <unknowns>.
+      // This is really where we ought to be validating the input matches <T>.
+      handle.update(() => loadedDoc as Automerge.Doc<T>)
+      handle.doneLoading()
+    } else {
+      // Because the network subsystem might still be booting up, we wait
+      // here so that we don't immediately give up loading because we're still
+      // making our initial connection to a sync server.
+      await this.networkSubsystem.whenReady()
+      handle.request()
+    }
+
+    this.#registerHandleWithSubsystems(handle)
     return handle
   }
 
+  /**
+   * Retrieves a document by id. It gets data from the local system, but also emits a `document`
+   * event to advertise interest in the document.
+   */
+  async findClassic<T>(
+    /** The url or documentId of the handle to retrieve */
+    id: AnyDocumentId,
+    options: RepoFindOptions & AbortOptions = {}
+  ): Promise<DocHandle<T>> {
+    const documentId = interpretAsDocumentId(id)
+    const { allowableStates, signal } = options
+
+    return Promise.race([
+      (async () => {
+        const handle = await this.#loadDocument<T>(documentId)
+        if (!allowableStates) {
+          await handle.whenReady([READY, UNAVAILABLE])
+          if (handle.state === UNAVAILABLE && !signal?.aborted) {
+            throw new Error(`Document ${id} is unavailable`)
+          }
+        }
+        return handle
+      })(),
+      abortable(signal),
+    ])
+  }
+
   delete(
     /** The url or documentId of the handle to delete */
     id: AnyDocumentId
@@ -461,6 +694,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     handle.delete()
 
     delete this.#handleCache[documentId]
+    delete this.#progressCache[documentId]
     this.emit("delete-document", { documentId })
   }
 
@@ -475,8 +709,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     const documentId = interpretAsDocumentId(id)
 
     const handle = this.#getHandle({ documentId })
-    const doc = await handle.doc()
-    if (!doc) return undefined
+    const doc = handle.doc()
     return Automerge.save(doc)
   }
 
@@ -530,15 +763,46 @@ export class Repo extends EventEmitter<RepoEvents> {
       : Object.values(this.#handleCache)
     await Promise.all(
       handles.map(async handle => {
-        const doc = handle.docSync()
-        if (!doc) {
-          return
-        }
-        return this.storageSubsystem!.saveDoc(handle.documentId, doc)
+        return this.storageSubsystem!.saveDoc(handle.documentId, handle.doc())
       })
     )
   }
 
+  /**
+   * Removes a DocHandle from the handleCache.
+   * @hidden this API is experimental and may change.
+   * @param documentId - documentId of the DocHandle to remove from handleCache, if present in cache.
+   * @returns Promise<void>
+   */
+  async removeFromCache(documentId: DocumentId) {
+    if (!this.#handleCache[documentId]) {
+      this.#log(
+        `WARN: removeFromCache called but handle not found in handleCache for documentId: ${documentId}`
+      )
+      return
+    }
+    const handle = this.#getHandle({ documentId })
+    await handle.whenReady([READY, UNLOADED, DELETED, UNAVAILABLE])
+    const doc = handle.doc()
+    // because this is an internal-ish function, we'll be extra careful about undefined docs here
+    if (doc) {
+      if (handle.isReady()) {
+        handle.unload()
+      } else {
+        this.#log(
+          `WARN: removeFromCache called but handle for documentId: ${documentId} in unexpected state: ${handle.state}`
+        )
+      }
+      delete this.#handleCache[documentId]
+      // TODO: remove document from synchronizer when removeDocument is implemented
+      // this.synchronizer.removeDocument(documentId)
+    } else {
+      this.#log(
+        `WARN: removeFromCache called but doc undefined for documentId: ${documentId}`
+      )
+    }
+  }
+
   shutdown(): Promise<void> {
     this.networkSubsystem.adapters.forEach(adapter => {
       adapter.disconnect()
@@ -575,6 +839,13 @@ export interface RepoConfig {
    * Whether to enable the experimental remote heads gossiping feature
    */
   enableRemoteHeadsGossiping?: boolean
+
+  /**
+   * A list of automerge URLs which should never be loaded regardless of what
+   * messages are received or what the share policy is. This is useful to avoid
+   * loading documents that are known to be too resource intensive.
+   */
+  denylist?: AutomergeUrl[]
 }
 
 /** A function that determines whether we should share a document with a peer
@@ -598,6 +869,11 @@ export interface RepoEvents {
   "delete-document": (arg: DeleteDocumentPayload) => void
   /** A document was marked as unavailable (we don't have it and none of our peers have it) */
   "unavailable-document": (arg: DeleteDocumentPayload) => void
+  "doc-metrics": (arg: DocMetrics) => void
+}
+
+export interface RepoFindOptions {
+  allowableStates?: string[]
 }
 
 export interface DocumentPayload {
@@ -607,3 +883,17 @@ export interface DocumentPayload {
 export interface DeleteDocumentPayload {
   documentId: DocumentId
 }
+
+export type DocMetrics =
+  | DocSyncMetrics
+  | {
+      type: "doc-loaded"
+      documentId: DocumentId
+      durationMillis: number
+      numOps: number
+      numChanges: number
+    }
+  | {
+      type: "doc-denied"
+      documentId: DocumentId
+    }

package/src/helpers/abortable.ts

@@ -0,0 +1,61 @@
+/**
+ * Creates a promise that rejects when the signal is aborted.
+ *
+ * @remarks
+ * This utility creates a promise that rejects when the provided AbortSignal is aborted.
+ * It's designed to be used with Promise.race() to make operations abortable.
+ *
+ * @example
+ * ```typescript
+ * const controller = new AbortController();
+ *
+ * try {
+ *   const result = await Promise.race([
+ *     fetch('https://api.example.com/data'),
+ *     abortable(controller.signal)
+ *   ]);
+ * } catch (err) {
+ *   if (err.name === 'AbortError') {
+ *     console.log('The operation was aborted');
+ *   }
+ * }
+ *
+ * // Later, to abort:
+ * controller.abort();
+ * ```
+ *
+ * @param signal - An AbortSignal that can be used to abort the operation
+ * @param cleanup - Optional cleanup function that will be called if aborted
+ * @returns A promise that rejects with AbortError when the signal is aborted
+ * @throws {DOMException} With name "AbortError" when aborted
+ */
+export function abortable(
+  signal?: AbortSignal,
+  cleanup?: () => void
+): Promise<never> {
+  if (signal?.aborted) {
+    throw new DOMException("Operation aborted", "AbortError")
+  }
+
+  if (!signal) {
+    return new Promise(() => {}) // Never resolves
+  }
+
+  return new Promise((_, reject) => {
+    signal.addEventListener(
+      "abort",
+      () => {
+        cleanup?.()
+        reject(new DOMException("Operation aborted", "AbortError"))
+      },
+      { once: true }
+    )
+  })
+}
+
+/**
+ * Include this type in an options object to pass an AbortSignal to a function.
+ */
+export interface AbortOptions {
+  signal?: AbortSignal
+}

package/src/helpers/bufferFromHex.ts

@@ -0,0 +1,14 @@
+export const uint8ArrayFromHexString = (hexString: string): Uint8Array => {
+  if (hexString.length % 2 !== 0) {
+    throw new Error("Hex string must have an even length")
+  }
+  const bytes = new Uint8Array(hexString.length / 2)
+  for (let i = 0; i < hexString.length; i += 2) {
+    bytes[i >> 1] = parseInt(hexString.slice(i, i + 2), 16)
+  }
+  return bytes
+}
+
+export const uint8ArrayToHexString = (data: Uint8Array): string => {
+  return Array.from(data, byte => byte.toString(16).padStart(2, "0")).join("")
+}

package/src/helpers/headsAreSame.ts

@@ -1,6 +1,6 @@
-import { Heads } from "@automerge/automerge/slim/next"
 import { arraysAreEqual } from "./arraysAreEqual.js"
+import type { UrlHeads } from "../types.js"
 
-export const headsAreSame = (a: Heads, b: Heads) => {
+export const headsAreSame = (a: UrlHeads, b: UrlHeads) => {
   return arraysAreEqual(a, b)
 }