@automerge/automerge-repo 2.0.0-alpha.1 → 2.0.0-alpha.12

package/src/DocHandle.ts

@@ -72,6 +72,9 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
           this.emit("delete", { handle: this })
           return { doc: A.init() }
         }),
+        onUnload: assign(() => {
+          return { doc: A.init() }
+        }),
         onUnavailable: () => {
           this.emit("unavailable", { handle: this })
         },
@@ -86,6 +89,7 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
       context: { documentId, doc },
       on: {
         UPDATE: { actions: "onUpdate" },
+        UNLOAD: ".unloaded",
         DELETE: ".deleted",
       },
       states: {
@@ -113,6 +117,12 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
           on: { DOC_READY: "ready" },
         },
         ready: {},
+        unloaded: {
+          entry: "onUnload",
+          on: {
+            RELOAD: "loading",
+          },
+        },
         deleted: { entry: "onDelete", type: "final" },
       },
     })
@@ -131,7 +141,7 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
 
     // Start the machine, and send a create or find event to get things going
     this.#machine.start()
-    this.#machine.send({ type: BEGIN })
+    this.begin()
   }
 
   // PRIVATE
@@ -203,6 +213,14 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
    */
   isReady = () => this.inState(["ready"])
 
+  /**
+   * @returns true if the document has been unloaded.
+   *
+   * Unloaded documents are freed from memory but not removed from local storage. It's not currently
+   * possible at runtime to reload an unloaded document.
+   */
+  isUnloaded = () => this.inState(["unloaded"])
+
   /**
    * @returns true if the document has been marked as deleted.
    *
@@ -291,6 +309,94 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
     return A.getHeads(this.#doc)
   }
 
+  begin() {
+    this.#machine.send({ type: BEGIN })
+  }
+
+  /**
+   * Creates a fixed "view" of an automerge document at the given point in time represented
+   * by the `heads` passed in. The return value is the same type as docSync() and will return
+   * undefined if the object hasn't finished loading.
+   *
+   * @remarks
+   * A point-in-time in an automerge document is an *array* of heads since there may be
+   * concurrent edits. This API just returns a topologically sorted history of all edits
+   * so every previous entry will be (in some sense) before later ones, but the set of all possible
+   * history views would be quite large under concurrency (every thing in each branch against each other).
+   * There might be a clever way to think about this, but we haven't found it yet, so for now at least
+   * we present a single traversable view which excludes concurrency.
+   * @returns The individual heads for every change in the document.
+   */
+  history(): A.Heads[] | undefined {
+    if (!this.isReady()) {
+      return undefined
+    }
+    // This just returns all the heads as individual strings.
+
+    return A.topoHistoryTraversal(this.#doc).map(h => [h]) as A.Heads[]
+  }
+
+  /**
+   * Creates a fixed "view" of an automerge document at the given point in time represented
+   * by the `heads` passed in. The return value is the same type as docSync() and will return
+   * undefined if the object hasn't finished loading.
+   *
+   * @remarks
+   * Note that our Typescript types do not consider change over time and the current version
+   * of Automerge doesn't check types at runtime, so if you go back to an old set of heads
+   * that doesn't match the heads here, Typescript will not save you.
+   *
+   * @returns An Automerge.Doc<T> at the point in time.
+   */
+  view(heads: A.Heads): A.Doc<T> | undefined {
+    if (!this.isReady()) {
+      return undefined
+    }
+    return A.view(this.#doc, heads)
+  }
+
+  /**
+   * Returns a set of Patch operations that will move a materialized document from one state to another
+   * if applied.
+   *
+   * @remarks
+   * We allow specifying both a from/to heads or just a single comparison point, in which case
+   * the base will be the current document heads.
+   *
+   * @returns Automerge patches that go from one document state to the other. Use view() to get the full state.
+   */
+  diff(first: A.Heads, second?: A.Heads): A.Patch[] | undefined {
+    if (!this.isReady()) {
+      return undefined
+    }
+    // We allow only one set of heads to be specified, in which case we use the doc's heads
+    const from = second ? first : this.heads() || [] // because we guard above this should always have useful data
+    const to = second ? second : first
+    return A.diff(this.#doc, from, to)
+  }
+
+  /**
+   * `metadata(head?)` allows you to look at the metadata for a change
+   * this can be used to build history graphs to find commit messages and edit times.
+   * this interface.
+   *
+   * @remarks
+   * I'm really not convinced this is the right way to surface this information so
+   * I'm leaving this API "hidden".
+   *
+   * @hidden
+   */
+  metadata(change?: string): A.DecodedChange | undefined {
+    if (!this.isReady()) {
+      return undefined
+    }
+    if (!change) {
+      change = this.heads()![0]
+    }
+    // we return undefined instead of null by convention in this API
+    return A.inspectChange(this.#doc, change) || undefined
+  }
+
   /**
    * `update` is called any time we have a new document state; could be
    * from a local change, a remote change, or a new document from storage.
@@ -421,6 +527,16 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
     if (this.#state === "loading") this.#machine.send({ type: REQUEST })
   }
 
+  /** Called by the repo to free memory used by the document. */
+  unload() {
+    this.#machine.send({ type: UNLOAD })
+  }
+
+  /** Called by the repo to reuse an unloaded handle. */
+  reload() {
+    this.#machine.send({ type: RELOAD })
+  }
+
   /** Called by the repo when the document is deleted. */
   delete() {
     this.#machine.send({ type: DELETE })
@@ -439,6 +555,10 @@ export class DocHandle<T> extends EventEmitter<DocHandleEvents<T>> {
       data: encode(message),
     })
   }
+
+  metrics(): { numOps: number; numChanges: number } {
+    return A.stats(this.#doc)
+  }
 }
 
 //  TYPES
@@ -539,6 +659,8 @@ export const HandleState = {
   REQUESTING: "requesting",
   /** The document is available */
   READY: "ready",
+  /** The document has been unloaded from the handle, to free memory usage */
+  UNLOADED: "unloaded",
   /** The document has been deleted from the repo */
   DELETED: "deleted",
   /** The document was not available in storage or from any connected peers */
@@ -546,8 +668,15 @@ export const HandleState = {
 } as const
 export type HandleState = (typeof HandleState)[keyof typeof HandleState]
 
-export const { IDLE, LOADING, REQUESTING, READY, DELETED, UNAVAILABLE } =
-  HandleState
+export const {
+  IDLE,
+  LOADING,
+  REQUESTING,
+  READY,
+  UNLOADED,
+  DELETED,
+  UNAVAILABLE,
+} = HandleState
 
 // context
 
@@ -567,14 +696,18 @@ type DocHandleEvent<T> =
       type: typeof UPDATE
       payload: { callback: (doc: A.Doc<T>) => A.Doc<T> }
     }
-  | { type: typeof TIMEOUT }
+  | { type: typeof UNLOAD }
+  | { type: typeof RELOAD }
   | { type: typeof DELETE }
+  | { type: typeof TIMEOUT }
   | { type: typeof DOC_UNAVAILABLE }
 
 const BEGIN = "BEGIN"
 const REQUEST = "REQUEST"
 const DOC_READY = "DOC_READY"
 const UPDATE = "UPDATE"
+const UNLOAD = "UNLOAD"
+const RELOAD = "RELOAD"
 const DELETE = "DELETE"
 const TIMEOUT = "TIMEOUT"
 const DOC_UNAVAILABLE = "DOC_UNAVAILABLE"

package/src/Repo.ts

@@ -6,7 +6,14 @@ import {
   interpretAsDocumentId,
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
-import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
+import {
+  DELETED,
+  DocHandle,
+  DocHandleEncodedChangePayload,
+  READY,
+  UNAVAILABLE,
+  UNLOADED,
+} from "./DocHandle.js"
 import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
 import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
@@ -20,7 +27,10 @@ import { StorageAdapterInterface } from "./storage/StorageAdapterInterface.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { StorageId } from "./storage/types.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
-import { SyncStatePayload } from "./synchronizer/Synchronizer.js"
+import {
+  DocSyncMetrics,
+  SyncStatePayload,
+} from "./synchronizer/Synchronizer.js"
 import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
 
 function randomPeerId() {
@@ -49,7 +59,8 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   #handleCache: Record<DocumentId, DocHandle<any>> = {}
 
-  #synchronizer: CollectionSynchronizer
+  /** @hidden */
+  synchronizer: CollectionSynchronizer
 
   /** By default, we share generously with all peers. */
   /** @hidden */
@@ -75,33 +86,6 @@ export class Repo extends EventEmitter<RepoEvents> {
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
 
-    // DOC COLLECTION
-
-    // The `document` event is fired by the DocCollection any time we create a new document or look
-    // up a document by ID. We listen for it in order to wire up storage and network synchronization.
-    this.on("document", async ({ handle }) => {
-      if (storageSubsystem) {
-        // Save when the document changes, but no more often than saveDebounceRate.
-        const saveFn = ({
-          handle,
-          doc,
-        }: DocHandleEncodedChangePayload<any>) => {
-          void storageSubsystem.saveDoc(handle.documentId, doc)
-        }
-        handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
-      }
-
-      handle.on("unavailable", () => {
-        this.#log("document unavailable", { documentId: handle.documentId })
-        this.emit("unavailable-document", {
-          documentId: handle.documentId,
-        })
-      })
-
-      // Register the document with the synchronizer. This advertises our interest in the document.
-      this.#synchronizer.addDocument(handle.documentId)
-    })
-
     this.on("delete-document", ({ documentId }) => {
       // TODO Pass the delete on to the network
       // synchronizer.removeDocument(documentId)
@@ -115,16 +99,19 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     // SYNCHRONIZER
     // The synchronizer uses the network subsystem to keep documents in sync with peers.
-    this.#synchronizer = new CollectionSynchronizer(this)
+    this.synchronizer = new CollectionSynchronizer(this)
 
     // When the synchronizer emits messages, send them to peers
-    this.#synchronizer.on("message", message => {
+    this.synchronizer.on("message", message => {
       this.#log(`sending ${message.type} message to ${message.targetId}`)
       networkSubsystem.send(message)
     })
 
+    // Forward metrics from doc synchronizers
+    this.synchronizer.on("metrics", event => this.emit("doc-metrics", event))
+
     if (this.#remoteHeadsGossipingEnabled) {
-      this.#synchronizer.on("open-doc", ({ peerId, documentId }) => {
+      this.synchronizer.on("open-doc", ({ peerId, documentId }) => {
         this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
       })
     }
@@ -132,6 +119,12 @@ export class Repo extends EventEmitter<RepoEvents> {
     // STORAGE
     // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
     const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
+    if (storageSubsystem) {
+      storageSubsystem.on("document-loaded", event =>
+        this.emit("doc-metrics", { type: "doc-loaded", ...event })
+      )
+    }
+
     this.storageSubsystem = storageSubsystem
 
     // NETWORK
@@ -167,12 +160,12 @@ export class Repo extends EventEmitter<RepoEvents> {
           console.log("error in share policy", { err })
         })
 
-      this.#synchronizer.addPeer(peerId)
+      this.synchronizer.addPeer(peerId)
     })
 
     // When a peer disconnects, remove it from the synchronizer
     networkSubsystem.on("peer-disconnected", ({ peerId }) => {
-      this.#synchronizer.removePeer(peerId)
+      this.synchronizer.removePeer(peerId)
       this.#remoteHeadsSubscriptions.removePeer(peerId)
     })
 
@@ -181,7 +174,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       this.#receiveMessage(msg)
     })
 
-    this.#synchronizer.on("sync-state", message => {
+    this.synchronizer.on("sync-state", message => {
       this.#saveSyncState(message)
 
       const handle = this.#handleCache[message.documentId]
@@ -243,6 +236,32 @@ export class Repo extends EventEmitter<RepoEvents> {
     }
   }
 
+  // The `document` event is fired by the DocCollection any time we create a new document or look
+  // up a document by ID. We listen for it in order to wire up storage and network synchronization.
+  #registerHandleWithSubsystems(handle: DocHandle<any>) {
+    const { storageSubsystem } = this
+    if (storageSubsystem) {
+      // Save when the document changes, but no more often than saveDebounceRate.
+      const saveFn = ({ handle, doc }: DocHandleEncodedChangePayload<any>) => {
+        void storageSubsystem.saveDoc(handle.documentId, doc)
+      }
+      handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
+    }
+
+    handle.on("unavailable", () => {
+      this.#log("document unavailable", { documentId: handle.documentId })
+      this.emit("unavailable-document", {
+        documentId: handle.documentId,
+      })
+    })
+
+    // Register the document with the synchronizer. This advertises our interest in the document.
+    this.synchronizer.addDocument(handle.documentId)
+
+    // Preserve the old event in case anyone was using it.
+    this.emit("document", { handle })
+  }
+
   #receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "remote-subscription-change":
@@ -259,7 +278,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       case "request":
       case "ephemeral":
       case "doc-unavailable":
-        this.#synchronizer.receiveMessage(message).catch(err => {
+        this.synchronizer.receiveMessage(message).catch(err => {
           console.log("error receiving message", { err })
         })
     }
@@ -324,7 +343,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   /** Returns a list of all connected peer ids */
   get peers(): PeerId[] {
-    return this.#synchronizer.peers
+    return this.synchronizer.peers
   }
 
   getStorageIdOfPeer(peerId: PeerId): StorageId | undefined {
@@ -343,7 +362,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       documentId,
     }) as DocHandle<T>
 
-    this.emit("document", { handle })
+    this.#registerHandleWithSubsystems(handle)
 
     handle.update(() => {
       let nextDoc: Automerge.Doc<T>
@@ -378,7 +397,7 @@ export class Repo extends EventEmitter<RepoEvents> {
     if (!clonedHandle.isReady()) {
       throw new Error(
         `Cloned handle is not yet in ready state.
-        (Try await handle.waitForReady() first.)`
+        (Try await handle.whenReady() first.)`
       )
     }
 
@@ -425,29 +444,29 @@ export class Repo extends EventEmitter<RepoEvents> {
       documentId,
     }) as DocHandle<T>
 
-    // Try to load from disk before telling anyone else about it
-    if (this.storageSubsystem) {
-      void this.storageSubsystem.loadDoc(handle.documentId).then(loadedDoc => {
+    // Loading & network is going to be asynchronous no matter what,
+    // but we want to return the handle immediately.
+    const attemptLoad = this.storageSubsystem
+      ? this.storageSubsystem.loadDoc(handle.documentId)
+      : Promise.resolve(null)
+
+    attemptLoad
+      .then(async loadedDoc => {
         if (loadedDoc) {
           // uhhhh, sorry if you're reading this because we were lying to the type system
           handle.update(() => loadedDoc as Automerge.Doc<T>)
           handle.doneLoading()
         } else {
-          this.networkSubsystem
-            .whenReady()
-            .then(() => {
-              handle.request()
-            })
-            .catch(err => {
-              this.#log("error waiting for network", { err })
-            })
-          this.emit("document", { handle })
+          // we want to wait for the network subsystem to be ready before
+          // we request the document. this prevents entering unavailable during initialization.
+          await this.networkSubsystem.whenReady()
+          handle.request()
         }
+        this.#registerHandleWithSubsystems(handle)
+      })
+      .catch(err => {
+        this.#log("error waiting for network", { err })
       })
-    } else {
-      handle.request()
-      this.emit("document", { handle })
-    }
     return handle
   }
 
@@ -539,12 +558,49 @@ export class Repo extends EventEmitter<RepoEvents> {
     )
   }
 
+  /**
+   * Removes a DocHandle from the handleCache.
+   * @hidden this API is experimental and may change.
+   * @param documentId - documentId of the DocHandle to remove from handleCache, if present in cache.
+   * @returns Promise<void>
+   */
+  async removeFromCache(documentId: DocumentId) {
+    if (!this.#handleCache[documentId]) {
+      this.#log(
+        `WARN: removeFromCache called but handle not found in handleCache for documentId: ${documentId}`
+      )
+      return
+    }
+    const handle = this.#getHandle({ documentId })
+    const doc = await handle.doc([READY, UNLOADED, DELETED, UNAVAILABLE])
+    if (doc) {
+      if (handle.isReady()) {
+        handle.unload()
+      } else {
+        this.#log(
+          `WARN: removeFromCache called but handle for documentId: ${documentId} in unexpected state: ${handle.state}`
+        )
+      }
+      delete this.#handleCache[documentId]
+      // TODO: remove document from synchronizer when removeDocument is implemented
+      // this.synchronizer.removeDocument(documentId)
+    } else {
+      this.#log(
+        `WARN: removeFromCache called but doc undefined for documentId: ${documentId}`
+      )
+    }
+  }
+
   shutdown(): Promise<void> {
     this.networkSubsystem.adapters.forEach(adapter => {
       adapter.disconnect()
     })
     return this.flush()
   }
+
+  metrics(): { documents: { [key: string]: any } } {
+    return { documents: this.synchronizer.metrics() }
+  }
 }
 
 export interface RepoConfig {
@@ -594,6 +650,7 @@ export interface RepoEvents {
   "delete-document": (arg: DeleteDocumentPayload) => void
   /** A document was marked as unavailable (we don't have it and none of our peers have it) */
   "unavailable-document": (arg: DeleteDocumentPayload) => void
+  "doc-metrics": (arg: DocMetrics) => void
 }
 
 export interface DocumentPayload {
@@ -603,3 +660,13 @@ export interface DocumentPayload {
 export interface DeleteDocumentPayload {
   documentId: DocumentId
 }
+
+export type DocMetrics =
+  | DocSyncMetrics
+  | {
+      type: "doc-loaded"
+      documentId: DocumentId
+      durationMillis: number
+      numOps: number
+      numChanges: number
+    }

package/src/entrypoints/fullfat.ts

@@ -7,5 +7,4 @@ export * from "../index.js"
 // disable
 //
 // eslint-disable-next-line automerge-slimport/enforce-automerge-slim-import
-import { next as Am } from "@automerge/automerge"
-Am.init()
+import "@automerge/automerge"

package/src/entrypoints/slim.ts

@@ -1,2 +1,4 @@
 export * from "../index.js"
 export { initializeBase64Wasm, initializeWasm } from "@automerge/automerge/slim"
+// TODO: temporary work-around during alpha.
+export * as Automerge from "@automerge/automerge/slim"