@automerge/automerge-repo 1.1.0-alpha.7 → 1.1.0

package/src/Repo.ts

@@ -7,17 +7,18 @@ import {
   parseAutomergeUrl,
 } from "./AutomergeUrl.js"
 import { DocHandle, DocHandleEncodedChangePayload } from "./DocHandle.js"
+import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
+import { headsAreSame } from "./helpers/headsAreSame.js"
 import { throttle } from "./helpers/throttle.js"
 import { NetworkAdapter, type PeerMetadata } from "./network/NetworkAdapter.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
+import { RepoMessage } from "./network/messages.js"
 import { StorageAdapter } from "./storage/StorageAdapter.js"
 import { StorageSubsystem } from "./storage/StorageSubsystem.js"
+import { StorageId } from "./storage/types.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
+import { SyncStatePayload } from "./synchronizer/Synchronizer.js"
 import type { AnyDocumentId, DocumentId, PeerId } from "./types.js"
-import { RepoMessage, SyncStateMessage } from "./network/messages.js"
-import { StorageId } from "./storage/types.js"
-import { RemoteHeadsSubscriptions } from "./RemoteHeadsSubscriptions.js"
-import { headsAreSame } from "./helpers/headsAreSame.js"
 
 /** A Repo is a collection of documents with networking, syncing, and storage capabilities. */
 /** The `Repo` is the main entry point of this library
@@ -52,6 +53,7 @@ export class Repo extends EventEmitter<RepoEvents> {
   peerMetadataByPeerId: Record<PeerId, PeerMetadata> = {}
 
   #remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+  #remoteHeadsGossipingEnabled = false
 
   constructor({
     storage,
@@ -59,8 +61,10 @@ export class Repo extends EventEmitter<RepoEvents> {
     peerId,
     sharePolicy,
     isEphemeral = storage === undefined,
+    enableRemoteHeadsGossiping = false,
   }: RepoConfig) {
     super()
+    this.#remoteHeadsGossipingEnabled = enableRemoteHeadsGossiping
     this.#log = debug(`automerge-repo:repo`)
     this.sharePolicy = sharePolicy ?? this.sharePolicy
 
@@ -77,10 +81,7 @@ export class Repo extends EventEmitter<RepoEvents> {
         }: DocHandleEncodedChangePayload<any>) => {
           void storageSubsystem.saveDoc(handle.documentId, doc)
         }
-        handle.on(
-          "heads-changed",
-          throttle(saveFn, this.saveDebounceRate)
-        )
+        handle.on("heads-changed", throttle(saveFn, this.saveDebounceRate))
 
         if (isNew) {
           // this is a new document, immediately save it
@@ -140,9 +141,11 @@ export class Repo extends EventEmitter<RepoEvents> {
       networkSubsystem.send(message)
     })
 
-    this.#synchronizer.on("open-doc", ({ peerId, documentId }) => {
-      this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
-    })
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#synchronizer.on("open-doc", ({ peerId, documentId }) => {
+        this.#remoteHeadsSubscriptions.subscribePeerToDoc(peerId, documentId)
+      })
+    }
 
     // STORAGE
     // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
@@ -154,7 +157,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
     const myPeerMetadata: Promise<PeerMetadata> = new Promise(
       // eslint-disable-next-line no-async-promise-executor -- TODO: fix
-      async (resolve) =>
+      async resolve =>
         resolve({
           storageId: await storageSubsystem?.id(),
           isEphemeral,
@@ -178,7 +181,7 @@ export class Repo extends EventEmitter<RepoEvents> {
 
       this.sharePolicy(peerId)
         .then(shouldShare => {
-          if (shouldShare) {
+          if (shouldShare && this.#remoteHeadsGossipingEnabled) {
             this.#remoteHeadsSubscriptions.addGenerousPeer(peerId)
           }
         })
@@ -218,7 +221,7 @@ export class Repo extends EventEmitter<RepoEvents> {
       if (haveHeadsChanged) {
         handle.setRemoteHeads(storageId, message.syncState.theirHeads)
 
-        if (storageId) {
+        if (storageId && this.#remoteHeadsGossipingEnabled) {
           this.#remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
             message.documentId,
             storageId,
@@ -228,45 +231,51 @@ export class Repo extends EventEmitter<RepoEvents> {
       }
     })
 
-    this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
-      this.networkSubsystem.send({
-        type: "remote-heads-changed",
-        targetId: message.targetId,
-        documentId: message.documentId,
-        newHeads: {
-          [message.storageId]: {
-            heads: message.heads,
-            timestamp: message.timestamp,
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#remoteHeadsSubscriptions.on("notify-remote-heads", message => {
+        this.networkSubsystem.send({
+          type: "remote-heads-changed",
+          targetId: message.targetId,
+          documentId: message.documentId,
+          newHeads: {
+            [message.storageId]: {
+              heads: message.heads,
+              timestamp: message.timestamp,
+            },
           },
-        },
+        })
       })
-    })
 
-    this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
-      this.#log("change-remote-subs", message)
-      for (const peer of message.peers) {
-        this.networkSubsystem.send({
-          type: "remote-subscription-change",
-          targetId: peer,
-          add: message.add,
-          remove: message.remove,
-        })
-      }
-    })
+      this.#remoteHeadsSubscriptions.on("change-remote-subs", message => {
+        this.#log("change-remote-subs", message)
+        for (const peer of message.peers) {
+          this.networkSubsystem.send({
+            type: "remote-subscription-change",
+            targetId: peer,
+            add: message.add,
+            remove: message.remove,
+          })
+        }
+      })
 
-    this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
-      const handle = this.#handleCache[message.documentId]
-      handle.setRemoteHeads(message.storageId, message.remoteHeads)
-    })
+      this.#remoteHeadsSubscriptions.on("remote-heads-changed", message => {
+        const handle = this.#handleCache[message.documentId]
+        handle.setRemoteHeads(message.storageId, message.remoteHeads)
+      })
+    }
   }
 
   #receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "remote-subscription-change":
-        this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleControlMessage(message)
+        }
         break
       case "remote-heads-changed":
-        this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        if (this.#remoteHeadsGossipingEnabled) {
+          this.#remoteHeadsSubscriptions.handleRemoteHeads(message)
+        }
         break
       case "sync":
       case "request":
@@ -280,17 +289,17 @@ export class Repo extends EventEmitter<RepoEvents> {
 
   #throttledSaveSyncStateHandlers: Record<
     StorageId,
-    (message: SyncStateMessage) => void
+    (payload: SyncStatePayload) => void
   > = {}
 
   /** saves sync state throttled per storage id, if a peer doesn't have a storage id it's sync state is not persisted */
-  #saveSyncState(message: SyncStateMessage) {
+  #saveSyncState(payload: SyncStatePayload) {
     if (!this.storageSubsystem) {
       return
     }
 
     const { storageId, isEphemeral } =
-      this.peerMetadataByPeerId[message.peerId] || {}
+      this.peerMetadataByPeerId[payload.peerId] || {}
 
     if (!storageId || isEphemeral) {
       return
@@ -299,33 +308,37 @@ export class Repo extends EventEmitter<RepoEvents> {
     let handler = this.#throttledSaveSyncStateHandlers[storageId]
     if (!handler) {
       handler = this.#throttledSaveSyncStateHandlers[storageId] = throttle(
-        ({ documentId, syncState }: SyncStateMessage) => {
-          this.storageSubsystem!.saveSyncState(documentId, storageId, syncState)
-            .catch(err => {
-              this.#log("error saving sync state", { err })
-            })
+        ({ documentId, syncState }: SyncStatePayload) => {
+          void this.storageSubsystem!.saveSyncState(
+            documentId,
+            storageId,
+            syncState
+          )
         },
         this.saveDebounceRate
       )
     }
 
-    handler(message)
+    handler(payload)
   }
 
   /** Returns an existing handle if we have it; creates one otherwise. */
-  #getHandle<T>(
+  #getHandle<T>({
+    documentId,
+    isNew,
+    initialValue,
+  }: {
     /** The documentId of the handle to look up or create */
-    documentId: DocumentId,
-
-    /** If we know we're creating a new document, specify this so we can have access to it immediately */
+    documentId: DocumentId /** If we know we're creating a new document, specify this so we can have access to it immediately */
     isNew: boolean
-  ) {
+    initialValue?: T
+  }) {
     // If we have the handle cached, return it
     if (this.#handleCache[documentId]) return this.#handleCache[documentId]
 
     // If not, create a new handle, cache it, and return it
     if (!documentId) throw new Error(`Invalid documentId ${documentId}`)
-    const handle = new DocHandle<T>(documentId, { isNew })
+    const handle = new DocHandle<T>(documentId, { isNew, initialValue })
     this.#handleCache[documentId] = handle
     return handle
   }
@@ -345,32 +358,18 @@ export class Repo extends EventEmitter<RepoEvents> {
   }
 
   /**
-   * Creates a new document and returns a handle to it. The initial value of the document is
-   * an empty object `{}`. Its documentId is generated by the system. we emit a `document` event
-   * to advertise interest in the document.
+   * Creates a new document and returns a handle to it. The initial value of the document is an
+   * empty object `{}` unless an initial value is provided. Its documentId is generated by the
+   * system. we emit a `document` event to advertise interest in the document.
    */
-  create<T>(): DocHandle<T> {
-    // TODO:
-    // either
-    // - pass an initial value and do something like this to ensure that you get a valid initial value
-
-    // const myInitialValue = {
-    //   tasks: [],
-    //   filter: "all",
-    //
-    // const guaranteeInitialValue = (doc: any) => {
-    // if (!doc.tasks) doc.tasks = []
-    // if (!doc.filter) doc.filter = "all"
-
-    //   return { ...myInitialValue, ...doc }
-    // }
-
-    // or
-    // - pass a "reify" function that takes a `<any>` and returns `<T>`
-
+  create<T>(initialValue?: T): DocHandle<T> {
     // Generate a new UUID and store it in the buffer
     const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
-    const handle = this.#getHandle<T>(documentId, true) as DocHandle<T>
+    const handle = this.#getHandle<T>({
+      documentId,
+      isNew: true,
+      initialValue,
+    }) as DocHandle<T>
     this.emit("document", { handle, isNew: true })
     return handle
   }
@@ -436,7 +435,10 @@ export class Repo extends EventEmitter<RepoEvents> {
       return this.#handleCache[documentId]
     }
 
-    const handle = this.#getHandle<T>(documentId, false) as DocHandle<T>
+    const handle = this.#getHandle<T>({
+      documentId,
+      isNew: false,
+    }) as DocHandle<T>
     this.emit("document", { handle, isNew: false })
     return handle
   }
@@ -447,13 +449,29 @@ export class Repo extends EventEmitter<RepoEvents> {
   ) {
     const documentId = interpretAsDocumentId(id)
 
-    const handle = this.#getHandle(documentId, false)
+    const handle = this.#getHandle({ documentId, isNew: false })
     handle.delete()
 
     delete this.#handleCache[documentId]
     this.emit("delete-document", { documentId })
   }
 
+  /**
+   * Exports a document to a binary format.
+   * @param id - The url or documentId of the handle to export
+   *
+   * @returns Promise<Uint8Array | undefined> - A Promise containing the binary document,
+   * or undefined if the document is unavailable.
+   */
+  async export(id: AnyDocumentId): Promise<Uint8Array | undefined> {
+    const documentId = interpretAsDocumentId(id)
+
+    const handle = this.#getHandle({ documentId, isNew: false })
+    const doc = await handle.doc()
+    if (!doc) return undefined
+    return Automerge.save(doc)
+  }
+
   /**
    * Imports document binary into the repo.
    * @param binary - The binary to import
@@ -471,8 +489,14 @@ export class Repo extends EventEmitter<RepoEvents> {
   }
 
   subscribeToRemotes = (remotes: StorageId[]) => {
-    this.#log("subscribeToRemotes", { remotes })
-    this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+    if (this.#remoteHeadsGossipingEnabled) {
+      this.#log("subscribeToRemotes", { remotes })
+      this.#remoteHeadsSubscriptions.subscribeToRemotes(remotes)
+    } else {
+      this.#log(
+        "WARN: subscribeToRemotes called but remote heads gossiping is not enabled"
+      )
+    }
   }
 
   storageId = async (): Promise<StorageId | undefined> => {
@@ -503,6 +527,11 @@ export interface RepoConfig {
    * all peers). A server only syncs documents that a peer explicitly requests by ID.
    */
   sharePolicy?: SharePolicy
+
+  /**
+   * Whether to enable the experimental remote heads gossiping feature
+   */
+  enableRemoteHeadsGossiping?: boolean
 }
 
 /** A function that determines whether we should share a document with a peer

package/src/helpers/pause.ts

@@ -1,14 +1,6 @@
+/* c8 ignore start */
+
 export const pause = (t = 0) =>
   new Promise<void>(resolve => setTimeout(() => resolve(), t))
 
-export function rejectOnTimeout<T>(
-  promise: Promise<T>,
-  millis: number
-): Promise<T> {
-  return Promise.race([
-    promise,
-    pause(millis).then(() => {
-      throw new Error("timeout exceeded")
-    }),
-  ])
-}
+/* c8 ignore end */

package/src/helpers/withTimeout.ts

@@ -1,3 +1,4 @@
+/* c8 ignore start */
 /**
  * If `promise` is resolved before `t` ms elapse, the timeout is cleared and the result of the
  * promise is returned. If the timeout ends first, a `TimeoutError` is thrown.
@@ -26,3 +27,4 @@ export class TimeoutError extends Error {
     this.name = "TimeoutError"
   }
 }
+/* c8 ignore end */

package/src/index.ts

@@ -34,7 +34,7 @@ export {
 } from "./AutomergeUrl.js"
 export { Repo } from "./Repo.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
-export { isValidRepoMessage } from "./network/messages.js"
+export { isRepoMessage } from "./network/messages.js"
 export { StorageAdapter } from "./storage/StorageAdapter.js"
 
 /** @hidden **/

package/src/network/NetworkAdapter.ts

@@ -1,13 +1,15 @@
+/* c8 ignore start */
+
 import { EventEmitter } from "eventemitter3"
 import { PeerId } from "../types.js"
 import { Message } from "./messages.js"
 import { StorageId } from "../storage/types.js"
 
-/** 
+/**
  * Describes a peer intent to the system
  * storageId: the key for syncState to decide what the other peer already has
  * isEphemeral: to decide if we bother recording this peer's sync state
- * 
+ *
  */
 export interface PeerMetadata {
   storageId?: StorageId

package/src/network/NetworkSubsystem.ts

@@ -11,7 +11,7 @@ import {
   MessageContents,
   RepoMessage,
   isEphemeralMessage,
-  isValidRepoMessage,
+  isRepoMessage,
 } from "./messages.js"
 
 type EphemeralMessageSource = `${PeerId}:${SessionId}`
@@ -73,7 +73,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     })
 
     networkAdapter.on("message", msg => {
-      if (!isValidRepoMessage(msg)) {
+      if (!isRepoMessage(msg)) {
         this.#log(`invalid message: ${JSON.stringify(msg)}`)
         return
       }
@@ -146,7 +146,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     }
 
     const outbound = prepareMessage(message)
-    this.#log("sending message", outbound)
+    this.#log("sending message %o", outbound)
     peer.send(outbound as RepoMessage)
   }
 

package/src/network/messages.ts

@@ -1,17 +1,27 @@
 import { SyncState } from "@automerge/automerge"
-import { DocumentId, PeerId, SessionId } from "../types.js"
 import { StorageId } from "../storage/types.js"
+import { DocumentId, PeerId, SessionId } from "../types.js"
+
+export type Message = {
+  type: string
+
+  /** The peer ID of the sender of this message */
+  senderId: PeerId
+
+  /** The peer ID of the recipient of this message */
+  targetId: PeerId
+
+  data?: Uint8Array
+
+  documentId?: DocumentId
+}
 
 /**
  * A sync message for a particular document
  */
 export type SyncMessage = {
   type: "sync"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
   /** The automerge sync message */
@@ -21,53 +31,50 @@ export type SyncMessage = {
   documentId: DocumentId
 }
 
-/** An ephemeral message
+/**
+ * An ephemeral message.
  *
  * @remarks
- * Ephemeral messages are not persisted anywhere and have no particular
- * structure. `automerge-repo` will gossip them around, in order to avoid
- * eternal loops of ephemeral messages every message has a session ID, which
- * is a random number generated by the sender at startup time, and a sequence
- * number. The combination of these two things allows us to discard messages
- * we have already seen.
+ * Ephemeral messages are not persisted anywhere. The data property can be used by the application
+ * as needed. The repo gossips these around.
+ *
+ * In order to avoid infinite loops of ephemeral messages, every message has (a) a session ID, which
+ * is a random number generated by the sender at startup time; and (b) a sequence number. The
+ * combination of these two things allows us to discard messages we have already seen.
  * */
 export type EphemeralMessage = {
   type: "ephemeral"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** A sequence number which must be incremented for each message sent by this peer */
+  /** A sequence number which must be incremented for each message sent by this peer. */
   count: number
 
-  /** The ID of the session this message is part of. The sequence number for a given session always increases */
+  /** The ID of the session this message is part of. The sequence number for a given session always increases. */
   sessionId: SessionId
 
-  /** The document ID this message pertains to */
+  /** The document ID this message pertains to. */
   documentId: DocumentId
 
-  /** The actual data of the message */
+  /** The actual data of the message. */
   data: Uint8Array
 }
 
-/** Sent by a {@link Repo} to indicate that it does not have the document and none of it's connected peers do either */
+/**
+ * Sent by a {@link Repo} to indicate that it does not have the document and none of its connected
+ * peers do either.
+ */
 export type DocumentUnavailableMessage = {
   type: "doc-unavailable"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
   /** The document which the peer claims it doesn't have */
   documentId: DocumentId
 }
 
-/** Sent by a {@link Repo} to request a document from a peer
+/**
+ * Sent by a {@link Repo} to request a document from a peer.
  *
  * @remarks
  * This is identical to a {@link SyncMessage} except that it is sent by a {@link Repo}
@@ -75,47 +82,43 @@ export type DocumentUnavailableMessage = {
  * */
 export type RequestMessage = {
   type: "request"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** The initial automerge sync message */
+  /** The automerge sync message */
   data: Uint8Array
 
-  /** The document ID this message requests */
+  /** The document ID of the document this message is for */
   documentId: DocumentId
 }
 
-/** (anticipating work in progress) */
-export type AuthMessage<TPayload = any> = {
-  type: "auth"
-
-  /** The peer ID of the sender of this message */
-  senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
-  targetId: PeerId
-
-  /** The payload of the auth message (up to the specific auth provider) */
-  payload: TPayload
-}
-
+/**
+ * Sent by a {@link Repo} to add or remove storage IDs from a remote peer's subscription.
+ */
 export type RemoteSubscriptionControlMessage = {
   type: "remote-subscription-change"
   senderId: PeerId
   targetId: PeerId
+
+  /** The storage IDs to add to the subscription */
   add?: StorageId[]
+
+  /** The storage IDs to remove from the subscription */
   remove?: StorageId[]
 }
 
+/**
+ * Sent by a {@link Repo} to indicate that the heads of a document have changed on a remote peer.
+ */
 export type RemoteHeadsChanged = {
   type: "remote-heads-changed"
   senderId: PeerId
   targetId: PeerId
+
+  /** The document ID of the document that has changed */
   documentId: DocumentId
+
+  /** The document's new heads */
   newHeads: { [key: StorageId]: { heads: string[]; timestamp: number } }
 }
 
@@ -128,19 +131,17 @@ export type RepoMessage =
   | RemoteSubscriptionControlMessage
   | RemoteHeadsChanged
 
+/** These are message types that are handled by the {@link CollectionSynchronizer}.*/
 export type DocMessage =
   | SyncMessage
   | EphemeralMessage
   | RequestMessage
   | DocumentUnavailableMessage
 
-/** These are all the message types that a {@link NetworkAdapter} might see. */
-export type Message = RepoMessage | AuthMessage
-
 /**
  * The contents of a message, without the sender ID or other properties added by the {@link NetworkSubsystem})
  */
-export type MessageContents<T extends Message = Message> =
+export type MessageContents<T extends Message = RepoMessage> =
   T extends EphemeralMessage
     ? Omit<T, "senderId" | "count" | "sessionId">
     : Omit<T, "senderId">
@@ -160,16 +161,13 @@ export interface OpenDocMessage {
 
 // TYPE GUARDS
 
-export const isValidRepoMessage = (message: Message): message is RepoMessage =>
-  typeof message === "object" &&
-  typeof message.type === "string" &&
-  typeof message.senderId === "string" &&
-  (isSyncMessage(message) ||
-    isEphemeralMessage(message) ||
-    isRequestMessage(message) ||
-    isDocumentUnavailableMessage(message) ||
-    isRemoteSubscriptionControlMessage(message) ||
-    isRemoteHeadsChanged(message))
+export const isRepoMessage = (message: Message): message is RepoMessage =>
+  isSyncMessage(message) ||
+  isEphemeralMessage(message) ||
+  isRequestMessage(message) ||
+  isDocumentUnavailableMessage(message) ||
+  isRemoteSubscriptionControlMessage(message) ||
+  isRemoteHeadsChanged(message)
 
 // prettier-ignore
 export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage => 
@@ -184,9 +182,8 @@ export const isSyncMessage = (msg: Message): msg is SyncMessage =>
 export const isEphemeralMessage = (msg: Message): msg is EphemeralMessage =>
   msg.type === "ephemeral"
 
-export const isRemoteSubscriptionControlMessage = (
-  msg: Message
-): msg is RemoteSubscriptionControlMessage =>
+// prettier-ignore
+export const isRemoteSubscriptionControlMessage = (msg: Message): msg is RemoteSubscriptionControlMessage =>
   msg.type === "remote-subscription-change"
 
 export const isRemoteHeadsChanged = (msg: Message): msg is RemoteHeadsChanged =>

package/src/synchronizer/CollectionSynchronizer.ts

@@ -117,6 +117,7 @@ export class CollectionSynchronizer extends Synchronizer {
   }
 
   // TODO: implement this
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   removeDocument(documentId: DocumentId) {
     throw new Error("not implemented")
   }