@automerge/automerge-repo 1.0.6 → 1.0.8

package/src/network/messages.ts

@@ -1,161 +1,70 @@
-// utilities
-import { SessionId } from "../EphemeralData.js"
-export { type SessionId } from "../EphemeralData.js"
-import { DocumentId, PeerId } from "../types.js"
-
-export function isValidMessage(
-  message: NetworkAdapterMessage
-): message is
-  | SyncMessage
-  | EphemeralMessage
-  | RequestMessage
-  | DocumentUnavailableMessage {
-  return (
-    typeof message === "object" &&
-    typeof message.type === "string" &&
-    typeof message.senderId === "string" &&
-    (isSyncMessage(message) ||
-      isEphemeralMessage(message) ||
-      isRequestMessage(message) ||
-      isDocumentUnavailableMessage(message))
-  )
-}
-
-export function isDocumentUnavailableMessage(
-  message: NetworkAdapterMessage
-): message is DocumentUnavailableMessage {
-  return message.type === "doc-unavailable"
-}
-
-export function isRequestMessage(
-  message: NetworkAdapterMessage
-): message is RequestMessage {
-  return message.type === "request"
-}
-
-export function isSyncMessage(
-  message: NetworkAdapterMessage
-): message is SyncMessage {
-  return message.type === "sync"
-}
-
-export function isEphemeralMessage(
-  message: NetworkAdapterMessage | MessageContents
-): message is EphemeralMessage | EphemeralMessageContents {
-  return message.type === "ephemeral"
-}
-
-export interface SyncMessageEnvelope {
-  senderId: PeerId
-}
+import { DocumentId, PeerId, SessionId } from "../types.js"
 
-export interface SyncMessageContents {
-  type: "sync"
-  data: Uint8Array
-  targetId: PeerId
-  documentId: DocumentId
-}
-
-type static_assert<T extends true> = never
-
-// export type SyncMessage = SyncMessageEnvelope & SyncMessageContents
-// we inline the definitions here rather than using the above type alias because
-// otherwise typedoc can't produce nice docs. We use this `static_assert` thing
-// to make sure the types continue to line up though.
-type _check_sync_message = static_assert<SyncMessage extends SyncMessageEnvelope & SyncMessageContents ? true : false>
 /**
  * A sync message for a particular document
  */
 export type SyncMessage = {
+  type: "sync"
+
   /** The peer ID of the sender of this message */
   senderId: PeerId
-  type: "sync"
-  /** The automerge sync message */
-  data: Uint8Array
+
   /** The peer ID of the recipient of this message */
   targetId: PeerId
-  /** The document ID of the document this message is for */
-  documentId: DocumentId
-}
 
+  /** The automerge sync message */
+  data: Uint8Array
 
-export interface EphemeralMessageEnvelope {
-  senderId: PeerId
-  count: number
-  sessionId: SessionId
-}
-
-export interface EphemeralMessageContents {
-  type: "ephemeral"
-  targetId: PeerId
+  /** The document ID of the document this message is for */
   documentId: DocumentId
-  data: Uint8Array
 }
 
-// export type EphemeralMessage = EphemeralMessageEnvelope & EphemeralMessageContents
-// Inline definitions to get good docs, but check the types line up
-type _check_ephemeral_message = static_assert<EphemeralMessage extends EphemeralMessageEnvelope & EphemeralMessageContents ? true : false>
-
-/** An ephemeral message 
- * 
+/** An ephemeral message
+ *
  * @remarks
  * Ephemeral messages are not persisted anywhere and have no particular
- * structure. `automerge-repo` will gossip them around, in order to avoid 
+ * structure. `automerge-repo` will gossip them around, in order to avoid
  * eternal loops of ephemeral messages every message has a session ID, which
  * is a random number generated by the sender at startup time, and a sequence
  * number. The combination of these two things allows us to discard messages
  * we have already seen.
  * */
 export type EphemeralMessage = {
-  /** The ID of the peer who sent this message */
+  type: "ephemeral"
+
+  /** The peer ID of the sender of this message */
   senderId: PeerId
+
+  /** The peer ID of the recipient of this message */
+  targetId: PeerId
+
   /** A sequence number which must be incremented for each message sent by this peer */
   count: number
+
   /** The ID of the session this message is part of. The sequence number for a given session always increases */
   sessionId: SessionId
-  type: "ephemeral"
-  /** The peer this message is for */
-  targetId: PeerId
+
   /** The document ID this message pertains to */
   documentId: DocumentId
+
   /** The actual data of the message */
   data: Uint8Array
 }
 
-export interface DocumentUnavailableMessageContents {
-  type: "doc-unavailable"
-  documentId: DocumentId
-  targetId: PeerId
-}
-
-
-// export type DocumentUnavailableMessage = SyncMessageEnvelope & DocumentUnavailableMessageContents
-// Inline definitions to get good docs, but check the types line up
-type _check_doc_unavailable = static_assert<DocumentUnavailableMessage extends SyncMessageEnvelope & DocumentUnavailableMessageContents ? true : false>
 /** Sent by a {@link Repo} to indicate that it does not have the document and none of it's connected peers do either */
 export type DocumentUnavailableMessage = {
-  /** The peer who sent this message */
-  senderId: PeerId
   type: "doc-unavailable"
-  /** The document which the peer claims it doesn't have */
-  documentId: DocumentId
-  /** The peer this message is for */
-  targetId: PeerId
-}
 
-export interface RequestMessageContents {
-  type: "request"
-  data: Uint8Array
+  /** The peer ID of the sender of this message */
+  senderId: PeerId
+
+  /** The peer ID of the recipient of this message */
   targetId: PeerId
+
+  /** The document which the peer claims it doesn't have */
   documentId: DocumentId
 }
 
-// export type RequestMessage = SyncMessageEnvelope & RequestMessageContents
-// Inline definitions to get good docs, but check the types line up
-type _check_request_message = static_assert<RequestMessage extends SyncMessageEnvelope & RequestMessageContents ? true : false>
-// We inline the definitions here rather than using the above type alias because
-// otherwise typedoc can't produce nice docs without exporting SyncMessageEnvelope
-// and RequestMessageContents
 /** Sent by a {@link Repo} to request a document from a peer
  *
  * @remarks
@@ -163,58 +72,86 @@ type _check_request_message = static_assert<RequestMessage extends SyncMessageEn
  * as the initial sync message when asking the other peer if it has the document.
  * */
 export type RequestMessage = {
-  /** The peer who sent this message */
-  senderId: PeerId
   type: "request"
+
+  /** The peer ID of the sender of this message */
+  senderId: PeerId
+
+  /** The peer ID of the recipient of this message */
+  targetId: PeerId
+
   /** The initial automerge sync message */
   data: Uint8Array
-  /** The peer this message is for */
-  targetId: PeerId
+
   /** The document ID this message requests */
   documentId: DocumentId
 }
 
-export type MessageContents =
-  | SyncMessageContents
-  | EphemeralMessageContents
-  | RequestMessageContents
-  | DocumentUnavailableMessageContents
-
-/** The type of messages that {@link Repo} sends and receive to {@link NetworkAdapter}s */
-export type Message =
-  | SyncMessage
-  | EphemeralMessage
-  | RequestMessage
-  | DocumentUnavailableMessage
-
-export type SynchronizerMessage =
-  | SyncMessage
-  | RequestMessage
-  | DocumentUnavailableMessage
-  | EphemeralMessage
-
-
 /** Notify the network that we have arrived so everyone knows our peer ID */
 export type ArriveMessage = {
-  /** Our peer ID */
-  senderId: PeerId
   type: "arrive"
+
+  /** The peer ID of the sender of this message */
+  senderId: PeerId
+
+  /** Arrive messages don't have a targetId */
+  targetId: never
 }
 
 /** Respond to an arriving peer with our peer ID */
 export type WelcomeMessage = {
-  /** Our peer ID */
+  type: "welcome"
+
+  /** The peer ID of the recipient sender this message */
   senderId: PeerId
-  /** The ID of the peer who sent the {@link ArriveMessage} we are responding to */
+
+  /** The peer ID of the recipient of this message */
   targetId: PeerId
-  type: "welcome"
 }
 
-/** The type of messages that {@link NetworkAdapter}s send and receive to each other
+/** These are message types that a {@link NetworkAdapter} surfaces to a {@link Repo}. */
+export type RepoMessage =
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+
+/** These are all the message types that a {@link NetworkAdapter} might see.
  *
  * @remarks
- * It is not _required_ that a {@link NetworkAdapter} use this message type.
- * NetworkAdapters are free to use whatever message type makes sense for their
- * transport. However, this type is a useful default.
+ * It is not _required_ that a {@link NetworkAdapter} use these types: They are free to use
+ * whatever message type makes sense for their transport. However, this type is a useful default.
  * */
-export type NetworkAdapterMessage = ArriveMessage | WelcomeMessage | Message
+export type Message = RepoMessage | ArriveMessage | WelcomeMessage
+
+/**
+ * The contents of a message, without the sender ID or other properties added by the {@link NetworkSubsystem})
+ */
+export type MessageContents<T extends Message = Message> =
+  T extends EphemeralMessage
+    ? Omit<T, "senderId" | "count" | "sessionId">
+    : Omit<T, "senderId">
+
+// TYPE GUARDS
+
+export const isValidRepoMessage = (message: Message): message is RepoMessage =>
+  typeof message === "object" &&
+  typeof message.type === "string" &&
+  typeof message.senderId === "string" &&
+  (isSyncMessage(message) ||
+    isEphemeralMessage(message) ||
+    isRequestMessage(message) ||
+    isDocumentUnavailableMessage(message))
+
+// prettier-ignore
+export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage => 
+  msg.type === "doc-unavailable"
+
+export const isRequestMessage = (msg: Message): msg is RequestMessage =>
+  msg.type === "request"
+
+export const isSyncMessage = (msg: Message): msg is SyncMessage =>
+  msg.type === "sync"
+
+export const isEphemeralMessage = (msg: Message): msg is EphemeralMessage =>
+  msg.type === "ephemeral"

package/src/storage/StorageSubsystem.ts

@@ -1,10 +1,10 @@
 import * as A from "@automerge/automerge/next"
-import { StorageAdapter, StorageKey } from "./StorageAdapter.js"
-import * as sha256 from "fast-sha256"
-import { type DocumentId } from "../types.js"
-import { mergeArrays } from "../helpers/mergeArrays.js"
 import debug from "debug"
+import * as sha256 from "fast-sha256"
 import { headsAreSame } from "../helpers/headsAreSame.js"
+import { mergeArrays } from "../helpers/mergeArrays.js"
+import { type DocumentId } from "../types.js"
+import { StorageAdapter, StorageKey } from "./StorageAdapter.js"
 
 // Metadata about a chunk of data loaded from storage. This is stored on the
 // StorageSubsystem so when we are compacting we know what chunks we can safely delete
@@ -24,8 +24,8 @@ function keyHash(binary: Uint8Array) {
 }
 
 function headsHash(heads: A.Heads): string {
-  let encoder = new TextEncoder()
-  let headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
+  const encoder = new TextEncoder()
+  const headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
   return keyHash(headsbinary)
 }
 
@@ -53,7 +53,7 @@ export class StorageSubsystem {
       if (!this.#chunkInfos.has(documentId)) {
         this.#chunkInfos.set(documentId, [])
       }
-      this.#chunkInfos.get(documentId)!!.push({
+      this.#chunkInfos.get(documentId)!.push({
         key,
         type: "incremental",
         size: binary.length,
@@ -122,18 +122,18 @@ export class StorageSubsystem {
     if (!this.#shouldSave(documentId, doc)) {
       return
     }
-    let sourceChunks = this.#chunkInfos.get(documentId) ?? []
+    const sourceChunks = this.#chunkInfos.get(documentId) ?? []
     if (this.#shouldCompact(sourceChunks)) {
-      this.#saveTotal(documentId, doc, sourceChunks)
+      void this.#saveTotal(documentId, doc, sourceChunks)
     } else {
-      this.#saveIncremental(documentId, doc)
+      void this.#saveIncremental(documentId, doc)
     }
     this.#storedHeads.set(documentId, A.getHeads(doc))
   }
 
   async remove(documentId: DocumentId) {
-    this.#storageAdapter.removeRange([documentId, "snapshot"])
-    this.#storageAdapter.removeRange([documentId, "incremental"])
+    void this.#storageAdapter.removeRange([documentId, "snapshot"])
+    void this.#storageAdapter.removeRange([documentId, "incremental"])
   }
 
   #shouldSave(documentId: DocumentId, doc: A.Doc<unknown>): boolean {

package/src/synchronizer/CollectionSynchronizer.ts

@@ -1,21 +1,12 @@
-import { Repo } from "../Repo.js"
+import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
-import {
-  documentIdToBinary,
-  binaryToDocumentId,
-  stringifyAutomergeUrl,
-} from "../DocUrl.js"
-import { PeerId, DocumentId } from "../types.js"
+import { stringifyAutomergeUrl } from "../DocUrl.js"
+import { Repo } from "../Repo.js"
+import { RepoMessage } from "../network/messages.js"
+import { DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
 
-import debug from "debug"
-import {
-  DocumentUnavailableMessage,
-  RequestMessage,
-  SynchronizerMessage,
-  SyncMessage,
-} from "../network/messages.js"
 const log = debug("automerge-repo:collectionsync")
 
 /** A CollectionSynchronizer is responsible for synchronizing a DocCollection with peers. */
@@ -66,7 +57,7 @@ export class CollectionSynchronizer extends Synchronizer {
    * When we receive a sync message for a document we haven't got in memory, we
    * register it with the repo and start synchronizing
    */
-  async receiveMessage(message: SynchronizerMessage) {
+  async receiveMessage(message: RepoMessage) {
     log(
       `onSyncMessage: ${message.senderId}, ${message.documentId}, ${
         "data" in message ? message.data.byteLength + "bytes" : ""
@@ -121,7 +112,7 @@ export class CollectionSynchronizer extends Synchronizer {
     this.#peers.add(peerId)
     for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
       const { documentId } = docSynchronizer
-      this.repo.sharePolicy(peerId, documentId).then(okToShare => {
+      void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
         if (okToShare) docSynchronizer.beginSync([peerId])
       })
     }

package/src/synchronizer/DocSynchronizer.ts

@@ -1,28 +1,27 @@
 import * as A from "@automerge/automerge/next"
+import { decode } from "cbor-x"
+import debug from "debug"
 import {
-  AWAITING_NETWORK,
   DocHandle,
   DocHandleOutboundEphemeralMessagePayload,
   READY,
   REQUESTING,
   UNAVAILABLE,
 } from "../DocHandle.js"
-import { PeerId } from "../types.js"
-import { Synchronizer } from "./Synchronizer.js"
-
-import debug from "debug"
 import {
+  DocumentUnavailableMessage,
   EphemeralMessage,
-  isDocumentUnavailableMessage,
-  isRequestMessage,
-  Message,
+  MessageContents,
+  RepoMessage,
   RequestMessage,
-  SynchronizerMessage,
   SyncMessage,
+  isRequestMessage,
 } from "../network/messages.js"
+import { PeerId } from "../types.js"
+import { Synchronizer } from "./Synchronizer.js"
+import { throttle } from "../helpers/throttle.js"
 
 type PeerDocumentStatus = "unknown" | "has" | "unavailable" | "wants"
-import { decode } from "cbor-x"
 
 /**
  * DocSynchronizer takes a handle to an Automerge document, and receives & dispatches sync messages
@@ -30,8 +29,7 @@ import { decode } from "cbor-x"
  */
 export class DocSynchronizer extends Synchronizer {
   #log: debug.Debugger
-  #conciseLog: debug.Debugger
-  #opsLog: debug.Debugger
+  syncDebounceRate = 100
 
   /** Active peers */
   #peers: PeerId[] = []
@@ -45,14 +43,15 @@ export class DocSynchronizer extends Synchronizer {
 
   #syncStarted = false
 
-  constructor(private handle: DocHandle<any>) {
+  constructor(private handle: DocHandle<unknown>) {
     super()
     const docId = handle.documentId.slice(0, 5)
-    this.#conciseLog = debug(`automerge-repo:concise:docsync:${docId}`) // Only logs one line per receive/send
     this.#log = debug(`automerge-repo:docsync:${docId}`)
-    this.#opsLog = debug(`automerge-repo:ops:docsync:${docId}`) // Log list of ops of each message
 
-    handle.on("change", () => this.#syncWithPeers())
+    handle.on(
+      "change",
+      throttle(() => this.#syncWithPeers(), this.syncDebounceRate)
+    )
 
     handle.on("ephemeral-message-outbound", payload =>
       this.#broadcastToPeers(payload)
@@ -82,7 +81,9 @@ export class DocSynchronizer extends Synchronizer {
     this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
   }
 
-  async #broadcastToPeers({ data }: DocHandleOutboundEphemeralMessagePayload) {
+  async #broadcastToPeers({
+    data,
+  }: DocHandleOutboundEphemeralMessagePayload<unknown>) {
     this.#log(`broadcastToPeers`, this.#peers)
     this.#peers.forEach(peerId => this.#sendEphemeralMessage(peerId, data))
   }
@@ -90,12 +91,13 @@ export class DocSynchronizer extends Synchronizer {
   #sendEphemeralMessage(peerId: PeerId, data: Uint8Array) {
     this.#log(`sendEphemeralMessage ->${peerId}`)
 
-    this.emit("message", {
+    const message: MessageContents<EphemeralMessage> = {
       type: "ephemeral",
       targetId: peerId,
       documentId: this.handle.documentId,
       data,
-    })
+    }
+    this.emit("message", message)
   }
 
   #getSyncState(peerId: PeerId) {
@@ -128,13 +130,11 @@ export class DocSynchronizer extends Synchronizer {
     const [newSyncState, message] = A.generateSyncMessage(doc, syncState)
     this.#setSyncState(peerId, newSyncState)
     if (message) {
-      this.#logMessage(`sendSyncMessage 🡒 ${peerId}`, message)
-
-      const decoded = A.decodeSyncMessage(message)
+      const isNew = A.getHeads(doc).length === 0
 
       if (
         !this.handle.isReady() &&
-        decoded.heads.length === 0 &&
+        isNew &&
         newSyncState.sharedHeads.length === 0 &&
         !Object.values(this.#peerDocumentStatuses).includes("has") &&
         this.#peerDocumentStatuses[peerId] === "unknown"
@@ -145,43 +145,23 @@ export class DocSynchronizer extends Synchronizer {
           targetId: peerId,
           documentId: this.handle.documentId,
           data: message,
-        })
+        } as RequestMessage)
       } else {
         this.emit("message", {
           type: "sync",
           targetId: peerId,
           data: message,
           documentId: this.handle.documentId,
-        })
+        } as SyncMessage)
       }
 
       // if we have sent heads, then the peer now has or will have the document
-      if (decoded.heads.length > 0) {
+      if (!isNew) {
         this.#peerDocumentStatuses[peerId] = "has"
       }
     }
   }
 
-  #logMessage = (label: string, message: Uint8Array) => {
-    // This is real expensive...
-    return
-
-    const size = message.byteLength
-    const logText = `${label} ${size}b`
-    const decoded = A.decodeSyncMessage(message)
-
-    this.#conciseLog(logText)
-    this.#log(logText, decoded)
-
-    // expanding is expensive, so only do it if we're logging at this level
-    const expanded = this.#opsLog.enabled
-      ? decoded.changes.flatMap((change: A.Change) =>
-        A.decodeChange(change).ops.map((op: any) => JSON.stringify(op))
-      )
-      : null
-    this.#opsLog(logText, expanded)
-  }
-
   /// PUBLIC
 
   hasPeer(peerId: PeerId) {
@@ -189,6 +169,9 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   beginSync(peerIds: PeerId[]) {
+    const newPeers = new Set(
+      peerIds.filter(peerId => !this.#peers.includes(peerId))
+    )
     this.#log(`beginSync: ${peerIds.join(", ")}`)
 
     // HACK: if we have a sync state already, we round-trip it through the encoding system to make
@@ -204,15 +187,20 @@ export class DocSynchronizer extends Synchronizer {
     // At this point if we don't have anything in our storage, we need to use an empty doc to sync
     // with; but we don't want to surface that state to the front end
     void this.handle.doc([READY, REQUESTING, UNAVAILABLE]).then(doc => {
-
       // we register out peers first, then say that sync has started
       this.#syncStarted = true
       this.#checkDocUnavailable()
 
-      if (doc === undefined) return
+      const wasUnavailable = doc === undefined
+      if (wasUnavailable && newPeers.size == 0) {
+        return
+      }
+      // If the doc is unavailable we still need a blank document to generate
+      // the sync message from
+      const theDoc = doc ?? A.init<unknown>()
 
       peerIds.forEach(peerId => {
-        this.#sendSyncMessage(peerId, doc)
+        this.#sendSyncMessage(peerId, theDoc)
       })
     })
   }
@@ -222,7 +210,7 @@ export class DocSynchronizer extends Synchronizer {
     this.#peers = this.#peers.filter(p => p !== peerId)
   }
 
-  receiveMessage(message: SynchronizerMessage) {
+  receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "sync":
       case "request":
@@ -246,7 +234,7 @@ export class DocSynchronizer extends Synchronizer {
 
     const { senderId, data } = message
 
-    const contents = decode(data)
+    const contents = decode(new Uint8Array(data))
 
     this.handle.emit("ephemeral-message", {
       handle: this.handle,
@@ -320,11 +308,12 @@ export class DocSynchronizer extends Synchronizer {
       this.#peers
         .filter(peerId => this.#peerDocumentStatuses[peerId] === "wants")
         .forEach(peerId => {
-          this.emit("message", {
+          const message: MessageContents<DocumentUnavailableMessage> = {
             type: "doc-unavailable",
             documentId: this.handle.documentId,
             targetId: peerId,
-          })
+          }
+          this.emit("message", message)
         })
 
       this.handle.unavailable()

package/src/synchronizer/Synchronizer.ts

@@ -1,8 +1,8 @@
 import { EventEmitter } from "eventemitter3"
-import { Message, MessageContents } from "../network/messages.js"
+import { MessageContents, RepoMessage } from "../network/messages.js"
 
 export abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
-  abstract receiveMessage(message: Message): void
+  abstract receiveMessage(message: RepoMessage): void
 }
 
 export interface SynchronizerEvents {

package/src/types.ts

@@ -1,17 +1,22 @@
 /** The ID of a document. Typically you should use a {@link AutomergeUrl} instead.
  */
 export type DocumentId = string & { __documentId: true } // for logging
-/** A branded string representing a URL for a document 
+
+/** A branded string representing a URL for a document
  *
  * @remarks
- * An automerge URL has the form `automerge:<base58 encoded string>`. This 
+ * An automerge URL has the form `automerge:<base58 encoded string>`. This
  * type is returned from various routines which validate a url.
  *
  */
 export type AutomergeUrl = string & { __documentUrl: true } // for opening / linking
+
 /** A document ID as a Uint8Array instead of a bas58 encoded string. Typically you should use a {@link AutomergeUrl} instead.
  */
 export type BinaryDocumentId = Uint8Array & { __binaryDocumentId: true } // for storing / syncing
 
 /** A branded type for peer IDs */
-export type PeerId = string & { __peerId: false }
+export type PeerId = string & { __peerId: true }
+
+/** A randomly generated string created when the {@link Repo} starts up */
+export type SessionId = string & { __SessionId: true }