@automerge/automerge-repo 1.0.17 → 1.0.18

package/src/synchronizer/DocSynchronizer.ts

@@ -20,9 +20,20 @@ import {
 import { PeerId } from "../types.js"
 import { Synchronizer } from "./Synchronizer.js"
 import { throttle } from "../helpers/throttle.js"
+import { headsAreSame } from "../helpers/headsAreSame.js"
 
 type PeerDocumentStatus = "unknown" | "has" | "unavailable" | "wants"
 
+type PendingMessage = {
+  message: RequestMessage | SyncMessage
+  received: Date
+}
+
+interface DocSynchronizerConfig {
+  handle: DocHandle<unknown>
+  onLoadSyncState?: (peerId: PeerId) => A.SyncState | undefined
+}
+
 /**
  * DocSynchronizer takes a handle to an Automerge document, and receives & dispatches sync messages
  * to bring it inline with all other peers' versions.
@@ -34,17 +45,29 @@ export class DocSynchronizer extends Synchronizer {
   /** Active peers */
   #peers: PeerId[] = []
 
+  #pendingSyncStateCallbacks: Record<
+    PeerId,
+    ((syncState: A.SyncState) => void)[]
+  > = {}
+
   #peerDocumentStatuses: Record<PeerId, PeerDocumentStatus> = {}
 
   /** Sync state for each peer we've communicated with (including inactive peers) */
   #syncStates: Record<PeerId, A.SyncState> = {}
 
-  #pendingSyncMessages: Array<SyncMessage | RequestMessage> = []
+  #pendingSyncMessages: Array<PendingMessage> = []
 
   #syncStarted = false
 
-  constructor(private handle: DocHandle<unknown>) {
+  #handle: DocHandle<unknown>
+  #onLoadSyncState: (peerId: PeerId) => Promise<A.SyncState | undefined>
+
+  constructor({ handle, onLoadSyncState }: DocSynchronizerConfig) {
     super()
+    this.#handle = handle
+    this.#onLoadSyncState =
+      onLoadSyncState ?? (() => Promise.resolve(undefined))
+
     const docId = handle.documentId.slice(0, 5)
     this.#log = debug(`automerge-repo:docsync:${docId}`)
 
@@ -69,14 +92,14 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   get documentId() {
-    return this.handle.documentId
+    return this.#handle.documentId
   }
 
   /// PRIVATE
 
   async #syncWithPeers() {
     this.#log(`syncWithPeers`)
-    const doc = await this.handle.doc()
+    const doc = await this.#handle.doc()
     if (doc === undefined) return
     this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
   }
@@ -94,72 +117,111 @@ export class DocSynchronizer extends Synchronizer {
     const message: MessageContents<EphemeralMessage> = {
       type: "ephemeral",
       targetId: peerId,
-      documentId: this.handle.documentId,
+      documentId: this.#handle.documentId,
       data,
     }
     this.emit("message", message)
   }
 
-  #getSyncState(peerId: PeerId) {
+  #withSyncState(peerId: PeerId, callback: (syncState: A.SyncState) => void) {
     if (!this.#peers.includes(peerId)) {
-      this.#log("adding a new peer", peerId)
       this.#peers.push(peerId)
     }
 
-    // when a peer is added, we don't know if it has the document or not
     if (!(peerId in this.#peerDocumentStatuses)) {
       this.#peerDocumentStatuses[peerId] = "unknown"
     }
 
-    return this.#syncStates[peerId] ?? A.initSyncState()
+    const syncState = this.#syncStates[peerId]
+    if (syncState) {
+      callback(syncState)
+      return
+    }
+
+    let pendingCallbacks = this.#pendingSyncStateCallbacks[peerId]
+    if (!pendingCallbacks) {
+      this.#onLoadSyncState(peerId).then(syncState => {
+        this.#initSyncState(peerId, syncState ?? A.initSyncState())
+      })
+      pendingCallbacks = this.#pendingSyncStateCallbacks[peerId] = []
+    }
+
+    pendingCallbacks.push(callback)
   }
 
-  #setSyncState(peerId: PeerId, syncState: A.SyncState) {
-    // TODO: we maybe should be persisting sync states. But we want to be careful about how often we
-    // do that, because it can generate a lot of disk activity.
+  #initSyncState(peerId: PeerId, syncState: A.SyncState) {
+    const pendingCallbacks = this.#pendingSyncStateCallbacks[peerId]
+    if (pendingCallbacks) {
+      for (const callback of pendingCallbacks) {
+        callback(syncState)
+      }
+    }
 
-    // TODO: we only need to do this on reconnect
+    delete this.#pendingSyncStateCallbacks[peerId]
 
     this.#syncStates[peerId] = syncState
   }
 
+  #setSyncState(peerId: PeerId, syncState: A.SyncState) {
+    const previousSyncState = this.#syncStates[peerId]
+
+    this.#syncStates[peerId] = syncState
+
+    const haveTheirSyncedHeadsChanged =
+      syncState.theirHeads &&
+      (!previousSyncState ||
+        !previousSyncState.theirHeads ||
+        !headsAreSame(previousSyncState.theirHeads, syncState.theirHeads))
+
+    if (haveTheirSyncedHeadsChanged) {
+      this.#handle.setRemoteHeads(peerId, syncState.theirHeads)
+    }
+
+    this.emit("sync-state", {
+      peerId,
+      syncState,
+      documentId: this.#handle.documentId,
+    })
+  }
+
   #sendSyncMessage(peerId: PeerId, doc: A.Doc<unknown>) {
     this.#log(`sendSyncMessage ->${peerId}`)
 
-    const syncState = this.#getSyncState(peerId)
-    const [newSyncState, message] = A.generateSyncMessage(doc, syncState)
-    this.#setSyncState(peerId, newSyncState)
-    if (message) {
-      const isNew = A.getHeads(doc).length === 0
-
-      if (
-        !this.handle.isReady() &&
-        isNew &&
-        newSyncState.sharedHeads.length === 0 &&
-        !Object.values(this.#peerDocumentStatuses).includes("has") &&
-        this.#peerDocumentStatuses[peerId] === "unknown"
-      ) {
-        // we don't have the document (or access to it), so we request it
-        this.emit("message", {
-          type: "request",
-          targetId: peerId,
-          documentId: this.handle.documentId,
-          data: message,
-        } as RequestMessage)
-      } else {
-        this.emit("message", {
-          type: "sync",
-          targetId: peerId,
-          data: message,
-          documentId: this.handle.documentId,
-        } as SyncMessage)
-      }
-
-      // if we have sent heads, then the peer now has or will have the document
-      if (!isNew) {
-        this.#peerDocumentStatuses[peerId] = "has"
+    this.#withSyncState(peerId, syncState => {
+      const [newSyncState, message] = A.generateSyncMessage(doc, syncState)
+      if (message) {
+        this.#setSyncState(peerId, newSyncState)
+        const isNew = A.getHeads(doc).length === 0
+
+        if (
+          !this.#handle.isReady() &&
+          isNew &&
+          newSyncState.sharedHeads.length === 0 &&
+          !Object.values(this.#peerDocumentStatuses).includes("has") &&
+          this.#peerDocumentStatuses[peerId] === "unknown"
+        ) {
+          // we don't have the document (or access to it), so we request it
+          this.emit("message", {
+            type: "request",
+            targetId: peerId,
+            documentId: this.#handle.documentId,
+            data: message,
+          } as RequestMessage)
+        } else {
+          this.emit("message", {
+            type: "sync",
+            targetId: peerId,
+            data: message,
+            documentId: this.#handle.documentId,
+          } as SyncMessage)
+        }
+
+        // if we have sent heads, then the peer now has or will have the document
+        if (!isNew) {
+          this.#peerDocumentStatuses[peerId] = "has"
+        }
       }
-    }
+    })
   }
 
   /// PUBLIC
@@ -172,35 +234,45 @@ export class DocSynchronizer extends Synchronizer {
     const newPeers = new Set(
       peerIds.filter(peerId => !this.#peers.includes(peerId))
     )
-    this.#log(`beginSync: ${peerIds.join(", ")}`)
-
-    // HACK: if we have a sync state already, we round-trip it through the encoding system to make
-    // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
-    // messages during disconnection.
-    // TODO: cover that case with a test and remove this hack
-    peerIds.forEach(peerId => {
-      const syncStateRaw = this.#getSyncState(peerId)
-      const syncState = A.decodeSyncState(A.encodeSyncState(syncStateRaw))
-      this.#setSyncState(peerId, syncState)
-    })
 
     // At this point if we don't have anything in our storage, we need to use an empty doc to sync
     // with; but we don't want to surface that state to the front end
-    void this.handle.doc([READY, REQUESTING, UNAVAILABLE]).then(doc => {
-      // we register out peers first, then say that sync has started
-      this.#syncStarted = true
-      this.#checkDocUnavailable()
 
-      const wasUnavailable = doc === undefined
-      if (wasUnavailable && newPeers.size == 0) {
-        return
-      }
-      // If the doc is unavailable we still need a blank document to generate
-      // the sync message from
-      const theDoc = doc ?? A.init<unknown>()
+    const docPromise = this.#handle
+      .doc([READY, REQUESTING, UNAVAILABLE])
+      .then(doc => {
+        // we register out peers first, then say that sync has started
+        this.#syncStarted = true
+        this.#checkDocUnavailable()
+
+        const wasUnavailable = doc === undefined
+        if (wasUnavailable && newPeers.size == 0) {
+          return
+        }
 
-      peerIds.forEach(peerId => {
-        this.#sendSyncMessage(peerId, theDoc)
+        // If the doc is unavailable we still need a blank document to generate
+        // the sync message from
+        return doc ?? A.init<unknown>()
+      })
+
+    this.#log(`beginSync: ${peerIds.join(", ")}`)
+
+    peerIds.forEach(peerId => {
+      this.#withSyncState(peerId, syncState => {
+        // HACK: if we have a sync state already, we round-trip it through the encoding system to make
+        // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
+        // messages during disconnection.
+        // TODO: cover that case with a test and remove this hack
+        const reparsedSyncState = A.decodeSyncState(
+          A.encodeSyncState(syncState)
+        )
+        this.#setSyncState(peerId, reparsedSyncState)
+
+        docPromise.then(doc => {
+          if (doc) {
+            this.#sendSyncMessage(peerId, doc)
+          }
+        })
       })
     })
   }
@@ -229,15 +301,15 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   receiveEphemeralMessage(message: EphemeralMessage) {
-    if (message.documentId !== this.handle.documentId)
+    if (message.documentId !== this.#handle.documentId)
       throw new Error(`channelId doesn't match documentId`)
 
     const { senderId, data } = message
 
     const contents = decode(new Uint8Array(data))
 
-    this.handle.emit("ephemeral-message", {
-      handle: this.handle,
+    this.#handle.emit("ephemeral-message", {
+      handle: this.#handle,
       senderId,
       message: contents,
     })
@@ -252,20 +324,20 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   receiveSyncMessage(message: SyncMessage | RequestMessage) {
-    if (message.documentId !== this.handle.documentId)
+    if (message.documentId !== this.#handle.documentId)
       throw new Error(`channelId doesn't match documentId`)
 
     // We need to block receiving the syncMessages until we've checked local storage
-    if (!this.handle.inState([READY, REQUESTING, UNAVAILABLE])) {
-      this.#pendingSyncMessages.push(message)
+    if (!this.#handle.inState([READY, REQUESTING, UNAVAILABLE])) {
+      this.#pendingSyncMessages.push({ message, received: new Date() })
       return
     }
 
     this.#processAllPendingSyncMessages()
-    this.#processSyncMessage(message)
+    this.#processSyncMessage(message, new Date())
   }
 
-  #processSyncMessage(message: SyncMessage | RequestMessage) {
+  #processSyncMessage(message: SyncMessage | RequestMessage, received: Date) {
     if (isRequestMessage(message)) {
       this.#peerDocumentStatuses[message.senderId] = "wants"
     }
@@ -277,28 +349,30 @@ export class DocSynchronizer extends Synchronizer {
       this.#peerDocumentStatuses[message.senderId] = "has"
     }
 
-    this.handle.update(doc => {
-      const [newDoc, newSyncState] = A.receiveSyncMessage(
-        doc,
-        this.#getSyncState(message.senderId),
-        message.data
-      )
+    this.#withSyncState(message.senderId, syncState => {
+      this.#handle.update(doc => {
+        const [newDoc, newSyncState] = A.receiveSyncMessage(
+          doc,
+          syncState,
+          message.data
+        )
 
-      this.#setSyncState(message.senderId, newSyncState)
+        this.#setSyncState(message.senderId, newSyncState)
 
-      // respond to just this peer (as required)
-      this.#sendSyncMessage(message.senderId, doc)
-      return newDoc
-    })
+        // respond to just this peer (as required)
+        this.#sendSyncMessage(message.senderId, doc)
+        return newDoc
+      })
 
-    this.#checkDocUnavailable()
+      this.#checkDocUnavailable()
+    })
   }
 
   #checkDocUnavailable() {
     // if we know none of the peers have the document, tell all our peers that we don't either
     if (
       this.#syncStarted &&
-      this.handle.inState([REQUESTING]) &&
+      this.#handle.inState([REQUESTING]) &&
       this.#peers.every(
         peerId =>
           this.#peerDocumentStatuses[peerId] === "unavailable" ||
@@ -310,19 +384,19 @@ export class DocSynchronizer extends Synchronizer {
         .forEach(peerId => {
           const message: MessageContents<DocumentUnavailableMessage> = {
             type: "doc-unavailable",
-            documentId: this.handle.documentId,
+            documentId: this.#handle.documentId,
             targetId: peerId,
           }
           this.emit("message", message)
         })
 
-      this.handle.unavailable()
+      this.#handle.unavailable()
     }
   }
 
   #processAllPendingSyncMessages() {
     for (const message of this.#pendingSyncMessages) {
-      this.#processSyncMessage(message)
+      this.#processSyncMessage(message.message, message.received)
     }
 
     this.#pendingSyncMessages = []

package/src/synchronizer/Synchronizer.ts

@@ -1,5 +1,9 @@
 import { EventEmitter } from "eventemitter3"
-import { MessageContents, RepoMessage } from "../network/messages.js"
+import {
+  MessageContents,
+  RepoMessage,
+  SyncStateMessage,
+} from "../network/messages.js"
 
 export abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
   abstract receiveMessage(message: RepoMessage): void
@@ -7,4 +11,5 @@ export abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
 
 export interface SynchronizerEvents {
   message: (arg: MessageContents) => void
+  "sync-state": (arg: SyncStateMessage) => void
 }

package/test/DocHandle.test.ts

@@ -5,7 +5,7 @@ import { describe, it } from "vitest"
 import { generateAutomergeUrl, parseAutomergeUrl } from "../src/AutomergeUrl.js"
 import { eventPromise } from "../src/helpers/eventPromise.js"
 import { pause } from "../src/helpers/pause.js"
-import { DocHandle, DocHandleChangePayload } from "../src/index.js"
+import { DocHandle, DocHandleChangePayload, PeerId } from "../src/index.js"
 import { TestDoc } from "./types.js"
 
 describe("DocHandle", () => {
@@ -303,6 +303,23 @@ describe("DocHandle", () => {
     assert(wasBar, "foo should have been bar as we changed at the old heads")
   })
 
+  it("should allow to listen for remote head changes and manually read remote heads", async () => {
+    const handle = new DocHandle<TestDoc>(TEST_ID, { isNew: true })
+    const bob = "bob" as PeerId
+
+    const remoteHeadsMessagePromise = eventPromise(handle, "remote-heads")
+
+    handle.setRemoteHeads(bob, [])
+
+    const remoteHeadsMessage = await remoteHeadsMessagePromise
+
+    assert.strictEqual(remoteHeadsMessage.peerId, bob)
+    assert.deepStrictEqual(remoteHeadsMessage.heads, [])
+
+    // read remote heads manually
+    assert.deepStrictEqual(handle.getRemoteHeads(bob), [])
+  })
+
   describe("ephemeral messaging", () => {
     it("can broadcast a message for the network to send out", async () => {
       const handle = new DocHandle<TestDoc>(TEST_ID, { isNew: true })

package/test/DocSynchronizer.test.ts

@@ -1,7 +1,8 @@
 import assert from "assert"
+import * as A from "@automerge/automerge"
 import { describe, it } from "vitest"
-import { DocHandle } from "../src/DocHandle.js"
 import { generateAutomergeUrl, parseAutomergeUrl } from "../src/AutomergeUrl.js"
+import { DocHandle } from "../src/DocHandle.js"
 import { eventPromise } from "../src/helpers/eventPromise.js"
 import {
   DocumentUnavailableMessage,
@@ -22,7 +23,9 @@ describe("DocSynchronizer", () => {
   const setup = () => {
     const docId = parseAutomergeUrl(generateAutomergeUrl()).documentId
     handle = new DocHandle<TestDoc>(docId, { isNew: true })
-    docSynchronizer = new DocSynchronizer(handle)
+    docSynchronizer = new DocSynchronizer({
+      handle: handle as DocHandle<unknown>,
+    })
     return { handle, docSynchronizer }
   }
 
@@ -50,6 +53,27 @@ describe("DocSynchronizer", () => {
     assert.equal(type, "sync")
   })
 
+  it("emits a syncState message when the sync state is updated", async () => {
+    const { handle, docSynchronizer } = setup()
+    docSynchronizer.beginSync([alice])
+    handle.change(doc => {
+      doc.foo = "bar"
+    })
+    const message1 = await eventPromise(docSynchronizer, "sync-state")
+    const message2 = await eventPromise(docSynchronizer, "sync-state")
+
+    assert.equal(message1.peerId, "alice")
+    assert.equal(message1.documentId, handle.documentId)
+    assert.deepEqual(message1.syncState.lastSentHeads, [])
+
+    assert.equal(message2.peerId, "alice")
+    assert.equal(message2.documentId, handle.documentId)
+    assert.deepEqual(
+      message2.syncState.lastSentHeads,
+      A.getHeads(handle.docSync())
+    )
+  })
+
   it("still syncs with a peer after it disconnects and reconnects", async () => {
     const { handle, docSynchronizer } = setup()
 
@@ -81,7 +105,9 @@ describe("DocSynchronizer", () => {
     const docId = parseAutomergeUrl(generateAutomergeUrl()).documentId
 
     const handle = new DocHandle<TestDoc>(docId, { isNew: false })
-    docSynchronizer = new DocSynchronizer(handle)
+    docSynchronizer = new DocSynchronizer({
+      handle: handle as DocHandle<unknown>,
+    })
     docSynchronizer.beginSync([alice])
     handle.request()
     const message = await eventPromise(docSynchronizer, "message")
@@ -93,29 +119,34 @@ describe("DocSynchronizer", () => {
     const docId = parseAutomergeUrl(generateAutomergeUrl()).documentId
 
     const bobHandle = new DocHandle<TestDoc>(docId, { isNew: false })
-    const bobDocSynchronizer = new DocSynchronizer(bobHandle)
+    const bobDocSynchronizer = new DocSynchronizer({
+      handle: bobHandle as DocHandle<unknown>,
+    })
     bobDocSynchronizer.beginSync([alice])
     bobHandle.request()
     const message = await eventPromise(bobDocSynchronizer, "message")
 
     const aliceHandle = new DocHandle<TestDoc>(docId, { isNew: false })
-    const aliceDocSynchronizer = new DocSynchronizer(aliceHandle)
+    const aliceDocSynchronizer = new DocSynchronizer({
+      handle: aliceHandle as DocHandle<unknown>,
+    })
     aliceHandle.request()
 
     aliceDocSynchronizer.receiveSyncMessage({ ...message, senderId: bob })
     aliceDocSynchronizer.beginSync([charlie, bob])
 
-    const [charlieMessage, bobMessage] = await new Promise<MessageContents[]>(
-      resolve => {
-        const messages: MessageContents[] = []
-        aliceDocSynchronizer.on("message", message => {
-          messages.push(message)
-          if (messages.length === 2) {
-            resolve(messages)
-          }
-        })
-      }
-    )
+    const messages = await new Promise<MessageContents[]>(resolve => {
+      const messages: MessageContents[] = []
+      aliceDocSynchronizer.on("message", message => {
+        messages.push(message)
+        if (messages.length === 2) {
+          resolve(messages)
+        }
+      })
+    })
+
+    const bobMessage = messages.find(m => m.targetId === bob)
+    const charlieMessage = messages.find(m => m.targetId === charlie)
 
     // the response should be a sync message, not a request message
     assert.equal(charlieMessage.targetId, "charlie")