@automerge/automerge-repo 2.0.0-collectionsync-alpha.1 → 2.0.0

package/src/synchronizer/DocSynchronizer.ts

@@ -1,4 +1,4 @@
-import * as A from "@automerge/automerge/slim/next"
+import { next as A } from "@automerge/automerge/slim"
 import { decode } from "cbor-x"
 import debug from "debug"
 import {
@@ -17,10 +17,9 @@ import {
   SyncMessage,
   isRequestMessage,
 } from "../network/messages.js"
-import { AutomergeUrl, DocumentId, PeerId } from "../types.js"
+import { PeerId } from "../types.js"
 import { Synchronizer } from "./Synchronizer.js"
 import { throttle } from "../helpers/throttle.js"
-import { parseAutomergeUrl } from "../AutomergeUrl.js"
 
 type PeerDocumentStatus = "unknown" | "has" | "unavailable" | "wants"
 
@@ -31,7 +30,8 @@ type PendingMessage = {
 
 interface DocSynchronizerConfig {
   handle: DocHandle<unknown>
-  beelay: A.beelay.Beelay
+  peerId: PeerId
+  onLoadSyncState?: (peerId: PeerId) => Promise<A.SyncState | undefined>
 }
 
 /**
@@ -45,56 +45,71 @@ export class DocSynchronizer extends Synchronizer {
   /** Active peers */
   #peers: PeerId[] = []
 
+  #pendingSyncStateCallbacks: Record<
+    PeerId,
+    ((syncState: A.SyncState) => void)[]
+  > = {}
+
   #peerDocumentStatuses: Record<PeerId, PeerDocumentStatus> = {}
-  #lastSaveOffset: string | null = null
+
+  /** Sync state for each peer we've communicated with (including inactive peers) */
+  #syncStates: Record<PeerId, A.SyncState> = {}
+
+  #pendingSyncMessages: Array<PendingMessage> = []
+
+  // We keep this around at least in part for debugging.
+  // eslint-disable-next-line no-unused-private-class-members
+  #peerId: PeerId
   #syncStarted = false
-  #beelay: A.beelay.Beelay
 
   #handle: DocHandle<unknown>
-  #docId: DocumentId
+  #onLoadSyncState: (peerId: PeerId) => Promise<A.SyncState | undefined>
 
-  constructor({ handle, beelay }: DocSynchronizerConfig) {
+  constructor({ handle, peerId, onLoadSyncState }: DocSynchronizerConfig) {
     super()
+    this.#peerId = peerId
     this.#handle = handle
-    this.#beelay = beelay
-    this.#docId = this.#handle.documentId
+    this.#onLoadSyncState =
+      onLoadSyncState ?? (() => Promise.resolve(undefined))
+
+    const docId = handle.documentId.slice(0, 5)
+    this.#log = debug(`automerge-repo:docsync:${docId}`)
 
-    this.#log = debug(`automerge-repo:docsync:${this.#handle.documentId}`)
+    handle.on(
+      "change",
+      throttle(() => this.#syncWithPeers(), this.syncDebounceRate)
+    )
 
     handle.on("ephemeral-message-outbound", payload =>
       this.#broadcastToPeers(payload)
     )
 
-    handle.on("change", changeInfo => {
-      const newLinks = changeInfo.patches
-        .map(patch => {
-          if (patch.action === "put") {
-            if (patch.value instanceof A.Link) {
-              return patch.value
-            }
-          }
-          return null
-        })
-        .filter(v => v != null)
-      for (const link of newLinks) {
-        const { documentId: target } = parseAutomergeUrl(
-          link.target as AutomergeUrl
-        )
-        this.#beelay.addLink({ from: this.#handle.documentId, to: target })
-      }
-    })
+    // Process pending sync messages immediately after the handle becomes ready.
+    void (async () => {
+      this.#processAllPendingSyncMessages()
+    })()
   }
 
   get peerStates() {
     return this.#peerDocumentStatuses
   }
 
-  get documentId(): DocumentId {
-    return this.#docId
+  get documentId() {
+    return this.#handle.documentId
   }
 
   /// PRIVATE
 
+  async #syncWithPeers() {
+    try {
+      await this.#handle.whenReady()
+      const doc = this.#handle.doc() // XXX THIS ONE IS WEIRD
+      this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
+    } catch (e) {
+      console.log("sync with peers threw an exception")
+    }
+  }
+
   async #broadcastToPeers({
     data,
   }: DocHandleOutboundEphemeralMessagePayload<unknown>) {
@@ -108,85 +123,188 @@ export class DocSynchronizer extends Synchronizer {
     const message: MessageContents<EphemeralMessage> = {
       type: "ephemeral",
       targetId: peerId,
-      documentId: this.documentId,
+      documentId: this.#handle.documentId,
       data,
     }
     this.emit("message", message)
   }
 
+  #withSyncState(peerId: PeerId, callback: (syncState: A.SyncState) => void) {
+    this.#addPeer(peerId)
+
+    if (!(peerId in this.#peerDocumentStatuses)) {
+      this.#peerDocumentStatuses[peerId] = "unknown"
+    }
+
+    const syncState = this.#syncStates[peerId]
+    if (syncState) {
+      callback(syncState)
+      return
+    }
+
+    let pendingCallbacks = this.#pendingSyncStateCallbacks[peerId]
+    if (!pendingCallbacks) {
+      this.#onLoadSyncState(peerId)
+        .then(syncState => {
+          this.#initSyncState(peerId, syncState ?? A.initSyncState())
+        })
+        .catch(err => {
+          this.#log(`Error loading sync state for ${peerId}: ${err}`)
+        })
+      pendingCallbacks = this.#pendingSyncStateCallbacks[peerId] = []
+    }
+
+    pendingCallbacks.push(callback)
+  }
+
+  #addPeer(peerId: PeerId) {
+    if (!this.#peers.includes(peerId)) {
+      this.#peers.push(peerId)
+      this.emit("open-doc", { documentId: this.documentId, peerId })
+    }
+  }
+
+  #initSyncState(peerId: PeerId, syncState: A.SyncState) {
+    const pendingCallbacks = this.#pendingSyncStateCallbacks[peerId]
+    if (pendingCallbacks) {
+      for (const callback of pendingCallbacks) {
+        callback(syncState)
+      }
+    }
+
+    delete this.#pendingSyncStateCallbacks[peerId]
+
+    this.#syncStates[peerId] = syncState
+  }
+
+  #setSyncState(peerId: PeerId, syncState: A.SyncState) {
+    this.#syncStates[peerId] = syncState
+
+    this.emit("sync-state", {
+      peerId,
+      syncState,
+      documentId: this.#handle.documentId,
+    })
+  }
+
+  #sendSyncMessage(peerId: PeerId, doc: A.Doc<unknown>) {
+    this.#log(`sendSyncMessage ->${peerId}`)
+
+    this.#withSyncState(peerId, syncState => {
+      const [newSyncState, message] = A.generateSyncMessage(doc, syncState)
+      if (message) {
+        this.#setSyncState(peerId, newSyncState)
+        const isNew = A.getHeads(doc).length === 0
+
+        if (
+          !this.#handle.isReady() &&
+          isNew &&
+          newSyncState.sharedHeads.length === 0 &&
+          !Object.values(this.#peerDocumentStatuses).includes("has") &&
+          this.#peerDocumentStatuses[peerId] === "unknown"
+        ) {
+          // we don't have the document (or access to it), so we request it
+          this.emit("message", {
+            type: "request",
+            targetId: peerId,
+            documentId: this.#handle.documentId,
+            data: message,
+          } as RequestMessage)
+        } else {
+          this.emit("message", {
+            type: "sync",
+            targetId: peerId,
+            data: message,
+            documentId: this.#handle.documentId,
+          } as SyncMessage)
+        }
+
+        // if we have sent heads, then the peer now has or will have the document
+        if (!isNew) {
+          this.#peerDocumentStatuses[peerId] = "has"
+        }
+      }
+    })
+  }
+
   /// PUBLIC
 
   hasPeer(peerId: PeerId) {
     return this.#peers.includes(peerId)
   }
 
-  beginSync(peerIds: PeerId[]) {
-    this.#log(`beginSync: ${peerIds.join(", ")}`)
-
-    const docPromise = this.#handle
+  async beginSync(peerIds: PeerId[]) {
+    void this.#handle
       .whenReady([READY, REQUESTING, UNAVAILABLE])
-      .then(doc => {
+      .then(() => {
+        this.#syncStarted = true
+        this.#checkDocUnavailable()
+      })
+      .catch(e => {
+        console.log("caught whenready", e)
         this.#syncStarted = true
         this.#checkDocUnavailable()
       })
-      // TODO: handle this error
-      .catch(() => {})
 
-    peerIds.forEach(peerId => {
-      if (!this.#peers.includes(peerId)) {
-        this.#peers.push(peerId)
-      } else {
-        return
-      }
-      this.#peerDocumentStatuses[peerId] = "unknown"
+    const peersWithDocument = this.#peers.some(peerId => {
+      return this.#peerDocumentStatuses[peerId] == "has"
+    })
 
-      docPromise.then(() => {
-        this.#syncStarted = true
-        this.#log(`beginning sync with ${peerId} for doc: ${this.documentId}`)
-        this.#beelay
-          .syncDoc(this.documentId, peerId)
-          .then(({ snapshot, found }) => {
-            this.#peerDocumentStatuses[peerId] = found ? "has" : "unavailable"
-            // this.#log("synced snapshot: ", snapshot)
-            if (found) {
-              this.#beelay.loadDocument(this.#docId).then(commitOrBundles => {
-                if (commitOrBundles != null) {
-                  this.#handle?.update(d => {
-                    let doc = d
-                    for (const commitOrBundle of commitOrBundles) {
-                      doc = A.loadIncremental(doc, commitOrBundle.contents)
-                    }
-                    return doc
-                  })
-                  this.#checkDocUnavailable()
-                }
-              })
-            } else {
-              this.#checkDocUnavailable()
+    if (peersWithDocument) {
+      await this.#handle.whenReady()
+    }
+
+    peerIds.forEach(peerId => {
+      this.#withSyncState(peerId, syncState => {
+        // HACK: if we have a sync state already, we round-trip it through the encoding system to make
+        // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
+        // messages during disconnection.
+        // TODO: cover that case with a test and remove this hack
+        const reparsedSyncState = A.decodeSyncState(
+          A.encodeSyncState(syncState)
+        )
+        this.#setSyncState(peerId, reparsedSyncState)
+
+        // At this point if we don't have anything in our storage, we need to use an empty doc to sync
+        // with; but we don't want to surface that state to the front end
+        this.#handle
+          .whenReady([READY, REQUESTING, UNAVAILABLE])
+          .then(() => {
+            const doc = this.#handle.isReady()
+              ? this.#handle.doc()
+              : A.init<unknown>()
+
+            const noPeersWithDocument = peerIds.every(
+              peerId =>
+                this.#peerDocumentStatuses[peerId] in ["unavailable", "wants"]
+            )
+
+            const wasUnavailable = doc === undefined
+            if (wasUnavailable && noPeersWithDocument) {
+              return
             }
-            this.#beelay.listen(peerId, snapshot)
+
+            // If the doc is unavailable we still need a blank document to generate
+            // the sync message from
+            this.#sendSyncMessage(peerId, doc ?? A.init<unknown>())
+          })
+          .catch(err => {
+            this.#log(`Error loading doc for ${peerId}: ${err}`)
           })
       })
     })
   }
 
-  peerWantsDocument(peerId: PeerId) {
-    this.#peerDocumentStatuses[peerId] = "wants"
-    if (!this.#peers.includes(peerId)) {
-      this.beginSync([peerId as PeerId])
-    }
-  }
-
   endSync(peerId: PeerId) {
     this.#log(`removing peer ${peerId}`)
     this.#peers = this.#peers.filter(p => p !== peerId)
-    this.#beelay.cancelListens(peerId)
   }
 
   receiveMessage(message: RepoMessage) {
     switch (message.type) {
       case "sync":
       case "request":
+        this.receiveSyncMessage(message)
         break
       case "ephemeral":
         this.receiveEphemeralMessage(message)
@@ -201,7 +319,7 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   receiveEphemeralMessage(message: EphemeralMessage) {
-    if (message.documentId !== this.documentId)
+    if (message.documentId !== this.#handle.documentId)
       throw new Error(`channelId doesn't match documentId`)
 
     const { senderId, data } = message
@@ -213,6 +331,7 @@ export class DocSynchronizer extends Synchronizer {
       senderId,
       message: contents,
     })
+
     this.#peers.forEach(peerId => {
       if (peerId === senderId) return
       this.emit("message", {
@@ -222,7 +341,59 @@ export class DocSynchronizer extends Synchronizer {
     })
   }
 
-  receiveSyncMessage(message: SyncMessage | RequestMessage) {}
+  receiveSyncMessage(message: SyncMessage | RequestMessage) {
+    if (message.documentId !== this.#handle.documentId)
+      throw new Error(`channelId doesn't match documentId`)
+
+    // We need to block receiving the syncMessages until we've checked local storage
+    if (!this.#handle.inState([READY, REQUESTING, UNAVAILABLE])) {
+      this.#pendingSyncMessages.push({ message, received: new Date() })
+      return
+    }
+
+    this.#processAllPendingSyncMessages()
+    this.#processSyncMessage(message)
+  }
+
+  #processSyncMessage(message: SyncMessage | RequestMessage) {
+    if (isRequestMessage(message)) {
+      this.#peerDocumentStatuses[message.senderId] = "wants"
+    }
+
+    this.#checkDocUnavailable()
+
+    // if the message has heads, then the peer has the document
+    if (A.decodeSyncMessage(message.data).heads.length > 0) {
+      this.#peerDocumentStatuses[message.senderId] = "has"
+    }
+
+    this.#withSyncState(message.senderId, syncState => {
+      this.#handle.update(doc => {
+        const start = performance.now()
+
+        const [newDoc, newSyncState] = A.receiveSyncMessage(
+          doc,
+          syncState,
+          message.data
+        )
+        const end = performance.now()
+        this.emit("metrics", {
+          type: "receive-sync-message",
+          documentId: this.#handle.documentId,
+          durationMillis: end - start,
+          ...A.stats(doc),
+        })
+
+        this.#setSyncState(message.senderId, newSyncState)
+
+        // respond to just this peer (as required)
+        this.#sendSyncMessage(message.senderId, doc)
+        return newDoc
+      })
+
+      this.#checkDocUnavailable()
+    })
+  }
 
   #checkDocUnavailable() {
     // if we know none of the peers have the document, tell all our peers that we don't either
@@ -240,25 +411,28 @@ export class DocSynchronizer extends Synchronizer {
         .forEach(peerId => {
           const message: MessageContents<DocumentUnavailableMessage> = {
             type: "doc-unavailable",
-            documentId: this.documentId,
+            documentId: this.#handle.documentId,
             targetId: peerId,
           }
           this.emit("message", message)
         })
 
-      if (this.#handle) {
-        this.#handle.unavailable()
-      }
+      this.#handle.unavailable()
     }
   }
 
-  metrics(): {
-    peers: PeerId[]
-    size: { numOps: number; numChanges: number } | undefined
-  } {
+  #processAllPendingSyncMessages() {
+    for (const message of this.#pendingSyncMessages) {
+      this.#processSyncMessage(message.message)
+    }
+
+    this.#pendingSyncMessages = []
+  }
+
+  metrics(): { peers: PeerId[]; size: { numOps: number; numChanges: number } } {
     return {
       peers: this.#peers,
-      size: this.#handle?.metrics(),
+      size: this.#handle.metrics(),
     }
   }
 }

package/src/types.ts

@@ -27,8 +27,11 @@ export type AnyDocumentId =
   | BinaryDocumentId
   | LegacyDocumentId
 
+// We need to define our own version of heads because the AutomergeHeads type is not bs58check encoded
+export type UrlHeads = string[] & { __automergeUrlHeads: unknown }
+
 /** A branded type for peer IDs */
 export type PeerId = string & { __peerId: true }
 
 /** A randomly generated string created when the {@link Repo} starts up */
-export type SessionId = string & { __SessionId: true }
+export type SessionId = string & { __sessionId: true }

package/test/AutomergeUrl.test.ts

@@ -3,9 +3,11 @@ import bs58check from "bs58check"
 import { describe, it } from "vitest"
 import {
   generateAutomergeUrl,
+  getHeadsFromUrl,
   isValidAutomergeUrl,
   parseAutomergeUrl,
   stringifyAutomergeUrl,
+  UrlHeads,
 } from "../src/AutomergeUrl.js"
 import type {
   AutomergeUrl,
@@ -102,3 +104,131 @@ describe("AutomergeUrl", () => {
     })
   })
 })
+
+describe("AutomergeUrl with heads", () => {
+  // Create some sample encoded heads for testing
+  const head1 = bs58check.encode(new Uint8Array([1, 2, 3, 4])) as string
+  const head2 = bs58check.encode(new Uint8Array([5, 6, 7, 8])) as string
+  const goodHeads = [head1, head2] as UrlHeads
+  const urlWithHeads = `${goodUrl}#${head1}|${head2}` as AutomergeUrl
+  const invalidHead = "not-base58-encoded"
+  const invalidHeads = [invalidHead] as UrlHeads
+
+  describe("stringifyAutomergeUrl", () => {
+    it("should stringify a url with heads", () => {
+      const url = stringifyAutomergeUrl({
+        documentId: goodDocumentId,
+        heads: goodHeads,
+      })
+      assert.strictEqual(url, urlWithHeads)
+    })
+
+    it("should throw if heads are not valid base58check", () => {
+      assert.throws(() =>
+        stringifyAutomergeUrl({
+          documentId: goodDocumentId,
+          heads: invalidHeads,
+        })
+      )
+    })
+  })
+
+  describe("parseAutomergeUrl", () => {
+    it("should parse a url with heads", () => {
+      const { documentId, heads } = parseAutomergeUrl(urlWithHeads)
+      assert.equal(documentId, goodDocumentId)
+      assert.deepEqual(heads, [head1, head2])
+    })
+
+    it("should parse a url without heads", () => {
+      const { documentId, heads } = parseAutomergeUrl(goodUrl)
+      assert.equal(documentId, goodDocumentId)
+      assert.equal(heads, undefined)
+    })
+
+    it("should throw on url with invalid heads encoding", () => {
+      const badUrl = `${goodUrl}#${invalidHead}` as AutomergeUrl
+      assert.throws(() => parseAutomergeUrl(badUrl))
+    })
+  })
+
+  describe("isValidAutomergeUrl", () => {
+    it("should return true for a valid url with heads", () => {
+      assert(isValidAutomergeUrl(urlWithHeads) === true)
+    })
+
+    it("should return false for a url with invalid heads", () => {
+      const badUrl = `${goodUrl}#${invalidHead}` as AutomergeUrl
+      assert(isValidAutomergeUrl(badUrl) === false)
+    })
+  })
+
+  describe("getHeadsFromUrl", () => {
+    it("should return heads from a valid url", () => {
+      const heads = getHeadsFromUrl(urlWithHeads)
+      assert.deepEqual(heads, [head1, head2])
+    })
+
+    it("should return undefined for url without heads", () => {
+      const heads = getHeadsFromUrl(goodUrl)
+      assert.equal(heads, undefined)
+    })
+  })
+  it("should handle a single head correctly", () => {
+    const urlWithOneHead = `${goodUrl}#${head1}` as AutomergeUrl
+    const { heads } = parseAutomergeUrl(urlWithOneHead)
+    assert.deepEqual(heads, [head1])
+  })
+
+  it("should round-trip urls with heads", () => {
+    const originalUrl = urlWithHeads
+    const parsed = parseAutomergeUrl(originalUrl)
+    const roundTripped = stringifyAutomergeUrl({
+      documentId: parsed.documentId,
+      heads: parsed.heads,
+    })
+    assert.equal(roundTripped, originalUrl)
+  })
+
+  describe("should reject malformed urls", () => {
+    it("should reject urls with trailing delimiter", () => {
+      assert(!isValidAutomergeUrl(`${goodUrl}#${head1}:` as AutomergeUrl))
+    })
+
+    it("should reject urls with empty head", () => {
+      assert(!isValidAutomergeUrl(`${goodUrl}#|${head1}` as AutomergeUrl))
+    })
+
+    it("should reject urls with multiple hash characters", () => {
+      assert(
+        !isValidAutomergeUrl(`${goodUrl}#${head1}#${head2}` as AutomergeUrl)
+      )
+    })
+  })
+})
+
+describe("empty heads section", () => {
+  it("should treat bare # as empty heads array", () => {
+    const urlWithEmptyHeads = `${goodUrl}#` as AutomergeUrl
+    const { heads } = parseAutomergeUrl(urlWithEmptyHeads)
+    assert.deepEqual(heads, [])
+  })
+
+  it("should round-trip empty heads array", () => {
+    const original = `${goodUrl}#` as AutomergeUrl
+    const parsed = parseAutomergeUrl(original)
+    const roundTripped = stringifyAutomergeUrl({
+      documentId: parsed.documentId,
+      heads: parsed.heads,
+    })
+    assert.equal(roundTripped, original)
+  })
+
+  it("should distinguish between no heads and empty heads", () => {
+    const noHeads = parseAutomergeUrl(goodUrl)
+    const emptyHeads = parseAutomergeUrl(`${goodUrl}#` as AutomergeUrl)
+
+    assert.equal(noHeads.heads, undefined)
+    assert.deepEqual(emptyHeads.heads, [])
+  })
+})

package/test/CollectionSynchronizer.test.ts

@@ -2,16 +2,14 @@ import assert from "assert"
 import { beforeEach, describe, it } from "vitest"
 import { PeerId, Repo, SyncMessage } from "../src/index.js"
 import { CollectionSynchronizer } from "../src/synchronizer/CollectionSynchronizer.js"
-import { next as Automerge } from "@automerge/automerge"
 
-describe.skip("CollectionSynchronizer", () => {
+describe("CollectionSynchronizer", () => {
   let repo: Repo
   let synchronizer: CollectionSynchronizer
-  let beelay: Automerge.beelay.Beelay
 
   beforeEach(() => {
     repo = new Repo()
-    synchronizer = new CollectionSynchronizer(beelay, repo)
+    synchronizer = new CollectionSynchronizer(repo)
   })
 
   it("is not null", async () => {
@@ -30,13 +28,13 @@ describe.skip("CollectionSynchronizer", () => {
         done()
       })
 
-      synchronizer.addDocument(handle.documentId)
+      synchronizer.addDocument(handle)
     }))
 
   it("starts synchronizing existing documents when a peer is added", () =>
     new Promise<void>(done => {
       const handle = repo.create()
-      synchronizer.addDocument(handle.documentId)
+      synchronizer.addDocument(handle)
       synchronizer.once("message", event => {
         const { targetId, documentId } = event as SyncMessage
         assert(targetId === "peer1")
@@ -52,7 +50,7 @@ describe.skip("CollectionSynchronizer", () => {
 
       repo.sharePolicy = async (peerId: PeerId) => peerId !== "peer1"
 
-      synchronizer.addDocument(handle.documentId)
+      synchronizer.addDocument(handle)
       synchronizer.once("message", () => {
         reject(new Error("Should not have sent a message"))
       })
@@ -73,7 +71,7 @@ describe.skip("CollectionSynchronizer", () => {
         reject(new Error("Should not have sent a message"))
       })
 
-      synchronizer.addDocument(handle.documentId)
+      synchronizer.addDocument(handle)
 
       setTimeout(done)
     }))