@automerge/automerge-repo 1.0.0-alpha.2 → 1.0.0-alpha.3

package/src/synchronizer/CollectionSynchronizer.ts

@@ -5,11 +5,17 @@ import {
   binaryToDocumentId,
   stringifyAutomergeUrl,
 } from "../DocUrl.js"
-import { ChannelId, BinaryDocumentId, PeerId, DocumentId } from "../types.js"
+import { PeerId, DocumentId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
 
 import debug from "debug"
+import {
+  DocumentUnavailableMessage,
+  RequestMessage,
+  SynchronizerMessage,
+  SyncMessage,
+} from "../network/messages.js"
 const log = debug("automerge-repo:collectionsync")
 
 /** A CollectionSynchronizer is responsible for synchronizing a DocCollection with peers. */
@@ -20,6 +26,9 @@ export class CollectionSynchronizer extends Synchronizer {
   /** A map of documentIds to their synchronizers */
   #docSynchronizers: Record<DocumentId, DocSynchronizer> = {}
 
+  /** Used to determine if the document is know to the Collection and a synchronizer exists or is being set up */
+  #docSetUp: Record<DocumentId, boolean> = {}
+
   constructor(private repo: DocCollection) {
     super()
   }
@@ -57,37 +66,42 @@ export class CollectionSynchronizer extends Synchronizer {
    * When we receive a sync message for a document we haven't got in memory, we
    * register it with the repo and start synchronizing
    */
-  async receiveSyncMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array
-  ) {
-    log(`onSyncMessage: ${peerId}, ${channelId}, ${message.byteLength}bytes`)
-
-    const documentId = channelId as unknown as DocumentId
+  async receiveMessage(message: SynchronizerMessage) {
+    log(
+      `onSyncMessage: ${message.senderId}, ${message.documentId}, ${
+        "data" in message ? message.data.byteLength + "bytes" : ""
+      }`
+    )
+
+    const documentId = message.documentId
     if (!documentId) {
       throw new Error("received a message with an invalid documentId")
     }
-    const docSynchronizer = await this.#fetchDocSynchronizer(documentId)
 
-    await docSynchronizer.receiveSyncMessage(peerId, channelId, message)
+    this.#docSetUp[documentId] = true
+
+    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
+
+    docSynchronizer.receiveMessage(message)
 
     // Initiate sync with any new peers
     const peers = await this.#documentGenerousPeers(documentId)
-    peers
-      .filter(peerId => !docSynchronizer.hasPeer(peerId))
-      .forEach(peerId => docSynchronizer.beginSync(peerId))
+    docSynchronizer.beginSync(
+      peers.filter(peerId => !docSynchronizer.hasPeer(peerId))
+    )
   }
 
   /**
    * Starts synchronizing the given document with all peers that we share it generously with.
    */
   addDocument(documentId: DocumentId) {
+    // HACK: this is a hack to prevent us from adding the same document twice
+    if (this.#docSetUp[documentId]) {
+      return
+    }
     const docSynchronizer = this.#fetchDocSynchronizer(documentId)
     void this.#documentGenerousPeers(documentId).then(peers => {
-      peers.forEach(peerId => {
-        docSynchronizer.beginSync(peerId)
-      })
+      docSynchronizer.beginSync(peers)
     })
   }
 
@@ -99,11 +113,16 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Adds a peer and maybe starts synchronizing with them */
   addPeer(peerId: PeerId) {
     log(`adding ${peerId} & synchronizing with them`)
+
+    if (this.#peers.has(peerId)) {
+      return
+    }
+
     this.#peers.add(peerId)
     for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
       const { documentId } = docSynchronizer
-      void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
-        if (okToShare) docSynchronizer.beginSync(peerId)
+      this.repo.sharePolicy(peerId, documentId).then(okToShare => {
+        if (okToShare) docSynchronizer.beginSync([peerId])
       })
     }
   }

package/src/synchronizer/DocSynchronizer.ts

@@ -1,9 +1,27 @@
 import * as A from "@automerge/automerge"
-import { DocHandle, READY, REQUESTING } from "../DocHandle.js"
-import { ChannelId, PeerId } from "../types.js"
+import {
+  DocHandle,
+  DocHandleOutboundEphemeralMessagePayload,
+  READY,
+  REQUESTING,
+  UNAVAILABLE,
+} from "../DocHandle.js"
+import { PeerId } from "../types.js"
 import { Synchronizer } from "./Synchronizer.js"
 
 import debug from "debug"
+import {
+  EphemeralMessage,
+  isDocumentUnavailableMessage,
+  isRequestMessage,
+  Message,
+  RequestMessage,
+  SynchronizerMessage,
+  SyncMessage,
+} from "../network/messages.js"
+
+type PeerDocumentStatus = "unknown" | "has" | "unavailable" | "wants"
+import { decode } from "cbor-x"
 
 /**
  * DocSynchronizer takes a handle to an Automerge document, and receives & dispatches sync messages
@@ -17,10 +35,14 @@ export class DocSynchronizer extends Synchronizer {
   /** Active peers */
   #peers: PeerId[] = []
 
+  #peerDocumentStatuses: Record<PeerId, PeerDocumentStatus> = {}
+
   /** Sync state for each peer we've communicated with (including inactive peers) */
   #syncStates: Record<PeerId, A.SyncState> = {}
 
-  #pendingSyncMessages: Array<{ peerId: PeerId; message: Uint8Array }> = []
+  #pendingSyncMessages: Array<SyncMessage | RequestMessage> = []
+
+  #syncStarted = false
 
   constructor(private handle: DocHandle<any>) {
     super()
@@ -31,6 +53,10 @@ export class DocSynchronizer extends Synchronizer {
 
     handle.on("change", () => this.#syncWithPeers())
 
+    handle.on("ephemeral-message-outbound", payload =>
+      this.#broadcastToPeers(payload)
+    )
+
     // Process pending sync messages immediately after the handle becomes ready.
     void (async () => {
       await handle.doc([READY, REQUESTING])
@@ -38,6 +64,10 @@ export class DocSynchronizer extends Synchronizer {
     })()
   }
 
+  get peerStates() {
+    return this.#peerDocumentStatuses
+  }
+
   get documentId() {
     return this.handle.documentId
   }
@@ -47,15 +77,37 @@ export class DocSynchronizer extends Synchronizer {
   async #syncWithPeers() {
     this.#log(`syncWithPeers`)
     const doc = await this.handle.doc()
+    if (doc === undefined) return
     this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
   }
 
+  async #broadcastToPeers({ data }: DocHandleOutboundEphemeralMessagePayload) {
+    this.#log(`broadcastToPeers`, this.#peers)
+    this.#peers.forEach(peerId => this.#sendEphemeralMessage(peerId, data))
+  }
+
+  #sendEphemeralMessage(peerId: PeerId, data: Uint8Array) {
+    this.#log(`sendEphemeralMessage ->${peerId}`)
+
+    this.emit("message", {
+      type: "ephemeral",
+      targetId: peerId,
+      documentId: this.handle.documentId,
+      data,
+    })
+  }
+
   #getSyncState(peerId: PeerId) {
     if (!this.#peers.includes(peerId)) {
       this.#log("adding a new peer", peerId)
       this.#peers.push(peerId)
     }
 
+    // when a peer is added, we don't know if it has the document or not
+    if (!(peerId in this.#peerDocumentStatuses)) {
+      this.#peerDocumentStatuses[peerId] = "unknown"
+    }
+
     return this.#syncStates[peerId] ?? A.initSyncState()
   }
 
@@ -77,16 +129,35 @@ export class DocSynchronizer extends Synchronizer {
     if (message) {
       this.#logMessage(`sendSyncMessage 🡒 ${peerId}`, message)
 
-      const channelId = this.handle.documentId as string as ChannelId
-
-      this.emit("message", {
-        targetId: peerId,
-        channelId,
-        message,
-        broadcast: false,
-      })
-    } else {
-      this.#log(`sendSyncMessage ->${peerId} [no message generated]`)
+      const decoded = A.decodeSyncMessage(message)
+
+      if (
+        !this.handle.isReady() &&
+        decoded.heads.length === 0 &&
+        newSyncState.sharedHeads.length === 0 &&
+        !Object.values(this.#peerDocumentStatuses).includes("has") &&
+        this.#peerDocumentStatuses[peerId] === "unknown"
+      ) {
+        // we don't have the document (or access to it), so we request it
+        this.emit("message", {
+          type: "request",
+          targetId: peerId,
+          documentId: this.handle.documentId,
+          data: message,
+        })
+      } else {
+        this.emit("message", {
+          type: "sync",
+          targetId: peerId,
+          data: message,
+          documentId: this.handle.documentId,
+        })
+      }
+
+      // if we have sent heads, then the peer now has or will have the document
+      if (decoded.heads.length > 0) {
+        this.#peerDocumentStatuses[peerId] = "has"
+      }
     }
   }
 
@@ -104,8 +175,8 @@ export class DocSynchronizer extends Synchronizer {
     // expanding is expensive, so only do it if we're logging at this level
     const expanded = this.#opsLog.enabled
       ? decoded.changes.flatMap(change =>
-          A.decodeChange(change).ops.map(op => JSON.stringify(op))
-        )
+        A.decodeChange(change).ops.map(op => JSON.stringify(op))
+      )
       : null
     this.#opsLog(logText, expanded)
   }
@@ -116,21 +187,33 @@ export class DocSynchronizer extends Synchronizer {
     return this.#peers.includes(peerId)
   }
 
-  beginSync(peerId: PeerId) {
-    this.#log(`beginSync: ${peerId}`)
+  beginSync(peerIds: PeerId[]) {
+    this.#log(`beginSync: ${peerIds.join(", ")}`)
 
     // At this point if we don't have anything in our storage, we need to use an empty doc to sync
     // with; but we don't want to surface that state to the front end
-    void this.handle.doc([READY, REQUESTING]).then(doc => {
+    void this.handle.doc([READY, REQUESTING, UNAVAILABLE]).then(doc => {
+      // if we don't have any peers, then we can say the document is unavailable
+
       // HACK: if we have a sync state already, we round-trip it through the encoding system to make
       // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
       // messages during disconnection.
       // TODO: cover that case with a test and remove this hack
-      const syncStateRaw = this.#getSyncState(peerId)
-      const syncState = A.decodeSyncState(A.encodeSyncState(syncStateRaw))
-      this.#setSyncState(peerId, syncState)
+      peerIds.forEach(peerId => {
+        const syncStateRaw = this.#getSyncState(peerId)
+        const syncState = A.decodeSyncState(A.encodeSyncState(syncStateRaw))
+        this.#setSyncState(peerId, syncState)
+      })
+
+      // we register out peers first, then say that sync has started
+      this.#syncStarted = true
+      this.#checkDocUnavailable()
 
-      this.#sendSyncMessage(peerId, doc)
+      if (doc === undefined) return
+
+      peerIds.forEach(peerId => {
+        this.#sendSyncMessage(peerId, doc)
+      })
     })
   }
 
@@ -139,43 +222,118 @@ export class DocSynchronizer extends Synchronizer {
     this.#peers = this.#peers.filter(p => p !== peerId)
   }
 
-  receiveSyncMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array
-  ) {
-    if ((channelId as string) !== (this.handle.documentId as string))
+  receiveMessage(message: SynchronizerMessage) {
+    switch (message.type) {
+      case "sync":
+      case "request":
+        this.receiveSyncMessage(message)
+        break
+      case "ephemeral":
+        this.receiveEphemeralMessage(message)
+        break
+      case "doc-unavailable":
+        this.#peerDocumentStatuses[message.senderId] = "unavailable"
+        this.#checkDocUnavailable()
+        break
+      default:
+        throw new Error(`unknown message type: ${message}`)
+    }
+  }
+
+  receiveEphemeralMessage(message: EphemeralMessage) {
+    if (message.documentId !== this.handle.documentId)
+      throw new Error(`channelId doesn't match documentId`)
+
+    const { senderId, data } = message
+
+    const contents = decode(data)
+
+    this.handle.emit("ephemeral-message", {
+      handle: this.handle,
+      senderId,
+      message: contents,
+    })
+
+    this.#peers.forEach(peerId => {
+      if (peerId === senderId) return
+      this.emit("message", {
+        ...message,
+        targetId: peerId,
+      })
+    })
+  }
+
+  receiveSyncMessage(message: SyncMessage | RequestMessage) {
+    if (message.documentId !== this.handle.documentId)
       throw new Error(`channelId doesn't match documentId`)
 
     // We need to block receiving the syncMessages until we've checked local storage
-    if (!this.handle.inState([READY, REQUESTING])) {
-      this.#pendingSyncMessages.push({ peerId, message })
+    if (!this.handle.inState([READY, REQUESTING, UNAVAILABLE])) {
+      this.#pendingSyncMessages.push(message)
       return
     }
 
     this.#processAllPendingSyncMessages()
-    this.#processSyncMessage(peerId, message)
+    this.#processSyncMessage(message)
   }
 
-  #processSyncMessage(peerId: PeerId, message: Uint8Array) {
+  #processSyncMessage(message: SyncMessage | RequestMessage) {
+    if (isRequestMessage(message)) {
+      this.#peerDocumentStatuses[message.senderId] = "wants"
+    }
+
+    this.#checkDocUnavailable()
+
+    // if the message has heads, then the peer has the document
+    if (A.decodeSyncMessage(message.data).heads.length > 0) {
+      this.#peerDocumentStatuses[message.senderId] = "has"
+    }
+
     this.handle.update(doc => {
       const [newDoc, newSyncState] = A.receiveSyncMessage(
         doc,
-        this.#getSyncState(peerId),
-        message
+        this.#getSyncState(message.senderId),
+        message.data
       )
 
-      this.#setSyncState(peerId, newSyncState)
+      this.#setSyncState(message.senderId, newSyncState)
 
       // respond to just this peer (as required)
-      this.#sendSyncMessage(peerId, doc)
+      this.#sendSyncMessage(message.senderId, doc)
       return newDoc
     })
+
+    this.#checkDocUnavailable()
+  }
+
+  #checkDocUnavailable() {
+    // if we know none of the peers have the document, tell all our peers that we don't either
+    if (
+      this.#syncStarted &&
+      this.handle.inState([REQUESTING]) &&
+      this.#peers.every(
+        peerId =>
+          this.#peerDocumentStatuses[peerId] === "unavailable" ||
+          this.#peerDocumentStatuses[peerId] === "wants"
+      )
+    ) {
+      this.#peers
+        .filter(peerId => this.#peerDocumentStatuses[peerId] === "wants")
+        .forEach(peerId => {
+          this.emit("message", {
+            type: "doc-unavailable",
+            documentId: this.handle.documentId,
+            targetId: peerId,
+          })
+        })
+
+      this.handle.unavailable()
+    }
   }
 
   #processAllPendingSyncMessages() {
-    for (const { peerId, message } of this.#pendingSyncMessages) {
-      this.#processSyncMessage(peerId, message)
+    for (const message of this.#pendingSyncMessages) {
+      this.#processSyncMessage(message)
     }
 
     this.#pendingSyncMessages = []

package/src/synchronizer/Synchronizer.ts

@@ -1,15 +1,10 @@
 import EventEmitter from "eventemitter3"
-import { ChannelId, PeerId } from "../types.js"
-import { MessagePayload } from "../network/NetworkAdapter.js"
+import { Message, MessageContents } from "../network/messages.js"
 
 export abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
-  abstract receiveSyncMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array
-  ): void
+  abstract receiveMessage(message: Message): void
 }
 
 export interface SynchronizerEvents {
-  message: (arg: MessagePayload) => void
+  message: (arg: MessageContents) => void
 }

package/src/types.ts

@@ -3,4 +3,7 @@ export type AutomergeUrl = string & { __documentUrl: true } // for opening / lin
 export type BinaryDocumentId = Uint8Array & { __binaryDocumentId: true } // for storing / syncing
 
 export type PeerId = string & { __peerId: false }
-export type ChannelId = string & { __channelId: false }
+
+export type DistributiveOmit<T, K extends keyof any> = T extends any
+  ? Omit<T, K>
+  : never

package/test/CollectionSynchronizer.test.ts

@@ -1,8 +1,7 @@
-import { CollectionSynchronizer } from "../src/synchronizer/CollectionSynchronizer.js"
-import { ChannelId, DocCollection, BinaryDocumentId, PeerId } from "../src"
 import assert from "assert"
 import { beforeEach } from "mocha"
-import { MessagePayload } from "../src/network/NetworkAdapter.js"
+import { DocCollection, PeerId } from "../src"
+import { CollectionSynchronizer } from "../src/synchronizer/CollectionSynchronizer.js"
 
 describe("CollectionSynchronizer", () => {
   let collection: DocCollection
@@ -21,9 +20,9 @@ describe("CollectionSynchronizer", () => {
     const handle = collection.create()
     synchronizer.addPeer("peer1" as PeerId)
 
-    synchronizer.once("message", (event: MessagePayload) => {
+    synchronizer.once("message", event => {
       assert(event.targetId === "peer1")
-      assert(event.channelId === (handle.documentId as unknown as ChannelId))
+      assert(event.documentId === handle.documentId)
       done()
     })
 
@@ -33,9 +32,9 @@ describe("CollectionSynchronizer", () => {
   it("starts synchronizing existing documents when a peer is added", done => {
     const handle = collection.create()
     synchronizer.addDocument(handle.documentId)
-    synchronizer.once("message", (event: MessagePayload) => {
+    synchronizer.once("message", event => {
       assert(event.targetId === "peer1")
-      assert(event.channelId === (handle.documentId as unknown as ChannelId))
+      assert(event.documentId === handle.documentId)
       done()
     })
     synchronizer.addPeer("peer1" as PeerId)

package/test/DocHandle.test.ts

@@ -5,10 +5,12 @@ import { DocHandle, DocHandleChangePayload } from "../src"
 import { pause } from "../src/helpers/pause"
 import { TestDoc } from "./types.js"
 import { generateAutomergeUrl, parseAutomergeUrl } from "../src/DocUrl"
+import { eventPromise } from "../src/helpers/eventPromise"
+import { decode } from "cbor-x"
 
 describe("DocHandle", () => {
-  const TEST_ID = parseAutomergeUrl(generateAutomergeUrl()).encodedDocumentId
-  const BOGUS_ID = parseAutomergeUrl(generateAutomergeUrl()).encodedDocumentId
+  const TEST_ID = parseAutomergeUrl(generateAutomergeUrl()).documentId
+  const BOGUS_ID = parseAutomergeUrl(generateAutomergeUrl()).documentId
 
   const docFromMockStorage = (doc: A.Doc<{ foo: string }>) => {
     return A.change<{ foo: string }>(doc, d => (d.foo = "bar"))
@@ -28,8 +30,7 @@ describe("DocHandle", () => {
 
     assert.equal(handle.isReady(), true)
     const doc = await handle.doc()
-    console.log("DOC", JSON.stringify(doc))
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
   })
 
   it("should allow sync access to the doc", async () => {
@@ -60,7 +61,7 @@ describe("DocHandle", () => {
     const doc = await handle.doc()
 
     assert.equal(handle.isReady(), true)
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
   })
 
   it("should block changes until ready()", async () => {
@@ -78,7 +79,7 @@ describe("DocHandle", () => {
     handle.change(d => (d.foo = "pizza"))
 
     const doc = await handle.doc()
-    assert.equal(doc.foo, "pizza")
+    assert.equal(doc?.foo, "pizza")
   })
 
   it("should not be ready while requesting from the network", async () => {
@@ -89,7 +90,7 @@ describe("DocHandle", () => {
 
     assert.equal(handle.docSync(), undefined)
     assert.equal(handle.isReady(), false)
-    assert.throws(() => handle.change(h => {}))
+    assert.throws(() => handle.change(_ => { }))
   })
 
   it("should become ready if the document is updated by the network", async () => {
@@ -105,7 +106,7 @@ describe("DocHandle", () => {
 
     const doc = await handle.doc()
     assert.equal(handle.isReady(), true)
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
   })
 
   it("should emit a change message when changes happen", async () => {
@@ -120,7 +121,7 @@ describe("DocHandle", () => {
     })
 
     const doc = await handle.doc()
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
 
     const changePayload = await p
     assert.deepStrictEqual(changePayload.doc, doc)
@@ -181,7 +182,7 @@ describe("DocHandle", () => {
     })
 
     const doc = await handle.doc()
-    assert.equal(doc.foo, "baz")
+    assert.equal(doc?.foo, "baz")
 
     return p
   })
@@ -196,7 +197,7 @@ describe("DocHandle", () => {
 
     await p
     const doc = await handle.doc()
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
   })
 
   it("should not emit a patch message if no change happens", done => {
@@ -232,7 +233,7 @@ describe("DocHandle", () => {
 
     // now it should not time out
     const doc = await handle.doc()
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
   })
 
   it("should time out if the document is not updated from the network", async () => {
@@ -265,7 +266,7 @@ describe("DocHandle", () => {
     await pause(5)
 
     const doc = await handle.doc()
-    assert.equal(doc.foo, "bar")
+    assert.equal(doc?.foo, "bar")
   })
 
   it("should emit a delete event when deleted", async () => {
@@ -301,4 +302,18 @@ describe("DocHandle", () => {
 
     assert(wasBar, "foo should have been bar as we changed at the old heads")
   })
+
+  describe("ephemeral messaging", () => {
+    it("can broadcast a message for the network to send out", async () => {
+      const handle = new DocHandle<TestDoc>(TEST_ID, { isNew: true })
+      const message = { foo: "bar" }
+
+      const promise = eventPromise(handle, "ephemeral-message-outbound")
+
+      handle.broadcast(message)
+
+      const { data } = await promise
+      assert.deepStrictEqual(decode(data), message)
+    })
+  })
 })