@automerge/automerge-repo 1.0.0-alpha.2 → 1.0.0-alpha.3

package/src/EphemeralData.ts

@@ -1,46 +1,16 @@
-import { decode, encode } from "cbor-x"
-import EventEmitter from "eventemitter3"
-import { ChannelId, PeerId } from "./index.js"
-import { MessagePayload } from "./network/NetworkAdapter.js"
-
-/**
- * EphemeralData provides a mechanism to broadcast short-lived data — cursor positions, presence,
- * heartbeats, etc. — that is useful in the moment but not worth persisting.
- */
-export class EphemeralData extends EventEmitter<EphemeralDataMessageEvents> {
-  /** Broadcast an ephemeral message */
-  broadcast(channelId: ChannelId, message: unknown) {
-    const messageBytes = encode(message)
-
-    this.emit("message", {
-      targetId: "*" as PeerId, // TODO: we don't really need a targetId for broadcast
-      channelId: ("m/" + channelId) as ChannelId,
-      message: messageBytes,
-      broadcast: true,
-    })
-  }
-
-  /** Receive an ephemeral message */
-  receive(senderId: PeerId, grossChannelId: ChannelId, message: Uint8Array) {
-    const data = decode(message)
-    const channelId = grossChannelId.slice(2) as ChannelId
-    this.emit("data", {
-      peerId: senderId,
-      channelId,
-      data,
-    })
-  }
-}
+import { DocumentId, PeerId } from "./index.js"
+import { EphemeralMessageContents } from "./network/messages.js"
 
 // types
+export type SessionId = string & { __SessionId: false }
 
 export interface EphemeralDataPayload {
-  channelId: ChannelId
+  documentId: DocumentId
   peerId: PeerId
-  data: { peerId: PeerId; channelId: ChannelId; data: unknown }
+  data: { peerId: PeerId; documentId: DocumentId; data: unknown }
 }
 
 export type EphemeralDataMessageEvents = {
-  message: (event: MessagePayload) => void
+  message: (event: EphemeralMessageContents) => void
   data: (event: EphemeralDataPayload) => void
 }

package/src/Repo.ts

@@ -1,5 +1,5 @@
+import debug from "debug"
 import { DocCollection } from "./DocCollection.js"
-import { EphemeralData } from "./EphemeralData.js"
 import { NetworkAdapter } from "./network/NetworkAdapter.js"
 import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
 import { StorageAdapter } from "./storage/StorageAdapter.js"
@@ -7,15 +7,12 @@ import { StorageSubsystem } from "./storage/StorageSubsystem.js"
 import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
 import { DocumentId, PeerId } from "./types.js"
 
-import debug from "debug"
-
 /** A Repo is a DocCollection with networking, syncing, and storage capabilities. */
 export class Repo extends DocCollection {
   #log: debug.Debugger
 
   networkSubsystem: NetworkSubsystem
   storageSubsystem?: StorageSubsystem
-  ephemeralData: EphemeralData
 
   constructor({ storage, network, peerId, sharePolicy }: RepoConfig) {
     super()
@@ -40,44 +37,45 @@ export class Repo extends DocCollection {
         }
       }
 
+      handle.on("unavailable", () => {
+        this.#log("document unavailable", { documentId: handle.documentId })
+        this.emit("unavailable-document", {
+          documentId: handle.documentId,
+        })
+      })
+
       handle.request()
 
       // Register the document with the synchronizer. This advertises our interest in the document.
       synchronizer.addDocument(handle.documentId)
     })
 
-    this.on("delete-document", ({ encodedDocumentId }) => {
+    this.on("delete-document", ({ documentId }) => {
       // TODO Pass the delete on to the network
       // synchronizer.removeDocument(documentId)
 
       if (storageSubsystem) {
-        storageSubsystem.remove(encodedDocumentId)
+        storageSubsystem.remove(documentId)
       }
     })
 
     // SYNCHRONIZER
     // The synchronizer uses the network subsystem to keep documents in sync with peers.
-
     const synchronizer = new CollectionSynchronizer(this)
 
     // When the synchronizer emits sync messages, send them to peers
-    synchronizer.on(
-      "message",
-      ({ targetId, channelId, message, broadcast }) => {
-        this.#log(`sending sync message to ${targetId}`)
-        networkSubsystem.sendMessage(targetId, channelId, message, broadcast)
-      }
-    )
+    synchronizer.on("message", message => {
+      this.#log(`sending sync message to ${message.targetId}`)
+      networkSubsystem.send(message)
+    })
 
     // STORAGE
     // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
-
     const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
     this.storageSubsystem = storageSubsystem
 
     // NETWORK
     // The network subsystem deals with sending and receiving messages to and from peers.
-
     const networkSubsystem = new NetworkSubsystem(network, peerId)
     this.networkSubsystem = networkSubsystem
 
@@ -94,40 +92,8 @@ export class Repo extends DocCollection {
 
     // Handle incoming messages
     networkSubsystem.on("message", async msg => {
-      const { senderId, channelId, message } = msg
-
-      // TODO: this demands a more principled way of associating channels with recipients
-
-      // Ephemeral channel ids start with "m/"
-      if (channelId.startsWith("m/")) {
-        // Ephemeral message
-        this.#log(`receiving ephemeral message from ${senderId}`)
-        ephemeralData.receive(senderId, channelId, message)
-      } else {
-        // Sync message
-        this.#log(`receiving sync message from ${senderId}`)
-        await synchronizer.receiveSyncMessage(senderId, channelId, message)
-      }
+      await synchronizer.receiveMessage(msg)
     })
-
-    // We establish a special channel for sync messages
-    networkSubsystem.join()
-
-    // EPHEMERAL DATA
-    // The ephemeral data subsystem uses the network to send and receive messages that are not
-    // persisted to storage, e.g. cursor position, presence, etc.
-
-    const ephemeralData = new EphemeralData()
-    this.ephemeralData = ephemeralData
-
-    // Send ephemeral messages to peers
-    ephemeralData.on(
-      "message",
-      ({ targetId, channelId, message, broadcast }) => {
-        this.#log(`sending ephemeral message to ${targetId}`)
-        networkSubsystem.sendMessage(targetId, channelId, message, broadcast)
-      }
-    )
   }
 }
 

package/src/helpers/tests/network-adapter-tests.ts

@@ -1,7 +1,8 @@
-import { PeerId, Repo, type NetworkAdapter, ChannelId } from "../../index.js"
+import { PeerId, Repo, type NetworkAdapter, DocumentId } from "../../index.js"
 import { eventPromise, eventPromises } from "../eventPromise.js"
 import { assert } from "chai"
 import { describe, it } from "mocha"
+import { pause } from "../pause.js"
 
 /**
  * Runs a series of tests against a set of three peers, each represented by one or more instantiated
@@ -46,7 +47,7 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Bob receives the change
         await eventPromise(bobHandle, "change")
-        assert.equal((await bobHandle.doc()).foo, "bar")
+        assert.equal((await bobHandle.doc())?.foo, "bar")
 
         // Bob changes the document
         bobHandle.change(d => {
@@ -55,7 +56,7 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Alice receives the change
         await eventPromise(aliceHandle, "change")
-        assert.equal((await aliceHandle.doc()).foo, "baz")
+        assert.equal((await aliceHandle.doc())?.foo, "baz")
       }
 
       // Run the test in both directions, in case they're different types of adapters
@@ -97,8 +98,8 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Bob and Charlie receive the change
       await eventPromises([bobHandle, charlieHandle], "change")
-      assert.equal((await bobHandle.doc()).foo, "bar")
-      assert.equal((await charlieHandle.doc()).foo, "bar")
+      assert.equal((await bobHandle.doc())?.foo, "bar")
+      assert.equal((await charlieHandle.doc())?.foo, "bar")
 
       // Charlie changes the document
       charlieHandle.change(d => {
@@ -107,15 +108,13 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Alice and Bob receive the change
       await eventPromises([aliceHandle, bobHandle], "change")
-      assert.equal((await bobHandle.doc()).foo, "baz")
-      assert.equal((await charlieHandle.doc()).foo, "baz")
+      assert.equal((await bobHandle.doc())?.foo, "baz")
+      assert.equal((await charlieHandle.doc())?.foo, "baz")
 
       teardown()
     })
 
-    // TODO: with BroadcastChannel, this test never ends, because it goes into an infinite loop,
-    // because the network has cycles (see #92)
-    it.skip("can broadcast a message", async () => {
+    it("can broadcast a message", async () => {
       const { adapters, teardown } = await setup()
       const [a, b, c] = adapters
 
@@ -128,13 +127,18 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
         "peer"
       )
 
-      const channelId = "broadcast" as ChannelId
+      const aliceHandle = aliceRepo.create<TestDoc>()
+      const charlieHandle = charlieRepo.find(aliceHandle.url)
+
+      // pause to give charlie a chance to let alice know it wants the doc
+      await pause(100)
+
       const alicePresenceData = { presence: "alice" }
+      aliceHandle.broadcast(alicePresenceData)
 
-      aliceRepo.ephemeralData.broadcast(channelId, alicePresenceData)
-      const { data } = await eventPromise(charlieRepo.ephemeralData, "data")
+      const { message } = await eventPromise(charlieHandle, "ephemeral-message")
 
-      assert.deepStrictEqual(data, alicePresenceData)
+      assert.deepStrictEqual(message, alicePresenceData)
       teardown()
     })
   })

package/src/index.ts

@@ -3,12 +3,22 @@ export { DocHandle, HandleState } from "./DocHandle.js"
 export type { DocHandleChangePayload } from "./DocHandle.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
 export type {
-  InboundMessagePayload,
-  MessagePayload,
   OpenPayload,
   PeerCandidatePayload,
   PeerDisconnectedPayload,
 } from "./network/NetworkAdapter.js"
+
+// This is a bit confusing right now, but:
+// Message is the type for messages used outside of the network adapters
+// there are some extra internal network adapter-only messages on NetworkAdapterMessage
+// and Message is (as of this writing) a union type for EphmeralMessage and SyncMessage
+export type {
+  Message,
+  NetworkAdapterMessage,
+  EphemeralMessage,
+  SyncMessage,
+} from "./network/messages.js"
+
 export { NetworkSubsystem } from "./network/NetworkSubsystem.js"
 export { Repo, type SharePolicy } from "./Repo.js"
 export { StorageAdapter, type StorageKey } from "./storage/StorageAdapter.js"

package/src/network/NetworkAdapter.ts

@@ -1,17 +1,13 @@
 import EventEmitter from "eventemitter3"
-import { PeerId, ChannelId } from "../types.js"
+import { PeerId } from "../types.js"
+import { Message } from "./messages.js"
 
 export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
   peerId?: PeerId // hmmm, maybe not
 
   abstract connect(url?: string): void
 
-  abstract sendMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array,
-    broadcast: boolean
-  ): void
+  abstract send(message: Message): void
 
   abstract join(): void
 
@@ -25,7 +21,7 @@ export interface NetworkAdapterEvents {
   close: () => void
   "peer-candidate": (payload: PeerCandidatePayload) => void
   "peer-disconnected": (payload: PeerDisconnectedPayload) => void
-  message: (payload: InboundMessagePayload) => void
+  message: (payload: Message) => void
 }
 
 export interface OpenPayload {
@@ -36,18 +32,6 @@ export interface PeerCandidatePayload {
   peerId: PeerId
 }
 
-export interface MessagePayload {
-  targetId: PeerId
-  channelId: ChannelId
-  message: Uint8Array
-  broadcast: boolean
-}
-
-export interface InboundMessagePayload extends MessagePayload {
-  type?: string
-  senderId: PeerId
-}
-
 export interface PeerDisconnectedPayload {
   peerId: PeerId
 }

package/src/network/NetworkSubsystem.ts

@@ -1,17 +1,31 @@
 import EventEmitter from "eventemitter3"
+import { PeerId } from "../types.js"
+import { NetworkAdapter, PeerDisconnectedPayload } from "./NetworkAdapter.js"
+
 import {
-  InboundMessagePayload,
-  NetworkAdapter,
-  PeerDisconnectedPayload,
-} from "./NetworkAdapter.js"
-import { ChannelId, PeerId } from "../types.js"
+  EphemeralMessage,
+  isEphemeralMessage,
+  isValidMessage,
+  Message,
+  MessageContents,
+} from "./messages.js"
 
 import debug from "debug"
+import { SessionId } from "../EphemeralData.js"
+
+type EphemeralMessageSource = `${PeerId}:${SessionId}`
+
+const getEphemeralMessageSource = (message: EphemeralMessage) =>
+  `${message.senderId}:${message.sessionId}` as EphemeralMessageSource
 
 export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #log: debug.Debugger
   #adaptersByPeer: Record<PeerId, NetworkAdapter> = {}
 
+  #count = 0
+  #sessionId: SessionId = Math.random().toString(36).slice(2) as SessionId
+  #ephemeralSessionCounts: Record<EphemeralMessageSource, number> = {}
+
   constructor(
     private adapters: NetworkAdapter[],
     public peerId = randomPeerId()
@@ -44,20 +58,24 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     })
 
     networkAdapter.on("message", msg => {
-      const { senderId, channelId, broadcast, message } = msg
-      this.#log(`message from ${senderId}`)
-
-      // If we receive a broadcast message from a network adapter we need to re-broadcast it to all
-      // our other peers. This is the world's worst gossip protocol.
-
-      // TODO: This relies on the network forming a tree! If there are cycles, this approach will
-      // loop messages around forever.
-      if (broadcast) {
-        Object.entries(this.#adaptersByPeer)
-          .filter(([id]) => id !== senderId)
-          .forEach(([id, peer]) => {
-            peer.sendMessage(id as PeerId, channelId, message, broadcast)
-          })
+      if (!isValidMessage(msg)) {
+        this.#log(`invalid message: ${JSON.stringify(msg)}`)
+        return
+      }
+
+      this.#log(`message from ${msg.senderId}`)
+
+      if (isEphemeralMessage(msg)) {
+        const source = getEphemeralMessageSource(msg)
+        if (
+          this.#ephemeralSessionCounts[source] === undefined ||
+          msg.count > this.#ephemeralSessionCounts[source]
+        ) {
+          this.#ephemeralSessionCounts[source] = msg.count
+          this.emit("message", msg)
+        }
+
+        return
       }
 
       this.emit("message", msg)
@@ -75,25 +93,30 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     networkAdapter.join()
   }
 
-  sendMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array,
-    broadcast: boolean
-  ) {
-    if (broadcast) {
-      Object.entries(this.#adaptersByPeer).forEach(([id, peer]) => {
-        this.#log(`sending broadcast to ${id}`)
-        peer.sendMessage(id as PeerId, channelId, message, true)
-      })
+  send(message: MessageContents) {
+    const peer = this.#adaptersByPeer[message.targetId]
+    if (!peer) {
+      this.#log(`Tried to send message but peer not found: ${message.targetId}`)
+      return
+    }
+    this.#log(`Sending message to ${message.targetId}`)
+
+    if (isEphemeralMessage(message)) {
+      const outbound =
+        "count" in message
+          ? message
+          : {
+              ...message,
+              count: ++this.#count,
+              sessionId: this.#sessionId,
+              senderId: this.peerId,
+            }
+      this.#log("Ephemeral message", outbound)
+      peer.send(outbound)
     } else {
-      const peer = this.#adaptersByPeer[peerId]
-      if (!peer) {
-        this.#log(`Tried to send message but peer not found: ${peerId}`)
-        return
-      }
-      this.#log(`Sending message to ${peerId}`)
-      peer.sendMessage(peerId, channelId, message, false)
+      const outbound = { ...message, senderId: this.peerId }
+      this.#log("Sync message", outbound)
+      peer.send(outbound)
     }
   }
 
@@ -117,7 +140,7 @@ function randomPeerId() {
 export interface NetworkSubsystemEvents {
   peer: (payload: PeerPayload) => void
   "peer-disconnected": (payload: PeerDisconnectedPayload) => void
-  message: (payload: InboundMessagePayload) => void
+  message: (payload: Message) => void
 }
 
 export interface PeerPayload {

package/src/network/messages.ts

@@ -0,0 +1,123 @@
+// utilities
+import { SessionId } from "../EphemeralData"
+import { DocumentId, PeerId } from "../types"
+
+export function isValidMessage(
+  message: NetworkAdapterMessage
+): message is
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage {
+  return (
+    typeof message === "object" &&
+    typeof message.type === "string" &&
+    typeof message.senderId === "string" &&
+    (isSyncMessage(message) ||
+      isEphemeralMessage(message) ||
+      isRequestMessage(message) ||
+      isDocumentUnavailableMessage(message))
+  )
+}
+
+export function isDocumentUnavailableMessage(
+  message: NetworkAdapterMessage
+): message is DocumentUnavailableMessage {
+  return message.type === "doc-unavailable"
+}
+
+export function isRequestMessage(
+  message: NetworkAdapterMessage
+): message is RequestMessage {
+  return message.type === "request"
+}
+
+export function isSyncMessage(
+  message: NetworkAdapterMessage
+): message is SyncMessage {
+  return message.type === "sync"
+}
+
+export function isEphemeralMessage(
+  message: NetworkAdapterMessage | MessageContents
+): message is EphemeralMessage | EphemeralMessageContents {
+  return message.type === "ephemeral"
+}
+
+export interface SyncMessageEnvelope {
+  senderId: PeerId
+}
+
+export interface SyncMessageContents {
+  type: "sync"
+  data: Uint8Array
+  targetId: PeerId
+  documentId: DocumentId
+}
+
+export type SyncMessage = SyncMessageEnvelope & SyncMessageContents
+
+export interface EphemeralMessageEnvelope {
+  senderId: PeerId
+  count: number
+  sessionId: SessionId
+}
+
+export interface EphemeralMessageContents {
+  type: "ephemeral"
+  targetId: PeerId
+  documentId: DocumentId
+  data: Uint8Array
+}
+
+export type EphemeralMessage = EphemeralMessageEnvelope &
+  EphemeralMessageContents
+
+export interface DocumentUnavailableMessageContents {
+  type: "doc-unavailable"
+  documentId: DocumentId
+  targetId: PeerId
+}
+
+export type DocumentUnavailableMessage = SyncMessageEnvelope &
+  DocumentUnavailableMessageContents
+
+export interface RequestMessageContents {
+  type: "request"
+  data: Uint8Array
+  targetId: PeerId
+  documentId: DocumentId
+}
+
+export type RequestMessage = SyncMessageEnvelope & RequestMessageContents
+
+export type MessageContents =
+  | SyncMessageContents
+  | EphemeralMessageContents
+  | RequestMessageContents
+  | DocumentUnavailableMessageContents
+
+export type Message =
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+
+export type SynchronizerMessage =
+  | SyncMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+  | EphemeralMessage
+
+type ArriveMessage = {
+  senderId: PeerId
+  type: "arrive"
+}
+
+type WelcomeMessage = {
+  senderId: PeerId
+  targetId: PeerId
+  type: "welcome"
+}
+
+export type NetworkAdapterMessage = ArriveMessage | WelcomeMessage | Message