@automerge/automerge-repo 1.0.0-alpha.2 → 1.0.0-alpha.4

package/src/network/NetworkAdapter.ts

@@ -1,17 +1,13 @@
-import EventEmitter from "eventemitter3"
-import { PeerId, ChannelId } from "../types.js"
+import { EventEmitter } from "eventemitter3"
+import { PeerId } from "../types.js"
+import { Message } from "./messages.js"
 
 export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
   peerId?: PeerId // hmmm, maybe not
 
   abstract connect(url?: string): void
 
-  abstract sendMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array,
-    broadcast: boolean
-  ): void
+  abstract send(message: Message): void
 
   abstract join(): void
 
@@ -21,11 +17,11 @@ export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents>
 // events & payloads
 
 export interface NetworkAdapterEvents {
-  open: (payload: OpenPayload) => void
+  ready: (payload: OpenPayload) => void
   close: () => void
   "peer-candidate": (payload: PeerCandidatePayload) => void
   "peer-disconnected": (payload: PeerDisconnectedPayload) => void
-  message: (payload: InboundMessagePayload) => void
+  message: (payload: Message) => void
 }
 
 export interface OpenPayload {
@@ -36,18 +32,6 @@ export interface PeerCandidatePayload {
   peerId: PeerId
 }
 
-export interface MessagePayload {
-  targetId: PeerId
-  channelId: ChannelId
-  message: Uint8Array
-  broadcast: boolean
-}
-
-export interface InboundMessagePayload extends MessagePayload {
-  type?: string
-  senderId: PeerId
-}
-
 export interface PeerDisconnectedPayload {
   peerId: PeerId
 }

package/src/network/NetworkSubsystem.ts

@@ -1,28 +1,51 @@
-import EventEmitter from "eventemitter3"
+import { EventEmitter } from "eventemitter3"
+import { PeerId } from "../types.js"
+import { NetworkAdapter, PeerDisconnectedPayload } from "./NetworkAdapter.js"
+
 import {
-  InboundMessagePayload,
-  NetworkAdapter,
-  PeerDisconnectedPayload,
-} from "./NetworkAdapter.js"
-import { ChannelId, PeerId } from "../types.js"
+  EphemeralMessage,
+  isEphemeralMessage,
+  isValidMessage,
+  Message,
+  MessageContents,
+} from "./messages.js"
 
 import debug from "debug"
+import { SessionId } from "../EphemeralData.js"
+
+type EphemeralMessageSource = `${PeerId}:${SessionId}`
+
+const getEphemeralMessageSource = (message: EphemeralMessage) =>
+  `${message.senderId}:${message.sessionId}` as EphemeralMessageSource
 
 export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #log: debug.Debugger
   #adaptersByPeer: Record<PeerId, NetworkAdapter> = {}
 
+  #count = 0
+  #sessionId: SessionId = Math.random().toString(36).slice(2) as SessionId
+  #ephemeralSessionCounts: Record<EphemeralMessageSource, number> = {}
+  #readyAdapterCount = 0 
+  #adapters: NetworkAdapter[] = []
+
   constructor(
-    private adapters: NetworkAdapter[],
+    adapters: NetworkAdapter[],
     public peerId = randomPeerId()
   ) {
     super()
     this.#log = debug(`automerge-repo:network:${this.peerId}`)
-    this.adapters.forEach(a => this.addNetworkAdapter(a))
+    adapters.forEach(a => this.addNetworkAdapter(a))
   }
 
   addNetworkAdapter(networkAdapter: NetworkAdapter) {
-    networkAdapter.connect(this.peerId)
+    this.#adapters.push(networkAdapter)
+    networkAdapter.once("ready", () => {
+      this.#readyAdapterCount++
+      this.#log("Adapters ready: ", this.#readyAdapterCount, "/", this.#adapters.length)
+      if (this.#readyAdapterCount === this.#adapters.length) {
+        this.emit("ready")
+      }
+    })
 
     networkAdapter.on("peer-candidate", ({ peerId }) => {
       this.#log(`peer candidate: ${peerId} `)
@@ -44,20 +67,24 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     })
 
     networkAdapter.on("message", msg => {
-      const { senderId, channelId, broadcast, message } = msg
-      this.#log(`message from ${senderId}`)
-
-      // If we receive a broadcast message from a network adapter we need to re-broadcast it to all
-      // our other peers. This is the world's worst gossip protocol.
-
-      // TODO: This relies on the network forming a tree! If there are cycles, this approach will
-      // loop messages around forever.
-      if (broadcast) {
-        Object.entries(this.#adaptersByPeer)
-          .filter(([id]) => id !== senderId)
-          .forEach(([id, peer]) => {
-            peer.sendMessage(id as PeerId, channelId, message, broadcast)
-          })
+      if (!isValidMessage(msg)) {
+        this.#log(`invalid message: ${JSON.stringify(msg)}`)
+        return
+      }
+
+      this.#log(`message from ${msg.senderId}`)
+
+      if (isEphemeralMessage(msg)) {
+        const source = getEphemeralMessageSource(msg)
+        if (
+          this.#ephemeralSessionCounts[source] === undefined ||
+          msg.count > this.#ephemeralSessionCounts[source]
+        ) {
+          this.#ephemeralSessionCounts[source] = msg.count
+          this.emit("message", msg)
+        }
+
+        return
       }
 
       this.emit("message", msg)
@@ -72,39 +99,61 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       })
     })
 
+    networkAdapter.connect(this.peerId)
     networkAdapter.join()
   }
 
-  sendMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array,
-    broadcast: boolean
-  ) {
-    if (broadcast) {
-      Object.entries(this.#adaptersByPeer).forEach(([id, peer]) => {
-        this.#log(`sending broadcast to ${id}`)
-        peer.sendMessage(id as PeerId, channelId, message, true)
-      })
+  send(message: MessageContents) {
+    const peer = this.#adaptersByPeer[message.targetId]
+    if (!peer) {
+      this.#log(`Tried to send message but peer not found: ${message.targetId}`)
+      return
+    }
+    this.#log(`Sending message to ${message.targetId}`)
+
+    if (isEphemeralMessage(message)) {
+      const outbound =
+        "count" in message
+          ? message
+          : {
+              ...message,
+              count: ++this.#count,
+              sessionId: this.#sessionId,
+              senderId: this.peerId,
+            }
+      this.#log("Ephemeral message", outbound)
+      peer.send(outbound)
     } else {
-      const peer = this.#adaptersByPeer[peerId]
-      if (!peer) {
-        this.#log(`Tried to send message but peer not found: ${peerId}`)
-        return
-      }
-      this.#log(`Sending message to ${peerId}`)
-      peer.sendMessage(peerId, channelId, message, false)
+      const outbound = { ...message, senderId: this.peerId }
+      this.#log("Sync message", outbound)
+      peer.send(outbound)
     }
   }
 
   join() {
     this.#log(`Joining network`)
-    this.adapters.forEach(a => a.join())
+    this.#adapters.forEach(a => a.join())
   }
 
   leave() {
     this.#log(`Leaving network`)
-    this.adapters.forEach(a => a.leave())
+    this.#adapters.forEach(a => a.leave())
+  }
+
+  isReady = () => {
+    return this.#readyAdapterCount === this.#adapters.length
+  }
+
+  whenReady = async () => {
+    if (this.isReady()) {
+      return
+    } else {
+      return new Promise<void>(resolve => {
+        this.once("ready", () => {
+          resolve()
+        })
+      })
+    }
   }
 }
 
@@ -117,7 +166,8 @@ function randomPeerId() {
 export interface NetworkSubsystemEvents {
   peer: (payload: PeerPayload) => void
   "peer-disconnected": (payload: PeerDisconnectedPayload) => void
-  message: (payload: InboundMessagePayload) => void
+  message: (payload: Message) => void
+  "ready": () => void
 }
 
 export interface PeerPayload {

package/src/network/messages.ts

@@ -0,0 +1,123 @@
+// utilities
+import { SessionId } from "../EphemeralData.js"
+import { DocumentId, PeerId } from "../types.js"
+
+export function isValidMessage(
+  message: NetworkAdapterMessage
+): message is
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage {
+  return (
+    typeof message === "object" &&
+    typeof message.type === "string" &&
+    typeof message.senderId === "string" &&
+    (isSyncMessage(message) ||
+      isEphemeralMessage(message) ||
+      isRequestMessage(message) ||
+      isDocumentUnavailableMessage(message))
+  )
+}
+
+export function isDocumentUnavailableMessage(
+  message: NetworkAdapterMessage
+): message is DocumentUnavailableMessage {
+  return message.type === "doc-unavailable"
+}
+
+export function isRequestMessage(
+  message: NetworkAdapterMessage
+): message is RequestMessage {
+  return message.type === "request"
+}
+
+export function isSyncMessage(
+  message: NetworkAdapterMessage
+): message is SyncMessage {
+  return message.type === "sync"
+}
+
+export function isEphemeralMessage(
+  message: NetworkAdapterMessage | MessageContents
+): message is EphemeralMessage | EphemeralMessageContents {
+  return message.type === "ephemeral"
+}
+
+export interface SyncMessageEnvelope {
+  senderId: PeerId
+}
+
+export interface SyncMessageContents {
+  type: "sync"
+  data: Uint8Array
+  targetId: PeerId
+  documentId: DocumentId
+}
+
+export type SyncMessage = SyncMessageEnvelope & SyncMessageContents
+
+export interface EphemeralMessageEnvelope {
+  senderId: PeerId
+  count: number
+  sessionId: SessionId
+}
+
+export interface EphemeralMessageContents {
+  type: "ephemeral"
+  targetId: PeerId
+  documentId: DocumentId
+  data: Uint8Array
+}
+
+export type EphemeralMessage = EphemeralMessageEnvelope &
+  EphemeralMessageContents
+
+export interface DocumentUnavailableMessageContents {
+  type: "doc-unavailable"
+  documentId: DocumentId
+  targetId: PeerId
+}
+
+export type DocumentUnavailableMessage = SyncMessageEnvelope &
+  DocumentUnavailableMessageContents
+
+export interface RequestMessageContents {
+  type: "request"
+  data: Uint8Array
+  targetId: PeerId
+  documentId: DocumentId
+}
+
+export type RequestMessage = SyncMessageEnvelope & RequestMessageContents
+
+export type MessageContents =
+  | SyncMessageContents
+  | EphemeralMessageContents
+  | RequestMessageContents
+  | DocumentUnavailableMessageContents
+
+export type Message =
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+
+export type SynchronizerMessage =
+  | SyncMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+  | EphemeralMessage
+
+type ArriveMessage = {
+  senderId: PeerId
+  type: "arrive"
+}
+
+type WelcomeMessage = {
+  senderId: PeerId
+  targetId: PeerId
+  type: "welcome"
+}
+
+export type NetworkAdapterMessage = ArriveMessage | WelcomeMessage | Message

package/src/storage/StorageSubsystem.ts

@@ -1,4 +1,4 @@
-import * as A from "@automerge/automerge"
+import * as A from "@automerge/automerge/next"
 import { StorageAdapter, StorageKey } from "./StorageAdapter.js"
 import * as sha256 from "fast-sha256"
 import { type DocumentId } from "../types.js"
@@ -25,7 +25,7 @@ function keyHash(binary: Uint8Array) {
 
 function headsHash(heads: A.Heads): string {
   let encoder = new TextEncoder()
-  let headsbinary = mergeArrays(heads.map(h => encoder.encode(h)))
+  let headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
   return keyHash(headsbinary)
 }
 

package/src/synchronizer/CollectionSynchronizer.ts

@@ -5,11 +5,17 @@ import {
   binaryToDocumentId,
   stringifyAutomergeUrl,
 } from "../DocUrl.js"
-import { ChannelId, BinaryDocumentId, PeerId, DocumentId } from "../types.js"
+import { PeerId, DocumentId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
 
 import debug from "debug"
+import {
+  DocumentUnavailableMessage,
+  RequestMessage,
+  SynchronizerMessage,
+  SyncMessage,
+} from "../network/messages.js"
 const log = debug("automerge-repo:collectionsync")
 
 /** A CollectionSynchronizer is responsible for synchronizing a DocCollection with peers. */
@@ -20,6 +26,9 @@ export class CollectionSynchronizer extends Synchronizer {
   /** A map of documentIds to their synchronizers */
   #docSynchronizers: Record<DocumentId, DocSynchronizer> = {}
 
+  /** Used to determine if the document is know to the Collection and a synchronizer exists or is being set up */
+  #docSetUp: Record<DocumentId, boolean> = {}
+
   constructor(private repo: DocCollection) {
     super()
   }
@@ -57,37 +66,42 @@ export class CollectionSynchronizer extends Synchronizer {
    * When we receive a sync message for a document we haven't got in memory, we
    * register it with the repo and start synchronizing
    */
-  async receiveSyncMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array
-  ) {
-    log(`onSyncMessage: ${peerId}, ${channelId}, ${message.byteLength}bytes`)
-
-    const documentId = channelId as unknown as DocumentId
+  async receiveMessage(message: SynchronizerMessage) {
+    log(
+      `onSyncMessage: ${message.senderId}, ${message.documentId}, ${
+        "data" in message ? message.data.byteLength + "bytes" : ""
+      }`
+    )
+
+    const documentId = message.documentId
     if (!documentId) {
       throw new Error("received a message with an invalid documentId")
     }
-    const docSynchronizer = await this.#fetchDocSynchronizer(documentId)
 
-    await docSynchronizer.receiveSyncMessage(peerId, channelId, message)
+    this.#docSetUp[documentId] = true
+
+    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
+
+    docSynchronizer.receiveMessage(message)
 
     // Initiate sync with any new peers
     const peers = await this.#documentGenerousPeers(documentId)
-    peers
-      .filter(peerId => !docSynchronizer.hasPeer(peerId))
-      .forEach(peerId => docSynchronizer.beginSync(peerId))
+    docSynchronizer.beginSync(
+      peers.filter(peerId => !docSynchronizer.hasPeer(peerId))
+    )
   }
 
   /**
    * Starts synchronizing the given document with all peers that we share it generously with.
    */
   addDocument(documentId: DocumentId) {
+    // HACK: this is a hack to prevent us from adding the same document twice
+    if (this.#docSetUp[documentId]) {
+      return
+    }
     const docSynchronizer = this.#fetchDocSynchronizer(documentId)
     void this.#documentGenerousPeers(documentId).then(peers => {
-      peers.forEach(peerId => {
-        docSynchronizer.beginSync(peerId)
-      })
+      docSynchronizer.beginSync(peers)
     })
   }
 
@@ -99,11 +113,16 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Adds a peer and maybe starts synchronizing with them */
   addPeer(peerId: PeerId) {
     log(`adding ${peerId} & synchronizing with them`)
+
+    if (this.#peers.has(peerId)) {
+      return
+    }
+
     this.#peers.add(peerId)
     for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
       const { documentId } = docSynchronizer
-      void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
-        if (okToShare) docSynchronizer.beginSync(peerId)
+      this.repo.sharePolicy(peerId, documentId).then(okToShare => {
+        if (okToShare) docSynchronizer.beginSync([peerId])
       })
     }
   }