@automerge/automerge-repo 1.0.0-alpha.0 → 1.0.0-alpha.3

package/src/network/NetworkSubsystem.ts

@@ -1,17 +1,31 @@
 import EventEmitter from "eventemitter3"
+import { PeerId } from "../types.js"
+import { NetworkAdapter, PeerDisconnectedPayload } from "./NetworkAdapter.js"
+
 import {
-  InboundMessagePayload,
-  NetworkAdapter,
-  PeerDisconnectedPayload,
-} from "./NetworkAdapter.js"
-import { ChannelId, PeerId } from "../types.js"
+  EphemeralMessage,
+  isEphemeralMessage,
+  isValidMessage,
+  Message,
+  MessageContents,
+} from "./messages.js"
 
 import debug from "debug"
+import { SessionId } from "../EphemeralData.js"
+
+type EphemeralMessageSource = `${PeerId}:${SessionId}`
+
+const getEphemeralMessageSource = (message: EphemeralMessage) =>
+  `${message.senderId}:${message.sessionId}` as EphemeralMessageSource
 
 export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #log: debug.Debugger
   #adaptersByPeer: Record<PeerId, NetworkAdapter> = {}
 
+  #count = 0
+  #sessionId: SessionId = Math.random().toString(36).slice(2) as SessionId
+  #ephemeralSessionCounts: Record<EphemeralMessageSource, number> = {}
+
   constructor(
     private adapters: NetworkAdapter[],
     public peerId = randomPeerId()
@@ -44,20 +58,24 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     })
 
     networkAdapter.on("message", msg => {
-      const { senderId, channelId, broadcast, message } = msg
-      this.#log(`message from ${senderId}`)
-
-      // If we receive a broadcast message from a network adapter we need to re-broadcast it to all
-      // our other peers. This is the world's worst gossip protocol.
-
-      // TODO: This relies on the network forming a tree! If there are cycles, this approach will
-      // loop messages around forever.
-      if (broadcast) {
-        Object.entries(this.#adaptersByPeer)
-          .filter(([id]) => id !== senderId)
-          .forEach(([id, peer]) => {
-            peer.sendMessage(id as PeerId, channelId, message, broadcast)
-          })
+      if (!isValidMessage(msg)) {
+        this.#log(`invalid message: ${JSON.stringify(msg)}`)
+        return
+      }
+
+      this.#log(`message from ${msg.senderId}`)
+
+      if (isEphemeralMessage(msg)) {
+        const source = getEphemeralMessageSource(msg)
+        if (
+          this.#ephemeralSessionCounts[source] === undefined ||
+          msg.count > this.#ephemeralSessionCounts[source]
+        ) {
+          this.#ephemeralSessionCounts[source] = msg.count
+          this.emit("message", msg)
+        }
+
+        return
       }
 
       this.emit("message", msg)
@@ -75,25 +93,30 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     networkAdapter.join()
   }
 
-  sendMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array,
-    broadcast: boolean
-  ) {
-    if (broadcast) {
-      Object.entries(this.#adaptersByPeer).forEach(([id, peer]) => {
-        this.#log(`sending broadcast to ${id}`)
-        peer.sendMessage(id as PeerId, channelId, message, true)
-      })
+  send(message: MessageContents) {
+    const peer = this.#adaptersByPeer[message.targetId]
+    if (!peer) {
+      this.#log(`Tried to send message but peer not found: ${message.targetId}`)
+      return
+    }
+    this.#log(`Sending message to ${message.targetId}`)
+
+    if (isEphemeralMessage(message)) {
+      const outbound =
+        "count" in message
+          ? message
+          : {
+              ...message,
+              count: ++this.#count,
+              sessionId: this.#sessionId,
+              senderId: this.peerId,
+            }
+      this.#log("Ephemeral message", outbound)
+      peer.send(outbound)
     } else {
-      const peer = this.#adaptersByPeer[peerId]
-      if (!peer) {
-        this.#log(`Tried to send message but peer not found: ${peerId}`)
-        return
-      }
-      this.#log(`Sending message to ${peerId}`)
-      peer.sendMessage(peerId, channelId, message, false)
+      const outbound = { ...message, senderId: this.peerId }
+      this.#log("Sync message", outbound)
+      peer.send(outbound)
     }
   }
 
@@ -117,7 +140,7 @@ function randomPeerId() {
 export interface NetworkSubsystemEvents {
   peer: (payload: PeerPayload) => void
   "peer-disconnected": (payload: PeerDisconnectedPayload) => void
-  message: (payload: InboundMessagePayload) => void
+  message: (payload: Message) => void
 }
 
 export interface PeerPayload {

package/src/network/messages.ts

@@ -0,0 +1,123 @@
+// utilities
+import { SessionId } from "../EphemeralData"
+import { DocumentId, PeerId } from "../types"
+
+export function isValidMessage(
+  message: NetworkAdapterMessage
+): message is
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage {
+  return (
+    typeof message === "object" &&
+    typeof message.type === "string" &&
+    typeof message.senderId === "string" &&
+    (isSyncMessage(message) ||
+      isEphemeralMessage(message) ||
+      isRequestMessage(message) ||
+      isDocumentUnavailableMessage(message))
+  )
+}
+
+export function isDocumentUnavailableMessage(
+  message: NetworkAdapterMessage
+): message is DocumentUnavailableMessage {
+  return message.type === "doc-unavailable"
+}
+
+export function isRequestMessage(
+  message: NetworkAdapterMessage
+): message is RequestMessage {
+  return message.type === "request"
+}
+
+export function isSyncMessage(
+  message: NetworkAdapterMessage
+): message is SyncMessage {
+  return message.type === "sync"
+}
+
+export function isEphemeralMessage(
+  message: NetworkAdapterMessage | MessageContents
+): message is EphemeralMessage | EphemeralMessageContents {
+  return message.type === "ephemeral"
+}
+
+export interface SyncMessageEnvelope {
+  senderId: PeerId
+}
+
+export interface SyncMessageContents {
+  type: "sync"
+  data: Uint8Array
+  targetId: PeerId
+  documentId: DocumentId
+}
+
+export type SyncMessage = SyncMessageEnvelope & SyncMessageContents
+
+export interface EphemeralMessageEnvelope {
+  senderId: PeerId
+  count: number
+  sessionId: SessionId
+}
+
+export interface EphemeralMessageContents {
+  type: "ephemeral"
+  targetId: PeerId
+  documentId: DocumentId
+  data: Uint8Array
+}
+
+export type EphemeralMessage = EphemeralMessageEnvelope &
+  EphemeralMessageContents
+
+export interface DocumentUnavailableMessageContents {
+  type: "doc-unavailable"
+  documentId: DocumentId
+  targetId: PeerId
+}
+
+export type DocumentUnavailableMessage = SyncMessageEnvelope &
+  DocumentUnavailableMessageContents
+
+export interface RequestMessageContents {
+  type: "request"
+  data: Uint8Array
+  targetId: PeerId
+  documentId: DocumentId
+}
+
+export type RequestMessage = SyncMessageEnvelope & RequestMessageContents
+
+export type MessageContents =
+  | SyncMessageContents
+  | EphemeralMessageContents
+  | RequestMessageContents
+  | DocumentUnavailableMessageContents
+
+export type Message =
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+
+export type SynchronizerMessage =
+  | SyncMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
+  | EphemeralMessage
+
+type ArriveMessage = {
+  senderId: PeerId
+  type: "arrive"
+}
+
+type WelcomeMessage = {
+  senderId: PeerId
+  targetId: PeerId
+  type: "welcome"
+}
+
+export type NetworkAdapterMessage = ArriveMessage | WelcomeMessage | Message

package/src/storage/StorageSubsystem.ts

@@ -3,6 +3,8 @@ import { StorageAdapter, StorageKey } from "./StorageAdapter.js"
 import * as sha256 from "fast-sha256"
 import { type DocumentId } from "../types.js"
 import { mergeArrays } from "../helpers/mergeArrays.js"
+import debug from "debug"
+import { headsAreSame } from "../helpers/headsAreSame.js"
 
 // Metadata about a chunk of data loaded from storage. This is stored on the
 // StorageSubsystem so when we are compacting we know what chunks we can safely delete
@@ -30,6 +32,8 @@ function headsHash(heads: A.Heads): string {
 export class StorageSubsystem {
   #storageAdapter: StorageAdapter
   #chunkInfos: Map<DocumentId, StorageChunkInfo[]> = new Map()
+  #storedHeads: Map<DocumentId, A.Heads> = new Map()
+  #log = debug(`automerge-repo:storage-subsystem`)
 
   constructor(storageAdapter: StorageAdapter) {
     this.#storageAdapter = storageAdapter
@@ -39,9 +43,10 @@ export class StorageSubsystem {
     documentId: DocumentId,
     doc: A.Doc<unknown>
   ): Promise<void> {
-    const binary = A.saveIncremental(doc)
+    const binary = A.saveSince(doc, this.#storedHeads.get(documentId) ?? [])
     if (binary && binary.length > 0) {
       const key = [documentId, "incremental", keyHash(binary)]
+      this.#log(`Saving incremental ${key} for document ${documentId}`)
       await this.#storageAdapter.save(key, binary)
       if (!this.#chunkInfos.has(documentId)) {
         this.#chunkInfos.set(documentId, [])
@@ -51,6 +56,7 @@ export class StorageSubsystem {
         type: "incremental",
         size: binary.length,
       })
+      this.#storedHeads.set(documentId, A.getHeads(doc))
     } else {
       return Promise.resolve()
     }
@@ -62,8 +68,14 @@ export class StorageSubsystem {
     sourceChunks: StorageChunkInfo[]
   ): Promise<void> {
     const binary = A.save(doc)
-    const key = [documentId, "snapshot", headsHash(A.getHeads(doc))]
-    const oldKeys = new Set(sourceChunks.map(c => c.key))
+    const snapshotHash = headsHash(A.getHeads(doc))
+    const key = [documentId, "snapshot", snapshotHash]
+    const oldKeys = new Set(
+      sourceChunks.map(c => c.key).filter(k => k[2] !== snapshotHash)
+    )
+
+    this.#log(`Saving snapshot ${key} for document ${documentId}`)
+    this.#log(`deleting old chunks ${Array.from(oldKeys)}`)
 
     await this.#storageAdapter.save(key, binary)
 
@@ -76,7 +88,7 @@ export class StorageSubsystem {
     this.#chunkInfos.set(documentId, newChunkInfos)
   }
 
-  async loadBinary(documentId: DocumentId): Promise<Uint8Array> {
+  async loadDoc(documentId: DocumentId): Promise<A.Doc<unknown> | null> {
     const loaded = await this.#storageAdapter.loadRange([documentId])
     const binaries = []
     const chunkInfos: StorageChunkInfo[] = []
@@ -93,16 +105,26 @@ export class StorageSubsystem {
       binaries.push(chunk.data)
     }
     this.#chunkInfos.set(documentId, chunkInfos)
-    return mergeArrays(binaries)
+    const binary = mergeArrays(binaries)
+    if (binary.length === 0) {
+      return null
+    }
+    const newDoc = A.loadIncremental(A.init(), binary)
+    this.#storedHeads.set(documentId, A.getHeads(newDoc))
+    return newDoc
   }
 
-  async save(documentId: DocumentId, doc: A.Doc<unknown>): Promise<void> {
+  async saveDoc(documentId: DocumentId, doc: A.Doc<unknown>): Promise<void> {
+    if (!this.#shouldSave(documentId, doc)) {
+      return
+    }
     let sourceChunks = this.#chunkInfos.get(documentId) ?? []
     if (this.#shouldCompact(sourceChunks)) {
       this.#saveTotal(documentId, doc, sourceChunks)
     } else {
       this.#saveIncremental(documentId, doc)
     }
+    this.#storedHeads.set(documentId, A.getHeads(doc))
   }
 
   async remove(documentId: DocumentId) {
@@ -110,6 +132,20 @@ export class StorageSubsystem {
     this.#storageAdapter.removeRange([documentId, "incremental"])
   }
 
+  #shouldSave(documentId: DocumentId, doc: A.Doc<unknown>): boolean {
+    const oldHeads = this.#storedHeads.get(documentId)
+    if (!oldHeads) {
+      return true
+    }
+
+    const newHeads = A.getHeads(doc)
+    if (headsAreSame(newHeads, oldHeads)) {
+      return false
+    }
+
+    return true
+  }
+
   #shouldCompact(sourceChunks: StorageChunkInfo[]) {
     // compact if the incremental size is greater than the snapshot size
     let snapshotSize = 0

package/src/synchronizer/CollectionSynchronizer.ts

@@ -5,11 +5,17 @@ import {
   binaryToDocumentId,
   stringifyAutomergeUrl,
 } from "../DocUrl.js"
-import { ChannelId, BinaryDocumentId, PeerId, DocumentId } from "../types.js"
+import { PeerId, DocumentId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
 
 import debug from "debug"
+import {
+  DocumentUnavailableMessage,
+  RequestMessage,
+  SynchronizerMessage,
+  SyncMessage,
+} from "../network/messages.js"
 const log = debug("automerge-repo:collectionsync")
 
 /** A CollectionSynchronizer is responsible for synchronizing a DocCollection with peers. */
@@ -20,6 +26,9 @@ export class CollectionSynchronizer extends Synchronizer {
   /** A map of documentIds to their synchronizers */
   #docSynchronizers: Record<DocumentId, DocSynchronizer> = {}
 
+  /** Used to determine if the document is know to the Collection and a synchronizer exists or is being set up */
+  #docSetUp: Record<DocumentId, boolean> = {}
+
   constructor(private repo: DocCollection) {
     super()
   }
@@ -57,37 +66,42 @@ export class CollectionSynchronizer extends Synchronizer {
    * When we receive a sync message for a document we haven't got in memory, we
    * register it with the repo and start synchronizing
    */
-  async receiveSyncMessage(
-    peerId: PeerId,
-    channelId: ChannelId,
-    message: Uint8Array
-  ) {
-    log(`onSyncMessage: ${peerId}, ${channelId}, ${message.byteLength}bytes`)
-
-    const documentId = channelId as unknown as DocumentId
+  async receiveMessage(message: SynchronizerMessage) {
+    log(
+      `onSyncMessage: ${message.senderId}, ${message.documentId}, ${
+        "data" in message ? message.data.byteLength + "bytes" : ""
+      }`
+    )
+
+    const documentId = message.documentId
     if (!documentId) {
       throw new Error("received a message with an invalid documentId")
     }
-    const docSynchronizer = await this.#fetchDocSynchronizer(documentId)
 
-    await docSynchronizer.receiveSyncMessage(peerId, channelId, message)
+    this.#docSetUp[documentId] = true
+
+    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
+
+    docSynchronizer.receiveMessage(message)
 
     // Initiate sync with any new peers
     const peers = await this.#documentGenerousPeers(documentId)
-    peers
-      .filter(peerId => !docSynchronizer.hasPeer(peerId))
-      .forEach(peerId => docSynchronizer.beginSync(peerId))
+    docSynchronizer.beginSync(
+      peers.filter(peerId => !docSynchronizer.hasPeer(peerId))
+    )
   }
 
   /**
    * Starts synchronizing the given document with all peers that we share it generously with.
    */
   addDocument(documentId: DocumentId) {
+    // HACK: this is a hack to prevent us from adding the same document twice
+    if (this.#docSetUp[documentId]) {
+      return
+    }
     const docSynchronizer = this.#fetchDocSynchronizer(documentId)
     void this.#documentGenerousPeers(documentId).then(peers => {
-      peers.forEach(peerId => {
-        docSynchronizer.beginSync(peerId)
-      })
+      docSynchronizer.beginSync(peers)
     })
   }
 
@@ -99,11 +113,16 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Adds a peer and maybe starts synchronizing with them */
   addPeer(peerId: PeerId) {
     log(`adding ${peerId} & synchronizing with them`)
+
+    if (this.#peers.has(peerId)) {
+      return
+    }
+
     this.#peers.add(peerId)
     for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
       const { documentId } = docSynchronizer
-      void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
-        if (okToShare) docSynchronizer.beginSync(peerId)
+      this.repo.sharePolicy(peerId, documentId).then(okToShare => {
+        if (okToShare) docSynchronizer.beginSync([peerId])
       })
     }
   }