@automerge/automerge-repo 0.0.1

package/src/network/NetworkAdapter.ts

@@ -0,0 +1,54 @@
+import EventEmitter from "eventemitter3"
+import { PeerId, ChannelId } from "../types.js"
+
+export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
+  peerId?: PeerId // hmmm, maybe not
+
+  abstract connect(url?: string): void
+
+  abstract sendMessage(
+    peerId: PeerId,
+    channelId: ChannelId,
+    message: Uint8Array,
+    broadcast: boolean
+  ): void
+
+  abstract join(channelId: ChannelId): void
+
+  abstract leave(channelId: ChannelId): void
+}
+
+// events & payloads
+
+export interface NetworkAdapterEvents {
+  open: (payload: OpenPayload) => void
+  close: () => void
+  "peer-candidate": (payload: PeerCandidatePayload) => void
+  "peer-disconnected": (payload: PeerDisconnectedPayload) => void
+  message: (payload: InboundMessagePayload) => void
+}
+
+export interface OpenPayload {
+  network: NetworkAdapter
+}
+
+export interface PeerCandidatePayload {
+  peerId: PeerId
+  channelId: ChannelId
+}
+
+export interface MessagePayload {
+  targetId: PeerId
+  channelId: ChannelId
+  message: Uint8Array
+  broadcast: boolean
+}
+
+export interface InboundMessagePayload extends MessagePayload {
+  type?: string
+  senderId: PeerId
+}
+
+export interface PeerDisconnectedPayload {
+  peerId: PeerId
+}

package/src/network/NetworkSubsystem.ts

@@ -0,0 +1,130 @@
+import EventEmitter from "eventemitter3"
+import {
+  InboundMessagePayload,
+  NetworkAdapter,
+  PeerDisconnectedPayload,
+} from "./NetworkAdapter.js"
+import { ChannelId, PeerId } from "../types.js"
+
+import debug from "debug"
+
+export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
+  #log: debug.Debugger
+  #adaptersByPeer: Record<PeerId, NetworkAdapter> = {}
+  #channels: ChannelId[]
+
+  constructor(
+    private adapters: NetworkAdapter[],
+    public peerId = randomPeerId()
+  ) {
+    super()
+    this.#log = debug(`automerge-repo:network:${this.peerId}`)
+    this.#channels = []
+    this.adapters.forEach(a => this.addNetworkAdapter(a))
+  }
+
+  addNetworkAdapter(networkAdapter: NetworkAdapter) {
+    networkAdapter.connect(this.peerId)
+
+    networkAdapter.on("peer-candidate", ({ peerId, channelId }) => {
+      this.#log(`peer candidate: ${peerId} `)
+
+      // TODO: This is where authentication would happen
+
+      if (!this.#adaptersByPeer[peerId]) {
+        // TODO: handle losing a server here
+        this.#adaptersByPeer[peerId] = networkAdapter
+      }
+
+      this.emit("peer", { peerId, channelId })
+    })
+
+    networkAdapter.on("peer-disconnected", ({ peerId }) => {
+      this.#log(`peer disconnected: ${peerId} `)
+      delete this.#adaptersByPeer[peerId]
+      this.emit("peer-disconnected", { peerId })
+    })
+
+    networkAdapter.on("message", msg => {
+      const { senderId, channelId, broadcast, message } = msg
+      this.#log(`message from ${senderId}`)
+
+      // If we receive a broadcast message from a network adapter we need to re-broadcast it to all
+      // our other peers. This is the world's worst gossip protocol.
+
+      // TODO: This relies on the network forming a tree! If there are cycles, this approach will
+      // loop messages around forever.
+      if (broadcast) {
+        Object.entries(this.#adaptersByPeer)
+          .filter(([id]) => id !== senderId)
+          .forEach(([id, peer]) => {
+            peer.sendMessage(id as PeerId, channelId, message, broadcast)
+          })
+      }
+
+      this.emit("message", msg)
+    })
+
+    networkAdapter.on("close", () => {
+      this.#log("adapter closed")
+      Object.entries(this.#adaptersByPeer).forEach(([peerId, other]) => {
+        if (other === networkAdapter) {
+          delete this.#adaptersByPeer[peerId as PeerId]
+        }
+      })
+    })
+
+    this.#channels.forEach(c => networkAdapter.join(c))
+  }
+
+  sendMessage(
+    peerId: PeerId,
+    channelId: ChannelId,
+    message: Uint8Array,
+    broadcast: boolean
+  ) {
+    if (broadcast) {
+      Object.entries(this.#adaptersByPeer).forEach(([id, peer]) => {
+        this.#log(`sending broadcast to ${id}`)
+        peer.sendMessage(id as PeerId, channelId, message, true)
+      })
+    } else {
+      const peer = this.#adaptersByPeer[peerId]
+      if (!peer) {
+        this.#log(`Tried to send message but peer not found: ${peerId}`)
+        return
+      }
+      this.#log(`Sending message to ${peerId}`)
+      peer.sendMessage(peerId, channelId, message, false)
+    }
+  }
+
+  join(channelId: ChannelId) {
+    this.#log(`Joining channel ${channelId}`)
+    this.#channels.push(channelId)
+    this.adapters.forEach(a => a.join(channelId))
+  }
+
+  leave(channelId: ChannelId) {
+    this.#log(`Leaving channel ${channelId}`)
+    this.#channels = this.#channels.filter(c => c !== channelId)
+    this.adapters.forEach(a => a.leave(channelId))
+  }
+}
+
+function randomPeerId() {
+  return `user-${Math.round(Math.random() * 100000)}` as PeerId
+}
+
+// events & payloads
+
+export interface NetworkSubsystemEvents {
+  peer: (payload: PeerPayload) => void
+  "peer-disconnected": (payload: PeerDisconnectedPayload) => void
+  message: (payload: InboundMessagePayload) => void
+}
+
+export interface PeerPayload {
+  peerId: PeerId
+  channelId: ChannelId
+}

package/src/storage/StorageAdapter.ts

@@ -0,0 +1,5 @@
+export abstract class StorageAdapter {
+  abstract load(docId: string): Promise<Uint8Array | null>
+  abstract save(docId: string, data: Uint8Array): void
+  abstract remove(docId: string): void
+}

package/src/storage/StorageSubsystem.ts

@@ -0,0 +1,91 @@
+import * as A from "@automerge/automerge"
+import { DocumentId } from "../types.js"
+import { mergeArrays } from "../helpers/mergeArrays.js"
+import { StorageAdapter } from "./StorageAdapter.js"
+
+export class StorageSubsystem {
+  #storageAdapter: StorageAdapter
+  #changeCount: Record<DocumentId, number> = {}
+
+  constructor(storageAdapter: StorageAdapter) {
+    this.#storageAdapter = storageAdapter
+  }
+
+  #saveIncremental(documentId: DocumentId, doc: A.Doc<unknown>) {
+    const binary = A.saveIncremental(doc)
+    if (binary && binary.length > 0) {
+      if (!this.#changeCount[documentId]) {
+        this.#changeCount[documentId] = 0
+      }
+
+      this.#storageAdapter.save(
+        `${documentId}.incremental.${this.#changeCount[documentId]}`,
+        binary
+      )
+
+      this.#changeCount[documentId]++
+    }
+  }
+
+  #saveTotal(documentId: DocumentId, doc: A.Doc<unknown>) {
+    const binary = A.save(doc)
+    this.#storageAdapter.save(`${documentId}.snapshot`, binary)
+
+    for (let i = 0; i < this.#changeCount[documentId]; i++) {
+      this.#storageAdapter.remove(`${documentId}.incremental.${i}`)
+    }
+
+    this.#changeCount[documentId] = 0
+  }
+
+  async loadBinary(documentId: DocumentId): Promise<Uint8Array> {
+    const result = []
+    let binary = await this.#storageAdapter.load(`${documentId}.snapshot`)
+    if (binary && binary.length > 0) {
+      result.push(binary)
+    }
+
+    let index = 0
+    while (
+      (binary = await this.#storageAdapter.load(
+        `${documentId}.incremental.${index}`
+      ))
+    ) {
+      this.#changeCount[documentId] = index + 1
+      if (binary && binary.length > 0) result.push(binary)
+      index += 1
+    }
+
+    return mergeArrays(result)
+  }
+
+  async load<T>(
+    documentId: DocumentId,
+    prevDoc: A.Doc<T> = A.init<T>()
+  ): Promise<A.Doc<T>> {
+    const doc = A.loadIncremental(prevDoc, await this.loadBinary(documentId))
+    A.saveIncremental(doc)
+    return doc
+  }
+
+  save(documentId: DocumentId, doc: A.Doc<unknown>) {
+    if (this.#shouldCompact(documentId)) {
+      this.#saveTotal(documentId, doc)
+    } else {
+      this.#saveIncremental(documentId, doc)
+    }
+  }
+
+  remove(documentId: DocumentId) {
+    this.#storageAdapter.remove(`${documentId}.snapshot`)
+
+    for (let i = 0; i < this.#changeCount[documentId]; i++) {
+      this.#storageAdapter.remove(`${documentId}.incremental.${i}`)
+    }
+  }
+
+  // TODO: make this, you know, good.
+  #shouldCompact(documentId: DocumentId) {
+    return this.#changeCount[documentId] >= 20
+  }
+}

package/src/synchronizer/CollectionSynchronizer.ts

@@ -0,0 +1,112 @@
+import { DocCollection } from "../DocCollection.js"
+import { DocHandle } from "../DocHandle.js"
+import { ChannelId, DocumentId, PeerId } from "../types.js"
+import { DocSynchronizer } from "./DocSynchronizer.js"
+import { Synchronizer } from "./Synchronizer.js"
+
+import debug from "debug"
+const log = debug("automerge-repo:collectionsync")
+
+/** A CollectionSynchronizer is responsible for synchronizing a DocCollection with peers. */
+export class CollectionSynchronizer extends Synchronizer {
+  /** The set of peers we are connected with */
+  #peers: Set<PeerId> = new Set()
+
+  /** A map of documentIds to their synchronizers */
+  #docSynchronizers: Record<DocumentId, DocSynchronizer> = {}
+
+  constructor(private repo: DocCollection) {
+    super()
+  }
+
+  /** Returns a synchronizer for the given document, creating one if it doesn't already exist.  */
+  #fetchDocSynchronizer(documentId: DocumentId) {
+    if (!this.#docSynchronizers[documentId]) {
+      const handle = this.repo.find(documentId)
+      this.#docSynchronizers[documentId] = this.#initDocSynchronizer(handle)
+    }
+    return this.#docSynchronizers[documentId]
+  }
+
+  /** Creates a new docSynchronizer and sets it up to propagate messages */
+  #initDocSynchronizer(handle: DocHandle<unknown>): DocSynchronizer {
+    const docSynchronizer = new DocSynchronizer(handle)
+    docSynchronizer.on("message", event => this.emit("message", event))
+    return docSynchronizer
+  }
+
+  /** returns an array of peerIds that we share this document generously with */
+  async #documentGenerousPeers(documentId: DocumentId): Promise<PeerId[]> {
+    const peers = Array.from(this.#peers)
+    const generousPeers: PeerId[] = []
+    for (const peerId of peers) {
+      const okToShare = await this.repo.sharePolicy(peerId, documentId)
+      if (okToShare) generousPeers.push(peerId)
+    }
+    return generousPeers
+  }
+
+  // PUBLIC
+
+  /**
+   * When we receive a sync message for a document we haven't got in memory, we
+   * register it with the repo and start synchronizing
+   */
+  async receiveSyncMessage(
+    peerId: PeerId,
+    channelId: ChannelId,
+    message: Uint8Array
+  ) {
+    log(`onSyncMessage: ${peerId}, ${channelId}, ${message.byteLength}bytes`)
+
+    const documentId = channelId as unknown as DocumentId
+    const docSynchronizer = await this.#fetchDocSynchronizer(documentId)
+
+    await docSynchronizer.receiveSyncMessage(peerId, channelId, message)
+
+    // Initiate sync with any new peers
+    const peers = await this.#documentGenerousPeers(documentId)
+    peers
+      .filter(peerId => !docSynchronizer.hasPeer(peerId))
+      .forEach(peerId => docSynchronizer.beginSync(peerId))
+  }
+
+  /**
+   * Starts synchronizing the given document with all peers that we share it generously with.
+   */
+  addDocument(documentId: DocumentId) {
+    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
+    void this.#documentGenerousPeers(documentId).then(peers => {
+      peers.forEach(peerId => {
+        docSynchronizer.beginSync(peerId)
+      })
+    })
+  }
+
+  // TODO: implement this
+  removeDocument(documentId: DocumentId) {
+    throw new Error("not implemented")
+  }
+
+  /** Adds a peer and maybe starts synchronizing with them */
+  addPeer(peerId: PeerId) {
+    log(`adding ${peerId} & synchronizing with them`)
+    this.#peers.add(peerId)
+    for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
+      const { documentId } = docSynchronizer
+      void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
+        if (okToShare) docSynchronizer.beginSync(peerId)
+      })
+    }
+  }
+
+  /** Removes a peer and stops synchronizing with them */
+  removePeer(peerId: PeerId) {
+    log(`removing peer ${peerId}`)
+    this.#peers.delete(peerId)
+
+    for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
+      docSynchronizer.endSync(peerId)
+    }
+  }
+}

package/src/synchronizer/DocSynchronizer.ts

@@ -0,0 +1,182 @@
+import * as A from "@automerge/automerge"
+import { DocHandle } from "../DocHandle.js"
+import { ChannelId, PeerId } from "../types.js"
+import { Synchronizer } from "./Synchronizer.js"
+
+import debug from "debug"
+
+/**
+ * DocSynchronizer takes a handle to an Automerge document, and receives & dispatches sync messages
+ * to bring it inline with all other peers' versions.
+ */
+export class DocSynchronizer extends Synchronizer {
+  #log: debug.Debugger
+  #conciseLog: debug.Debugger
+  #opsLog: debug.Debugger
+
+  /** Active peers */
+  #peers: PeerId[] = []
+
+  /** Sync state for each peer we've communicated with (including inactive peers) */
+  #syncStates: Record<PeerId, A.SyncState> = {}
+
+  #pendingSyncMessages: Array<{ peerId: PeerId; message: Uint8Array }> = []
+
+  constructor(private handle: DocHandle<any>) {
+    super()
+    const docId = handle.documentId.slice(0, 5)
+    this.#conciseLog = debug(`automerge-repo:concise:docsync:${docId}`) // Only logs one line per receive/send
+    this.#log = debug(`automerge-repo:docsync:${docId}`)
+    this.#opsLog = debug(`automerge-repo:ops:docsync:${docId}`) // Log list of ops of each message
+
+    handle.on("change", () => this.#syncWithPeers())
+
+    // Process pending sync messages immediately after the handle becomes ready.
+    void (async () => {
+      await handle.loadAttemptedValue()
+      this.#processAllPendingSyncMessages()
+    })()
+  }
+
+  get documentId() {
+    return this.handle.documentId
+  }
+
+  /// PRIVATE
+
+  async #syncWithPeers() {
+    this.#log(`syncWithPeers`)
+    const doc = await this.handle.value()
+    this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
+  }
+
+  #getSyncState(peerId: PeerId) {
+    if (!this.#peers.includes(peerId)) {
+      this.#log("adding a new peer", peerId)
+      this.#peers.push(peerId)
+    }
+
+    return this.#syncStates[peerId] ?? A.initSyncState()
+  }
+
+  #setSyncState(peerId: PeerId, syncState: A.SyncState) {
+    // TODO: we maybe should be persisting sync states. But we want to be careful about how often we
+    // do that, because it can generate a lot of disk activity.
+
+    // TODO: we only need to do this on reconnect
+
+    this.#syncStates[peerId] = syncState
+  }
+
+  #sendSyncMessage(peerId: PeerId, doc: A.Doc<unknown>) {
+    this.#log(`sendSyncMessage ->${peerId}`)
+
+    const syncState = this.#getSyncState(peerId)
+    const [newSyncState, message] = A.generateSyncMessage(doc, syncState)
+    this.#setSyncState(peerId, newSyncState)
+    if (message) {
+      this.#logMessage(`sendSyncMessage 🡒 ${peerId}`, message)
+
+      const channelId = this.handle.documentId as string as ChannelId
+      this.emit("message", {
+        targetId: peerId,
+        channelId,
+        message,
+        broadcast: false,
+      })
+    } else {
+      this.#log(`sendSyncMessage ->${peerId} [no message generated]`)
+    }
+  }
+
+  #logMessage = (label: string, message: Uint8Array) => {
+    // This is real expensive...
+    return
+
+    const size = message.byteLength
+    const logText = `${label} ${size}b`
+    const decoded = A.decodeSyncMessage(message)
+
+    this.#conciseLog(logText)
+    this.#log(logText, decoded)
+
+    // expanding is expensive, so only do it if we're logging at this level
+    const expanded = this.#opsLog.enabled
+      ? decoded.changes.flatMap(change =>
+          A.decodeChange(change).ops.map(op => JSON.stringify(op))
+        )
+      : null
+    this.#opsLog(logText, expanded)
+  }
+
+  /// PUBLIC
+
+  hasPeer(peerId: PeerId) {
+    return this.#peers.includes(peerId)
+  }
+
+  beginSync(peerId: PeerId) {
+    this.#log(`beginSync: ${peerId}`)
+
+    // At this point if we don't have anything in our storage, we need to use an empty doc to sync
+    // with; but we don't want to surface that state to the front end
+    void this.handle.loadAttemptedValue().then(doc => {
+      // HACK: if we have a sync state already, we round-trip it through the encoding system to make
+      // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
+      // messages during disconnection.
+      // TODO: cover that case with a test and remove this hack
+      const syncStateRaw = this.#getSyncState(peerId)
+      const syncState = A.decodeSyncState(A.encodeSyncState(syncStateRaw))
+      this.#setSyncState(peerId, syncState)
+
+      this.#sendSyncMessage(peerId, doc)
+    })
+  }
+
+  endSync(peerId: PeerId) {
+    this.#log(`removing peer ${peerId}`)
+    this.#peers = this.#peers.filter(p => p !== peerId)
+  }
+
+  receiveSyncMessage(
+    peerId: PeerId,
+    channelId: ChannelId,
+    message: Uint8Array
+  ) {
+    if ((channelId as string) !== (this.documentId as string))
+      throw new Error(`channelId doesn't match documentId`)
+
+    // We need to block receiving the syncMessages until we've checked local storage
+    if (!this.handle.isReadyOrRequesting()) {
+      this.#pendingSyncMessages.push({ peerId, message })
+      return
+    }
+
+    this.#processAllPendingSyncMessages()
+    this.#processSyncMessage(peerId, message)
+  }
+
+  #processSyncMessage(peerId: PeerId, message: Uint8Array) {
+    this.handle.update(doc => {
+      const [newDoc, newSyncState] = A.receiveSyncMessage(
+        doc,
+        this.#getSyncState(peerId),
+        message
+      )
+
+      this.#setSyncState(peerId, newSyncState)
+
+      // respond to just this peer (as required)
+      this.#sendSyncMessage(peerId, doc)
+      return newDoc
+    })
+  }
+
+  #processAllPendingSyncMessages() {
+    for (const { peerId, message } of this.#pendingSyncMessages) {
+      this.#processSyncMessage(peerId, message)
+    }
+
+    this.#pendingSyncMessages = []
+  }
+}

package/src/synchronizer/Synchronizer.ts

@@ -0,0 +1,15 @@
+import EventEmitter from "eventemitter3"
+import { ChannelId, PeerId } from "../types.js"
+import { MessagePayload } from "../network/NetworkAdapter.js"
+
+export abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
+  abstract receiveSyncMessage(
+    peerId: PeerId,
+    channelId: ChannelId,
+    message: Uint8Array
+  ): void
+}
+
+export interface SynchronizerEvents {
+  message: (arg: MessagePayload) => void
+}