@automerge/automerge-repo 2.0.0-collectionsync-alpha.1 → 2.0.0

package/src/storage/StorageSubsystem.ts

@@ -1,4 +1,4 @@
-import * as A from "@automerge/automerge/slim/next"
+import { next as A } from "@automerge/automerge/slim"
 import debug from "debug"
 import { headsAreSame } from "../helpers/headsAreSame.js"
 import { mergeArrays } from "../helpers/mergeArrays.js"
@@ -6,9 +6,9 @@ import { type DocumentId } from "../types.js"
 import { StorageAdapterInterface } from "./StorageAdapterInterface.js"
 import { ChunkInfo, StorageKey, StorageId } from "./types.js"
 import { keyHash, headsHash } from "./keyHash.js"
-import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
 import * as Uuid from "uuid"
 import { EventEmitter } from "eventemitter3"
+import { encodeHeads } from "../AutomergeUrl.js"
 
 type StorageSubsystemEvents = {
   "document-loaded": (arg: {
@@ -30,17 +30,17 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
   /** Record of the latest heads we've loaded or saved for each document  */
   #storedHeads: Map<DocumentId, A.Heads> = new Map()
 
-  #log = debug(`automerge-repo:storage-subsystem`)
+  /** Metadata on the chunks we've already loaded for each document */
+  #chunkInfos: Map<DocumentId, ChunkInfo[]> = new Map()
+
+  /** Flag to avoid compacting when a compaction is already underway */
+  #compacting = false
 
-  #beelay: A.beelay.Beelay
+  #log = debug(`automerge-repo:storage-subsystem`)
 
-  constructor(
-    beelay: A.beelay.Beelay,
-    storageAdapter: StorageAdapterInterface
-  ) {
+  constructor(storageAdapter: StorageAdapterInterface) {
     super()
     this.#storageAdapter = storageAdapter
-    this.#beelay = beelay
   }
 
   async id(): Promise<StorageId> {
@@ -112,16 +112,63 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
   // AUTOMERGE DOCUMENT STORAGE
 
   /**
-   * Loads the Automerge document with the given ID from storage.
+   * Loads and combines document chunks from storage, with snapshots first.
    */
-  async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
-    const doc = await this.#beelay.loadDocument(documentId)
-    if (doc == null) {
+  async loadDocData(documentId: DocumentId): Promise<Uint8Array | null> {
+    // Load snapshots first
+    const snapshotChunks = await this.#storageAdapter.loadRange([
+      documentId,
+      "snapshot",
+    ])
+    const incrementalChunks = await this.#storageAdapter.loadRange([
+      documentId,
+      "incremental",
+    ])
+
+    const binaries: Uint8Array[] = []
+    const chunkInfos: ChunkInfo[] = []
+
+    // Process snapshots first
+    for (const chunk of snapshotChunks) {
+      if (chunk.data === undefined) continue
+      chunkInfos.push({
+        key: chunk.key,
+        type: "snapshot",
+        size: chunk.data.length,
+      })
+      binaries.push(chunk.data)
+    }
+
+    // Then process incrementals
+    for (const chunk of incrementalChunks) {
+      if (chunk.data === undefined) continue
+      chunkInfos.push({
+        key: chunk.key,
+        type: "incremental",
+        size: chunk.data.length,
+      })
+      binaries.push(chunk.data)
+    }
+
+    // Store chunk infos for future reference
+    this.#chunkInfos.set(documentId, chunkInfos)
+
+    // If no chunks were found, return null
+    if (binaries.length === 0) {
       return null
     }
-    const binaries = doc.map(c => c.contents)
-    const binary = mergeArrays(binaries)
-    if (binary.length === 0) return null
+
+    // Merge the chunks into a single binary
+    return mergeArrays(binaries)
+  }
+
+  /**
+   * Loads the Automerge document with the given ID from storage.
+   */
+  async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
+    // Load and combine chunks
+    const binary = await this.loadDocData(documentId)
+    if (!binary) return null
 
     // Load into an Automerge document
     const start = performance.now()
@@ -135,6 +182,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
 
     // Record the latest heads for the document
     this.#storedHeads.set(documentId, A.getHeads(newDoc))
+
     return newDoc
   }
 
@@ -149,36 +197,15 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
     // Don't bother saving if the document hasn't changed
     if (!this.#shouldSave(documentId, doc)) return
 
-    const changes = A.getChanges(
-      A.view(doc, this.#storedHeads.get(documentId) ?? []),
-      doc
-    )
+    const sourceChunks = this.#chunkInfos.get(documentId) ?? []
+
+    if (this.#shouldCompact(sourceChunks)) {
+      await this.#saveTotal(documentId, doc, sourceChunks)
+    } else {
+      await this.#saveIncremental(documentId, doc)
+    }
 
-    const commits = changes.map(c => {
-      const decoded = A.decodeChange(c)
-      return {
-        parents: decoded.deps,
-        hash: decoded.hash,
-        contents: c,
-      }
-    })
-    let done = this.#beelay
-      .addCommits({
-        docId: documentId,
-        commits: changes.map(c => {
-          const decoded = A.decodeChange(c)
-          return {
-            parents: decoded.deps,
-            hash: decoded.hash,
-            contents: c,
-          }
-        }),
-      })
-      .catch(e => {
-        console.error(`Error saving document ${documentId}: ${e}`)
-      })
     this.#storedHeads.set(documentId, A.getHeads(doc))
-    await done
   }
 
   /**
@@ -190,6 +217,66 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
     await this.#storageAdapter.removeRange([documentId, "sync-state"])
   }
 
+  /**
+   * Saves just the incremental changes since the last save.
+   */
+  async #saveIncremental(
+    documentId: DocumentId,
+    doc: A.Doc<unknown>
+  ): Promise<void> {
+    const binary = A.saveSince(doc, this.#storedHeads.get(documentId) ?? [])
+    if (binary && binary.length > 0) {
+      const key = [documentId, "incremental", keyHash(binary)]
+      this.#log(`Saving incremental ${key} for document ${documentId}`)
+      await this.#storageAdapter.save(key, binary)
+      if (!this.#chunkInfos.has(documentId)) {
+        this.#chunkInfos.set(documentId, [])
+      }
+      this.#chunkInfos.get(documentId)!.push({
+        key,
+        type: "incremental",
+        size: binary.length,
+      })
+      this.#storedHeads.set(documentId, A.getHeads(doc))
+    } else {
+      return Promise.resolve()
+    }
+  }
+
+  /**
+   * Compacts the document storage into a single shapshot.
+   */
+  async #saveTotal(
+    documentId: DocumentId,
+    doc: A.Doc<unknown>,
+    sourceChunks: ChunkInfo[]
+  ): Promise<void> {
+    this.#compacting = true
+
+    const binary = A.save(doc)
+    const snapshotHash = headsHash(A.getHeads(doc))
+    const key = [documentId, "snapshot", snapshotHash]
+    const oldKeys = new Set(
+      sourceChunks.map(c => c.key).filter(k => k[2] !== snapshotHash)
+    )
+
+    this.#log(`Saving snapshot ${key} for document ${documentId}`)
+    this.#log(`deleting old chunks ${Array.from(oldKeys)}`)
+
+    await this.#storageAdapter.save(key, binary)
+
+    for (const key of oldKeys) {
+      await this.#storageAdapter.remove(key)
+    }
+
+    const newChunkInfos =
+      this.#chunkInfos.get(documentId)?.filter(c => !oldKeys.has(c.key)) ?? []
+    newChunkInfos.push({ key, type: "snapshot", size: binary.length })
+
+    this.#chunkInfos.set(documentId, newChunkInfos)
+    this.#compacting = false
+  }
+
   async loadSyncState(
     documentId: DocumentId,
     storageId: StorageId
@@ -224,11 +311,34 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
     }
 
     const newHeads = A.getHeads(doc)
-    if (headsAreSame(newHeads, oldHeads)) {
+    if (headsAreSame(encodeHeads(newHeads), encodeHeads(oldHeads))) {
       // the document hasn't changed
       return false
     }
 
     return true // the document has changed
   }
+
+  /**
+   * We only compact if the incremental size is greater than the snapshot size.
+   */
+  #shouldCompact(sourceChunks: ChunkInfo[]) {
+    if (this.#compacting) return false
+
+    let snapshotSize = 0
+    let incrementalSize = 0
+    for (const chunk of sourceChunks) {
+      if (chunk.type === "snapshot") {
+        snapshotSize += chunk.size
+      } else {
+        incrementalSize += chunk.size
+      }
+    }
+    // if the file is currently small, don't worry, just compact
+    // this might seem a bit arbitrary (1k is arbitrary) but is designed to ensure compaction
+    // for documents with only a single large change on top of an empty (or nearly empty) document
+    // for example: imported NPM modules, images, etc.
+    // if we have even more incrementals (so far) than the snapshot, compact
+    return snapshotSize < 1024 || incrementalSize >= snapshotSize
+  }
 }

package/src/storage/keyHash.ts

@@ -1,4 +1,4 @@
-import * as A from "@automerge/automerge/slim/next"
+import { next as A } from "@automerge/automerge/slim"
 import * as sha256 from "fast-sha256"
 import { mergeArrays } from "../helpers/mergeArrays.js"
 

package/src/synchronizer/CollectionSynchronizer.ts

@@ -1,12 +1,11 @@
 import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
-import { parseAutomergeUrl, stringifyAutomergeUrl } from "../AutomergeUrl.js"
+import { parseAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
 import { DocMessage } from "../network/messages.js"
 import { AutomergeUrl, DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
-import { next as A } from "@automerge/automerge"
 
 const log = debug("automerge-repo:collectionsync")
 
@@ -24,29 +23,41 @@ export class CollectionSynchronizer extends Synchronizer {
 
   #denylist: DocumentId[]
 
-  constructor(
-    private beelay: A.beelay.Beelay,
-    private repo: Repo,
-    denylist: AutomergeUrl[] = []
-  ) {
+  constructor(private repo: Repo, denylist: AutomergeUrl[] = []) {
     super()
     this.#denylist = denylist.map(url => parseAutomergeUrl(url).documentId)
   }
 
   /** Returns a synchronizer for the given document, creating one if it doesn't already exist.  */
-  #fetchDocSynchronizer(documentId: DocumentId) {
-    if (!this.docSynchronizers[documentId]) {
-      const handle = this.repo.find(stringifyAutomergeUrl({ documentId }))
-      this.docSynchronizers[documentId] = this.#initDocSynchronizer(handle)
+  #fetchDocSynchronizer(handle: DocHandle<unknown>) {
+    if (!this.docSynchronizers[handle.documentId]) {
+      this.docSynchronizers[handle.documentId] =
+        this.#initDocSynchronizer(handle)
     }
-    return this.docSynchronizers[documentId]
+    return this.docSynchronizers[handle.documentId]
   }
 
   /** Creates a new docSynchronizer and sets it up to propagate messages */
   #initDocSynchronizer(handle: DocHandle<unknown>): DocSynchronizer {
     const docSynchronizer = new DocSynchronizer({
-      beelay: this.beelay,
       handle,
+      peerId: this.repo.networkSubsystem.peerId,
+      onLoadSyncState: async peerId => {
+        if (!this.repo.storageSubsystem) {
+          return
+        }
+
+        const { storageId, isEphemeral } =
+          this.repo.peerMetadataByPeerId[peerId] || {}
+        if (!storageId || isEphemeral) {
+          return
+        }
+
+        return this.repo.storageSubsystem.loadSyncState(
+          handle.documentId,
+          storageId
+        )
+      },
     })
     docSynchronizer.on("message", event => this.emit("message", event))
     docSynchronizer.on("open-doc", event => this.emit("open-doc", event))
@@ -99,13 +110,16 @@ export class CollectionSynchronizer extends Synchronizer {
 
     this.#docSetUp[documentId] = true
 
-    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
+    const handle = await this.repo.find(documentId, {
+      allowableStates: ["ready", "unavailable", "requesting"],
+    })
+    const docSynchronizer = this.#fetchDocSynchronizer(handle)
 
     docSynchronizer.receiveMessage(message)
 
     // Initiate sync with any new peers
     const peers = await this.#documentGenerousPeers(documentId)
-    docSynchronizer.beginSync(
+    void docSynchronizer.beginSync(
       peers.filter(peerId => !docSynchronizer.hasPeer(peerId))
     )
   }
@@ -113,14 +127,14 @@ export class CollectionSynchronizer extends Synchronizer {
   /**
    * Starts synchronizing the given document with all peers that we share it generously with.
    */
-  addDocument(documentId: DocumentId) {
+  addDocument(handle: DocHandle<unknown>) {
     // HACK: this is a hack to prevent us from adding the same document twice
-    if (this.#docSetUp[documentId]) {
+    if (this.#docSetUp[handle.documentId]) {
       return
     }
-    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
-    void this.#documentGenerousPeers(documentId).then(peers => {
-      docSynchronizer.beginSync(peers)
+    const docSynchronizer = this.#fetchDocSynchronizer(handle)
+    void this.#documentGenerousPeers(handle.documentId).then(peers => {
+      void docSynchronizer.beginSync(peers)
     })
   }
 
@@ -142,7 +156,7 @@ export class CollectionSynchronizer extends Synchronizer {
     for (const docSynchronizer of Object.values(this.docSynchronizers)) {
       const { documentId } = docSynchronizer
       void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
-        if (okToShare) docSynchronizer.beginSync([peerId])
+        if (okToShare) void docSynchronizer.beginSync([peerId])
       })
     }
   }
@@ -168,13 +182,12 @@ export class CollectionSynchronizer extends Synchronizer {
       size: { numOps: number; numChanges: number }
     }
   } {
-    return {}
-    // return Object.fromEntries(
-    //   Object.entries(this.docSynchronizers).map(
-    //     ([documentId, synchronizer]) => {
-    //       return [documentId, synchronizer.metrics()]
-    //     }
-    //   )
-    // )
+    return Object.fromEntries(
+      Object.entries(this.docSynchronizers).map(
+        ([documentId, synchronizer]) => {
+          return [documentId, synchronizer.metrics()]
+        }
+      )
+    )
   }
 }