@automerge/automerge-repo 2.0.0-alpha.7 → 2.0.0-collectionsync-alpha.1

package/src/storage/StorageSubsystem.ts

@@ -8,28 +8,39 @@ import { ChunkInfo, StorageKey, StorageId } from "./types.js"
 import { keyHash, headsHash } from "./keyHash.js"
 import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
 import * as Uuid from "uuid"
+import { EventEmitter } from "eventemitter3"
+
+type StorageSubsystemEvents = {
+  "document-loaded": (arg: {
+    documentId: DocumentId
+    durationMillis: number
+    numOps: number
+    numChanges: number
+  }) => void
+}
 
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
  * storage adapter. It also provides a generic key/value storage interface for other uses.
  */
-export class StorageSubsystem {
+export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
   /** The storage adapter to use for saving and loading documents */
   #storageAdapter: StorageAdapterInterface
 
   /** Record of the latest heads we've loaded or saved for each document  */
   #storedHeads: Map<DocumentId, A.Heads> = new Map()
 
-  /** Metadata on the chunks we've already loaded for each document */
-  #chunkInfos: Map<DocumentId, ChunkInfo[]> = new Map()
-
-  /** Flag to avoid compacting when a compaction is already underway */
-  #compacting = false
-
   #log = debug(`automerge-repo:storage-subsystem`)
 
-  constructor(storageAdapter: StorageAdapterInterface) {
+  #beelay: A.beelay.Beelay
+
+  constructor(
+    beelay: A.beelay.Beelay,
+    storageAdapter: StorageAdapterInterface
+  ) {
+    super()
     this.#storageAdapter = storageAdapter
+    this.#beelay = beelay
   }
 
   async id(): Promise<StorageId> {
@@ -104,37 +115,26 @@ export class StorageSubsystem {
    * Loads the Automerge document with the given ID from storage.
    */
   async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
-    // Load all the chunks for this document
-    const chunks = await this.#storageAdapter.loadRange([documentId])
-    const binaries = []
-    const chunkInfos: ChunkInfo[] = []
-
-    for (const chunk of chunks) {
-      // chunks might have been deleted in the interim
-      if (chunk.data === undefined) continue
-
-      const chunkType = chunkTypeFromKey(chunk.key)
-      if (chunkType == null) continue
-
-      chunkInfos.push({
-        key: chunk.key,
-        type: chunkType,
-        size: chunk.data.length,
-      })
-      binaries.push(chunk.data)
+    const doc = await this.#beelay.loadDocument(documentId)
+    if (doc == null) {
+      return null
     }
-    this.#chunkInfos.set(documentId, chunkInfos)
-
-    // Merge the chunks into a single binary
+    const binaries = doc.map(c => c.contents)
     const binary = mergeArrays(binaries)
     if (binary.length === 0) return null
 
     // Load into an Automerge document
+    const start = performance.now()
     const newDoc = A.loadIncremental(A.init(), binary) as A.Doc<T>
+    const end = performance.now()
+    this.emit("document-loaded", {
+      documentId,
+      durationMillis: end - start,
+      ...A.stats(newDoc),
+    })
 
     // Record the latest heads for the document
     this.#storedHeads.set(documentId, A.getHeads(newDoc))
-
     return newDoc
   }
 
@@ -149,13 +149,36 @@ export class StorageSubsystem {
     // Don't bother saving if the document hasn't changed
     if (!this.#shouldSave(documentId, doc)) return
 
-    const sourceChunks = this.#chunkInfos.get(documentId) ?? []
-    if (this.#shouldCompact(sourceChunks)) {
-      await this.#saveTotal(documentId, doc, sourceChunks)
-    } else {
-      await this.#saveIncremental(documentId, doc)
-    }
+    const changes = A.getChanges(
+      A.view(doc, this.#storedHeads.get(documentId) ?? []),
+      doc
+    )
+
+    const commits = changes.map(c => {
+      const decoded = A.decodeChange(c)
+      return {
+        parents: decoded.deps,
+        hash: decoded.hash,
+        contents: c,
+      }
+    })
+    let done = this.#beelay
+      .addCommits({
+        docId: documentId,
+        commits: changes.map(c => {
+          const decoded = A.decodeChange(c)
+          return {
+            parents: decoded.deps,
+            hash: decoded.hash,
+            contents: c,
+          }
+        }),
+      })
+      .catch(e => {
+        console.error(`Error saving document ${documentId}: ${e}`)
+      })
     this.#storedHeads.set(documentId, A.getHeads(doc))
+    await done
   }
 
   /**
@@ -167,66 +190,6 @@ export class StorageSubsystem {
     await this.#storageAdapter.removeRange([documentId, "sync-state"])
   }
 
-  /**
-   * Saves just the incremental changes since the last save.
-   */
-  async #saveIncremental(
-    documentId: DocumentId,
-    doc: A.Doc<unknown>
-  ): Promise<void> {
-    const binary = A.saveSince(doc, this.#storedHeads.get(documentId) ?? [])
-    if (binary && binary.length > 0) {
-      const key = [documentId, "incremental", keyHash(binary)]
-      this.#log(`Saving incremental ${key} for document ${documentId}`)
-      await this.#storageAdapter.save(key, binary)
-      if (!this.#chunkInfos.has(documentId)) {
-        this.#chunkInfos.set(documentId, [])
-      }
-      this.#chunkInfos.get(documentId)!.push({
-        key,
-        type: "incremental",
-        size: binary.length,
-      })
-      this.#storedHeads.set(documentId, A.getHeads(doc))
-    } else {
-      return Promise.resolve()
-    }
-  }
-
-  /**
-   * Compacts the document storage into a single shapshot.
-   */
-  async #saveTotal(
-    documentId: DocumentId,
-    doc: A.Doc<unknown>,
-    sourceChunks: ChunkInfo[]
-  ): Promise<void> {
-    this.#compacting = true
-
-    const binary = A.save(doc)
-    const snapshotHash = headsHash(A.getHeads(doc))
-    const key = [documentId, "snapshot", snapshotHash]
-    const oldKeys = new Set(
-      sourceChunks.map(c => c.key).filter(k => k[2] !== snapshotHash)
-    )
-
-    this.#log(`Saving snapshot ${key} for document ${documentId}`)
-    this.#log(`deleting old chunks ${Array.from(oldKeys)}`)
-
-    await this.#storageAdapter.save(key, binary)
-
-    for (const key of oldKeys) {
-      await this.#storageAdapter.remove(key)
-    }
-
-    const newChunkInfos =
-      this.#chunkInfos.get(documentId)?.filter(c => !oldKeys.has(c.key)) ?? []
-    newChunkInfos.push({ key, type: "snapshot", size: binary.length })
-
-    this.#chunkInfos.set(documentId, newChunkInfos)
-    this.#compacting = false
-  }
-
   async loadSyncState(
     documentId: DocumentId,
     storageId: StorageId
@@ -268,27 +231,4 @@ export class StorageSubsystem {
 
     return true // the document has changed
   }
-
-  /**
-   * We only compact if the incremental size is greater than the snapshot size.
-   */
-  #shouldCompact(sourceChunks: ChunkInfo[]) {
-    if (this.#compacting) return false
-
-    let snapshotSize = 0
-    let incrementalSize = 0
-    for (const chunk of sourceChunks) {
-      if (chunk.type === "snapshot") {
-        snapshotSize += chunk.size
-      } else {
-        incrementalSize += chunk.size
-      }
-    }
-    // if the file is currently small, don't worry, just compact
-    // this might seem a bit arbitrary (1k is arbitrary) but is designed to ensure compaction
-    // for documents with only a single large change on top of an empty (or nearly empty) document
-    // for example: imported NPM modules, images, etc.
-    // if we have even more incrementals (so far) than the snapshot, compact
-    return snapshotSize < 1024 || incrementalSize >= snapshotSize
-  }
 }

package/src/synchronizer/CollectionSynchronizer.ts

@@ -1,11 +1,12 @@
 import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
-import { stringifyAutomergeUrl } from "../AutomergeUrl.js"
+import { parseAutomergeUrl, stringifyAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
 import { DocMessage } from "../network/messages.js"
-import { DocumentId, PeerId } from "../types.js"
+import { AutomergeUrl, DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
+import { next as A } from "@automerge/automerge"
 
 const log = debug("automerge-repo:collectionsync")
 
@@ -21,8 +22,15 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Used to determine if the document is know to the Collection and a synchronizer exists or is being set up */
   #docSetUp: Record<DocumentId, boolean> = {}
 
-  constructor(private repo: Repo) {
+  #denylist: DocumentId[]
+
+  constructor(
+    private beelay: A.beelay.Beelay,
+    private repo: Repo,
+    denylist: AutomergeUrl[] = []
+  ) {
     super()
+    this.#denylist = denylist.map(url => parseAutomergeUrl(url).documentId)
   }
 
   /** Returns a synchronizer for the given document, creating one if it doesn't already exist.  */
@@ -37,27 +45,13 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Creates a new docSynchronizer and sets it up to propagate messages */
   #initDocSynchronizer(handle: DocHandle<unknown>): DocSynchronizer {
     const docSynchronizer = new DocSynchronizer({
+      beelay: this.beelay,
       handle,
-      onLoadSyncState: async peerId => {
-        if (!this.repo.storageSubsystem) {
-          return
-        }
-
-        const { storageId, isEphemeral } =
-          this.repo.peerMetadataByPeerId[peerId] || {}
-        if (!storageId || isEphemeral) {
-          return
-        }
-
-        return this.repo.storageSubsystem.loadSyncState(
-          handle.documentId,
-          storageId
-        )
-      },
     })
     docSynchronizer.on("message", event => this.emit("message", event))
     docSynchronizer.on("open-doc", event => this.emit("open-doc", event))
     docSynchronizer.on("sync-state", event => this.emit("sync-state", event))
+    docSynchronizer.on("metrics", event => this.emit("metrics", event))
     return docSynchronizer
   }
 
@@ -90,6 +84,19 @@ export class CollectionSynchronizer extends Synchronizer {
       throw new Error("received a message with an invalid documentId")
     }
 
+    if (this.#denylist.includes(documentId)) {
+      this.emit("metrics", {
+        type: "doc-denied",
+        documentId,
+      })
+      this.emit("message", {
+        type: "doc-unavailable",
+        documentId,
+        targetId: message.senderId,
+      })
+      return
+    }
+
     this.#docSetUp[documentId] = true
 
     const docSynchronizer = this.#fetchDocSynchronizer(documentId)
@@ -161,12 +168,13 @@ export class CollectionSynchronizer extends Synchronizer {
       size: { numOps: number; numChanges: number }
     }
   } {
-    return Object.fromEntries(
-      Object.entries(this.docSynchronizers).map(
-        ([documentId, synchronizer]) => {
-          return [documentId, synchronizer.metrics()]
-        }
-      )
-    )
+    return {}
+    // return Object.fromEntries(
+    //   Object.entries(this.docSynchronizers).map(
+    //     ([documentId, synchronizer]) => {
+    //       return [documentId, synchronizer.metrics()]
+    //     }
+    //   )
+    // )
   }
 }