@automerge/automerge-repo 2.0.0-alpha.6 → 2.0.0-collectionsync-alpha.1

package/dist/src/synchronizer/DocSynchronizer.js

@@ -0,0 +1,263 @@
+import * as A from "@automerge/automerge/next"
+import { READY, REQUESTING, UNAVAILABLE } from "../DocHandle.js"
+import { Synchronizer } from "./Synchronizer.js"
+import debug from "debug"
+import { isRequestMessage } from "../network/messages.js"
+import { decode } from "cbor-x"
+/**
+ * DocSynchronizer takes a handle to an Automerge document, and receives & dispatches sync messages
+ * to bring it inline with all other peers' versions.
+ */
+export class DocSynchronizer extends Synchronizer {
+  handle
+  #log
+  #conciseLog
+  #opsLog
+  /** Active peers */
+  #peers = []
+  #peerDocumentStatuses = {}
+  /** Sync state for each peer we've communicated with (including inactive peers) */
+  #syncStates = {}
+  #pendingSyncMessages = []
+  #syncStarted = false
+  constructor(handle) {
+    super()
+    this.handle = handle
+    const docId = handle.documentId.slice(0, 5)
+    this.#conciseLog = debug(`automerge-repo:concise:docsync:${docId}`) // Only logs one line per receive/send
+    this.#log = debug(`automerge-repo:docsync:${docId}`)
+    this.#opsLog = debug(`automerge-repo:ops:docsync:${docId}`) // Log list of ops of each message
+    handle.on("change", () => this.#syncWithPeers())
+    handle.on("ephemeral-message-outbound", payload =>
+      this.#broadcastToPeers(payload)
+    )
+    // Process pending sync messages immediately after the handle becomes ready.
+    void (async () => {
+      await handle.doc([READY, REQUESTING])
+      this.#processAllPendingSyncMessages()
+    })()
+  }
+  get peerStates() {
+    return this.#peerDocumentStatuses
+  }
+  get documentId() {
+    return this.handle.documentId
+  }
+  /// PRIVATE
+  async #syncWithPeers() {
+    this.#log(`syncWithPeers`)
+    const doc = await this.handle.doc()
+    if (doc === undefined) return
+    this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
+  }
+  async #broadcastToPeers({ data }) {
+    this.#log(`broadcastToPeers`, this.#peers)
+    this.#peers.forEach(peerId => this.#sendEphemeralMessage(peerId, data))
+  }
+  #sendEphemeralMessage(peerId, data) {
+    this.#log(`sendEphemeralMessage ->${peerId}`)
+    this.emit("message", {
+      type: "ephemeral",
+      targetId: peerId,
+      documentId: this.handle.documentId,
+      data,
+    })
+  }
+  #getSyncState(peerId) {
+    if (!this.#peers.includes(peerId)) {
+      this.#log("adding a new peer", peerId)
+      this.#peers.push(peerId)
+    }
+    // when a peer is added, we don't know if it has the document or not
+    if (!(peerId in this.#peerDocumentStatuses)) {
+      this.#peerDocumentStatuses[peerId] = "unknown"
+    }
+    return this.#syncStates[peerId] ?? A.initSyncState()
+  }
+  #setSyncState(peerId, syncState) {
+    // TODO: we maybe should be persisting sync states. But we want to be careful about how often we
+    // do that, because it can generate a lot of disk activity.
+    // TODO: we only need to do this on reconnect
+    this.#syncStates[peerId] = syncState
+  }
+  #sendSyncMessage(peerId, doc) {
+    this.#log(`sendSyncMessage ->${peerId}`)
+    const syncState = this.#getSyncState(peerId)
+    const [newSyncState, message] = A.generateSyncMessage(doc, syncState)
+    this.#setSyncState(peerId, newSyncState)
+    if (message) {
+      this.#logMessage(`sendSyncMessage 🡒 ${peerId}`, message)
+      const decoded = A.decodeSyncMessage(message)
+      if (
+        !this.handle.isReady() &&
+        decoded.heads.length === 0 &&
+        newSyncState.sharedHeads.length === 0 &&
+        !Object.values(this.#peerDocumentStatuses).includes("has") &&
+        this.#peerDocumentStatuses[peerId] === "unknown"
+      ) {
+        // we don't have the document (or access to it), so we request it
+        this.emit("message", {
+          type: "request",
+          targetId: peerId,
+          documentId: this.handle.documentId,
+          data: message,
+        })
+      } else {
+        this.emit("message", {
+          type: "sync",
+          targetId: peerId,
+          data: message,
+          documentId: this.handle.documentId,
+        })
+      }
+      // if we have sent heads, then the peer now has or will have the document
+      if (decoded.heads.length > 0) {
+        this.#peerDocumentStatuses[peerId] = "has"
+      }
+    }
+  }
+  #logMessage = (label, message) => {
+    // This is real expensive...
+    return
+    const size = message.byteLength
+    const logText = `${label} ${size}b`
+    const decoded = A.decodeSyncMessage(message)
+    this.#conciseLog(logText)
+    this.#log(logText, decoded)
+    // expanding is expensive, so only do it if we're logging at this level
+    const expanded = this.#opsLog.enabled
+      ? decoded.changes.flatMap(change =>
+          A.decodeChange(change).ops.map(op => JSON.stringify(op))
+        )
+      : null
+    this.#opsLog(logText, expanded)
+  }
+  /// PUBLIC
+  hasPeer(peerId) {
+    return this.#peers.includes(peerId)
+  }
+  beginSync(peerIds) {
+    this.#log(`beginSync: ${peerIds.join(", ")}`)
+    // HACK: if we have a sync state already, we round-trip it through the encoding system to make
+    // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
+    // messages during disconnection.
+    // TODO: cover that case with a test and remove this hack
+    peerIds.forEach(peerId => {
+      const syncStateRaw = this.#getSyncState(peerId)
+      const syncState = A.decodeSyncState(A.encodeSyncState(syncStateRaw))
+      this.#setSyncState(peerId, syncState)
+    })
+    // At this point if we don't have anything in our storage, we need to use an empty doc to sync
+    // with; but we don't want to surface that state to the front end
+    void this.handle.doc([READY, REQUESTING, UNAVAILABLE]).then(doc => {
+      // we register out peers first, then say that sync has started
+      this.#syncStarted = true
+      this.#checkDocUnavailable()
+      if (doc === undefined) return
+      peerIds.forEach(peerId => {
+        this.#sendSyncMessage(peerId, doc)
+      })
+    })
+  }
+  endSync(peerId) {
+    this.#log(`removing peer ${peerId}`)
+    this.#peers = this.#peers.filter(p => p !== peerId)
+  }
+  receiveMessage(message) {
+    switch (message.type) {
+      case "sync":
+      case "request":
+        this.receiveSyncMessage(message)
+        break
+      case "ephemeral":
+        this.receiveEphemeralMessage(message)
+        break
+      case "doc-unavailable":
+        this.#peerDocumentStatuses[message.senderId] = "unavailable"
+        this.#checkDocUnavailable()
+        break
+      default:
+        throw new Error(`unknown message type: ${message}`)
+    }
+  }
+  receiveEphemeralMessage(message) {
+    if (message.documentId !== this.handle.documentId)
+      throw new Error(`channelId doesn't match documentId`)
+    const { senderId, data } = message
+    const contents = decode(data)
+    this.handle.emit("ephemeral-message", {
+      handle: this.handle,
+      senderId,
+      message: contents,
+    })
+    this.#peers.forEach(peerId => {
+      if (peerId === senderId) return
+      this.emit("message", {
+        ...message,
+        targetId: peerId,
+      })
+    })
+  }
+  receiveSyncMessage(message) {
+    if (message.documentId !== this.handle.documentId)
+      throw new Error(`channelId doesn't match documentId`)
+    // We need to block receiving the syncMessages until we've checked local storage
+    if (!this.handle.inState([READY, REQUESTING, UNAVAILABLE])) {
+      this.#pendingSyncMessages.push(message)
+      return
+    }
+    this.#processAllPendingSyncMessages()
+    this.#processSyncMessage(message)
+  }
+  #processSyncMessage(message) {
+    if (isRequestMessage(message)) {
+      this.#peerDocumentStatuses[message.senderId] = "wants"
+    }
+    this.#checkDocUnavailable()
+    // if the message has heads, then the peer has the document
+    if (A.decodeSyncMessage(message.data).heads.length > 0) {
+      this.#peerDocumentStatuses[message.senderId] = "has"
+    }
+    this.handle.update(doc => {
+      const [newDoc, newSyncState] = A.receiveSyncMessage(
+        doc,
+        this.#getSyncState(message.senderId),
+        message.data
+      )
+      this.#setSyncState(message.senderId, newSyncState)
+      // respond to just this peer (as required)
+      this.#sendSyncMessage(message.senderId, doc)
+      return newDoc
+    })
+    this.#checkDocUnavailable()
+  }
+  #checkDocUnavailable() {
+    // if we know none of the peers have the document, tell all our peers that we don't either
+    if (
+      this.#syncStarted &&
+      this.handle.inState([REQUESTING]) &&
+      this.#peers.every(
+        peerId =>
+          this.#peerDocumentStatuses[peerId] === "unavailable" ||
+          this.#peerDocumentStatuses[peerId] === "wants"
+      )
+    ) {
+      this.#peers
+        .filter(peerId => this.#peerDocumentStatuses[peerId] === "wants")
+        .forEach(peerId => {
+          this.emit("message", {
+            type: "doc-unavailable",
+            documentId: this.handle.documentId,
+            targetId: peerId,
+          })
+        })
+      this.handle.unavailable()
+    }
+  }
+  #processAllPendingSyncMessages() {
+    for (const message of this.#pendingSyncMessages) {
+      this.#processSyncMessage(message)
+    }
+    this.#pendingSyncMessages = []
+  }
+}

package/dist/src/synchronizer/Synchronizer.d.ts

@@ -0,0 +1,9 @@
+import { EventEmitter } from "eventemitter3"
+import { Message, MessageContents } from "../network/messages.js"
+export declare abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
+  abstract receiveMessage(message: Message): void
+}
+export interface SynchronizerEvents {
+  message: (arg: MessageContents) => void
+}
+//# sourceMappingURL=Synchronizer.d.ts.map

package/dist/src/synchronizer/Synchronizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Synchronizer.d.ts","sourceRoot":"","sources":["../../../src/synchronizer/Synchronizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAEjE,8BAAsB,YAAa,SAAQ,YAAY,CAAC,kBAAkB,CAAC;IACzE,QAAQ,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;CAChD;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,IAAI,CAAA;CACxC"}

package/dist/src/synchronizer/Synchronizer.js

@@ -0,0 +1,2 @@
+import { EventEmitter } from "eventemitter3"
+export class Synchronizer extends EventEmitter {}

package/dist/src/types.d.ts

@@ -0,0 +1,16 @@
+export type DocumentId = string & {
+  __documentId: true
+}
+export type AutomergeUrl = string & {
+  __documentUrl: true
+}
+export type BinaryDocumentId = Uint8Array & {
+  __binaryDocumentId: true
+}
+export type PeerId = string & {
+  __peerId: false
+}
+export type DistributiveOmit<T, K extends keyof any> = T extends any
+  ? Omit<T, K>
+  : never
+//# sourceMappingURL=types.d.ts.map

package/dist/src/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG;IAAE,YAAY,EAAE,IAAI,CAAA;CAAE,CAAA;AACxD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG;IAAE,aAAa,EAAE,IAAI,CAAA;CAAE,CAAA;AAC3D,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG;IAAE,kBAAkB,EAAE,IAAI,CAAA;CAAE,CAAA;AAExE,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG;IAAE,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAA;AAEjD,MAAM,MAAM,gBAAgB,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,GAAG,IAAI,CAAC,SAAS,GAAG,GAChE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GACV,KAAK,CAAA"}

package/dist/src/types.js

@@ -0,0 +1 @@
+export {}

package/dist/storage/StorageAdapter.d.ts

@@ -29,4 +29,13 @@ export declare abstract class StorageAdapter implements StorageAdapterInterface
     /** Remove all values with keys that start with `keyPrefix` */
     abstract removeRange(keyPrefix: StorageKey): Promise<void>;
 }
+export declare class InMemoryStorageAdapter extends StorageAdapter {
+    #private;
+    loadRange(keyPrefix: StorageKey): Promise<Chunk[]>;
+    removeRange(keyPrefix: string[]): Promise<void>;
+    load(key: string[]): Promise<Uint8Array | undefined>;
+    save(key: string[], binary: Uint8Array): Promise<void>;
+    remove(key: string[]): Promise<void>;
+    keys(): string[];
+}
 //# sourceMappingURL=StorageAdapter.d.ts.map

package/dist/storage/StorageAdapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"StorageAdapter.d.ts","sourceRoot":"","sources":["../../src/storage/StorageAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAE9C;;;;;;GAMG;AACH,8BAAsB,cAAe,YAAW,uBAAuB;IACrE,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAE/D,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/D,8CAA8C;IAC9C,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/C;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAE3D,8DAA8D;IAC9D,QAAQ,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;CAC3D"}
+{"version":3,"file":"StorageAdapter.d.ts","sourceRoot":"","sources":["../../src/storage/StorageAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAE9C;;;;;;GAMG;AACH,8BAAsB,cAAe,YAAW,uBAAuB;IACrE,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAE/D,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/D,8CAA8C;IAC9C,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/C;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAE3D,8DAA8D;IAC9D,QAAQ,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;CAC3D;AAED,qBAAa,sBAAuB,SAAQ,cAAc;;IAWlD,SAAS,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAOlD,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAIpD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU;IAKtC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE;IAI1B,IAAI;CAGL"}

package/dist/storage/StorageAdapter.js

@@ -7,3 +7,36 @@
  */
 export class StorageAdapter {
 }
+export class InMemoryStorageAdapter extends StorageAdapter {
+    #data = {};
+    #keyToString(key) {
+        return key.join(".");
+    }
+    #stringToKey(key) {
+        return key.split(".");
+    }
+    async loadRange(keyPrefix) {
+        const range = Object.entries(this.#data)
+            .filter(([key, _]) => key.startsWith(this.#keyToString(keyPrefix)))
+            .map(([key, data]) => ({ key: this.#stringToKey(key), data }));
+        return Promise.resolve(range);
+    }
+    async removeRange(keyPrefix) {
+        Object.entries(this.#data)
+            .filter(([key, _]) => key.startsWith(this.#keyToString(keyPrefix)))
+            .forEach(([key, _]) => delete this.#data[key]);
+    }
+    async load(key) {
+        return new Promise(resolve => resolve(this.#data[this.#keyToString(key)]));
+    }
+    async save(key, binary) {
+        this.#data[this.#keyToString(key)] = binary;
+        return Promise.resolve();
+    }
+    async remove(key) {
+        delete this.#data[this.#keyToString(key)];
+    }
+    keys() {
+        return Object.keys(this.#data);
+    }
+}

package/dist/storage/StorageSubsystem.d.ts

@@ -2,13 +2,22 @@ import * as A from "@automerge/automerge/slim/next";
 import { type DocumentId } from "../types.js";
 import { StorageAdapterInterface } from "./StorageAdapterInterface.js";
 import { StorageId } from "./types.js";
+import { EventEmitter } from "eventemitter3";
+type StorageSubsystemEvents = {
+    "document-loaded": (arg: {
+        documentId: DocumentId;
+        durationMillis: number;
+        numOps: number;
+        numChanges: number;
+    }) => void;
+};
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
  * storage adapter. It also provides a generic key/value storage interface for other uses.
  */
-export declare class StorageSubsystem {
+export declare class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
     #private;
-    constructor(storageAdapter: StorageAdapterInterface);
+    constructor(beelay: A.beelay.Beelay, storageAdapter: StorageAdapterInterface);
     id(): Promise<StorageId>;
     /** Loads a value from storage. */
     load(
@@ -49,4 +58,5 @@ export declare class StorageSubsystem {
     loadSyncState(documentId: DocumentId, storageId: StorageId): Promise<A.SyncState | undefined>;
     saveSyncState(documentId: DocumentId, storageId: StorageId, syncState: A.SyncState): Promise<void>;
 }
+export {};
 //# sourceMappingURL=StorageSubsystem.d.ts.map

package/dist/storage/StorageSubsystem.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"StorageSubsystem.d.ts","sourceRoot":"","sources":["../../src/storage/StorageSubsystem.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,gCAAgC,CAAA;AAInD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAyB,SAAS,EAAE,MAAM,YAAY,CAAA;AAK7D;;;GAGG;AACH,qBAAa,gBAAgB;;gBAef,cAAc,EAAE,uBAAuB;IAI7C,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;IA2B9B,kCAAkC;IAC5B,IAAI;IACR,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,yFAAyF;IACzF,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAKlC,gCAAgC;IAC1B,IAAI;IACR,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,yFAAyF;IACzF,GAAG,EAAE,MAAM;IAEX,sCAAsC;IACtC,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC;IAKhB,oCAAoC;IAC9B,MAAM;IACV,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,2FAA2F;IAC3F,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC;IAOhB;;OAEG;IACG,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAmClE;;;;;;OAMG;IACG,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAazE;;OAEG;IACG,SAAS,CAAC,UAAU,EAAE,UAAU;IAkEhC,aAAa,CACjB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;IAW7B,aAAa,CACjB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,SAAS,EAAE,CAAC,CAAC,SAAS,GACrB,OAAO,CAAC,IAAI,CAAC;CA8CjB"}
+{"version":3,"file":"StorageSubsystem.d.ts","sourceRoot":"","sources":["../../src/storage/StorageSubsystem.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,gCAAgC,CAAA;AAInD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAyB,SAAS,EAAE,MAAM,YAAY,CAAA;AAI7D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,iBAAiB,EAAE,CAAC,GAAG,EAAE;QACvB,UAAU,EAAE,UAAU,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,MAAM,EAAE,MAAM,CAAA;QACd,UAAU,EAAE,MAAM,CAAA;KACnB,KAAK,IAAI,CAAA;CACX,CAAA;AAED;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,YAAY,CAAC,sBAAsB,CAAC;;gBAYtE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EACvB,cAAc,EAAE,uBAAuB;IAOnC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;IA2B9B,kCAAkC;IAC5B,IAAI;IACR,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,yFAAyF;IACzF,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAKlC,gCAAgC;IAC1B,IAAI;IACR,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,yFAAyF;IACzF,GAAG,EAAE,MAAM;IAEX,sCAAsC;IACtC,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC;IAKhB,oCAAoC;IAC9B,MAAM;IACV,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,2FAA2F;IAC3F,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC;IAOhB;;OAEG;IACG,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAwBlE;;;;;;OAMG;IACG,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCzE;;OAEG;IACG,SAAS,CAAC,UAAU,EAAE,UAAU;IAMhC,aAAa,CACjB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;IAW7B,aAAa,CACjB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,SAAS,EAAE,CAAC,CAAC,SAAS,GACrB,OAAO,CAAC,IAAI,CAAC;CAuBjB"}

package/dist/storage/StorageSubsystem.js

@@ -2,25 +2,23 @@ import * as A from "@automerge/automerge/slim/next";
 import debug from "debug";
 import { headsAreSame } from "../helpers/headsAreSame.js";
 import { mergeArrays } from "../helpers/mergeArrays.js";
-import { keyHash, headsHash } from "./keyHash.js";
-import { chunkTypeFromKey } from "./chunkTypeFromKey.js";
 import * as Uuid from "uuid";
+import { EventEmitter } from "eventemitter3";
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
  * storage adapter. It also provides a generic key/value storage interface for other uses.
  */
-export class StorageSubsystem {
+export class StorageSubsystem extends EventEmitter {
     /** The storage adapter to use for saving and loading documents */
     #storageAdapter;
     /** Record of the latest heads we've loaded or saved for each document  */
     #storedHeads = new Map();
-    /** Metadata on the chunks we've already loaded for each document */
-    #chunkInfos = new Map();
-    /** Flag to avoid compacting when a compaction is already underway */
-    #compacting = false;
     #log = debug(`automerge-repo:storage-subsystem`);
-    constructor(storageAdapter) {
+    #beelay;
+    constructor(beelay, storageAdapter) {
+        super();
         this.#storageAdapter = storageAdapter;
+        this.#beelay = beelay;
     }
     async id() {
         const storedId = await this.#storageAdapter.load(["storage-adapter-id"]);
@@ -76,31 +74,23 @@ export class StorageSubsystem {
      * Loads the Automerge document with the given ID from storage.
      */
     async loadDoc(documentId) {
-        // Load all the chunks for this document
-        const chunks = await this.#storageAdapter.loadRange([documentId]);
-        const binaries = [];
-        const chunkInfos = [];
-        for (const chunk of chunks) {
-            // chunks might have been deleted in the interim
-            if (chunk.data === undefined)
-                continue;
-            const chunkType = chunkTypeFromKey(chunk.key);
-            if (chunkType == null)
-                continue;
-            chunkInfos.push({
-                key: chunk.key,
-                type: chunkType,
-                size: chunk.data.length,
-            });
-            binaries.push(chunk.data);
+        const doc = await this.#beelay.loadDocument(documentId);
+        if (doc == null) {
+            return null;
         }
-        this.#chunkInfos.set(documentId, chunkInfos);
-        // Merge the chunks into a single binary
+        const binaries = doc.map(c => c.contents);
         const binary = mergeArrays(binaries);
         if (binary.length === 0)
             return null;
         // Load into an Automerge document
+        const start = performance.now();
         const newDoc = A.loadIncremental(A.init(), binary);
+        const end = performance.now();
+        this.emit("document-loaded", {
+            documentId,
+            durationMillis: end - start,
+            ...A.stats(newDoc),
+        });
         // Record the latest heads for the document
         this.#storedHeads.set(documentId, A.getHeads(newDoc));
         return newDoc;
@@ -116,14 +106,32 @@ export class StorageSubsystem {
         // Don't bother saving if the document hasn't changed
         if (!this.#shouldSave(documentId, doc))
             return;
-        const sourceChunks = this.#chunkInfos.get(documentId) ?? [];
-        if (this.#shouldCompact(sourceChunks)) {
-            await this.#saveTotal(documentId, doc, sourceChunks);
-        }
-        else {
-            await this.#saveIncremental(documentId, doc);
-        }
+        const changes = A.getChanges(A.view(doc, this.#storedHeads.get(documentId) ?? []), doc);
+        const commits = changes.map(c => {
+            const decoded = A.decodeChange(c);
+            return {
+                parents: decoded.deps,
+                hash: decoded.hash,
+                contents: c,
+            };
+        });
+        let done = this.#beelay
+            .addCommits({
+            docId: documentId,
+            commits: changes.map(c => {
+                const decoded = A.decodeChange(c);
+                return {
+                    parents: decoded.deps,
+                    hash: decoded.hash,
+                    contents: c,
+                };
+            }),
+        })
+            .catch(e => {
+            console.error(`Error saving document ${documentId}: ${e}`);
+        });
         this.#storedHeads.set(documentId, A.getHeads(doc));
+        await done;
     }
     /**
      * Removes the Automerge document with the given ID from storage
@@ -133,49 +141,6 @@ export class StorageSubsystem {
         await this.#storageAdapter.removeRange([documentId, "incremental"]);
         await this.#storageAdapter.removeRange([documentId, "sync-state"]);
     }
-    /**
-     * Saves just the incremental changes since the last save.
-     */
-    async #saveIncremental(documentId, doc) {
-        const binary = A.saveSince(doc, this.#storedHeads.get(documentId) ?? []);
-        if (binary && binary.length > 0) {
-            const key = [documentId, "incremental", keyHash(binary)];
-            this.#log(`Saving incremental ${key} for document ${documentId}`);
-            await this.#storageAdapter.save(key, binary);
-            if (!this.#chunkInfos.has(documentId)) {
-                this.#chunkInfos.set(documentId, []);
-            }
-            this.#chunkInfos.get(documentId).push({
-                key,
-                type: "incremental",
-                size: binary.length,
-            });
-            this.#storedHeads.set(documentId, A.getHeads(doc));
-        }
-        else {
-            return Promise.resolve();
-        }
-    }
-    /**
-     * Compacts the document storage into a single shapshot.
-     */
-    async #saveTotal(documentId, doc, sourceChunks) {
-        this.#compacting = true;
-        const binary = A.save(doc);
-        const snapshotHash = headsHash(A.getHeads(doc));
-        const key = [documentId, "snapshot", snapshotHash];
-        const oldKeys = new Set(sourceChunks.map(c => c.key).filter(k => k[2] !== snapshotHash));
-        this.#log(`Saving snapshot ${key} for document ${documentId}`);
-        this.#log(`deleting old chunks ${Array.from(oldKeys)}`);
-        await this.#storageAdapter.save(key, binary);
-        for (const key of oldKeys) {
-            await this.#storageAdapter.remove(key);
-        }
-        const newChunkInfos = this.#chunkInfos.get(documentId)?.filter(c => !oldKeys.has(c.key)) ?? [];
-        newChunkInfos.push({ key, type: "snapshot", size: binary.length });
-        this.#chunkInfos.set(documentId, newChunkInfos);
-        this.#compacting = false;
-    }
     async loadSyncState(documentId, storageId) {
         const key = [documentId, "sync-state", storageId];
         try {
@@ -207,27 +172,4 @@ export class StorageSubsystem {
         }
         return true; // the document has changed
     }
-    /**
-     * We only compact if the incremental size is greater than the snapshot size.
-     */
-    #shouldCompact(sourceChunks) {
-        if (this.#compacting)
-            return false;
-        let snapshotSize = 0;
-        let incrementalSize = 0;
-        for (const chunk of sourceChunks) {
-            if (chunk.type === "snapshot") {
-                snapshotSize += chunk.size;
-            }
-            else {
-                incrementalSize += chunk.size;
-            }
-        }
-        // if the file is currently small, don't worry, just compact
-        // this might seem a bit arbitrary (1k is arbitrary) but is designed to ensure compaction
-        // for documents with only a single large change on top of an empty (or nearly empty) document
-        // for example: imported NPM modules, images, etc.
-        // if we have even more incrementals (so far) than the snapshot, compact
-        return snapshotSize < 1024 || incrementalSize >= snapshotSize;
-    }
 }

package/dist/synchronizer/CollectionSynchronizer.d.ts

@@ -1,16 +1,18 @@
 import { Repo } from "../Repo.js";
 import { DocMessage } from "../network/messages.js";
-import { DocumentId, PeerId } from "../types.js";
+import { AutomergeUrl, DocumentId, PeerId } from "../types.js";
 import { DocSynchronizer } from "./DocSynchronizer.js";
 import { Synchronizer } from "./Synchronizer.js";
+import { next as A } from "@automerge/automerge";
 /** A CollectionSynchronizer is responsible for synchronizing a DocCollection with peers. */
 export declare class CollectionSynchronizer extends Synchronizer {
     #private;
+    private beelay;
     private repo;
     /** A map of documentIds to their synchronizers */
     /** @hidden */
     docSynchronizers: Record<DocumentId, DocSynchronizer>;
-    constructor(repo: Repo);
+    constructor(beelay: A.beelay.Beelay, repo: Repo, denylist?: AutomergeUrl[]);
     /**
      * When we receive a sync message for a document we haven't got in memory, we
      * register it with the repo and start synchronizing

package/dist/synchronizer/CollectionSynchronizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"CollectionSynchronizer.d.ts","sourceRoot":"","sources":["../../src/synchronizer/CollectionSynchronizer.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAA;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAIhD,4FAA4F;AAC5F,qBAAa,sBAAuB,SAAQ,YAAY;;IAW1C,OAAO,CAAC,IAAI;IAPxB,kDAAkD;IAClD,cAAc;IACd,gBAAgB,EAAE,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAK;gBAKtC,IAAI,EAAE,IAAI;IAqD9B;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,UAAU;IAyBxC;;OAEG;IACH,WAAW,CAAC,UAAU,EAAE,UAAU;IAalC,cAAc,CAAC,UAAU,EAAE,UAAU;IAIrC,2DAA2D;IAC3D,OAAO,CAAC,MAAM,EAAE,MAAM;IAgBtB,uDAAuD;IACvD,UAAU,CAAC,MAAM,EAAE,MAAM;IASzB,+CAA+C;IAC/C,IAAI,KAAK,IAAI,MAAM,EAAE,CAEpB;IAED,OAAO,IAAI;QACT,CAAC,GAAG,EAAE,MAAM,GAAG;YACb,KAAK,EAAE,MAAM,EAAE,CAAA;YACf,IAAI,EAAE;gBAAE,MAAM,EAAE,MAAM,CAAC;gBAAC,UAAU,EAAE,MAAM,CAAA;aAAE,CAAA;SAC7C,CAAA;KACF;CASF"}
+{"version":3,"file":"CollectionSynchronizer.d.ts","sourceRoot":"","sources":["../../src/synchronizer/CollectionSynchronizer.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAA;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,sBAAsB,CAAA;AAIhD,4FAA4F;AAC5F,qBAAa,sBAAuB,SAAQ,YAAY;;IAcpD,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,IAAI;IAXd,kDAAkD;IAClD,cAAc;IACd,gBAAgB,EAAE,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAK;gBAQhD,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EACvB,IAAI,EAAE,IAAI,EAClB,QAAQ,GAAE,YAAY,EAAO;IAyC/B;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,UAAU;IAsCxC;;OAEG;IACH,WAAW,CAAC,UAAU,EAAE,UAAU;IAalC,cAAc,CAAC,UAAU,EAAE,UAAU;IAIrC,2DAA2D;IAC3D,OAAO,CAAC,MAAM,EAAE,MAAM;IAgBtB,uDAAuD;IACvD,UAAU,CAAC,MAAM,EAAE,MAAM;IASzB,+CAA+C;IAC/C,IAAI,KAAK,IAAI,MAAM,EAAE,CAEpB;IAED,OAAO,IAAI;QACT,CAAC,GAAG,EAAE,MAAM,GAAG;YACb,KAAK,EAAE,MAAM,EAAE,CAAA;YACf,IAAI,EAAE;gBAAE,MAAM,EAAE,MAAM,CAAC;gBAAC,UAAU,EAAE,MAAM,CAAA;aAAE,CAAA;SAC7C,CAAA;KACF;CAUF"}