@automerge/automerge-repo 2.0.0-alpha.7 → 2.0.0-beta.2

package/src/helpers/tests/network-adapter-tests.ts

@@ -49,9 +49,10 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
         // Alice creates a document
         const aliceHandle = aliceRepo.create<TestDoc>()
 
-        // Bob receives the document
-        await eventPromise(bobRepo, "document")
-        const bobHandle = bobRepo.find<TestDoc>(aliceHandle.url)
+        // TODO: ... let connections complete. this shouldn't be necessary.
+        await pause(50)
+
+        const bobHandle = await bobRepo.find<TestDoc>(aliceHandle.url)
 
         // Alice changes the document
         aliceHandle.change(d => {
@@ -60,7 +61,7 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Bob receives the change
         await eventPromise(bobHandle, "change")
-        assert.equal((await bobHandle.doc())?.foo, "bar")
+        assert.equal((await bobHandle).doc()?.foo, "bar")
 
         // Bob changes the document
         bobHandle.change(d => {
@@ -69,7 +70,7 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Alice receives the change
         await eventPromise(aliceHandle, "change")
-        assert.equal((await aliceHandle.doc())?.foo, "baz")
+        assert.equal(aliceHandle.doc().foo, "baz")
       }
 
       // Run the test in both directions, in case they're different types of adapters
@@ -100,9 +101,9 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
       const docUrl = aliceHandle.url
 
       // Bob and Charlie receive the document
-      await eventPromises([bobRepo, charlieRepo], "document")
-      const bobHandle = bobRepo.find<TestDoc>(docUrl)
-      const charlieHandle = charlieRepo.find<TestDoc>(docUrl)
+      await pause(50)
+      const bobHandle = await bobRepo.find<TestDoc>(docUrl)
+      const charlieHandle = await charlieRepo.find<TestDoc>(docUrl)
 
       // Alice changes the document
       aliceHandle.change(d => {
@@ -111,8 +112,8 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Bob and Charlie receive the change
       await eventPromises([bobHandle, charlieHandle], "change")
-      assert.equal((await bobHandle.doc())?.foo, "bar")
-      assert.equal((await charlieHandle.doc())?.foo, "bar")
+      assert.equal(bobHandle.doc().foo, "bar")
+      assert.equal(charlieHandle.doc().foo, "bar")
 
       // Charlie changes the document
       charlieHandle.change(d => {
@@ -121,8 +122,8 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Alice and Bob receive the change
       await eventPromises([aliceHandle, bobHandle], "change")
-      assert.equal((await bobHandle.doc())?.foo, "baz")
-      assert.equal((await charlieHandle.doc())?.foo, "baz")
+      assert.equal(bobHandle.doc().foo, "baz")
+      assert.equal(charlieHandle.doc().foo, "baz")
 
       teardown()
     })
@@ -141,7 +142,7 @@ export function runNetworkAdapterTests(_setup: SetupFn, title?: string): void {
       )
 
       const aliceHandle = aliceRepo.create<TestDoc>()
-      const charlieHandle = charlieRepo.find(aliceHandle.url)
+      const charlieHandle = await charlieRepo.find(aliceHandle.url)
 
       // pause to give charlie a chance to let alice know it wants the doc
       await pause(100)

package/src/helpers/tests/storage-adapter-tests.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest"
+import { describe, expect, beforeEach, it as _it } from "vitest"
 
 import type { StorageAdapterInterface } from "../../storage/StorageAdapterInterface.js"
 
@@ -8,120 +8,90 @@ const PAYLOAD_C = () => new Uint8Array([2, 111, 74, 131, 236, 96, 142, 193])
 
 const LARGE_PAYLOAD = new Uint8Array(100000).map(() => Math.random() * 256)
 
-export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
-  const setup = async () => {
-    const { adapter, teardown = NO_OP } = await _setup()
-    return { adapter, teardown }
-  }
+type AdapterTestContext = {
+  adapter: StorageAdapterInterface
+}
+
+const it = _it<AdapterTestContext>
+
+export function runStorageAdapterTests(setup: SetupFn, title?: string): void {
+  beforeEach<AdapterTestContext>(async ctx => {
+    const { adapter, teardown = NO_OP } = await setup()
+    ctx.adapter = adapter
+    return teardown
+  })
 
   describe(`Storage adapter acceptance tests ${
     title ? `(${title})` : ""
   }`, () => {
     describe("load", () => {
-      it("should return undefined if there is no data", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return undefined if there is no data", async ({ adapter }) => {
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toBeUndefined()
-
-        teardown()
       })
     })
 
     describe("save and load", () => {
-      it("should return data that was saved", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return data that was saved", async ({ adapter }) => {
         await adapter.save(["storage-adapter-id"], PAYLOAD_A())
         const actual = await adapter.load(["storage-adapter-id"])
         expect(actual).toStrictEqual(PAYLOAD_A())
-
-        teardown()
       })
 
-      it("should work with composite keys", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should work with composite keys", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toStrictEqual(PAYLOAD_A())
-
-        teardown()
       })
 
-      it("should work with a large payload", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should work with a large payload", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], LARGE_PAYLOAD)
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toStrictEqual(LARGE_PAYLOAD)
-
-        teardown()
       })
     })
 
     describe("loadRange", () => {
-      it("should return an empty array if there is no data", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return an empty array if there is no data", async ({
+        adapter,
+      }) => {
         expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([])
-
-        teardown()
       })
     })
 
     describe("save and loadRange", () => {
-      it("should return all the data that matches the key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return all the data that matches the key", async ({
+        adapter,
+      }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "snapshot", "yyyyy"], PAYLOAD_B())
         await adapter.save(["AAAAA", "sync-state", "zzzzz"], PAYLOAD_C())
 
-        expect(await adapter.loadRange(["AAAAA"])).toStrictEqual(
-          expect.arrayContaining([
-            { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_A() },
-            { key: ["AAAAA", "snapshot", "yyyyy"], data: PAYLOAD_B() },
-            { key: ["AAAAA", "sync-state", "zzzzz"], data: PAYLOAD_C() },
-          ])
-        )
-
-        expect(await adapter.loadRange(["AAAAA", "sync-state"])).toStrictEqual(
-          expect.arrayContaining([
-            { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_A() },
-            { key: ["AAAAA", "sync-state", "zzzzz"], data: PAYLOAD_C() },
-          ])
-        )
-
-        teardown()
-      })
+        expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([
+          { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_A() },
+          { key: ["AAAAA", "snapshot", "yyyyy"], data: PAYLOAD_B() },
+          { key: ["AAAAA", "sync-state", "zzzzz"], data: PAYLOAD_C() },
+        ])
 
-      it("should only load values that match they key", async () => {
-        const { adapter, teardown } = await setup()
+        expect(await adapter.loadRange(["AAAAA", "sync-state"])).toStrictEqual([
+          { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_A() },
+          { key: ["AAAAA", "sync-state", "zzzzz"], data: PAYLOAD_C() },
+        ])
+      })
 
+      it("should only load values that match they key", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["BBBBB", "sync-state", "zzzzz"], PAYLOAD_C())
 
         const actual = await adapter.loadRange(["AAAAA"])
-        expect(actual).toStrictEqual(
-          expect.arrayContaining([
-            { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_A() },
-          ])
-        )
-        expect(actual).toStrictEqual(
-          expect.not.arrayContaining([
-            { key: ["BBBBB", "sync-state", "zzzzz"], data: PAYLOAD_C() },
-          ])
-        )
-
-        teardown()
+        expect(actual).toStrictEqual([
+          { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_A() },
+        ])
       })
     })
 
     describe("save and remove", () => {
-      it("after removing, should be empty", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("after removing, should be empty", async ({ adapter }) => {
         await adapter.save(["AAAAA", "snapshot", "xxxxx"], PAYLOAD_A())
         await adapter.remove(["AAAAA", "snapshot", "xxxxx"])
 
@@ -129,30 +99,24 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(
           await adapter.load(["AAAAA", "snapshot", "xxxxx"])
         ).toBeUndefined()
-
-        teardown()
       })
     })
 
     describe("save and save", () => {
-      it("should overwrite data saved with the same key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should overwrite data saved with the same key", async ({
+        adapter,
+      }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_B())
 
         expect(await adapter.loadRange(["AAAAA", "sync-state"])).toStrictEqual([
           { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
     })
 
     describe("removeRange", () => {
-      it("should remove a range of records", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should remove a range of records", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "snapshot", "yyyyy"], PAYLOAD_B())
         await adapter.save(["AAAAA", "sync-state", "zzzzz"], PAYLOAD_C())
@@ -162,13 +126,9 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([
           { key: ["AAAAA", "snapshot", "yyyyy"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
 
-      it("should not remove records that don't match", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should not remove records that don't match", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["BBBBB", "sync-state", "zzzzz"], PAYLOAD_B())
 
@@ -178,8 +138,6 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(actual).toStrictEqual([
           { key: ["BBBBB", "sync-state", "zzzzz"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
     })
   })
@@ -189,5 +147,5 @@ const NO_OP = () => {}
 
 export type SetupFn = () => Promise<{
   adapter: StorageAdapterInterface
-  teardown?: () => void
+  teardown?: () => void | Promise<void>
 }>

package/src/index.ts

@@ -34,6 +34,8 @@ export {
   stringifyAutomergeUrl,
   interpretAsDocumentId,
   generateAutomergeUrl,
+  encodeHeads,
+  decodeHeads,
 } from "./AutomergeUrl.js"
 export { Repo } from "./Repo.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
@@ -75,6 +77,11 @@ export type {
   PeerMetadata,
 } from "./network/NetworkAdapterInterface.js"
 
+export type {
+  NetworkSubsystemEvents,
+  PeerPayload,
+} from "./network/NetworkSubsystem.js"
+
 export type {
   DocumentUnavailableMessage,
   EphemeralMessage,

package/src/storage/StorageSubsystem.ts

@@ -6,14 +6,24 @@ import { type DocumentId } from "../types.js"
 import { StorageAdapterInterface } from "./StorageAdapterInterface.js"
 import { ChunkInfo, StorageKey, StorageId } from "./types.js"
 import { keyHash, headsHash } from "./keyHash.js"
-import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
 import * as Uuid from "uuid"
+import { EventEmitter } from "eventemitter3"
+import { encodeHeads } from "../AutomergeUrl.js"
+
+type StorageSubsystemEvents = {
+  "document-loaded": (arg: {
+    documentId: DocumentId
+    durationMillis: number
+    numOps: number
+    numChanges: number
+  }) => void
+}
 
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
  * storage adapter. It also provides a generic key/value storage interface for other uses.
  */
-export class StorageSubsystem {
+export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
   /** The storage adapter to use for saving and loading documents */
   #storageAdapter: StorageAdapterInterface
 
@@ -29,6 +39,7 @@ export class StorageSubsystem {
   #log = debug(`automerge-repo:storage-subsystem`)
 
   constructor(storageAdapter: StorageAdapterInterface) {
+    super()
     this.#storageAdapter = storageAdapter
   }
 
@@ -101,36 +112,73 @@ export class StorageSubsystem {
   // AUTOMERGE DOCUMENT STORAGE
 
   /**
-   * Loads the Automerge document with the given ID from storage.
+   * Loads and combines document chunks from storage, with snapshots first.
    */
-  async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
-    // Load all the chunks for this document
-    const chunks = await this.#storageAdapter.loadRange([documentId])
-    const binaries = []
+  async loadDocData(documentId: DocumentId): Promise<Uint8Array | null> {
+    // Load snapshots first
+    const snapshotChunks = await this.#storageAdapter.loadRange([
+      documentId,
+      "snapshot",
+    ])
+    const incrementalChunks = await this.#storageAdapter.loadRange([
+      documentId,
+      "incremental",
+    ])
+
+    const binaries: Uint8Array[] = []
     const chunkInfos: ChunkInfo[] = []
 
-    for (const chunk of chunks) {
-      // chunks might have been deleted in the interim
+    // Process snapshots first
+    for (const chunk of snapshotChunks) {
       if (chunk.data === undefined) continue
+      chunkInfos.push({
+        key: chunk.key,
+        type: "snapshot",
+        size: chunk.data.length,
+      })
+      binaries.push(chunk.data)
+    }
 
-      const chunkType = chunkTypeFromKey(chunk.key)
-      if (chunkType == null) continue
-
+    // Then process incrementals
+    for (const chunk of incrementalChunks) {
+      if (chunk.data === undefined) continue
       chunkInfos.push({
         key: chunk.key,
-        type: chunkType,
+        type: "incremental",
         size: chunk.data.length,
       })
       binaries.push(chunk.data)
     }
+
+    // Store chunk infos for future reference
     this.#chunkInfos.set(documentId, chunkInfos)
 
+    // If no chunks were found, return null
+    if (binaries.length === 0) {
+      return null
+    }
+
     // Merge the chunks into a single binary
-    const binary = mergeArrays(binaries)
-    if (binary.length === 0) return null
+    return mergeArrays(binaries)
+  }
+
+  /**
+   * Loads the Automerge document with the given ID from storage.
+   */
+  async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
+    // Load and combine chunks
+    const binary = await this.loadDocData(documentId)
+    if (!binary) return null
 
     // Load into an Automerge document
+    const start = performance.now()
     const newDoc = A.loadIncremental(A.init(), binary) as A.Doc<T>
+    const end = performance.now()
+    this.emit("document-loaded", {
+      documentId,
+      durationMillis: end - start,
+      ...A.stats(newDoc),
+    })
 
     // Record the latest heads for the document
     this.#storedHeads.set(documentId, A.getHeads(newDoc))
@@ -150,11 +198,13 @@ export class StorageSubsystem {
     if (!this.#shouldSave(documentId, doc)) return
 
     const sourceChunks = this.#chunkInfos.get(documentId) ?? []
+
     if (this.#shouldCompact(sourceChunks)) {
       await this.#saveTotal(documentId, doc, sourceChunks)
     } else {
       await this.#saveIncremental(documentId, doc)
     }
+
     this.#storedHeads.set(documentId, A.getHeads(doc))
   }
 
@@ -261,7 +311,7 @@ export class StorageSubsystem {
     }
 
     const newHeads = A.getHeads(doc)
-    if (headsAreSame(newHeads, oldHeads)) {
+    if (headsAreSame(encodeHeads(newHeads), encodeHeads(oldHeads))) {
       // the document hasn't changed
       return false
     }

package/src/synchronizer/CollectionSynchronizer.ts

@@ -1,9 +1,9 @@
 import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
-import { stringifyAutomergeUrl } from "../AutomergeUrl.js"
+import { parseAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
 import { DocMessage } from "../network/messages.js"
-import { DocumentId, PeerId } from "../types.js"
+import { AutomergeUrl, DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
 
@@ -21,23 +21,27 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Used to determine if the document is know to the Collection and a synchronizer exists or is being set up */
   #docSetUp: Record<DocumentId, boolean> = {}
 
-  constructor(private repo: Repo) {
+  #denylist: DocumentId[]
+
+  constructor(private repo: Repo, denylist: AutomergeUrl[] = []) {
     super()
+    this.#denylist = denylist.map(url => parseAutomergeUrl(url).documentId)
   }
 
   /** Returns a synchronizer for the given document, creating one if it doesn't already exist.  */
-  #fetchDocSynchronizer(documentId: DocumentId) {
-    if (!this.docSynchronizers[documentId]) {
-      const handle = this.repo.find(stringifyAutomergeUrl({ documentId }))
-      this.docSynchronizers[documentId] = this.#initDocSynchronizer(handle)
+  #fetchDocSynchronizer(handle: DocHandle<unknown>) {
+    if (!this.docSynchronizers[handle.documentId]) {
+      this.docSynchronizers[handle.documentId] =
+        this.#initDocSynchronizer(handle)
     }
-    return this.docSynchronizers[documentId]
+    return this.docSynchronizers[handle.documentId]
   }
 
   /** Creates a new docSynchronizer and sets it up to propagate messages */
   #initDocSynchronizer(handle: DocHandle<unknown>): DocSynchronizer {
     const docSynchronizer = new DocSynchronizer({
       handle,
+      peerId: this.repo.networkSubsystem.peerId,
       onLoadSyncState: async peerId => {
         if (!this.repo.storageSubsystem) {
           return
@@ -58,6 +62,7 @@ export class CollectionSynchronizer extends Synchronizer {
     docSynchronizer.on("message", event => this.emit("message", event))
     docSynchronizer.on("open-doc", event => this.emit("open-doc", event))
     docSynchronizer.on("sync-state", event => this.emit("sync-state", event))
+    docSynchronizer.on("metrics", event => this.emit("metrics", event))
     return docSynchronizer
   }
 
@@ -90,15 +95,31 @@ export class CollectionSynchronizer extends Synchronizer {
       throw new Error("received a message with an invalid documentId")
     }
 
+    if (this.#denylist.includes(documentId)) {
+      this.emit("metrics", {
+        type: "doc-denied",
+        documentId,
+      })
+      this.emit("message", {
+        type: "doc-unavailable",
+        documentId,
+        targetId: message.senderId,
+      })
+      return
+    }
+
     this.#docSetUp[documentId] = true
 
-    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
+    const handle = await this.repo.find(documentId, {
+      allowableStates: ["ready", "unavailable", "requesting"],
+    })
+    const docSynchronizer = this.#fetchDocSynchronizer(handle)
 
     docSynchronizer.receiveMessage(message)
 
     // Initiate sync with any new peers
     const peers = await this.#documentGenerousPeers(documentId)
-    docSynchronizer.beginSync(
+    void docSynchronizer.beginSync(
       peers.filter(peerId => !docSynchronizer.hasPeer(peerId))
     )
   }
@@ -106,14 +127,14 @@ export class CollectionSynchronizer extends Synchronizer {
   /**
    * Starts synchronizing the given document with all peers that we share it generously with.
    */
-  addDocument(documentId: DocumentId) {
+  addDocument(handle: DocHandle<unknown>) {
     // HACK: this is a hack to prevent us from adding the same document twice
-    if (this.#docSetUp[documentId]) {
+    if (this.#docSetUp[handle.documentId]) {
       return
     }
-    const docSynchronizer = this.#fetchDocSynchronizer(documentId)
-    void this.#documentGenerousPeers(documentId).then(peers => {
-      docSynchronizer.beginSync(peers)
+    const docSynchronizer = this.#fetchDocSynchronizer(handle)
+    void this.#documentGenerousPeers(handle.documentId).then(peers => {
+      void docSynchronizer.beginSync(peers)
     })
   }
 
@@ -135,7 +156,7 @@ export class CollectionSynchronizer extends Synchronizer {
     for (const docSynchronizer of Object.values(this.docSynchronizers)) {
       const { documentId } = docSynchronizer
       void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
-        if (okToShare) docSynchronizer.beginSync([peerId])
+        if (okToShare) void docSynchronizer.beginSync([peerId])
       })
     }
   }