@automerge/automerge-repo 2.0.0-alpha.1 → 2.0.0-alpha.12

package/src/helpers/tests/storage-adapter-tests.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest"
+import { describe, expect, beforeEach, it as _it } from "vitest"
 
 import type { StorageAdapterInterface } from "../../storage/StorageAdapterInterface.js"
 
@@ -8,72 +8,61 @@ const PAYLOAD_C = () => new Uint8Array([2, 111, 74, 131, 236, 96, 142, 193])
 
 const LARGE_PAYLOAD = new Uint8Array(100000).map(() => Math.random() * 256)
 
-export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
-  const setup = async () => {
-    const { adapter, teardown = NO_OP } = await _setup()
-    return { adapter, teardown }
-  }
+type AdapterTestContext = {
+  adapter: StorageAdapterInterface
+}
+
+const it = _it<AdapterTestContext>
+
+export function runStorageAdapterTests(setup: SetupFn, title?: string): void {
+  beforeEach<AdapterTestContext>(async ctx => {
+    const { adapter, teardown = NO_OP } = await setup()
+    ctx.adapter = adapter
+    return teardown
+  })
 
   describe(`Storage adapter acceptance tests ${
     title ? `(${title})` : ""
   }`, () => {
     describe("load", () => {
-      it("should return undefined if there is no data", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return undefined if there is no data", async ({ adapter }) => {
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toBeUndefined()
-
-        teardown()
       })
     })
 
     describe("save and load", () => {
-      it("should return data that was saved", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return data that was saved", async ({ adapter }) => {
         await adapter.save(["storage-adapter-id"], PAYLOAD_A())
         const actual = await adapter.load(["storage-adapter-id"])
         expect(actual).toStrictEqual(PAYLOAD_A())
-
-        teardown()
       })
 
-      it("should work with composite keys", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should work with composite keys", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toStrictEqual(PAYLOAD_A())
-
-        teardown()
       })
 
-      it("should work with a large payload", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should work with a large payload", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], LARGE_PAYLOAD)
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toStrictEqual(LARGE_PAYLOAD)
-
-        teardown()
       })
     })
 
     describe("loadRange", () => {
-      it("should return an empty array if there is no data", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return an empty array if there is no data", async ({
+        adapter,
+      }) => {
         expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([])
-
-        teardown()
       })
     })
 
     describe("save and loadRange", () => {
-      it("should return all the data that matches the key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return all the data that matches the key", async ({
+        adapter,
+      }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "snapshot", "yyyyy"], PAYLOAD_B())
         await adapter.save(["AAAAA", "sync-state", "zzzzz"], PAYLOAD_C())
@@ -92,13 +81,9 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
             { key: ["AAAAA", "sync-state", "zzzzz"], data: PAYLOAD_C() },
           ])
         )
-
-        teardown()
       })
 
-      it("should only load values that match they key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should only load values that match they key", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["BBBBB", "sync-state", "zzzzz"], PAYLOAD_C())
 
@@ -113,15 +98,11 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
             { key: ["BBBBB", "sync-state", "zzzzz"], data: PAYLOAD_C() },
           ])
         )
-
-        teardown()
       })
     })
 
     describe("save and remove", () => {
-      it("after removing, should be empty", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("after removing, should be empty", async ({ adapter }) => {
         await adapter.save(["AAAAA", "snapshot", "xxxxx"], PAYLOAD_A())
         await adapter.remove(["AAAAA", "snapshot", "xxxxx"])
 
@@ -129,30 +110,24 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(
           await adapter.load(["AAAAA", "snapshot", "xxxxx"])
         ).toBeUndefined()
-
-        teardown()
       })
     })
 
     describe("save and save", () => {
-      it("should overwrite data saved with the same key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should overwrite data saved with the same key", async ({
+        adapter,
+      }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_B())
 
         expect(await adapter.loadRange(["AAAAA", "sync-state"])).toStrictEqual([
           { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
     })
 
     describe("removeRange", () => {
-      it("should remove a range of records", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should remove a range of records", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "snapshot", "yyyyy"], PAYLOAD_B())
         await adapter.save(["AAAAA", "sync-state", "zzzzz"], PAYLOAD_C())
@@ -162,13 +137,9 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([
           { key: ["AAAAA", "snapshot", "yyyyy"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
 
-      it("should not remove records that don't match", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should not remove records that don't match", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["BBBBB", "sync-state", "zzzzz"], PAYLOAD_B())
 
@@ -178,8 +149,6 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(actual).toStrictEqual([
           { key: ["BBBBB", "sync-state", "zzzzz"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
     })
   })
@@ -189,5 +158,5 @@ const NO_OP = () => {}
 
 export type SetupFn = () => Promise<{
   adapter: StorageAdapterInterface
-  teardown?: () => void
+  teardown?: () => void | Promise<void>
 }>

package/src/storage/StorageSubsystem.ts

@@ -8,12 +8,22 @@ import { ChunkInfo, StorageKey, StorageId } from "./types.js"
 import { keyHash, headsHash } from "./keyHash.js"
 import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
 import * as Uuid from "uuid"
+import { EventEmitter } from "eventemitter3"
+
+type StorageSubsystemEvents = {
+  "document-loaded": (arg: {
+    documentId: DocumentId
+    durationMillis: number
+    numOps: number
+    numChanges: number
+  }) => void
+}
 
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
  * storage adapter. It also provides a generic key/value storage interface for other uses.
  */
-export class StorageSubsystem {
+export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
   /** The storage adapter to use for saving and loading documents */
   #storageAdapter: StorageAdapterInterface
 
@@ -29,6 +39,7 @@ export class StorageSubsystem {
   #log = debug(`automerge-repo:storage-subsystem`)
 
   constructor(storageAdapter: StorageAdapterInterface) {
+    super()
     this.#storageAdapter = storageAdapter
   }
 
@@ -130,7 +141,14 @@ export class StorageSubsystem {
     if (binary.length === 0) return null
 
     // Load into an Automerge document
+    const start = performance.now()
     const newDoc = A.loadIncremental(A.init(), binary) as A.Doc<T>
+    const end = performance.now()
+    this.emit("document-loaded", {
+      documentId,
+      durationMillis: end - start,
+      ...A.stats(newDoc),
+    })
 
     // Record the latest heads for the document
     this.#storedHeads.set(documentId, A.getHeads(newDoc))
@@ -232,8 +250,13 @@ export class StorageSubsystem {
     storageId: StorageId
   ): Promise<A.SyncState | undefined> {
     const key = [documentId, "sync-state", storageId]
-    const loaded = await this.#storageAdapter.load(key)
-    return loaded ? A.decodeSyncState(loaded) : undefined
+    try {
+      const loaded = await this.#storageAdapter.load(key)
+      return loaded ? A.decodeSyncState(loaded) : undefined
+    } catch (e) {
+      this.#log(`Error loading sync state for ${documentId} from ${storageId}`)
+      return undefined
+    }
   }
 
   async saveSyncState(

package/src/synchronizer/CollectionSynchronizer.ts

@@ -15,7 +15,8 @@ export class CollectionSynchronizer extends Synchronizer {
   #peers: Set<PeerId> = new Set()
 
   /** A map of documentIds to their synchronizers */
-  #docSynchronizers: Record<DocumentId, DocSynchronizer> = {}
+  /** @hidden */
+  docSynchronizers: Record<DocumentId, DocSynchronizer> = {}
 
   /** Used to determine if the document is know to the Collection and a synchronizer exists or is being set up */
   #docSetUp: Record<DocumentId, boolean> = {}
@@ -26,11 +27,11 @@ export class CollectionSynchronizer extends Synchronizer {
 
   /** Returns a synchronizer for the given document, creating one if it doesn't already exist.  */
   #fetchDocSynchronizer(documentId: DocumentId) {
-    if (!this.#docSynchronizers[documentId]) {
+    if (!this.docSynchronizers[documentId]) {
       const handle = this.repo.find(stringifyAutomergeUrl({ documentId }))
-      this.#docSynchronizers[documentId] = this.#initDocSynchronizer(handle)
+      this.docSynchronizers[documentId] = this.#initDocSynchronizer(handle)
     }
-    return this.#docSynchronizers[documentId]
+    return this.docSynchronizers[documentId]
   }
 
   /** Creates a new docSynchronizer and sets it up to propagate messages */
@@ -57,6 +58,7 @@ export class CollectionSynchronizer extends Synchronizer {
     docSynchronizer.on("message", event => this.emit("message", event))
     docSynchronizer.on("open-doc", event => this.emit("open-doc", event))
     docSynchronizer.on("sync-state", event => this.emit("sync-state", event))
+    docSynchronizer.on("metrics", event => this.emit("metrics", event))
     return docSynchronizer
   }
 
@@ -131,7 +133,7 @@ export class CollectionSynchronizer extends Synchronizer {
     }
 
     this.#peers.add(peerId)
-    for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
+    for (const docSynchronizer of Object.values(this.docSynchronizers)) {
       const { documentId } = docSynchronizer
       void this.repo.sharePolicy(peerId, documentId).then(okToShare => {
         if (okToShare) docSynchronizer.beginSync([peerId])
@@ -144,7 +146,7 @@ export class CollectionSynchronizer extends Synchronizer {
     log(`removing peer ${peerId}`)
     this.#peers.delete(peerId)
 
-    for (const docSynchronizer of Object.values(this.#docSynchronizers)) {
+    for (const docSynchronizer of Object.values(this.docSynchronizers)) {
       docSynchronizer.endSync(peerId)
     }
   }
@@ -153,4 +155,19 @@ export class CollectionSynchronizer extends Synchronizer {
   get peers(): PeerId[] {
     return Array.from(this.#peers)
   }
+
+  metrics(): {
+    [key: string]: {
+      peers: PeerId[]
+      size: { numOps: number; numChanges: number }
+    }
+  } {
+    return Object.fromEntries(
+      Object.entries(this.docSynchronizers).map(
+        ([documentId, synchronizer]) => {
+          return [documentId, synchronizer.metrics()]
+        }
+      )
+    )
+  }
 }

package/src/synchronizer/DocSynchronizer.ts

@@ -351,11 +351,19 @@ export class DocSynchronizer extends Synchronizer {
 
     this.#withSyncState(message.senderId, syncState => {
       this.#handle.update(doc => {
+        const start = performance.now()
         const [newDoc, newSyncState] = A.receiveSyncMessage(
           doc,
           syncState,
           message.data
         )
+        const end = performance.now()
+        this.emit("metrics", {
+          type: "receive-sync-message",
+          documentId: this.#handle.documentId,
+          durationMillis: end - start,
+          ...A.stats(doc),
+        })
 
         this.#setSyncState(message.senderId, newSyncState)
 
@@ -401,4 +409,11 @@ export class DocSynchronizer extends Synchronizer {
 
     this.#pendingSyncMessages = []
   }
+
+  metrics(): { peers: PeerId[]; size: { numOps: number; numChanges: number } } {
+    return {
+      peers: this.#peers,
+      size: this.#handle.metrics(),
+    }
+  }
 }

package/src/synchronizer/Synchronizer.ts

@@ -15,6 +15,7 @@ export interface SynchronizerEvents {
   message: (payload: MessageContents) => void
   "sync-state": (payload: SyncStatePayload) => void
   "open-doc": (arg: OpenDocMessage) => void
+  metrics: (arg: DocSyncMetrics) => void
 }
 
 /** Notify the repo that the sync state has changed  */
@@ -23,3 +24,11 @@ export interface SyncStatePayload {
   documentId: DocumentId
   syncState: SyncState
 }
+
+export type DocSyncMetrics = {
+  type: "receive-sync-message"
+  documentId: DocumentId
+  durationMillis: number
+  numOps: number
+  numChanges: number
+}

package/test/DocHandle.test.ts

@@ -7,6 +7,7 @@ import { eventPromise } from "../src/helpers/eventPromise.js"
 import { pause } from "../src/helpers/pause.js"
 import { DocHandle, DocHandleChangePayload } from "../src/index.js"
 import { TestDoc } from "./types.js"
+import { UNLOADED } from "../src/DocHandle.js"
 
 describe("DocHandle", () => {
   const TEST_ID = parseAutomergeUrl(generateAutomergeUrl()).documentId
@@ -68,6 +69,15 @@ describe("DocHandle", () => {
     assert.equal(doc?.foo, "bar")
   })
 
+  /** HISTORY TRAVERSAL
+   * This API is relatively alpha-ish but we're already
+   * doing things in our own apps that are fairly ambitious
+   * by routing around to a lower-level API.
+   * This is an attempt to wrap up the existing practice
+   * in a slightly more supportable set of APIs but should be
+   * considered provisional: expect further improvements.
+   */
+
   it("should return the heads when requested", async () => {
     const handle = setup()
     handle.change(d => (d.foo = "bar"))
@@ -84,6 +94,94 @@ describe("DocHandle", () => {
     assert.deepEqual(handle.heads(), undefined)
   })
 
+  it("should return the history when requested", async () => {
+    const handle = setup()
+    handle.change(d => (d.foo = "bar"))
+    handle.change(d => (d.foo = "baz"))
+    assert.equal(handle.isReady(), true)
+
+    const history = handle.history()
+    assert.deepEqual(handle.history().length, 2)
+  })
+
+  it("should return a commit from the history", async () => {
+    const handle = setup()
+    handle.change(d => (d.foo = "zero"))
+    handle.change(d => (d.foo = "one"))
+    handle.change(d => (d.foo = "two"))
+    handle.change(d => (d.foo = "three"))
+    assert.equal(handle.isReady(), true)
+
+    const history = handle.history()
+    const view = handle.view(history[1])
+    assert.deepEqual(view, { foo: "one" })
+  })
+
+  it("should return diffs", async () => {
+    const handle = setup()
+    handle.change(d => (d.foo = "zero"))
+    handle.change(d => (d.foo = "one"))
+    handle.change(d => (d.foo = "two"))
+    handle.change(d => (d.foo = "three"))
+    assert.equal(handle.isReady(), true)
+
+    const history = handle.history()
+    const patches = handle.diff(history[1])
+    assert.deepEqual(patches, [
+      { action: "put", path: ["foo"], value: "" },
+      { action: "splice", path: ["foo", 0], value: "one" },
+    ])
+  })
+
+  it("should support arbitrary diffs too", async () => {
+    const handle = setup()
+    handle.change(d => (d.foo = "zero"))
+    handle.change(d => (d.foo = "one"))
+    handle.change(d => (d.foo = "two"))
+    handle.change(d => (d.foo = "three"))
+    assert.equal(handle.isReady(), true)
+
+    const history = handle.history()
+    const patches = handle.diff(history[1], history[3])
+    assert.deepEqual(patches, [
+      { action: "put", path: ["foo"], value: "" },
+      { action: "splice", path: ["foo", 0], value: "three" },
+    ])
+    const backPatches = handle.diff(history[3], history[1])
+    assert.deepEqual(backPatches, [
+      { action: "put", path: ["foo"], value: "" },
+      { action: "splice", path: ["foo", 0], value: "one" },
+    ])
+  })
+
+  it("should allow direct access to decoded changes", async () => {
+    const handle = setup()
+    const time = Date.now()
+    handle.change(d => (d.foo = "foo"), { message: "commitMessage" })
+    assert.equal(handle.isReady(), true)
+
+    const metadata = handle.metadata()
+    assert.deepEqual(metadata.message, "commitMessage")
+    // NOTE: I'm not testing time because of https://github.com/automerge/automerge/issues/965
+    // but it does round-trip successfully!
+  })
+
+  it("should allow direct access to a specific decoded change", async () => {
+    const handle = setup()
+    const time = Date.now()
+    handle.change(d => (d.foo = "foo"), { message: "commitMessage" })
+    handle.change(d => (d.foo = "foo"), { message: "commitMessage2" })
+    handle.change(d => (d.foo = "foo"), { message: "commitMessage3" })
+    handle.change(d => (d.foo = "foo"), { message: "commitMessage4" })
+    assert.equal(handle.isReady(), true)
+
+    const history = handle.history()
+    const metadata = handle.metadata(history[0][0])
+    assert.deepEqual(metadata.message, "commitMessage")
+    // NOTE: I'm not testing time because of https://github.com/automerge/automerge/issues/965
+    // but it does round-trip successfully!
+  })
+
   /**
    * Once there's a Repo#stop API this case should be covered in accompanying
    * tests and the following test removed.
@@ -325,6 +423,49 @@ describe("DocHandle", () => {
     assert.equal(handle.isDeleted(), true)
   })
 
+  it("should clear document reference when unloaded", async () => {
+    const handle = setup()
+
+    handle.change(doc => {
+      doc.foo = "bar"
+    })
+    const doc = await handle.doc()
+    assert.equal(doc?.foo, "bar")
+
+    handle.unload()
+    assert.equal(handle.isUnloaded(), true)
+
+    const clearedDoc = await handle.doc([UNLOADED])
+    assert.notEqual(clearedDoc?.foo, "bar")
+  })
+
+  it("should allow reloading after unloading", async () => {
+    const handle = setup()
+
+    handle.change(doc => {
+      doc.foo = "bar"
+    })
+    const doc = await handle.doc()
+    assert.equal(doc?.foo, "bar")
+
+    handle.unload()
+
+    // reload to transition from unloaded to loading
+    handle.reload()
+
+    // simulate requesting from the network
+    handle.request()
+
+    // simulate updating from the network
+    handle.update(doc => {
+      return A.change(doc, d => (d.foo = "bar"))
+    })
+
+    const reloadedDoc = await handle.doc()
+    assert.equal(handle.isReady(), true)
+    assert.equal(reloadedDoc?.foo, "bar")
+  })
+
   it("should allow changing at old heads", async () => {
     const handle = setup()
 

package/test/Repo.test.ts

@@ -486,6 +486,39 @@ describe("Repo", () => {
       const doc = await handle.doc()
       expect(doc).toEqual({})
     })
+
+    describe("handle cache", () => {
+      it("contains doc handle", async () => {
+        const { repo } = setup()
+        const handle = repo.create({ foo: "bar" })
+        await handle.doc()
+        assert(repo.handles[handle.documentId])
+      })
+
+      it("delete removes doc handle", async () => {
+        const { repo } = setup()
+        const handle = repo.create({ foo: "bar" })
+        await handle.doc()
+        await repo.delete(handle.documentId)
+        assert(repo.handles[handle.documentId] === undefined)
+      })
+
+      it("removeFromCache removes doc handle", async () => {
+        const { repo } = setup()
+        const handle = repo.create({ foo: "bar" })
+        await handle.doc()
+        await repo.removeFromCache(handle.documentId)
+        assert(repo.handles[handle.documentId] === undefined)
+      })
+
+      it("removeFromCache for documentId not found", async () => {
+        const { repo } = setup()
+        const badDocumentId = "badbadbad" as DocumentId
+        const handleCacheSize = Object.keys(repo.handles).length
+        await repo.removeFromCache(badDocumentId)
+        assert(Object.keys(repo.handles).length === handleCacheSize)
+      })
+    })
   })
 
   describe("flush behaviour", () => {
@@ -832,6 +865,46 @@ describe("Repo", () => {
       teardown()
     })
 
+    it("synchronizes changes from bobRepo to charlieRepo when loading from storage", async () => {
+      const { bobRepo, bobStorage, teardown } = await setup()
+
+      // We create a repo that uses bobStorage to put a document into its imaginary disk
+      // without it knowing about it
+      const bobRepo2 = new Repo({
+        storage: bobStorage,
+      })
+      const inStorageHandle = bobRepo2.create<TestDoc>({
+        foo: "foundOnFakeDisk",
+      })
+      await bobRepo2.flush()
+
+      // Now, let's load it on the original bob repo (which shares a "disk")
+      const bobFoundIt = bobRepo.find<TestDoc>(inStorageHandle.url)
+      await bobFoundIt.whenReady()
+
+      // Before checking if it syncs, make sure we have it!
+      // (This behaviour is mostly test-validation, we are already testing load/save elsewhere.)
+      assert.deepStrictEqual(await bobFoundIt.doc(), { foo: "foundOnFakeDisk" })
+
+      await pause(10)
+
+      // We should have a docSynchronizer and its peers should be alice and charlie
+      assert.strictEqual(
+        bobRepo.synchronizer.docSynchronizers[bobFoundIt.documentId]?.hasPeer(
+          "alice" as PeerId
+        ),
+        true
+      )
+      assert.strictEqual(
+        bobRepo.synchronizer.docSynchronizers[bobFoundIt.documentId]?.hasPeer(
+          "charlie" as PeerId
+        ),
+        true
+      )
+
+      teardown()
+    })
+
     it("charlieRepo doesn't have a document it's not supposed to have", async () => {
       const { aliceRepo, bobRepo, charlieRepo, notForCharlie, teardown } =
         await setup()

package/test/StorageSubsystem.test.ts

@@ -211,6 +211,23 @@ describe("StorageSubsystem", () => {
           )
           assert.strictEqual(loadedSyncState, undefined)
         })
+
+        it("returns a undefined if loading an existing sync state fails", async () => {
+          const storage = new StorageSubsystem(adapter)
+
+          const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
+          const bobStorageId = Uuid.v4() as StorageId
+
+          const syncStateKey = [documentId, "sync-state", bobStorageId]
+          // Save garbage data to simulate a corrupted sync state
+          await adapter.save(syncStateKey, Buffer.from("invalid data"))
+
+          const loadedSyncState = await storage.loadSyncState(
+            documentId,
+            bobStorageId
+          )
+          assert.strictEqual(loadedSyncState, undefined)
+        })
       })
 
       describe("storage id", () => {