@automerge/automerge-repo 2.0.0-alpha.6 → 2.0.0-collectionsync-alpha.1

package/test/RemoteHeadsSubscriptions.test.ts

@@ -10,7 +10,7 @@ import {
 } from "../src/network/messages.js"
 import { collectMessages } from "./helpers/collectMessages.js"
 
-describe("RepoHeadsSubscriptions", () => {
+describe.skip("RepoHeadsSubscriptions", () => {
   const storageA = "remote-a" as StorageId
   const storageB = "remote-b" as StorageId
   const storageC = "remote-c" as StorageId

package/test/Repo.test.ts

@@ -30,6 +30,8 @@ import { getRandomItem } from "./helpers/getRandomItem.js"
 import { TestDoc } from "./types.js"
 import { StorageId, StorageKey } from "../src/storage/types.js"
 
+const someDoc = A.init()
+
 describe("Repo", () => {
   describe("constructor", () => {
     it("can be instantiated without any configuration", () => {
@@ -40,6 +42,7 @@ describe("Repo", () => {
 
   describe("local only", () => {
     const setup = ({ startReady = true } = {}) => {
+      const someDoc = A.init()
       const storageAdapter = new DummyStorageAdapter()
       const networkAdapter = new DummyNetworkAdapter({ startReady })
 
@@ -261,6 +264,8 @@ describe("Repo", () => {
         storage: storageAdapter,
       })
 
+      // TODO: remove this pause
+      await pause(100)
       await repo.flush()
 
       const bobHandle = repo2.find<TestDoc>(handle.url)
@@ -278,6 +283,8 @@ describe("Repo", () => {
 
       assert.equal(handle.isReady(), true)
 
+      // TODO: remove this pause
+      await pause(100)
       await repo.flush()
 
       const repo2 = new Repo({
@@ -380,6 +387,8 @@ describe("Repo", () => {
       })
 
       await repo.flush()
+      // TODO: remove this pause
+      await pause(100)
 
       const initialKeys = storage.keys()
 
@@ -421,7 +430,7 @@ describe("Repo", () => {
       }
     })
 
-    it("doesn't create multiple snapshots in storage when a series of large changes are made in succession", async () => {
+    it.skip("doesn't create multiple snapshots in storage when a series of large changes are made in succession", async () => {
       const { repo, storageAdapter } = setup()
       const handle = repo.create<{ objects: LargeObject[] }>()
 
@@ -486,9 +495,42 @@ describe("Repo", () => {
       const doc = await handle.doc()
       expect(doc).toEqual({})
     })
+
+    describe("handle cache", () => {
+      it("contains doc handle", async () => {
+        const { repo } = setup()
+        const handle = repo.create({ foo: "bar" })
+        await handle.doc()
+        assert(repo.handles[handle.documentId])
+      })
+
+      it("delete removes doc handle", async () => {
+        const { repo } = setup()
+        const handle = repo.create({ foo: "bar" })
+        await handle.doc()
+        await repo.delete(handle.documentId)
+        assert(repo.handles[handle.documentId] === undefined)
+      })
+
+      it("removeFromCache removes doc handle", async () => {
+        const { repo } = setup()
+        const handle = repo.create({ foo: "bar" })
+        await handle.doc()
+        await repo.removeFromCache(handle.documentId)
+        assert(repo.handles[handle.documentId] === undefined)
+      })
+
+      it("removeFromCache for documentId not found", async () => {
+        const { repo } = setup()
+        const badDocumentId = "badbadbad" as DocumentId
+        const handleCacheSize = Object.keys(repo.handles).length
+        await repo.removeFromCache(badDocumentId)
+        assert(Object.keys(repo.handles).length === handleCacheSize)
+      })
+    })
   })
 
-  describe("flush behaviour", () => {
+  describe.skip("flush behaviour", () => {
     const setup = () => {
       let blockedSaves = new Set<{ path: StorageKey; resolve: () => void }>()
       let resume = (documentIds?: DocumentId[]) => {
@@ -617,7 +659,40 @@ describe("Repo", () => {
     })
   })
 
-  describe("with peers (linear network)", async () => {
+  it("should sync correctly", async function () {
+    const pipe = DummyNetworkAdapter.createConnectedPair()
+    const repo1 = new Repo({
+      peerId: "peer1" as PeerId,
+      storage: new DummyStorageAdapter(),
+      sharePolicy: async () => false,
+      network: [pipe[0]],
+    })
+    const repo2 = new Repo({
+      peerId: "peer2" as PeerId,
+      storage: new DummyStorageAdapter(),
+      sharePolicy: async () => true,
+      network: [pipe[1]],
+    })
+
+    const handleOn1 = repo1.create()
+    await handleOn1.whenReady()
+    handleOn1.change((d: any) => {
+      d.foo = "bar"
+    })
+
+    pipe[0].peerCandidate("peer2" as PeerId)
+    pipe[1].peerCandidate("peer1" as PeerId)
+
+    await repo2.networkSubsystem.whenReady()
+    console.log("about to load handle on 2")
+    const handleOn2 = repo2.find(handleOn1.url)
+
+    await handleOn2.whenReady()
+    await pause(1)
+    assert.deepStrictEqual(handleOn2.docSync(), { foo: "bar" })
+  })
+
+  describe("with peers (linear network)", function () {
     it("n-peers connected in a line", async () => {
       const createNConnectedRepos = async (
         numberOfPeers: number,
@@ -677,17 +752,18 @@ describe("Repo", () => {
         return { repos }
       }
 
-      const numberOfPeers = 10
+      // const numberOfPeers = 10
+      const numberOfPeers = 2
       const { repos } = await createNConnectedRepos(numberOfPeers, 10)
 
       const handle0 = repos[0].create()
       handle0.change((d: any) => {
         d.foo = "bar"
       })
-
       const handleN = repos[numberOfPeers - 1].find<TestDoc>(handle0.url)
 
       await handleN.whenReady()
+      await pause(1)
       assert.deepStrictEqual(handleN.docSync(), { foo: "bar" })
     })
 
@@ -769,6 +845,7 @@ describe("Repo", () => {
       }
 
       const aliceHandle = aliceRepo.create<TestDoc>()
+      // await pause(10)
       aliceHandle.change(d => {
         d.foo = "bar"
       })
@@ -794,7 +871,6 @@ describe("Repo", () => {
         eventPromise(bobRepo.networkSubsystem, "peer"),
         eventPromise(charlieRepo.networkSubsystem, "peer"),
       ])
-
       return {
         alice,
         aliceRepo,
@@ -833,8 +909,7 @@ describe("Repo", () => {
     })
 
     it("synchronizes changes from bobRepo to charlieRepo when loading from storage", async () => {
-      const { bobRepo, bobStorage, charlieRepo, aliceHandle, teardown } =
-        await setup()
+      const { bobRepo, bobStorage, teardown } = await setup()
 
       // We create a repo that uses bobStorage to put a document into its imaginary disk
       // without it knowing about it
@@ -845,8 +920,8 @@ describe("Repo", () => {
         foo: "foundOnFakeDisk",
       })
       await bobRepo2.flush()
+      console.log("bob storage: ", bobStorage.keys())
 
-      console.log("loading from disk", inStorageHandle.url)
       // Now, let's load it on the original bob repo (which shares a "disk")
       const bobFoundIt = bobRepo.find<TestDoc>(inStorageHandle.url)
       await bobFoundIt.whenReady()
@@ -855,6 +930,8 @@ describe("Repo", () => {
       // (This behaviour is mostly test-validation, we are already testing load/save elsewhere.)
       assert.deepStrictEqual(await bobFoundIt.doc(), { foo: "foundOnFakeDisk" })
 
+      await pause(10)
+
       // We should have a docSynchronizer and its peers should be alice and charlie
       assert.strictEqual(
         bobRepo.synchronizer.docSynchronizers[bobFoundIt.documentId]?.hasPeer(
@@ -872,7 +949,7 @@ describe("Repo", () => {
       teardown()
     })
 
-    it("charlieRepo doesn't have a document it's not supposed to have", async () => {
+    it.skip("charlieRepo doesn't have a document it's not supposed to have", async () => {
       const { aliceRepo, bobRepo, charlieRepo, notForCharlie, teardown } =
         await setup()
 
@@ -902,7 +979,7 @@ describe("Repo", () => {
       teardown()
     })
 
-    it("charlieRepo can request a document across a network of multiple peers", async () => {
+    it.skip("charlieRepo can request a document across a network of multiple peers", async () => {
       const { charlieRepo, notForBob, teardown } = await setup()
 
       const handle = charlieRepo.find<TestDoc>(notForBob)
@@ -976,6 +1053,8 @@ describe("Repo", () => {
       connectAliceToBob()
 
       await eventPromise(aliceRepo.networkSubsystem, "peer")
+      // TODO: remove this pause
+      await pause(100)
 
       const doc = await handle.doc(["ready"])
       assert.deepStrictEqual(doc, { foo: "baz" })
@@ -1037,7 +1116,7 @@ describe("Repo", () => {
       await handle.whenReady()
     })
 
-    it("a deleted document from charlieRepo can be refetched", async () => {
+    it.skip("a deleted document from charlieRepo can be refetched", async () => {
       const { charlieRepo, aliceHandle, teardown } = await setup()
 
       const deletePromise = eventPromise(charlieRepo, "delete-document")
@@ -1049,6 +1128,8 @@ describe("Repo", () => {
         d.foo = "baz"
       })
       await changePromise
+      // TODO: remove this pause
+      await pause(100)
 
       const handle3 = charlieRepo.find<TestDoc>(aliceHandle.url)
       await eventPromise(handle3, "change")
@@ -1070,7 +1151,7 @@ describe("Repo", () => {
         const repo = getRandomItem([aliceRepo, bobRepo, charlieRepo])
         const docs = Object.values(repo.handles)
         const doc =
-          Math.random() < 0.5
+          Math.random() < 0.5 || docs.length == 0
             ? // heads, create a new doc
               repo.create<TestDoc>()
             : // tails, pick a random doc
@@ -1115,8 +1196,11 @@ describe("Repo", () => {
         setTimeout(resolve, 100)
       })
 
+      const messagePromise = eventPromise(bobHandle, "ephemeral-message")
       aliceHandle.broadcast(data)
-      const { message } = await eventPromise(bobHandle, "ephemeral-message")
+
+      // const { message } = await eventPromise(bobHandle, "ephemeral-message")
+      const { message } = await messagePromise
 
       assert.deepStrictEqual(message, data)
       assert.equal(charlieRepo.handles[notForCharlie], undefined, "charlie no")
@@ -1125,108 +1209,7 @@ describe("Repo", () => {
       teardown()
     })
 
-    it("should save sync state of other peers", async () => {
-      const { bobRepo, teardown, charlieRepo } = await setup({
-        connectAlice: false,
-      })
-
-      const bobHandle = bobRepo.create<TestDoc>()
-      bobHandle.change(d => {
-        d.foo = "bar"
-      })
-
-      await pause(200)
-
-      // bob should store the sync state of charlie
-      const storedSyncState = await bobRepo.storageSubsystem.loadSyncState(
-        bobHandle.documentId,
-        await charlieRepo!.storageSubsystem.id()
-      )
-      assert.deepStrictEqual(storedSyncState.sharedHeads, bobHandle.heads())
-
-      teardown()
-    })
-
-    it("should not save sync state of ephemeral peers", async () => {
-      const { bobRepo, teardown, charlieRepo } = await setup({
-        connectAlice: false,
-        isCharlieEphemeral: true,
-      })
-
-      const bobHandle = bobRepo.create<TestDoc>()
-      bobHandle.change(d => {
-        d.foo = "bar"
-      })
-
-      await pause(200)
-
-      // bob should not store the sync state for charlie because charly is an ephemeral peer
-      const storedSyncState = await bobRepo.storageSubsystem.loadSyncState(
-        bobHandle.documentId,
-        await charlieRepo!.storageSubsystem.id()
-      )
-      assert.deepStrictEqual(storedSyncState, undefined)
-
-      teardown()
-    })
-
-    it("should load sync state from storage", async () => {
-      const { bobRepo, teardown, charlie, charlieRepo, bobStorage, bob } =
-        await setup({
-          connectAlice: false,
-        })
-
-      // create a new doc and count sync messages
-      const bobHandle = bobRepo.create<TestDoc>()
-      bobHandle.change(d => {
-        d.foo = "bar"
-      })
-      let bobSyncMessages = 0
-      bobRepo.networkSubsystem.on("message", message => {
-        if (message.type === "sync") {
-          bobSyncMessages++
-        }
-      })
-      await pause(500)
-
-      // repo has no stored sync state for charlie so we should see 2 sync messages
-      assert.strictEqual(bobSyncMessages, 2)
-
-      await bobRepo.flush()
-
-      // setup new repo which uses bob's storage
-      const bob2Repo = new Repo({
-        storage: bobStorage,
-        peerId: "bob-2" as PeerId,
-      })
-
-      // connnect it with charlie
-      const channel = new MessageChannel()
-      bob2Repo.networkSubsystem.addNetworkAdapter(
-        new MessageChannelNetworkAdapter(channel.port2)
-      )
-      charlieRepo.networkSubsystem.addNetworkAdapter(
-        new MessageChannelNetworkAdapter(channel.port1)
-      )
-
-      // lookup doc we've previously created and count the messages
-      bob2Repo.find(bobHandle.documentId)
-      let bob2SyncMessages = 0
-      bob2Repo.networkSubsystem.on("message", message => {
-        if (message.type === "sync") {
-          bob2SyncMessages++
-        }
-      })
-      await pause(100)
-
-      // repo has stored sync state for charlie so we should see one sync messages
-      assert.strictEqual(bob2SyncMessages, 1)
-
-      channel.port1.close()
-      teardown()
-    })
-
-    it("should report the remote heads when they change", async () => {
+    it.skip("should report the remote heads when they change", async () => {
       const { bobRepo, charlieRepo, teardown } = await setup({
         connectAlice: false,
       })
@@ -1315,7 +1298,7 @@ describe("Repo", () => {
     assert.equal(bobDoc.isReady(), true)
   })
 
-  describe("with peers (mesh network)", () => {
+  describe.skip("with peers (mesh network)", () => {
     const setup = async () => {
       // Set up three repos; connect Alice to Bob, Bob to Charlie, and Alice to Charlie
 
@@ -1453,6 +1436,131 @@ describe("Repo", () => {
       teardown()
     })
   })
+
+  describe.skip("the denylist", () => {
+    it("should immediately return an unavailable message in response to a request for a denylisted document", async () => {
+      const storage = new DummyStorageAdapter()
+
+      // first create the document in storage
+      const dummyRepo = new Repo({ network: [], storage })
+      const doc = dummyRepo.create({ foo: "bar" })
+      await dummyRepo.flush()
+
+      // Check that the document actually is in storage
+      let docId = doc.documentId
+      assert(storage.keys().some((k: string) => k.includes(docId)))
+
+      const channel = new MessageChannel()
+      const { port1: clientToServer, port2: serverToClient } = channel
+      const server = new Repo({
+        network: [new MessageChannelNetworkAdapter(serverToClient)],
+        storage,
+        denylist: [doc.url],
+      })
+      const client = new Repo({
+        network: [new MessageChannelNetworkAdapter(clientToServer)],
+      })
+
+      await Promise.all([
+        eventPromise(server.networkSubsystem, "peer"),
+        eventPromise(client.networkSubsystem, "peer"),
+      ])
+
+      const clientDoc = client.find(doc.url)
+      await pause(100)
+      assert.strictEqual(clientDoc.docSync(), undefined)
+
+      const openDocs = Object.keys(server.metrics().documents).length
+      assert.deepEqual(openDocs, 0)
+    })
+  })
+  describe("beelay bundle requests", () => {
+    function endsWithTwoZerosInDecimal(hexString: string) {
+      // Validate input is 32 bytes (64 characters) of hex
+      if (!/^[0-9a-fA-F]+$/.test(hexString)) {
+        throw new Error("Input must be a hexadecimal string, got " + hexString)
+      }
+      // Convert hex to decimal string
+      const decimal = BigInt("0x" + hexString).toString()
+      // Check if the last two digits are zeros
+      return decimal.endsWith("00")
+    }
+
+    it("creates new bundles", () => {
+      const repo = new Repo({ network: [] })
+      const handle = repo.create<TestDoc>()
+      let heads = handle.heads()
+      let boundaryHashGenerated = false
+      let iteration = 0
+      while (!boundaryHashGenerated) {
+        handle.change(d => {
+          d.foo = `foo${iteration}`
+        })
+        const changeHash = A.decodeChange(
+          A.getLastLocalChange(handle.docSync()!)
+        ).hash
+        boundaryHashGenerated = endsWithTwoZerosInDecimal(changeHash)
+      }
+    })
+  })
+
+  describe("when syncing collections", () => {
+    function setup() {
+      const abChannel = new MessageChannel()
+
+      const { port1: ab, port2: ba } = abChannel
+
+      const alice = new Repo({
+        network: [new MessageChannelNetworkAdapter(ab)],
+        peerId: "alice" as PeerId,
+        sharePolicy: async () => false,
+      })
+      const bob = new Repo({
+        network: [new MessageChannelNetworkAdapter(ba)],
+        peerId: "bob" as PeerId,
+      })
+      return { alice, bob }
+    }
+
+    it("should sync linked documents", async () => {
+      let { alice, bob } = setup()
+      let doc1 = alice.create<{ linked: A.Link | null }>({ linked: null })
+      let doc2 = alice.create<{ foo: string }>({ foo: "bar" })
+      doc1.change(d => {
+        d.linked = new A.Link(doc2.url)
+      })
+
+      let doc1OnBob = bob.find(doc1.url)
+      // TODO remove this pause
+      await pause(100)
+
+      await doc1OnBob.whenReady()
+      bob.networkSubsystem.disconnect()
+
+      let doc2OnBob = bob.find(doc2.url)
+      await doc2OnBob.whenReady()
+      assert.deepStrictEqual(doc2OnBob.docSync(), { foo: "bar" })
+    })
+
+    it("should sync linked documents when a document is initialized contaiing links", async () => {
+      let { alice, bob } = setup()
+      let doc2 = alice.create<{ foo: string }>({ foo: "bar" })
+      let doc1 = alice.create<{ linked: A.Link | null }>({
+        linked: new A.Link(doc2.url),
+      })
+
+      let doc1OnBob = bob.find(doc1.url)
+      // TODO remove this pause
+      await pause(100)
+
+      await doc1OnBob.whenReady()
+      bob.networkSubsystem.disconnect()
+
+      let doc2OnBob = bob.find(doc2.url)
+      await doc2OnBob.whenReady()
+      assert.deepStrictEqual(doc2OnBob.docSync(), { foo: "bar" })
+    })
+  })
 })
 
 const warn = console.warn

package/test/StorageSubsystem.test.ts

@@ -1,5 +1,5 @@
 import { NodeFSStorageAdapter } from "../../automerge-repo-storage-nodefs/src/index.js"
-import * as A from "@automerge/automerge/next"
+import { next as A } from "@automerge/automerge"
 import assert from "assert"
 import fs from "fs"
 import os from "os"
@@ -21,10 +21,14 @@ describe("StorageSubsystem", () => {
   }
 
   for (const [adapterName, adapter] of Object.entries(adaptersToTest)) {
+    const beelay = new A.beelay.Beelay({
+      peerId: "somepeer",
+      storage: adapter,
+    })
     describe(adapterName, () => {
       describe("Automerge document storage", () => {
         it("stores and retrieves an Automerge document", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const doc = A.change(A.init<any>(), "test", d => {
             d.foo = "bar"
@@ -42,7 +46,7 @@ describe("StorageSubsystem", () => {
         })
 
         it("retrieves an Automerge document following lots of changes", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           type TestDoc = { foo: number }
 
@@ -60,7 +64,7 @@ describe("StorageSubsystem", () => {
           }
 
           // reload it from storage, simulating a new process
-          const storage2 = new StorageSubsystem(adapter)
+          const storage2 = new StorageSubsystem(beelay, adapter)
           const reloadedDoc = await storage2.loadDoc<TestDoc>(key)
 
           // check that the doc has the right value
@@ -68,7 +72,7 @@ describe("StorageSubsystem", () => {
         })
 
         it("stores incremental changes following a load", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const doc = A.change(A.init<any>(), "test", d => {
             d.foo = "bar"
@@ -76,10 +80,10 @@ describe("StorageSubsystem", () => {
 
           // save it to storage
           const key = parseAutomergeUrl(generateAutomergeUrl()).documentId
-          storage.saveDoc(key, doc)
+          await storage.saveDoc(key, doc)
 
           // reload it from storage, simulating a new process
-          const storage2 = new StorageSubsystem(adapter)
+          const storage2 = new StorageSubsystem(beelay, adapter)
           const reloadedDoc = await storage2.loadDoc(key)
 
           assert(reloadedDoc, "doc should be loaded")
@@ -93,8 +97,8 @@ describe("StorageSubsystem", () => {
           storage2.saveDoc(key, changedDoc)
         })
 
-        it("removes an Automerge document", async () => {
-          const storage = new StorageSubsystem(adapter)
+        it.skip("removes an Automerge document", async () => {
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const doc = A.change(A.init<any>(), "test", d => {
             d.foo = "bar"
@@ -123,7 +127,7 @@ describe("StorageSubsystem", () => {
 
       describe("Arbitrary key/value storage", () => {
         it("stores and retrieves a blob", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const value = cbor.encode({ foo: "bar" })
 
@@ -137,7 +141,7 @@ describe("StorageSubsystem", () => {
         })
 
         it("keeps namespaces separate", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const key = "ABC123"
 
@@ -159,7 +163,7 @@ describe("StorageSubsystem", () => {
         })
 
         it("removes a blob", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const value = cbor.encode({ foo: "bar" })
 
@@ -180,7 +184,7 @@ describe("StorageSubsystem", () => {
 
       describe("sync state", () => {
         it("stores and retrieve sync state", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
           const syncState = A.initSyncState()
@@ -197,7 +201,7 @@ describe("StorageSubsystem", () => {
         })
 
         it("delete sync state if document is deleted", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
           const syncState = A.initSyncState()
@@ -213,7 +217,7 @@ describe("StorageSubsystem", () => {
         })
 
         it("returns a undefined if loading an existing sync state fails", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
           const bobStorageId = Uuid.v4() as StorageId
@@ -232,7 +236,7 @@ describe("StorageSubsystem", () => {
 
       describe("storage id", () => {
         it("generates a unique id", async () => {
-          const storage = new StorageSubsystem(adapter)
+          const storage = new StorageSubsystem(beelay, adapter)
 
           // generate unique id and return same id on subsequence calls
           const id1 = await storage.id()

package/test/remoteHeads.test.ts

@@ -14,7 +14,7 @@ import { DummyStorageAdapter } from "../src/helpers/DummyStorageAdapter.js"
 import { collectMessages } from "./helpers/collectMessages.js"
 import { TestDoc } from "./types.js"
 
-describe("DocHandle.remoteHeads", () => {
+describe.skip("DocHandle.remoteHeads", () => {
   const TEST_ID = parseAutomergeUrl(generateAutomergeUrl()).documentId
 
   it("should allow to listen for remote head changes and manually read remote heads", async () => {