@automerge/automerge-repo 1.0.19 → 1.1.0-alpha.13

package/test/Repo.test.ts

@@ -1,9 +1,9 @@
 import { next as A } from "@automerge/automerge"
-import { MessageChannelNetworkAdapter } from "@automerge/automerge-repo-network-messagechannel"
+import { MessageChannelNetworkAdapter } from "../../automerge-repo-network-messagechannel/src/index.js"
 import assert from "assert"
 import * as Uuid from "uuid"
-import { describe, it } from "vitest"
-import { DocHandleRemoteHeadsPayload, READY } from "../src/DocHandle.js"
+import { describe, expect, it } from "vitest"
+import { READY } from "../src/DocHandle.js"
 import { parseAutomergeUrl } from "../src/AutomergeUrl.js"
 import {
   generateAutomergeUrl,
@@ -13,6 +13,7 @@ import { Repo } from "../src/Repo.js"
 import { eventPromise } from "../src/helpers/eventPromise.js"
 import { pause } from "../src/helpers/pause.js"
 import {
+  AnyDocumentId,
   AutomergeUrl,
   DocHandle,
   DocumentId,
@@ -28,8 +29,18 @@ import {
 } from "./helpers/generate-large-object.js"
 import { getRandomItem } from "./helpers/getRandomItem.js"
 import { TestDoc } from "./types.js"
+import { StorageId } from "../src/storage/types.js"
 
 describe("Repo", () => {
+  describe("constructor", () => {
+    it("can be instantiated without network adapters", () => {
+      const repo = new Repo({
+        network: [],
+      })
+      expect(repo).toBeInstanceOf(Repo)
+    })
+  })
+
   describe("local only", () => {
     const setup = ({ startReady = true } = {}) => {
       const storageAdapter = new DummyStorageAdapter()
@@ -320,6 +331,27 @@ describe("Repo", () => {
         repo.delete(handle.documentId)
       }))
 
+    it("exports a document", async () => {
+      const { repo } = setup()
+      const handle = repo.create<TestDoc>()
+      handle.change(d => {
+        d.foo = "bar"
+      })
+      assert.equal(handle.isReady(), true)
+
+      const exported = await repo.export(handle.documentId)
+      const loaded = A.load(exported)
+      const doc = await handle.doc()
+      assert.deepEqual(doc, loaded)
+    })
+
+    it("rejects when exporting a document that does not exist", async () => {
+      const { repo } = setup()
+      assert.rejects(async () => {
+        await repo.export("foo" as AnyDocumentId)
+      })
+    })
+
     it("storage state doesn't change across reloads when the document hasn't changed", async () => {
       const storage = new DummyStorageAdapter()
 
@@ -337,6 +369,9 @@ describe("Repo", () => {
         d.count = 1
       })
 
+      // wait because storage id is not initialized immediately
+      await pause()
+
       const initialKeys = storage.keys()
 
       const repo2 = new Repo({
@@ -392,10 +427,37 @@ describe("Repo", () => {
       const storageKeyTypes = storageAdapter.keys().map(k => k.split(".")[1])
       assert(storageKeyTypes.filter(k => k === "snapshot").length === 1)
     })
+
+    it("can import an existing document", async () => {
+      const { repo } = setup()
+      const doc = A.init<TestDoc>()
+      const updatedDoc = A.change(doc, d => {
+        d.foo = "bar"
+      })
+
+      const saved = A.save(updatedDoc)
+
+      const handle = repo.import<TestDoc>(saved)
+      assert.equal(handle.isReady(), true)
+      const v = await handle.doc()
+      assert.equal(v?.foo, "bar")
+
+      expect(A.getHistory(v)).toEqual(A.getHistory(updatedDoc))
+    })
+
+    it("throws an error if we try to import an invalid document", async () => {
+      const { repo } = setup()
+      expect(() => {
+        repo.import<TestDoc>(A.init<TestDoc> as unknown as Uint8Array)
+      }).toThrow()
+    })
   })
 
   describe("with peers (linear network)", async () => {
-    const setup = async ({ connectAlice = true } = {}) => {
+    const setup = async ({
+      connectAlice = true,
+      isCharlieEphemeral = false,
+    } = {}) => {
       const charlieExcludedDocuments: DocumentId[] = []
       const bobExcludedDocuments: DocumentId[] = []
 
@@ -451,6 +513,7 @@ describe("Repo", () => {
         storage: charlieStorage,
         network: [new MessageChannelNetworkAdapter(cb)],
         peerId: charlie,
+        isEphemeral: isCharlieEphemeral,
       })
 
       const teardown = () => {
@@ -789,7 +852,7 @@ describe("Repo", () => {
     })
 
     it("should save sync state of other peers", async () => {
-      const { bobRepo, teardown, charlie } = await setup({
+      const { bobRepo, teardown, charlieRepo } = await setup({
         connectAlice: false,
       })
 
@@ -803,7 +866,7 @@ describe("Repo", () => {
       // bob should store the sync state of charlie
       const storedSyncState = await bobRepo.storageSubsystem.loadSyncState(
         bobHandle.documentId,
-        charlie
+        await charlieRepo!.storageSubsystem.id()
       )
       const docHeads = A.getHeads(bobHandle.docSync())
       assert.deepStrictEqual(storedSyncState.sharedHeads, docHeads)
@@ -811,6 +874,29 @@ describe("Repo", () => {
       teardown()
     })
 
+    it("should not save sync state of ephemeral peers", async () => {
+      const { bobRepo, teardown, charlieRepo } = await setup({
+        connectAlice: false,
+        isCharlieEphemeral: true,
+      })
+
+      const bobHandle = bobRepo.create<TestDoc>()
+      bobHandle.change(d => {
+        d.foo = "bar"
+      })
+
+      await pause(200)
+
+      // bob should not store the sync state for charlie because charly is an ephemeral peer
+      const storedSyncState = await bobRepo.storageSubsystem.loadSyncState(
+        bobHandle.documentId,
+        await charlieRepo!.storageSubsystem.id()
+      )
+      assert.deepStrictEqual(storedSyncState, undefined)
+
+      teardown()
+    })
+
     it("should load sync state from storage", async () => {
       const { bobRepo, teardown, charlie, charlieRepo, bobStorage, bob } =
         await setup({
@@ -823,13 +909,15 @@ describe("Repo", () => {
         d.foo = "bar"
       })
       let bobSyncMessages = 0
-      bobRepo.networkSubsystem.on("message", () => {
-        bobSyncMessages++
+      bobRepo.networkSubsystem.on("message", message => {
+        if (message.type === "sync") {
+          bobSyncMessages++
+        }
       })
       await pause(500)
 
-      // repo has no stored sync state for charlie so we should see two sync messages
-      assert.strictEqual(bobSyncMessages, 2)
+      // repo has no stored sync state for charlie so we should see 3 sync messages
+      assert.strictEqual(bobSyncMessages, 3)
 
       // setup new repo which uses bob's storage
       const bob2Repo = new Repo({
@@ -850,8 +938,10 @@ describe("Repo", () => {
       // lookup doc we've previously created and count the messages
       bob2Repo.find(bobHandle.documentId)
       let bob2SyncMessages = 0
-      bob2Repo.networkSubsystem.on("message", m => {
-        bob2SyncMessages++
+      bob2Repo.networkSubsystem.on("message", message => {
+        if (message.type === "sync") {
+          bob2SyncMessages++
+        }
       })
       await pause(100)
 
@@ -866,6 +956,7 @@ describe("Repo", () => {
       const { bobRepo, charlieRepo, teardown } = await setup({
         connectAlice: false,
       })
+      const charliedStorageId = await charlieRepo.storageSubsystem.id()
 
       const handle = bobRepo.create<TestDoc>()
       handle.change(d => {
@@ -876,11 +967,11 @@ describe("Repo", () => {
       await pause(50)
 
       const nextRemoteHeadsPromise = new Promise<{
-        peerId: PeerId
+        storageId: StorageId
         heads: A.Heads
       }>(resolve => {
-        handle.on("remote-heads", ({ peerId, heads }) => {
-          resolve({ peerId, heads })
+        handle.on("remote-heads", ({ storageId, heads }) => {
+          resolve({ storageId, heads })
         })
       })
 
@@ -901,11 +992,11 @@ describe("Repo", () => {
       assert.deepStrictEqual(charlieHeads, bobHeads)
 
       const nextRemoteHeads = await nextRemoteHeadsPromise
-      assert.deepStrictEqual(nextRemoteHeads.peerId, "charlie")
+      assert.deepStrictEqual(nextRemoteHeads.storageId, charliedStorageId)
       assert.deepStrictEqual(nextRemoteHeads.heads, charlieHeads)
 
       assert.deepStrictEqual(
-        handle.getRemoteHeads("charlie" as PeerId),
+        handle.getRemoteHeads(charliedStorageId),
         A.getHeads(charlieHandle.docSync())
       )
 

package/test/StorageSubsystem.test.ts

@@ -1,4 +1,4 @@
-import { NodeFSStorageAdapter } from "@automerge/automerge-repo-storage-nodefs"
+import { NodeFSStorageAdapter } from "../../automerge-repo-storage-nodefs/src/index.js"
 import * as A from "@automerge/automerge/next"
 import assert from "assert"
 import fs from "fs"
@@ -8,7 +8,9 @@ import { describe, it } from "vitest"
 import { generateAutomergeUrl, parseAutomergeUrl } from "../src/AutomergeUrl.js"
 import { PeerId, cbor } from "../src/index.js"
 import { StorageSubsystem } from "../src/storage/StorageSubsystem.js"
+import { StorageId } from "../src/storage/types.js"
 import { DummyStorageAdapter } from "./helpers/DummyStorageAdapter.js"
+import * as Uuid from "uuid"
 
 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "automerge-repo-tests"))
 
@@ -182,12 +184,15 @@ describe("StorageSubsystem", () => {
 
           const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
           const syncState = A.initSyncState()
-          const bob = "bob" as PeerId
+          const bobStorageId = Uuid.v4() as StorageId
 
           const rawSyncState = A.decodeSyncState(A.encodeSyncState(syncState))
 
-          await storage.saveSyncState(documentId, bob, syncState)
-          const loadedSyncState = await storage.loadSyncState(documentId, bob)
+          await storage.saveSyncState(documentId, bobStorageId, syncState)
+          const loadedSyncState = await storage.loadSyncState(
+            documentId,
+            bobStorageId
+          )
           assert.deepStrictEqual(loadedSyncState, rawSyncState)
         })
 
@@ -196,14 +201,31 @@ describe("StorageSubsystem", () => {
 
           const { documentId } = parseAutomergeUrl(generateAutomergeUrl())
           const syncState = A.initSyncState()
-          const bob = "bob" as PeerId
+          const bobStorageId = Uuid.v4() as StorageId
 
-          await storage.saveSyncState(documentId, bob, syncState)
+          await storage.saveSyncState(documentId, bobStorageId, syncState)
           await storage.removeDoc(documentId)
-          const loadedSyncState = await storage.loadSyncState(documentId, bob)
+          const loadedSyncState = await storage.loadSyncState(
+            documentId,
+            bobStorageId
+          )
           assert.strictEqual(loadedSyncState, undefined)
         })
       })
+
+      describe("storage id", () => {
+        it("generates a unique id", async () => {
+          const storage = new StorageSubsystem(adapter)
+
+          // generate unique id and return same id on subsequence calls
+          const id1 = await storage.id()
+          const id2 = await storage.id()
+
+          assert.strictEqual(Uuid.validate(id1), true)
+          assert.strictEqual(Uuid.validate(id2), true)
+          assert.strictEqual(id1, id2)
+        })
+      })
     })
   })
 })

package/test/helpers/waitForMessages.ts

@@ -0,0 +1,22 @@
+import { EventEmitter } from "eventemitter3"
+import { pause } from "../../src/helpers/pause.js"
+
+export async function waitForMessages(
+  emitter: EventEmitter,
+  event: string,
+  timeout: number = 100
+): Promise<any[]> {
+  const messages = []
+
+  const onEvent = message => {
+    messages.push(message)
+  }
+
+  emitter.on(event, onEvent)
+
+  await pause(timeout)
+
+  emitter.off(event)
+
+  return messages
+}

package/test/remoteHeads.test.ts

@@ -0,0 +1,260 @@
+import { MessageChannelNetworkAdapter } from "../../automerge-repo-network-messagechannel/dist/index.js"
+import * as A from "@automerge/automerge/next"
+import assert from "assert"
+import { setTimeout } from "timers/promises"
+import { describe, it } from "vitest"
+import { generateAutomergeUrl, parseAutomergeUrl } from "../src/AutomergeUrl.js"
+import { eventPromise } from "../src/helpers/eventPromise.js"
+import {
+  DocHandle,
+  DocHandleRemoteHeadsPayload,
+  PeerId,
+  Repo,
+} from "../src/index.js"
+import { DummyStorageAdapter } from "./helpers/DummyStorageAdapter.js"
+import { waitForMessages } from "./helpers/waitForMessages.js"
+import { TestDoc } from "./types.js"
+
+describe("DocHandle.remoteHeads", () => {
+  const TEST_ID = parseAutomergeUrl(generateAutomergeUrl()).documentId
+
+  it("should allow to listen for remote head changes and manually read remote heads", async () => {
+    const handle = new DocHandle<TestDoc>(TEST_ID, { isNew: true })
+    const bobRepo = new Repo({
+      peerId: "bob" as PeerId,
+      network: [],
+      storage: new DummyStorageAdapter(),
+      enableRemoteHeadsGossiping: true,
+    })
+    const bobStorageId = await bobRepo.storageId()
+
+    const remoteHeadsMessagePromise = eventPromise(handle, "remote-heads")
+    handle.setRemoteHeads(bobStorageId, [])
+
+    const remoteHeadsMessage = await remoteHeadsMessagePromise
+
+    assert.strictEqual(remoteHeadsMessage.storageId, bobStorageId)
+    assert.deepStrictEqual(remoteHeadsMessage.heads, [])
+
+    // read remote heads manually
+    assert.deepStrictEqual(handle.getRemoteHeads(bobStorageId), [])
+  })
+
+  describe("multi hop sync", () => {
+    async function setup() {
+      // setup topology: tab -> service worker -> sync server <- service worker <- tab
+      const leftTab1 = new Repo({
+        peerId: "left-tab-1" as PeerId,
+        network: [],
+        sharePolicy: async () => true,
+        enableRemoteHeadsGossiping: true,
+      })
+      const leftTab2 = new Repo({
+        peerId: "left-tab-2" as PeerId,
+        network: [],
+        sharePolicy: async () => true,
+        enableRemoteHeadsGossiping: true,
+      })
+      const leftServiceWorker = new Repo({
+        peerId: "left-service-worker" as PeerId,
+        network: [],
+        sharePolicy: async peer => peer === "sync-server",
+        storage: new DummyStorageAdapter(),
+        isEphemeral: false,
+        enableRemoteHeadsGossiping: true,
+      })
+      const syncServer = new Repo({
+        peerId: "sync-server" as PeerId,
+        network: [],
+        isEphemeral: false,
+        sharePolicy: async () => false,
+        storage: new DummyStorageAdapter(),
+        enableRemoteHeadsGossiping: true,
+      })
+      const rightServiceWorker = new Repo({
+        peerId: "right-service-worker" as PeerId,
+        network: [],
+        sharePolicy: async peer => peer === "sync-server",
+        isEphemeral: false,
+        storage: new DummyStorageAdapter(),
+        enableRemoteHeadsGossiping: true,
+      })
+      const rightTab = new Repo({
+        peerId: "right-tab" as PeerId,
+        network: [],
+        sharePolicy: async () => true,
+        enableRemoteHeadsGossiping: true,
+      })
+
+      // connect them all up
+      connectRepos(leftTab1, leftServiceWorker)
+      connectRepos(leftTab2, leftServiceWorker)
+      connectRepos(leftServiceWorker, syncServer)
+      connectRepos(syncServer, rightServiceWorker)
+      connectRepos(rightServiceWorker, rightTab)
+
+      await setTimeout(100)
+
+      return {
+        leftTab1,
+        leftTab2,
+        leftServiceWorker,
+        syncServer,
+        rightServiceWorker,
+        rightTab,
+      }
+    }
+
+    it("should report remoteHeads for peers", async () => {
+      const { rightTab, rightServiceWorker, leftServiceWorker, leftTab1 } =
+        await setup()
+
+      // subscribe to the left service worker storage ID on the right tab
+      rightTab.subscribeToRemotes([await leftServiceWorker.storageId()!])
+
+      await setTimeout(100)
+
+      // create a doc in the left tab
+      const leftTabDoc = leftTab1.create<TestDoc>()
+      leftTabDoc.change(d => (d.foo = "bar"))
+
+      // wait for the document to arrive on the right tab
+      const rightTabDoc = rightTab.find<TestDoc>(leftTabDoc.url)
+      await rightTabDoc.whenReady()
+
+      // wait for the document to arrive in the left service worker
+      const leftServiceWorkerDoc = leftServiceWorker.find(leftTabDoc.documentId)
+      await leftServiceWorkerDoc.whenReady()
+
+      const leftServiceWorkerStorageId = await leftServiceWorker.storageId()
+      let leftSeenByRightPromise = new Promise<DocHandleRemoteHeadsPayload>(
+        resolve => {
+          rightTabDoc.on("remote-heads", message => {
+            if (message.storageId === leftServiceWorkerStorageId) {
+              resolve(message)
+            }
+          })
+        }
+      )
+
+      // make a change on the right
+      rightTabDoc.change(d => (d.foo = "baz"))
+
+      // wait for the change to be acknolwedged by the left
+      const leftSeenByRight = await leftSeenByRightPromise
+
+      assert.deepStrictEqual(
+        leftSeenByRight.heads,
+        A.getHeads(leftServiceWorkerDoc.docSync())
+      )
+    })
+
+    it("should report remoteHeads only for documents the subscriber has open", async () => {
+      const { leftTab1, rightTab, rightServiceWorker } = await setup()
+
+      // subscribe leftTab to storageId of rightServiceWorker
+      leftTab1.subscribeToRemotes([await rightServiceWorker.storageId()!])
+
+      await setTimeout(100)
+
+      // create 2 docs in right tab
+      const rightTabDocA = rightTab.create<TestDoc>()
+      rightTabDocA.change(d => (d.foo = "A"))
+
+      const rightTabDocB = rightTab.create<TestDoc>()
+      rightTabDocB.change(d => (d.foo = "B"))
+
+      // open doc b in left tab 1
+      const leftTabDocA = leftTab1.find<TestDoc>(rightTabDocA.url)
+
+      const remoteHeadsChangedMessages = (
+        await waitForMessages(leftTab1.networkSubsystem, "message")
+      ).filter(({ type }) => type === "remote-heads-changed")
+
+      // we should only be notified of the head changes of doc A
+      const docIds = remoteHeadsChangedMessages.map(d => d.documentId)
+      const uniqueDocIds = [...new Set(docIds)]
+      assert.deepStrictEqual(uniqueDocIds, [leftTabDocA.documentId])
+    })
+
+    it("should report remote heads for doc on subscribe if peer already knows them", async () => {
+      const { leftTab1, leftTab2, rightTab, rightServiceWorker } = await setup()
+
+      // create 2 docs in right tab
+      const rightTabDocA = rightTab.create<TestDoc>()
+      rightTabDocA.change(d => (d.foo = "A"))
+
+      const rightTabDocB = rightTab.create<TestDoc>()
+      rightTabDocB.change(d => (d.foo = "B"))
+
+      // open docs in left tab 1
+      const leftTab1DocA = leftTab1.find<TestDoc>(rightTabDocA.url)
+      const leftTab1DocB = leftTab1.find<TestDoc>(rightTabDocB.url)
+
+      // subscribe leftTab 1 to storageId of rightServiceWorker
+      leftTab1.subscribeToRemotes([await rightServiceWorker.storageId()!])
+
+      await setTimeout(200)
+
+      // now the left service worker has the remote heads of the right service worker for both doc A and doc B
+      // if we subscribe from left tab 1 the left service workers should send it's stored remote heads immediately
+
+      // open doc and subscribe leftTab 2 to storageId of rightServiceWorker
+      const leftTab2DocA = leftTab2.find<TestDoc>(rightTabDocA.url)
+      leftTab2.subscribeToRemotes([await rightServiceWorker.storageId()!])
+
+      const remoteHeadsChangedMessages = (
+        await waitForMessages(leftTab2.networkSubsystem, "message")
+      ).filter(({ type }) => type === "remote-heads-changed")
+
+      // we should only be notified of the head changes of doc A
+      assert.strictEqual(remoteHeadsChangedMessages.length, 1)
+      assert.strictEqual(
+        remoteHeadsChangedMessages[0].documentId,
+        leftTab1DocA.documentId
+      )
+    })
+
+    it("should report remote heads for subscribed storage id once we open a new doc", async () => {
+      const { leftTab1, leftTab2, rightTab, rightServiceWorker } = await setup()
+
+      // create 2 docs in right tab
+      const rightTabDocA = rightTab.create<TestDoc>()
+      rightTabDocA.change(d => (d.foo = "A"))
+
+      const rightTabDocB = rightTab.create<TestDoc>()
+      rightTabDocB.change(d => (d.foo = "B"))
+
+      await setTimeout(200)
+
+      // subscribe leftTab 1 to storageId of rightServiceWorker
+      leftTab1.subscribeToRemotes([await rightServiceWorker.storageId()!])
+
+      // in leftTab 1 open doc A
+      const leftTab1DocA = leftTab1.find<TestDoc>(rightTabDocA.url)
+
+      const remoteHeadsChangedMessages = (
+        await waitForMessages(leftTab1.networkSubsystem, "message")
+      ).filter(({ type }) => type === "remote-heads-changed")
+
+      // console.log(JSON.stringify(remoteHeadsChangedMessages, null, 2))
+
+      assert.strictEqual(remoteHeadsChangedMessages.length, 1)
+      assert.strictEqual(
+        remoteHeadsChangedMessages[0].documentId,
+        leftTab1DocA.documentId
+      )
+    })
+  })
+})
+
+function connectRepos(repo1: Repo, repo2: Repo) {
+  const { port1: leftToRight, port2: rightToLeft } = new MessageChannel()
+
+  repo1.networkSubsystem.addNetworkAdapter(
+    new MessageChannelNetworkAdapter(leftToRight)
+  )
+  repo2.networkSubsystem.addNetworkAdapter(
+    new MessageChannelNetworkAdapter(rightToLeft)
+  )
+}

package/.eslintrc

@@ -1,28 +0,0 @@
-{
-  "env": {
-    "browser": true,
-    "es2021": true
-  },
-  "extends": [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/eslint-recommended",
-    "plugin:@typescript-eslint/recommended"
-  ],
-  "ignorePatterns": ["dist/**", "test/**", "node_modules/**"],
-  "parser": "@typescript-eslint/parser",
-  "plugins": ["@typescript-eslint"],
-  "parserOptions": {
-    "project": "./tsconfig.json",
-    "ecmaVersion": "latest",
-    "sourceType": "module"
-  },
-  "rules": {
-    "semi": ["error", "never"],
-    "import/extensions": 0,
-    "lines-between-class-members": 0,
-    "@typescript-eslint/no-floating-promises": "error",
-    "@typescript-eslint/no-empty-function": ["warn", { "allow": ["methods"] }],
-    "no-param-reassign": 0,
-    "no-use-before-define": 0
-  }
-}