@automerge/automerge-repo 1.0.19 → 1.1.0-alpha.2

package/src/network/messages.ts

@@ -1,5 +1,6 @@
 import { SyncState } from "@automerge/automerge"
 import { DocumentId, PeerId, SessionId } from "../types.js"
+import { StorageId } from "../storage/types.js"
 
 /**
  * A sync message for a particular document
@@ -102,12 +103,32 @@ export type AuthMessage<TPayload = any> = {
   payload: TPayload
 }
 
+export type RemoteSubscriptionControlMessage = {
+  type: "remote-subscription-change",
+  senderId: PeerId,
+  targetId: PeerId,
+  add?: StorageId[],
+  remove?: StorageId[],
+}
+
+export type RemoteHeadsChanged = {
+  type: "remote-heads-changed",
+  senderId: PeerId,
+  targetId: PeerId,
+  documentId: DocumentId,
+  newHeads: {[key: StorageId]: {heads: string[], timestamp: number}},
+}
+
 /** These are message types that a {@link NetworkAdapter} surfaces to a {@link Repo}. */
 export type RepoMessage =
   | SyncMessage
   | EphemeralMessage
   | RequestMessage
   | DocumentUnavailableMessage
+  | RemoteSubscriptionControlMessage
+  | RemoteHeadsChanged
+
+export type DocMessage = SyncMessage | EphemeralMessage | RequestMessage | DocumentUnavailableMessage
 
 /** These are all the message types that a {@link NetworkAdapter} might see. */
 export type Message = RepoMessage | AuthMessage
@@ -136,7 +157,9 @@ export const isValidRepoMessage = (message: Message): message is RepoMessage =>
   (isSyncMessage(message) ||
     isEphemeralMessage(message) ||
     isRequestMessage(message) ||
-    isDocumentUnavailableMessage(message))
+    isDocumentUnavailableMessage(message) ||
+    isRemoteSubscriptionControlMessage(message) ||
+    isRemoteHeadsChanged(message))
 
 // prettier-ignore
 export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage => 
@@ -150,3 +173,9 @@ export const isSyncMessage = (msg: Message): msg is SyncMessage =>
 
 export const isEphemeralMessage = (msg: Message): msg is EphemeralMessage =>
   msg.type === "ephemeral"
+
+export const isRemoteSubscriptionControlMessage = (msg: Message): msg is RemoteSubscriptionControlMessage =>
+  msg.type === "remote-subscription-change"
+
+export const isRemoteHeadsChanged = (msg: Message): msg is RemoteHeadsChanged =>
+  msg.type === "remote-heads-changed"

package/src/storage/StorageSubsystem.ts

@@ -2,11 +2,12 @@ import * as A from "@automerge/automerge/next"
 import debug from "debug"
 import { headsAreSame } from "../helpers/headsAreSame.js"
 import { mergeArrays } from "../helpers/mergeArrays.js"
-import { PeerId, type DocumentId } from "../types.js"
+import { type DocumentId } from "../types.js"
 import { StorageAdapter } from "./StorageAdapter.js"
-import { ChunkInfo, StorageKey } from "./types.js"
+import { ChunkInfo, StorageKey, StorageId } from "./types.js"
 import { keyHash, headsHash } from "./keyHash.js"
 import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
+import * as Uuid from "uuid"
 
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
@@ -31,6 +32,23 @@ export class StorageSubsystem {
     this.#storageAdapter = storageAdapter
   }
 
+  async id(): Promise<StorageId> {
+    let storedId = await this.#storageAdapter.load(["storage-adapter-id"])
+
+    let id: StorageId
+    if (storedId) {
+      id = new TextDecoder().decode(storedId) as StorageId
+    } else {
+      id = Uuid.v4() as StorageId
+      await this.#storageAdapter.save(
+        ["storage-adapter-id"],
+        new TextEncoder().encode(id)
+      )
+    }
+
+    return id
+  }
+
   // ARBITRARY KEY/VALUE STORAGE
 
   // The `load`, `save`, and `remove` methods are for generic key/value storage, as opposed to
@@ -211,19 +229,19 @@ export class StorageSubsystem {
 
   async loadSyncState(
     documentId: DocumentId,
-    peerId: PeerId
+    storageId: StorageId
   ): Promise<A.SyncState | undefined> {
-    const key = [documentId, "sync-state", peerId]
+    const key = [documentId, "sync-state", storageId]
     const loaded = await this.#storageAdapter.load(key)
     return loaded ? A.decodeSyncState(loaded) : undefined
   }
 
   async saveSyncState(
     documentId: DocumentId,
-    peerId: PeerId,
+    storageId: StorageId,
     syncState: A.SyncState
   ): Promise<void> {
-    const key = [documentId, "sync-state", peerId]
+    const key = [documentId, "sync-state", storageId]
     await this.#storageAdapter.save(key, A.encodeSyncState(syncState))
   }
 
@@ -261,6 +279,11 @@ export class StorageSubsystem {
         incrementalSize += chunk.size
       }
     }
-    return incrementalSize >= snapshotSize
+    // if the file is currently small, don't worry, just compact
+    // this might seem a bit arbitrary (1k is arbitrary) but is designed to ensure compaction
+    // for documents with only a single large change on top of an empty (or nearly empty) document
+    // for example: imported NPM modules, images, etc.
+    // if we have even more incrementals (so far) than the snapshot, compact
+    return snapshotSize < 1024 || incrementalSize >= snapshotSize
   }
 }

package/src/storage/types.ts

@@ -37,3 +37,6 @@ export type ChunkType = "snapshot" | "incremental"
  * should not assume any particular structure.
  **/
 export type StorageKey = string[]
+
+/** A branded type for storage IDs */
+export type StorageId = string & { __storageId: true }

package/src/synchronizer/CollectionSynchronizer.ts

@@ -2,7 +2,7 @@ import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
 import { stringifyAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
-import { RepoMessage } from "../network/messages.js"
+import { DocMessage, RepoMessage } from "../network/messages.js"
 import { DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
@@ -37,14 +37,20 @@ export class CollectionSynchronizer extends Synchronizer {
   #initDocSynchronizer(handle: DocHandle<unknown>): DocSynchronizer {
     const docSynchronizer = new DocSynchronizer({
       handle,
-      onLoadSyncState: peerId => {
+      onLoadSyncState: async peerId => {
         if (!this.repo.storageSubsystem) {
-          return Promise.resolve(undefined)
+          return
+        }
+
+        const { storageId, isEphemeral } =
+          this.repo.peerMetadataByPeerId[peerId] || {}
+        if (!storageId || isEphemeral) {
+          return
         }
 
         return this.repo.storageSubsystem.loadSyncState(
           handle.documentId,
-          peerId
+          storageId
         )
       },
     })
@@ -70,7 +76,7 @@ export class CollectionSynchronizer extends Synchronizer {
    * When we receive a sync message for a document we haven't got in memory, we
    * register it with the repo and start synchronizing
    */
-  async receiveMessage(message: RepoMessage) {
+  async receiveMessage(message: DocMessage) {
     log(
       `onSyncMessage: ${message.senderId}, ${message.documentId}, ${
         "data" in message ? message.data.byteLength + "bytes" : ""

package/src/synchronizer/DocSynchronizer.ts

@@ -163,20 +163,8 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   #setSyncState(peerId: PeerId, syncState: A.SyncState) {
-    const previousSyncState = this.#syncStates[peerId]
-
     this.#syncStates[peerId] = syncState
 
-    const haveTheirSyncedHeadsChanged =
-      syncState.theirHeads &&
-      (!previousSyncState ||
-        !previousSyncState.theirHeads ||
-        !headsAreSame(previousSyncState.theirHeads, syncState.theirHeads))
-
-    if (haveTheirSyncedHeadsChanged) {
-      this.#handle.setRemoteHeads(peerId, syncState.theirHeads)
-    }
-
     this.emit("sync-state", {
       peerId,
       syncState,

package/test/DocHandle.test.ts

@@ -303,23 +303,6 @@ describe("DocHandle", () => {
     assert(wasBar, "foo should have been bar as we changed at the old heads")
   })
 
-  it("should allow to listen for remote head changes and manually read remote heads", async () => {
-    const handle = new DocHandle<TestDoc>(TEST_ID, { isNew: true })
-    const bob = "bob" as PeerId
-
-    const remoteHeadsMessagePromise = eventPromise(handle, "remote-heads")
-
-    handle.setRemoteHeads(bob, [])
-
-    const remoteHeadsMessage = await remoteHeadsMessagePromise
-
-    assert.strictEqual(remoteHeadsMessage.peerId, bob)
-    assert.deepStrictEqual(remoteHeadsMessage.heads, [])
-
-    // read remote heads manually
-    assert.deepStrictEqual(handle.getRemoteHeads(bob), [])
-  })
-
   describe("ephemeral messaging", () => {
     it("can broadcast a message for the network to send out", async () => {
       const handle = new DocHandle<TestDoc>(TEST_ID, { isNew: true })

package/test/RemoteHeadsSubscriptions.test.ts

@@ -0,0 +1,343 @@
+import * as A from "@automerge/automerge"
+import assert from "assert"
+import { describe, it } from "vitest"
+import { RemoteHeadsSubscriptions } from "../src/RemoteHeadsSubscriptions.js"
+import { PeerId, StorageId } from "../src/index.js"
+import { generateAutomergeUrl, parseAutomergeUrl } from "../src/AutomergeUrl.js"
+import { pause } from "../src/helpers/pause.js"
+import { EventEmitter } from "eventemitter3"
+import {
+  RemoteHeadsChanged,
+  RemoteSubscriptionControlMessage,
+} from "../src/network/messages.js"
+
+describe("RepoHeadsSubscriptions", () => {
+  const storageA = "remote-a" as StorageId
+  const storageB = "remote-b" as StorageId
+  const storageC = "remote-c" as StorageId
+  const storageD = "remote-d" as StorageId
+  const peerA = "peer-a" as PeerId
+  const peerB = "peer-b" as PeerId
+  const peerC = "peer-c" as PeerId
+  const peerD = "peer-d" as PeerId
+
+  const { documentId: docA } = parseAutomergeUrl(generateAutomergeUrl())
+  const { documentId: docB } = parseAutomergeUrl(generateAutomergeUrl())
+  const { documentId: docC } = parseAutomergeUrl(generateAutomergeUrl())
+
+  const docAHeadsChangedForStorageB: RemoteHeadsChanged = {
+    type: "remote-heads-changed",
+    senderId: peerD,
+    targetId: peerA,
+    documentId: docA,
+    newHeads: {
+      [storageB]: {
+        heads: [],
+        timestamp: Date.now(),
+      },
+    },
+  }
+
+  const docBHeadsChangedForStorageB: RemoteHeadsChanged = {
+    type: "remote-heads-changed",
+    senderId: peerD,
+    targetId: peerA,
+    documentId: docB,
+    newHeads: {
+      [storageB]: {
+        heads: [],
+        timestamp: Date.now(),
+      },
+    },
+  }
+
+  const docBHeads = A.getHeads(
+    A.change(A.init(), doc => {
+      ;(doc as any).foo = "123"
+    })
+  )
+
+  const docBHeadsChangedForStorageB2: RemoteHeadsChanged = {
+    type: "remote-heads-changed",
+    senderId: peerD,
+    targetId: peerA,
+    documentId: docB,
+    newHeads: {
+      [storageB]: {
+        heads: docBHeads,
+        timestamp: Date.now() + 1,
+      },
+    },
+  }
+
+  const subscribePeerCToStorageB: RemoteSubscriptionControlMessage = {
+    type: "remote-subscription-change",
+    senderId: peerC,
+    targetId: peerA,
+    add: [storageB],
+  }
+
+  const unsubscribePeerCFromStorageB: RemoteSubscriptionControlMessage = {
+    type: "remote-subscription-change",
+    senderId: peerC,
+    targetId: peerA,
+    remove: [storageB],
+  }
+
+  it("should allow to subscribe and unsubscribe to storage ids", async () => {
+    const remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+
+    const remoteHeadsMessages = waitForMessages(
+      remoteHeadsSubscriptions,
+      "remote-heads-changed"
+    )
+
+    const changeRemoteSubsAfterSubscribe = waitForMessages(
+      remoteHeadsSubscriptions,
+      "change-remote-subs"
+    )
+
+    // subscribe to storageB and change storageB heads
+    remoteHeadsSubscriptions.subscribeToRemotes([storageB])
+    remoteHeadsSubscriptions.handleRemoteHeads(docAHeadsChangedForStorageB)
+
+    // receive event for new heads of storageB
+    let messages = await remoteHeadsMessages
+    assert.strictEqual(messages.length, 1)
+    assert.strictEqual(messages[0].storageId, storageB)
+    assert.strictEqual(messages[0].documentId, docA)
+    assert.deepStrictEqual(messages[0].remoteHeads, [])
+
+    // receive event for add sub to storageB
+    messages = await changeRemoteSubsAfterSubscribe
+    assert.strictEqual(messages.length, 1)
+    assert.deepStrictEqual(messages[0].add, [storageB])
+    assert.deepStrictEqual(messages[0].remove, undefined)
+    assert.deepStrictEqual(messages[0].peers, [])
+
+    const remoteHeadsMessagesAfterUnsub = waitForMessages(
+      remoteHeadsSubscriptions,
+      "change-remote-subs"
+    )
+
+    // unsubscribe from storageB
+    remoteHeadsSubscriptions.unsubscribeFromRemotes([storageB])
+
+    // receive event for remove sub from storageB
+    messages = await remoteHeadsMessagesAfterUnsub
+    assert.strictEqual(messages.length, 1)
+    assert.deepStrictEqual(messages[0].add, undefined)
+    assert.deepStrictEqual(messages[0].remove, [storageB])
+    assert.deepStrictEqual(messages[0].peers, [])
+  })
+
+  it("should forward all changes to generous peers", async () => {
+    const remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+
+    const notifyRemoteHeadsMessagesPromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "notify-remote-heads"
+    )
+
+    const changeRemoteSubsMessagesPromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "change-remote-subs"
+    )
+
+    remoteHeadsSubscriptions.addGenerousPeer(peerC)
+    remoteHeadsSubscriptions.subscribeToRemotes([storageB])
+
+    // change message for docA in storageB
+    remoteHeadsSubscriptions.handleRemoteHeads(docAHeadsChangedForStorageB)
+
+    // change heads directly, are not forwarded
+    remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
+      docC,
+      storageB,
+      []
+    )
+
+    // should forward remote-heads events
+    let messages = await notifyRemoteHeadsMessagesPromise
+    assert.strictEqual(messages.length, 1)
+    assert.strictEqual(messages[0].documentId, docA)
+    assert.strictEqual(messages[0].storageId, storageB)
+    assert.deepStrictEqual(messages[0].heads, [])
+
+    // should forward subscriptions to generous peer
+    messages = await changeRemoteSubsMessagesPromise
+    assert.strictEqual(messages.length, 1)
+    assert.deepStrictEqual(messages[0].add, [storageB])
+    assert.deepStrictEqual(messages[0].remove, undefined)
+    assert.deepStrictEqual(messages[0].peers, [peerC])
+
+    const changeRemoteSubsMessagesAfterUnsubPromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "change-remote-subs"
+    )
+
+    // unsubsscribe from storage B
+    remoteHeadsSubscriptions.unsubscribeFromRemotes([storageB])
+
+    // should forward unsubscribe to generous peer
+    messages = await changeRemoteSubsMessagesAfterUnsubPromise
+    assert.strictEqual(messages.length, 1)
+    assert.deepStrictEqual(messages[0].add, undefined)
+    assert.deepStrictEqual(messages[0].remove, [storageB])
+    assert.deepStrictEqual(messages[0].peers, [peerC])
+  })
+
+  it("should not notify generous peers of changed remote heads, if they send the heads originally", async () => {
+    const remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+
+    const messagesPromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "notify-remote-heads"
+    )
+
+    remoteHeadsSubscriptions.addGenerousPeer(peerC)
+    remoteHeadsSubscriptions.subscribeToRemotes([storageB])
+    remoteHeadsSubscriptions.handleRemoteHeads({
+      type: "remote-heads-changed",
+      senderId: peerC,
+      targetId: peerA,
+      documentId: docA,
+      newHeads: {
+        [storageB]: {
+          heads: [],
+          timestamp: Date.now(),
+        },
+      },
+    })
+
+    const messages = await messagesPromise
+    assert.strictEqual(messages.length, 0)
+  })
+
+  it("should allow peers to subscribe and unsubscribe to storageIds", async () => {
+    const remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+    remoteHeadsSubscriptions.subscribeToRemotes([storageB])
+
+    // subscribe peer c to storage b
+    remoteHeadsSubscriptions.handleControlMessage(subscribePeerCToStorageB)
+    const messagesAfterSubscribePromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "notify-remote-heads"
+    )
+
+    // change message for docA in storageB
+    remoteHeadsSubscriptions.handleRemoteHeads(docAHeadsChangedForStorageB)
+
+    // change heads directly
+    remoteHeadsSubscriptions.handleImmediateRemoteHeadsChanged(
+      docC,
+      storageB,
+      []
+    )
+
+    // expect peer c to be notified both changes
+    let messages = await messagesAfterSubscribePromise
+    assert.strictEqual(messages.length, 2)
+    assert.strictEqual(messages[0].documentId, docA)
+    assert.strictEqual(messages[0].storageId, storageB)
+    assert.deepStrictEqual(messages[0].heads, [])
+    assert.strictEqual(messages[1].documentId, docC)
+    assert.strictEqual(messages[1].storageId, storageB)
+    assert.deepStrictEqual(messages[1].heads, [])
+
+    // unsubscribe peer C
+    remoteHeadsSubscriptions.handleControlMessage(unsubscribePeerCFromStorageB)
+    const messagesAfteUnsubscribePromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "notify-remote-heads"
+    )
+
+    // heads of docB for storageB change
+    remoteHeadsSubscriptions.handleRemoteHeads(docBHeadsChangedForStorageB)
+
+    // expect not to be be notified
+    messages = await messagesAfteUnsubscribePromise
+    assert.strictEqual(messages.length, 0)
+  })
+
+  it("should ignore sync states with an older timestamp", async () => {
+    const remoteHeadsSubscription = new RemoteHeadsSubscriptions()
+
+    const messagesPromise = waitForMessages(
+      remoteHeadsSubscription,
+      "remote-heads-changed"
+    )
+
+    remoteHeadsSubscription.subscribeToRemotes([storageB])
+    remoteHeadsSubscription.handleRemoteHeads(docBHeadsChangedForStorageB2)
+
+    // send message with old heads
+    remoteHeadsSubscription.handleRemoteHeads(docBHeadsChangedForStorageB)
+
+    const messages = await messagesPromise
+    assert.strictEqual(messages.length, 1)
+    assert.strictEqual(messages[0].storageId, storageB)
+    assert.strictEqual(messages[0].documentId, docB)
+    assert.deepStrictEqual(messages[0].remoteHeads, docBHeads)
+  })
+
+  it("should remove subs of disconnected peers", async () => {
+    const remoteHeadsSubscriptions = new RemoteHeadsSubscriptions()
+
+    const messagesPromise = waitForMessages(
+      remoteHeadsSubscriptions,
+      "change-remote-subs"
+    )
+
+    remoteHeadsSubscriptions.handleControlMessage({
+      type: "remote-subscription-change",
+      senderId: peerB,
+      targetId: peerA,
+      add: [storageA, storageC],
+    })
+
+    remoteHeadsSubscriptions.handleControlMessage({
+      type: "remote-subscription-change",
+      senderId: peerC,
+      targetId: peerA,
+      add: [storageA, storageD],
+    })
+
+    remoteHeadsSubscriptions.removePeer(peerB)
+
+    const messages = await messagesPromise
+    assert.deepStrictEqual(messages.length, 3)
+
+    assert.deepStrictEqual(messages[0].add, [storageA, storageC])
+    assert.deepStrictEqual(messages[0].remove, [])
+    assert.deepStrictEqual(messages[0].peers, [])
+
+    assert.deepStrictEqual(messages[1].add, [storageD])
+    assert.deepStrictEqual(messages[1].remove, [])
+    assert.deepStrictEqual(messages[1].peers, [])
+
+    assert.deepStrictEqual(messages[2].add, undefined)
+    assert.deepStrictEqual(messages[2].remove, [storageC])
+    assert.deepStrictEqual(messages[2].peers, [])
+  })
+})
+
+async function waitForMessages(
+  emitter: EventEmitter,
+  event: string,
+  timeout: number = 100
+): Promise<any[]> {
+  const messages = []
+
+  const onEvent = message => {
+    messages.push(message)
+  }
+
+  emitter.on(event, onEvent)
+
+  await pause(timeout)
+
+  emitter.off(event)
+
+  return messages
+}