@automerge/automerge-repo 0.2.1 → 1.0.0-alpha.0

package/src/helpers/tests/network-adapter-tests.ts

@@ -37,7 +37,7 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Bob receives the document
         await eventPromise(bobRepo, "document")
-        const bobHandle = bobRepo.find<TestDoc>(aliceHandle.documentId)
+        const bobHandle = bobRepo.find<TestDoc>(aliceHandle.url)
 
         // Alice changes the document
         aliceHandle.change(d => {
@@ -46,7 +46,7 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Bob receives the change
         await eventPromise(bobHandle, "change")
-        assert.equal((await bobHandle.value()).foo, "bar")
+        assert.equal((await bobHandle.doc()).foo, "bar")
 
         // Bob changes the document
         bobHandle.change(d => {
@@ -55,7 +55,7 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
         // Alice receives the change
         await eventPromise(aliceHandle, "change")
-        assert.equal((await aliceHandle.value()).foo, "baz")
+        assert.equal((await aliceHandle.doc()).foo, "baz")
       }
 
       // Run the test in both directions, in case they're different types of adapters
@@ -83,12 +83,12 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Alice creates a document
       const aliceHandle = aliceRepo.create<TestDoc>()
-      const documentId = aliceHandle.documentId
+      const docUrl = aliceHandle.url
 
       // Bob and Charlie receive the document
       await eventPromises([bobRepo, charlieRepo], "document")
-      const bobHandle = bobRepo.find<TestDoc>(documentId)
-      const charlieHandle = charlieRepo.find<TestDoc>(documentId)
+      const bobHandle = bobRepo.find<TestDoc>(docUrl)
+      const charlieHandle = charlieRepo.find<TestDoc>(docUrl)
 
       // Alice changes the document
       aliceHandle.change(d => {
@@ -97,8 +97,8 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Bob and Charlie receive the change
       await eventPromises([bobHandle, charlieHandle], "change")
-      assert.equal((await bobHandle.value()).foo, "bar")
-      assert.equal((await charlieHandle.value()).foo, "bar")
+      assert.equal((await bobHandle.doc()).foo, "bar")
+      assert.equal((await charlieHandle.doc()).foo, "bar")
 
       // Charlie changes the document
       charlieHandle.change(d => {
@@ -107,8 +107,8 @@ export function runAdapterTests(_setup: SetupFn, title?: string): void {
 
       // Alice and Bob receive the change
       await eventPromises([aliceHandle, bobHandle], "change")
-      assert.equal((await bobHandle.value()).foo, "baz")
-      assert.equal((await charlieHandle.value()).foo, "baz")
+      assert.equal((await bobHandle.doc()).foo, "baz")
+      assert.equal((await charlieHandle.doc()).foo, "baz")
 
       teardown()
     })

package/src/index.ts

@@ -1,9 +1,6 @@
 export { DocCollection } from "./DocCollection.js"
 export { DocHandle, HandleState } from "./DocHandle.js"
-export type {
-  DocHandleChangePayload,
-  DocHandlePatchPayload,
-} from "./DocHandle.js"
+export type { DocHandleChangePayload } from "./DocHandle.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
 export type {
   InboundMessagePayload,
@@ -14,7 +11,12 @@ export type {
 } from "./network/NetworkAdapter.js"
 export { NetworkSubsystem } from "./network/NetworkSubsystem.js"
 export { Repo, type SharePolicy } from "./Repo.js"
-export { StorageAdapter } from "./storage/StorageAdapter.js"
+export { StorageAdapter, type StorageKey } from "./storage/StorageAdapter.js"
 export { StorageSubsystem } from "./storage/StorageSubsystem.js"
 export { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
+export {
+  parseAutomergeUrl,
+  isValidAutomergeUrl,
+  stringifyAutomergeUrl as generateAutomergeUrl,
+} from "./DocUrl.js"
 export * from "./types.js"

package/src/network/NetworkAdapter.ts

@@ -13,9 +13,9 @@ export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents>
     broadcast: boolean
   ): void
 
-  abstract join(channelId: ChannelId): void
+  abstract join(): void
 
-  abstract leave(channelId: ChannelId): void
+  abstract leave(): void
 }
 
 // events & payloads
@@ -34,7 +34,6 @@ export interface OpenPayload {
 
 export interface PeerCandidatePayload {
   peerId: PeerId
-  channelId: ChannelId
 }
 
 export interface MessagePayload {

package/src/network/NetworkSubsystem.ts

@@ -11,7 +11,6 @@ import debug from "debug"
 export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #log: debug.Debugger
   #adaptersByPeer: Record<PeerId, NetworkAdapter> = {}
-  #channels: ChannelId[]
 
   constructor(
     private adapters: NetworkAdapter[],
@@ -19,14 +18,13 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   ) {
     super()
     this.#log = debug(`automerge-repo:network:${this.peerId}`)
-    this.#channels = []
     this.adapters.forEach(a => this.addNetworkAdapter(a))
   }
 
   addNetworkAdapter(networkAdapter: NetworkAdapter) {
     networkAdapter.connect(this.peerId)
 
-    networkAdapter.on("peer-candidate", ({ peerId, channelId }) => {
+    networkAdapter.on("peer-candidate", ({ peerId }) => {
       this.#log(`peer candidate: ${peerId} `)
 
       // TODO: This is where authentication would happen
@@ -36,7 +34,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
         this.#adaptersByPeer[peerId] = networkAdapter
       }
 
-      this.emit("peer", { peerId, channelId })
+      this.emit("peer", { peerId })
     })
 
     networkAdapter.on("peer-disconnected", ({ peerId }) => {
@@ -74,7 +72,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       })
     })
 
-    this.#channels.forEach(c => networkAdapter.join(c))
+    networkAdapter.join()
   }
 
   sendMessage(
@@ -99,16 +97,14 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     }
   }
 
-  join(channelId: ChannelId) {
-    this.#log(`Joining channel ${channelId}`)
-    this.#channels.push(channelId)
-    this.adapters.forEach(a => a.join(channelId))
+  join() {
+    this.#log(`Joining network`)
+    this.adapters.forEach(a => a.join())
   }
 
-  leave(channelId: ChannelId) {
-    this.#log(`Leaving channel ${channelId}`)
-    this.#channels = this.#channels.filter(c => c !== channelId)
-    this.adapters.forEach(a => a.leave(channelId))
+  leave() {
+    this.#log(`Leaving network`)
+    this.adapters.forEach(a => a.leave())
   }
 }
 
@@ -126,5 +122,4 @@ export interface NetworkSubsystemEvents {
 
 export interface PeerPayload {
   peerId: PeerId
-  channelId: ChannelId
 }

package/src/storage/StorageAdapter.ts

@@ -4,15 +4,17 @@ export abstract class StorageAdapter {
   // [documentId, "snapshot"] or [documentId, "incremental", "0"]
   // but the storage adapter is agnostic to the meaning of the key
   // and we expect to store other data in the future such as syncstates
-  abstract load(key: string[]): Promise<Uint8Array | undefined>
-  abstract save(key: string[], data: Uint8Array): Promise<void>
-  abstract remove(key: string[]): Promise<void>
+  abstract load(key: StorageKey): Promise<Uint8Array | undefined>
+  abstract save(key: StorageKey, data: Uint8Array): Promise<void>
+  abstract remove(key: StorageKey): Promise<void>
 
   // the keyprefix will match any key that starts with the given array
   // for example, [documentId, "incremental"] will match all incremental saves
   // or [documentId] will match all data for a given document
   // be careful! this will also match [documentId, "syncState"]!
   // (we aren't using this yet but keep it in mind.)
-  abstract loadRange(keyPrefix: string[]): Promise<Uint8Array[]>
-  abstract removeRange(keyPrefix: string[]): Promise<void>
+  abstract loadRange(keyPrefix: StorageKey): Promise<{key: StorageKey, data: Uint8Array}[]>
+  abstract removeRange(keyPrefix: StorageKey): Promise<void>
 }
+
+export type  StorageKey = string[]

package/src/storage/StorageSubsystem.ts

@@ -1,8 +1,18 @@
 import * as A from "@automerge/automerge"
-import { DocumentId } from "../types.js"
-import { StorageAdapter } from "./StorageAdapter.js"
-import { mergeArrays } from "../helpers/mergeArrays.js"
+import { StorageAdapter, StorageKey } from "./StorageAdapter.js"
 import * as sha256 from "fast-sha256"
+import { type DocumentId } from "../types.js"
+import { mergeArrays } from "../helpers/mergeArrays.js"
+
+// Metadata about a chunk of data loaded from storage. This is stored on the
+// StorageSubsystem so when we are compacting we know what chunks we can safely delete
+type StorageChunkInfo = {
+  key: StorageKey
+  type: ChunkType
+  size: number
+}
+
+export type ChunkType = "snapshot" | "incremental"
 
 function keyHash(binary: Uint8Array) {
   const hash = sha256.hash(binary)
@@ -11,57 +21,87 @@ function keyHash(binary: Uint8Array) {
   return hashHex
 }
 
+function headsHash(heads: A.Heads): string {
+  let encoder = new TextEncoder()
+  let headsbinary = mergeArrays(heads.map(h => encoder.encode(h)))
+  return keyHash(headsbinary)
+}
+
 export class StorageSubsystem {
   #storageAdapter: StorageAdapter
+  #chunkInfos: Map<DocumentId, StorageChunkInfo[]> = new Map()
 
   constructor(storageAdapter: StorageAdapter) {
     this.#storageAdapter = storageAdapter
   }
 
-  async #saveIncremental(documentId: DocumentId, doc: A.Doc<unknown>) {
+  async #saveIncremental(
+    documentId: DocumentId,
+    doc: A.Doc<unknown>
+  ): Promise<void> {
     const binary = A.saveIncremental(doc)
     if (binary && binary.length > 0) {
       const key = [documentId, "incremental", keyHash(binary)]
-      return await this.#storageAdapter.save(key, binary)
+      await this.#storageAdapter.save(key, binary)
+      if (!this.#chunkInfos.has(documentId)) {
+        this.#chunkInfos.set(documentId, [])
+      }
+      this.#chunkInfos.get(documentId)!!.push({
+        key,
+        type: "incremental",
+        size: binary.length,
+      })
     } else {
-      Promise.resolve()
+      return Promise.resolve()
     }
   }
 
-  async #saveTotal(documentId: DocumentId, doc: A.Doc<unknown>) {
+  async #saveTotal(
+    documentId: DocumentId,
+    doc: A.Doc<unknown>,
+    sourceChunks: StorageChunkInfo[]
+  ): Promise<void> {
     const binary = A.save(doc)
+    const key = [documentId, "snapshot", headsHash(A.getHeads(doc))]
+    const oldKeys = new Set(sourceChunks.map(c => c.key))
 
-    // TODO: this is still racy if two nodes are both writing to the store
-    await this.#storageAdapter.save([documentId, "snapshot"], binary)
+    await this.#storageAdapter.save(key, binary)
 
-    // don't start deleting the incremental keys until save is done!
-    return this.#storageAdapter.removeRange([documentId, "incremental"])
+    for (const key of oldKeys) {
+      await this.#storageAdapter.remove(key)
+    }
+    const newChunkInfos =
+      this.#chunkInfos.get(documentId)?.filter(c => !oldKeys.has(c.key)) ?? []
+    newChunkInfos.push({ key, type: "snapshot", size: binary.length })
+    this.#chunkInfos.set(documentId, newChunkInfos)
   }
 
   async loadBinary(documentId: DocumentId): Promise<Uint8Array> {
-    // it would probably be best to ensure .snapshot comes back first
-    // prevent the race condition with saveIncremental
-    const binaries: Uint8Array[] = await this.#storageAdapter.loadRange([
-      documentId,
-    ])
-
+    const loaded = await this.#storageAdapter.loadRange([documentId])
+    const binaries = []
+    const chunkInfos: StorageChunkInfo[] = []
+    for (const chunk of loaded) {
+      const chunkType = chunkTypeFromKey(chunk.key)
+      if (chunkType == null) {
+        continue
+      }
+      chunkInfos.push({
+        key: chunk.key,
+        type: chunkType,
+        size: chunk.data.length,
+      })
+      binaries.push(chunk.data)
+    }
+    this.#chunkInfos.set(documentId, chunkInfos)
     return mergeArrays(binaries)
   }
 
-  async load<T>(
-    documentId: DocumentId,
-    prevDoc: A.Doc<T> = A.init<T>()
-  ): Promise<A.Doc<T>> {
-    const doc = A.loadIncremental(prevDoc, await this.loadBinary(documentId))
-    A.saveIncremental(doc)
-    return doc
-  }
-
-  async save(documentId: DocumentId, doc: A.Doc<unknown>) {
-    if (this.#shouldCompact(documentId)) {
-      return this.#saveTotal(documentId, doc)
+  async save(documentId: DocumentId, doc: A.Doc<unknown>): Promise<void> {
+    let sourceChunks = this.#chunkInfos.get(documentId) ?? []
+    if (this.#shouldCompact(sourceChunks)) {
+      this.#saveTotal(documentId, doc, sourceChunks)
     } else {
-      return this.#saveIncremental(documentId, doc)
+      this.#saveIncremental(documentId, doc)
     }
   }
 
@@ -70,9 +110,30 @@ export class StorageSubsystem {
     this.#storageAdapter.removeRange([documentId, "incremental"])
   }
 
-  // TODO: make this, you know, good.
-  // this is probably fine
-  #shouldCompact(documentId: DocumentId) {
-    return Math.random() < 0.05 // this.#changeCount[documentId] >= 20
+  #shouldCompact(sourceChunks: StorageChunkInfo[]) {
+    // compact if the incremental size is greater than the snapshot size
+    let snapshotSize = 0
+    let incrementalSize = 0
+    for (const chunk of sourceChunks) {
+      if (chunk.type === "snapshot") {
+        snapshotSize += chunk.size
+      } else {
+        incrementalSize += chunk.size
+      }
+    }
+    return incrementalSize > snapshotSize
+  }
+}
+
+function chunkTypeFromKey(key: StorageKey): ChunkType | null {
+  if (key.length < 2) {
+    return null
+  }
+  const chunkTypeStr = key[key.length - 2]
+  if (chunkTypeStr === "snapshot" || chunkTypeStr === "incremental") {
+    const chunkType: ChunkType = chunkTypeStr
+    return chunkType
+  } else {
+    return null
   }
 }

package/src/synchronizer/CollectionSynchronizer.ts

@@ -1,6 +1,11 @@
 import { DocCollection } from "../DocCollection.js"
 import { DocHandle } from "../DocHandle.js"
-import { ChannelId, DocumentId, PeerId } from "../types.js"
+import {
+  documentIdToBinary,
+  binaryToDocumentId,
+  stringifyAutomergeUrl,
+} from "../DocUrl.js"
+import { ChannelId, BinaryDocumentId, PeerId, DocumentId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
 
@@ -22,7 +27,7 @@ export class CollectionSynchronizer extends Synchronizer {
   /** Returns a synchronizer for the given document, creating one if it doesn't already exist.  */
   #fetchDocSynchronizer(documentId: DocumentId) {
     if (!this.#docSynchronizers[documentId]) {
-      const handle = this.repo.find(documentId)
+      const handle = this.repo.find(stringifyAutomergeUrl({ documentId }))
       this.#docSynchronizers[documentId] = this.#initDocSynchronizer(handle)
     }
     return this.#docSynchronizers[documentId]
@@ -60,6 +65,9 @@ export class CollectionSynchronizer extends Synchronizer {
     log(`onSyncMessage: ${peerId}, ${channelId}, ${message.byteLength}bytes`)
 
     const documentId = channelId as unknown as DocumentId
+    if (!documentId) {
+      throw new Error("received a message with an invalid documentId")
+    }
     const docSynchronizer = await this.#fetchDocSynchronizer(documentId)
 
     await docSynchronizer.receiveSyncMessage(peerId, channelId, message)

package/src/synchronizer/DocSynchronizer.ts

@@ -1,5 +1,5 @@
 import * as A from "@automerge/automerge"
-import { DocHandle } from "../DocHandle.js"
+import { DocHandle, READY, REQUESTING } from "../DocHandle.js"
 import { ChannelId, PeerId } from "../types.js"
 import { Synchronizer } from "./Synchronizer.js"
 
@@ -33,7 +33,7 @@ export class DocSynchronizer extends Synchronizer {
 
     // Process pending sync messages immediately after the handle becomes ready.
     void (async () => {
-      await handle.loadAttemptedValue()
+      await handle.doc([READY, REQUESTING])
       this.#processAllPendingSyncMessages()
     })()
   }
@@ -46,7 +46,7 @@ export class DocSynchronizer extends Synchronizer {
 
   async #syncWithPeers() {
     this.#log(`syncWithPeers`)
-    const doc = await this.handle.value()
+    const doc = await this.handle.doc()
     this.#peers.forEach(peerId => this.#sendSyncMessage(peerId, doc))
   }
 
@@ -78,6 +78,7 @@ export class DocSynchronizer extends Synchronizer {
       this.#logMessage(`sendSyncMessage 🡒 ${peerId}`, message)
 
       const channelId = this.handle.documentId as string as ChannelId
+
       this.emit("message", {
         targetId: peerId,
         channelId,
@@ -120,7 +121,7 @@ export class DocSynchronizer extends Synchronizer {
 
     // At this point if we don't have anything in our storage, we need to use an empty doc to sync
     // with; but we don't want to surface that state to the front end
-    void this.handle.loadAttemptedValue().then(doc => {
+    void this.handle.doc([READY, REQUESTING]).then(doc => {
       // HACK: if we have a sync state already, we round-trip it through the encoding system to make
       // sure state is preserved. This prevents an infinite loop caused by failed attempts to send
       // messages during disconnection.
@@ -143,11 +144,11 @@ export class DocSynchronizer extends Synchronizer {
     channelId: ChannelId,
     message: Uint8Array
   ) {
-    if ((channelId as string) !== (this.documentId as string))
+    if ((channelId as string) !== (this.handle.documentId as string))
       throw new Error(`channelId doesn't match documentId`)
 
     // We need to block receiving the syncMessages until we've checked local storage
-    if (!this.handle.isReadyOrRequesting()) {
+    if (!this.handle.inState([READY, REQUESTING])) {
       this.#pendingSyncMessages.push({ peerId, message })
       return
     }

package/src/types.ts

@@ -1,3 +1,6 @@
-export type DocumentId = string & { __documentId: true }
+export type DocumentId = string & { __documentId: true } // for logging
+export type AutomergeUrl = string & { __documentUrl: true } // for opening / linking
+export type BinaryDocumentId = Uint8Array & { __binaryDocumentId: true } // for storing / syncing
+
 export type PeerId = string & { __peerId: false }
 export type ChannelId = string & { __channelId: false }

package/test/CollectionSynchronizer.test.ts

@@ -1,5 +1,5 @@
 import { CollectionSynchronizer } from "../src/synchronizer/CollectionSynchronizer.js"
-import { ChannelId, DocCollection, DocumentId, PeerId } from "../src"
+import { ChannelId, DocCollection, BinaryDocumentId, PeerId } from "../src"
 import assert from "assert"
 import { beforeEach } from "mocha"
 import { MessagePayload } from "../src/network/NetworkAdapter.js"

package/test/DocCollection.test.ts

@@ -1,8 +1,9 @@
 import assert from "assert"
-import { DocCollection, DocumentId } from "../src"
+import { DocCollection, BinaryDocumentId } from "../src"
 import { TestDoc } from "./types.js"
+import { generateAutomergeUrl, stringifyAutomergeUrl } from "../src/DocUrl"
 
-const MISSING_DOCID = "non-existent-docID" as DocumentId
+const MISSING_DOCID = generateAutomergeUrl()
 
 describe("DocCollection", () => {
   it("can create documents which are ready to go", async () => {