@automerge/automerge-repo 1.0.12 → 1.0.14

package/src/helpers/cbor.ts

@@ -1,7 +1,7 @@
 import { Encoder, decode as cborXdecode } from "cbor-x"
 
 export function encode(obj: unknown): Buffer {
-  const encoder = new Encoder({ tagUint8Array: false })
+  const encoder = new Encoder({ tagUint8Array: false, useRecords: false })
   return encoder.encode(obj)
 }
 

package/src/index.ts

@@ -31,7 +31,7 @@ export {
   isValidAutomergeUrl,
   parseAutomergeUrl,
   stringifyAutomergeUrl,
-} from "./DocUrl.js"
+} from "./AutomergeUrl.js"
 export { Repo } from "./Repo.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
 export { isValidRepoMessage } from "./network/messages.js"
@@ -52,6 +52,7 @@ export type {
   DocHandleOutboundEphemeralMessagePayload,
   HandleState,
 } from "./DocHandle.js"
+
 export type {
   DeleteDocumentPayload,
   DocumentPayload,
@@ -59,21 +60,28 @@ export type {
   RepoEvents,
   SharePolicy,
 } from "./Repo.js"
+
 export type {
   NetworkAdapterEvents,
   OpenPayload,
   PeerCandidatePayload,
   PeerDisconnectedPayload,
 } from "./network/NetworkAdapter.js"
+
 export type {
-  ArriveMessage,
   DocumentUnavailableMessage,
   EphemeralMessage,
   Message,
   RepoMessage,
   RequestMessage,
   SyncMessage,
-  WelcomeMessage,
 } from "./network/messages.js"
-export type { StorageKey } from "./storage/StorageAdapter.js"
+
+export type {
+  Chunk,
+  ChunkInfo,
+  ChunkType,
+  StorageKey,
+} from "./storage/types.js"
+
 export * from "./types.js"

package/src/network/NetworkAdapter.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from "eventemitter3"
 import { PeerId } from "../types.js"
-import { RepoMessage } from "./messages.js"
+import { Message } from "./messages.js"
 
 /** An interface representing some way to connect to other peers
  *
@@ -22,7 +22,7 @@ export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents>
    *
    * @argument message - the message to send
    */
-  abstract send(message: RepoMessage): void
+  abstract send(message: Message): void
 
   /** Called by the {@link Repo} to disconnect from the network */
   abstract disconnect(): void
@@ -44,7 +44,7 @@ export interface NetworkAdapterEvents {
   "peer-disconnected": (payload: PeerDisconnectedPayload) => void
 
   /** Emitted when the network adapter receives a message from a peer */
-  message: (payload: RepoMessage) => void
+  message: (payload: Message) => void
 }
 
 export interface OpenPayload {

package/src/network/messages.ts

@@ -87,26 +87,18 @@ export type RequestMessage = {
   documentId: DocumentId
 }
 
-/** Notify the network that we have arrived so everyone knows our peer ID */
-export type ArriveMessage = {
-  type: "arrive"
+/** (anticipating work in progress) */
+export type AuthMessage<TPayload = any> = {
+  type: "auth"
 
   /** The peer ID of the sender of this message */
   senderId: PeerId
 
-  /** Arrive messages don't have a targetId */
-  targetId: never
-}
-
-/** Respond to an arriving peer with our peer ID */
-export type WelcomeMessage = {
-  type: "welcome"
-
-  /** The peer ID of the recipient sender this message */
-  senderId: PeerId
-
   /** The peer ID of the recipient of this message */
   targetId: PeerId
+
+  /** The payload of the auth message (up to the specific auth provider) */
+  payload: TPayload
 }
 
 /** These are message types that a {@link NetworkAdapter} surfaces to a {@link Repo}. */
@@ -116,13 +108,8 @@ export type RepoMessage =
   | RequestMessage
   | DocumentUnavailableMessage
 
-/** These are all the message types that a {@link NetworkAdapter} might see.
- *
- * @remarks
- * It is not _required_ that a {@link NetworkAdapter} use these types: They are free to use
- * whatever message type makes sense for their transport. However, this type is a useful default.
- * */
-export type Message = RepoMessage | ArriveMessage | WelcomeMessage
+/** These are all the message types that a {@link NetworkAdapter} might see. */
+export type Message = RepoMessage | AuthMessage
 
 /**
  * The contents of a message, without the sender ID or other properties added by the {@link NetworkSubsystem})

package/src/storage/StorageAdapter.ts

@@ -1,41 +1,34 @@
+import { StorageKey, Chunk } from "./types.js"
+
 /** A storage adapter represents some way of storing binary data for a {@link Repo}
  *
  * @remarks
- * `StorageAdapter`s are a little like a key/value store. The keys are arrays
- * of strings ({@link StorageKey}) and the values are binary blobs.
+ * `StorageAdapter`s provide a key/value storage interface. The keys are arrays of strings
+ * ({@link StorageKey}) and the values are binary blobs.
  */
 export abstract class StorageAdapter {
-  // load, store, or remove a single binary blob based on an array key
-  // automerge-repo mostly uses keys in the following form:
-  // [documentId, "snapshot"] or [documentId, "incremental", "0"]
-  // but the storage adapter is agnostic to the meaning of the key
-  // and we expect to store other data in the future such as syncstates
-  /** Load the single blob correspongind to `key` */
+  /** Load the single value corresponding to `key` */
   abstract load(key: StorageKey): Promise<Uint8Array | undefined>
-  /** save the blod `data` to the key `key` */
+
+  /** Save the value `data` to the key `key` */
   abstract save(key: StorageKey, data: Uint8Array): Promise<void>
-  /** remove the blob corresponding to `key` */
+
+  /** Remove the value corresponding to `key` */
   abstract remove(key: StorageKey): Promise<void>
 
-  // the keyprefix will match any key that starts with the given array
-  // for example, [documentId, "incremental"] will match all incremental saves
-  // or [documentId] will match all data for a given document
-  // be careful! this will also match [documentId, "syncState"]!
-  // (we aren't using this yet but keep it in mind.)
-  /** Load all blobs with keys that start with `keyPrefix` */
-  abstract loadRange(keyPrefix: StorageKey): Promise<{key: StorageKey, data: Uint8Array}[]>
-  /** Remove all blobs with keys that start with `keyPrefix` */
+  /**
+   * Load all values with keys that start with `keyPrefix`.
+   *
+   * @remarks
+   * The `keyprefix` will match any key that starts with the given array. For example:
+   * - `[documentId, "incremental"]` will match all incremental saves
+   * - `[documentId]` will match all data for a given document.
+   *
+   * Be careful! `[documentId]` would also match something like `[documentId, "syncState"]`! We
+   * aren't using this yet but keep it in mind.)
+   */
+  abstract loadRange(keyPrefix: StorageKey): Promise<Chunk[]>
+
+  /** Remove all values with keys that start with `keyPrefix` */
   abstract removeRange(keyPrefix: StorageKey): Promise<void>
 }
-
-/** The type of keys for a {@link StorageAdapter}
- *
- * @remarks
- * Storage keys are arrays because they are hierarchical and the storage 
- * subsystem will need to be able to do range queries for all keys that
- * have a particular prefix. For example, incremental changes for a given
- * document might be stored under `[<documentId>, "incremental", <SHA256>]`.
- * `StorageAdapter` implementations should not assume any particular structure
- * though.
- **/
-export type  StorageKey = string[]

package/src/storage/StorageSubsystem.ts

@@ -1,46 +1,156 @@
 import * as A from "@automerge/automerge/next"
 import debug from "debug"
-import * as sha256 from "fast-sha256"
 import { headsAreSame } from "../helpers/headsAreSame.js"
 import { mergeArrays } from "../helpers/mergeArrays.js"
 import { type DocumentId } from "../types.js"
-import { StorageAdapter, StorageKey } from "./StorageAdapter.js"
-
-// Metadata about a chunk of data loaded from storage. This is stored on the
-// StorageSubsystem so when we are compacting we know what chunks we can safely delete
-type StorageChunkInfo = {
-  key: StorageKey
-  type: ChunkType
-  size: number
-}
+import { StorageAdapter } from "./StorageAdapter.js"
+import { ChunkInfo, StorageKey } from "./types.js"
+import { keyHash, headsHash } from "./keyHash.js"
+import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
+
+/**
+ * The storage subsystem is responsible for saving and loading Automerge documents to and from
+ * storage adapter. It also provides a generic key/value storage interface for other uses.
+ */
+export class StorageSubsystem {
+  /** The storage adapter to use for saving and loading documents */
+  #storageAdapter: StorageAdapter
 
-export type ChunkType = "snapshot" | "incremental"
+  /** Record of the latest heads we've loaded or saved for each document  */
+  #storedHeads: Map<DocumentId, A.Heads> = new Map()
 
-function keyHash(binary: Uint8Array) {
-  const hash = sha256.hash(binary)
-  const hashArray = Array.from(new Uint8Array(hash)) // convert buffer to byte array
-  const hashHex = hashArray.map(b => ("00" + b.toString(16)).slice(-2)).join("") // convert bytes to hex string
-  return hashHex
-}
+  /** Metadata on the chunks we've already loaded for each document */
+  #chunkInfos: Map<DocumentId, ChunkInfo[]> = new Map()
 
-function headsHash(heads: A.Heads): string {
-  const encoder = new TextEncoder()
-  const headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
-  return keyHash(headsbinary)
-}
+  /** Flag to avoid compacting when a compaction is already underway */
+  #compacting = false
 
-export class StorageSubsystem {
-  #storageAdapter: StorageAdapter
-  #chunkInfos: Map<DocumentId, StorageChunkInfo[]> = new Map()
-  #storedHeads: Map<DocumentId, A.Heads> = new Map()
   #log = debug(`automerge-repo:storage-subsystem`)
 
-  #snapshotting = false
-
   constructor(storageAdapter: StorageAdapter) {
     this.#storageAdapter = storageAdapter
   }
 
+  // ARBITRARY KEY/VALUE STORAGE
+
+  // The `load`, `save`, and `remove` methods are for generic key/value storage, as opposed to
+  // Automerge documents. For example, they're used by the LocalFirstAuthProvider to persist the
+  // encrypted team graph that encodes group membership and permissions.
+  //
+  // The namespace parameter is to prevent collisions with other users of the storage subsystem.
+  // Typically this will be the name of the plug-in, adapter, or other system that is using it. For
+  // example, the LocalFirstAuthProvider uses the namespace `LocalFirstAuthProvider`.
+
+  /** Loads a value from storage. */
+  async load(
+    /** Namespace to prevent collisions with other users of the storage subsystem. */
+    namespace: string,
+
+    /** Key to load. Typically a UUID or other unique identifier, but could be any string. */
+    key: string
+  ): Promise<Uint8Array | undefined> {
+    const storageKey = [namespace, key] as StorageKey
+    return await this.#storageAdapter.load(storageKey)
+  }
+
+  /** Saves a value in storage. */
+  async save(
+    /** Namespace to prevent collisions with other users of the storage subsystem. */
+    namespace: string,
+
+    /** Key to load. Typically a UUID or other unique identifier, but could be any string. */
+    key: string,
+
+    /** Data to save, as a binary blob. */
+    data: Uint8Array
+  ): Promise<void> {
+    const storageKey = [namespace, key] as StorageKey
+    await this.#storageAdapter.save(storageKey, data)
+  }
+
+  /** Removes a value from storage. */
+  async remove(
+    /** Namespace to prevent collisions with other users of the storage subsystem. */
+    namespace: string,
+
+    /** Key to remove. Typically a UUID or other unique identifier, but could be any string. */
+    key: string
+  ): Promise<void> {
+    const storageKey = [namespace, key] as StorageKey
+    await this.#storageAdapter.remove(storageKey)
+  }
+
+  // AUTOMERGE DOCUMENT STORAGE
+
+  /**
+   * Loads the Automerge document with the given ID from storage.
+   */
+  async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
+    // Load all the chunks for this document
+    const chunks = await this.#storageAdapter.loadRange([documentId])
+    const binaries = []
+    const chunkInfos: ChunkInfo[] = []
+
+    for (const chunk of chunks) {
+      // chunks might have been deleted in the interim
+      if (chunk.data === undefined) continue
+
+      const chunkType = chunkTypeFromKey(chunk.key)
+      if (chunkType == null) continue
+
+      chunkInfos.push({
+        key: chunk.key,
+        type: chunkType,
+        size: chunk.data.length,
+      })
+      binaries.push(chunk.data)
+    }
+    this.#chunkInfos.set(documentId, chunkInfos)
+
+    // Merge the chunks into a single binary
+    const binary = mergeArrays(binaries)
+    if (binary.length === 0) return null
+
+    // Load into an Automerge document
+    const newDoc = A.loadIncremental(A.init(), binary) as A.Doc<T>
+
+    // Record the latest heads for the document
+    this.#storedHeads.set(documentId, A.getHeads(newDoc))
+
+    return newDoc
+  }
+
+  /**
+   * Saves the provided Automerge document to storage.
+   *
+   * @remarks
+   * Under the hood this makes incremental saves until the incremental size is greater than the
+   * snapshot size, at which point the document is compacted into a single snapshot.
+   */
+  async saveDoc(documentId: DocumentId, doc: A.Doc<unknown>): Promise<void> {
+    // Don't bother saving if the document hasn't changed
+    if (!this.#shouldSave(documentId, doc)) return
+
+    const sourceChunks = this.#chunkInfos.get(documentId) ?? []
+    if (this.#shouldCompact(sourceChunks)) {
+      await this.#saveTotal(documentId, doc, sourceChunks)
+    } else {
+      await this.#saveIncremental(documentId, doc)
+    }
+    this.#storedHeads.set(documentId, A.getHeads(doc))
+  }
+
+  /**
+   * Removes the Automerge document with the given ID from storage
+   */
+  async removeDoc(documentId: DocumentId) {
+    await this.#storageAdapter.removeRange([documentId, "snapshot"])
+    await this.#storageAdapter.removeRange([documentId, "incremental"])
+  }
+
+  /**
+   * Saves just the incremental changes since the last save.
+   */
   async #saveIncremental(
     documentId: DocumentId,
     doc: A.Doc<unknown>
@@ -64,12 +174,16 @@ export class StorageSubsystem {
     }
   }
 
+  /**
+   * Compacts the document storage into a single shapshot.
+   */
   async #saveTotal(
     documentId: DocumentId,
     doc: A.Doc<unknown>,
-    sourceChunks: StorageChunkInfo[]
+    sourceChunks: ChunkInfo[]
   ): Promise<void> {
-    this.#snapshotting = true
+    this.#compacting = true
+
     const binary = A.save(doc)
     const snapshotHash = headsHash(A.getHeads(doc))
     const key = [documentId, "snapshot", snapshotHash]
@@ -85,76 +199,41 @@ export class StorageSubsystem {
     for (const key of oldKeys) {
       await this.#storageAdapter.remove(key)
     }
+
     const newChunkInfos =
       this.#chunkInfos.get(documentId)?.filter(c => !oldKeys.has(c.key)) ?? []
     newChunkInfos.push({ key, type: "snapshot", size: binary.length })
-    this.#chunkInfos.set(documentId, newChunkInfos)
-    this.#snapshotting = false
-  }
-
-  async loadDoc(documentId: DocumentId): Promise<A.Doc<unknown> | null> {
-    const loaded = await this.#storageAdapter.loadRange([documentId])
-    const binaries = []
-    const chunkInfos: StorageChunkInfo[] = []
-    for (const chunk of loaded) {
-      const chunkType = chunkTypeFromKey(chunk.key)
-      if (chunkType == null) {
-        continue
-      }
-      chunkInfos.push({
-        key: chunk.key,
-        type: chunkType,
-        size: chunk.data.length,
-      })
-      binaries.push(chunk.data)
-    }
-    this.#chunkInfos.set(documentId, chunkInfos)
-    const binary = mergeArrays(binaries)
-    if (binary.length === 0) {
-      return null
-    }
-    const newDoc = A.loadIncremental(A.init(), binary)
-    this.#storedHeads.set(documentId, A.getHeads(newDoc))
-    return newDoc
-  }
 
-  async saveDoc(documentId: DocumentId, doc: A.Doc<unknown>): Promise<void> {
-    if (!this.#shouldSave(documentId, doc)) {
-      return
-    }
-    const sourceChunks = this.#chunkInfos.get(documentId) ?? []
-    if (this.#shouldCompact(sourceChunks)) {
-      void this.#saveTotal(documentId, doc, sourceChunks)
-    } else {
-      void this.#saveIncremental(documentId, doc)
-    }
-    this.#storedHeads.set(documentId, A.getHeads(doc))
-  }
+    this.#chunkInfos.set(documentId, newChunkInfos)
 
-  async remove(documentId: DocumentId) {
-    void this.#storageAdapter.removeRange([documentId, "snapshot"])
-    void this.#storageAdapter.removeRange([documentId, "incremental"])
+    this.#compacting = false
   }
 
+  /**
+   * Returns true if the document has changed since the last time it was saved.
+   */
   #shouldSave(documentId: DocumentId, doc: A.Doc<unknown>): boolean {
     const oldHeads = this.#storedHeads.get(documentId)
     if (!oldHeads) {
+      // we haven't saved this document before
       return true
     }
 
     const newHeads = A.getHeads(doc)
     if (headsAreSame(newHeads, oldHeads)) {
+      // the document hasn't changed
       return false
     }
 
-    return true
+    return true // the document has changed
   }
 
-  #shouldCompact(sourceChunks: StorageChunkInfo[]) {
-    if (this.#snapshotting) {
-      return false
-    }
-    // compact if the incremental size is greater than the snapshot size
+  /**
+   * We only compact if the incremental size is greater than the snapshot size.
+   */
+  #shouldCompact(sourceChunks: ChunkInfo[]) {
+    if (this.#compacting) return false
+
     let snapshotSize = 0
     let incrementalSize = 0
     for (const chunk of sourceChunks) {
@@ -167,16 +246,3 @@ export class StorageSubsystem {
     return incrementalSize >= snapshotSize
   }
 }
-
-function chunkTypeFromKey(key: StorageKey): ChunkType | null {
-  if (key.length < 2) {
-    return null
-  }
-  const chunkTypeStr = key[key.length - 2]
-  if (chunkTypeStr === "snapshot" || chunkTypeStr === "incremental") {
-    const chunkType: ChunkType = chunkTypeStr
-    return chunkType
-  } else {
-    return null
-  }
-}

package/src/storage/chunkTypeFromKey.ts

@@ -0,0 +1,22 @@
+import { StorageKey } from "./types.js"
+import { ChunkType } from "./types.js"
+
+/**
+ * Keys for storing Automerge documents are of the form:
+ * ```ts
+ * [documentId, "snapshot", hash]  // OR
+ * [documentId, "incremental", hash]
+ * ```
+ * This function returns the chunk type ("snapshot" or "incremental") if the key is in one of these
+ * forms.
+ */
+export function chunkTypeFromKey(key: StorageKey): ChunkType | null {
+  if (key.length < 2) return null
+
+  const chunkTypeStr = key[key.length - 2] // next-to-last element in key
+  if (chunkTypeStr === "snapshot" || chunkTypeStr === "incremental") {
+    return chunkTypeStr as ChunkType
+  }
+
+  return null
+}

package/src/storage/keyHash.ts

@@ -0,0 +1,17 @@
+import * as A from "@automerge/automerge/next"
+import * as sha256 from "fast-sha256"
+import { mergeArrays } from "../helpers/mergeArrays.js"
+
+export function keyHash(binary: Uint8Array) {
+  // calculate hash
+  const hash = sha256.hash(binary)
+  return bufferToHexString(hash)
+}
+export function headsHash(heads: A.Heads): string {
+  const encoder = new TextEncoder()
+  const headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
+  return keyHash(headsbinary)
+}
+function bufferToHexString(data: Uint8Array) {
+  return Array.from(data, byte => byte.toString(16).padStart(2, "0")).join("")
+}

package/src/storage/types.ts

@@ -0,0 +1,39 @@
+/**
+ * A chunk is a snapshot or incremental change that is stored in a {@link StorageAdapter}.
+ */
+export type Chunk = {
+  key: StorageKey
+  data: Uint8Array | undefined
+}
+
+/**
+ * Metadata about a chunk of data loaded from storage. This is stored on the StorageSubsystem so
+ * when we are compacting we know what chunks we can safely delete.
+ */
+export type ChunkInfo = {
+  key: StorageKey
+  type: ChunkType
+  size: number
+}
+
+export type ChunkType = "snapshot" | "incremental"
+
+/**
+ * A storage key is an array of strings that represents a path to a value in a
+ * {@link StorageAdapter}.
+ *
+ * @remarks
+ * Storage keys are arrays because they are hierarchical and they allow the storage subsystem to do
+ * range queries for all keys that have a particular prefix. For example, incremental changes for a
+ * given document might be stored under `[<documentId>, "incremental", <SHA256>]`.
+ *
+ * automerge-repo mostly uses keys in the following form:
+ * ```ts
+ * [documentId, "snapshot", hash]  // OR
+ * [documentId, "incremental", hash]
+ * ```
+ *
+ * However, the storage adapter implementation should be agnostic to the meaning of the key and
+ * should not assume any particular structure.
+ **/
+export type StorageKey = string[]

package/src/synchronizer/CollectionSynchronizer.ts

@@ -1,6 +1,6 @@
 import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
-import { stringifyAutomergeUrl } from "../DocUrl.js"
+import { stringifyAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
 import { RepoMessage } from "../network/messages.js"
 import { DocumentId, PeerId } from "../types.js"

package/src/types.ts

@@ -1,20 +1,32 @@
-/** The ID of a document. Typically you should use a {@link AutomergeUrl} instead.
- */
-export type DocumentId = string & { __documentId: true } // for logging
-
-/** A branded string representing a URL for a document
- *
- * @remarks
- * An automerge URL has the form `automerge:<base58 encoded string>`. This
- * type is returned from various routines which validate a url.
- *
+/**
+ * A branded string representing a URL for a document, in the form `automerge:<base58check encoded
+ * string>`; for example, `automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu`.
  */
 export type AutomergeUrl = string & { __documentUrl: true } // for opening / linking
 
-/** A document ID as a Uint8Array instead of a bas58 encoded string. Typically you should use a {@link AutomergeUrl} instead.
+/**
+ * The base58check-encoded UUID of a document. This is the string following the `automerge:`
+ * protocol prefix in an AutomergeUrl; for example, `4NMNnkMhL8jXrdJ9jamS58PAVdXu`. When recording
+ * links to an Automerge document in another Automerge document, you should store a
+ * {@link AutomergeUrl} instead.
  */
+export type DocumentId = string & { __documentId: true } // for logging
+
+/** The unencoded UUID of a document. Typically you should use a {@link AutomergeUrl} instead. */
 export type BinaryDocumentId = Uint8Array & { __binaryDocumentId: true } // for storing / syncing
 
+/**
+ * A UUID encoded as a hex string. As of v1.0, a {@link DocumentID} is stored as a base58-encoded string with a checksum.
+ * Support for this format will be removed in a future version.
+ */
+export type LegacyDocumentId = string & { __legacyDocumentId: true }
+
+export type AnyDocumentId =
+  | AutomergeUrl
+  | DocumentId
+  | BinaryDocumentId
+  | LegacyDocumentId
+
 /** A branded type for peer IDs */
 export type PeerId = string & { __peerId: true }