@automerge/automerge-repo 0.0.1

package/src/DocHandle.ts

@@ -0,0 +1,386 @@
+import * as A from "@automerge/automerge"
+import debug from "debug"
+import EventEmitter from "eventemitter3"
+import {
+  assign,
+  BaseActionObject,
+  createMachine,
+  interpret,
+  Interpreter,
+  ResolveTypegenMeta,
+  ServiceMap,
+  StateSchema,
+  StateValue,
+  TypegenDisabled,
+} from "xstate"
+import { waitFor } from "xstate/lib/waitFor.js"
+import { headsAreSame } from "./helpers/headsAreSame.js"
+import { pause } from "./helpers/pause.js"
+import type { ChannelId, DocumentId, PeerId } from "./types.js"
+import { withTimeout, TimeoutError } from "./helpers/withTimeout.js"
+
+/** DocHandle is a wrapper around a single Automerge document that lets us listen for changes. */
+export class DocHandle<T> //
+  extends EventEmitter<DocHandleEvents<T>>
+{
+  #log: debug.Debugger
+
+  #machine: DocHandleXstateMachine<T>
+  #timeoutDelay: number
+
+  constructor(
+    public documentId: DocumentId,
+    { isNew = false, timeoutDelay = 700000 }: DocHandleOptions = {}
+  ) {
+    super()
+    this.#timeoutDelay = timeoutDelay
+    this.#log = debug(`automerge-repo:dochandle:${documentId.slice(0, 5)}`)
+
+    // initial doc
+    const doc = A.init<T>({
+      patchCallback: (patches, { before, after }) =>
+        this.emit("patch", { handle: this, patches, before, after }),
+    })
+
+    /**
+     * Internally we use a state machine to orchestrate document loading and/or syncing, in order to
+     * avoid requesting data we already have, or surfacing intermediate values to the consumer.
+     *
+     *                      ┌─────────┐           ┌────────────┐
+     *  ┌───────┐  ┌──FIND──┤ loading ├─REQUEST──►│ requesting ├─UPDATE──┐
+     *  │ idle  ├──┤        └───┬─────┘           └────────────┘         │
+     *  └───────┘  │            │                                        └─►┌─────────┐
+     *             │            └───────LOAD───────────────────────────────►│  ready  │
+     *             └──CREATE───────────────────────────────────────────────►└─────────┘
+     */
+    this.#machine = interpret(
+      createMachine<DocHandleContext<T>, DocHandleEvent<T>>(
+        {
+          predictableActionArguments: true,
+
+          id: "docHandle",
+          initial: IDLE,
+          context: { documentId, doc },
+          states: {
+            idle: {
+              on: {
+                // If we're creating a new document, we don't need to load anything
+                CREATE: { target: READY },
+                // If we're accessing an existing document, we need to request it from storage
+                // and/or the network
+                FIND: { target: LOADING },
+                DELETE: { actions: "onDelete", target: DELETED },
+              },
+            },
+            loading: {
+              on: {
+                // LOAD is called by the Repo if the document is found in storage
+                LOAD: { actions: "onLoad", target: READY },
+                // REQUEST is called by the Repo if the document is not found in storage
+                REQUEST: { target: REQUESTING },
+                DELETE: { actions: "onDelete", target: DELETED },
+              },
+            },
+            requesting: {
+              on: {
+                // UPDATE is called by the Repo when we receive changes from the network
+                UPDATE: { actions: "onUpdate" },
+                // REQUEST_COMPLETE is called from `onUpdate` when the doc has been fully loaded from the network
+                REQUEST_COMPLETE: { target: READY },
+                DELETE: { actions: "onDelete", target: DELETED },
+              },
+            },
+            ready: {
+              on: {
+                // UPDATE is called by the Repo when we receive changes from the network
+                UPDATE: { actions: "onUpdate", target: READY },
+                DELETE: { actions: "onDelete", target: DELETED },
+              },
+            },
+            error: {},
+            deleted: {},
+          },
+        },
+
+        {
+          actions: {
+            /** Apply the binary changes from storage and put the updated doc on context */
+            onLoad: assign((context, { payload }: LoadEvent) => {
+              const { binary } = payload
+              const { doc } = context
+              const newDoc = A.loadIncremental(doc, binary)
+              return { doc: newDoc }
+            }),
+
+            /** Put the updated doc on context; if it's different, emit a `change` event */
+            onUpdate: assign((context, { payload }: UpdateEvent<T>) => {
+              const { doc: oldDoc } = context
+
+              const { callback } = payload
+              const newDoc = callback(oldDoc)
+
+              const docChanged = !headsAreSame(newDoc, oldDoc)
+              if (docChanged) {
+                this.emit("change", { handle: this, doc: newDoc })
+                if (!this.isReady()) {
+                  this.#machine.send(REQUEST_COMPLETE)
+                }
+              }
+              return { doc: newDoc }
+            }),
+            onDelete: assign(() => {
+              this.emit("delete", { handle: this })
+              return { doc: undefined }
+            }),
+          },
+        }
+      )
+    )
+      .onTransition(({ value: state }, { type: event }) =>
+        this.#log(`${event} → ${state}`, this.#doc)
+      )
+      .start()
+
+    this.#machine.send(isNew ? CREATE : FIND)
+  }
+
+  get doc() {
+    if (!this.isReady()) {
+      throw new Error(
+        `DocHandle#${this.documentId} is not ready. Check \`handle.isReady()\` before accessing the document.`
+      )
+    }
+
+    return this.#doc
+  }
+
+  // PRIVATE
+
+  /** Returns the current document */
+  get #doc() {
+    return this.#machine?.getSnapshot().context.doc
+  }
+
+  /** Returns the docHandle's state (READY, etc.) */
+  get #state() {
+    return this.#machine?.getSnapshot().value
+  }
+
+  /** Returns a promise that resolves when the docHandle is in one of the given states */
+  #statePromise(awaitStates: HandleState | HandleState[]) {
+    if (!Array.isArray(awaitStates)) awaitStates = [awaitStates]
+    return Promise.any(
+      awaitStates.map(state => waitFor(this.#machine, s => s.matches(state)))
+    )
+  }
+
+  // PUBLIC
+
+  isReady = () => this.#state === READY
+  isReadyOrRequesting = () =>
+    this.#state === READY || this.#state === REQUESTING
+  isDeleted = () => this.#state === DELETED
+
+  /**
+   * Returns the current document, waiting for the handle to be ready if necessary.
+   */
+  async value(awaitStates: HandleState[] = [READY]) {
+    await pause() // yield one tick because reasons
+    try {
+      // wait for the document to enter one of the desired states
+      await withTimeout(this.#statePromise(awaitStates), this.#timeoutDelay)
+    } catch (error) {
+      if (error instanceof TimeoutError)
+        throw new Error(`DocHandle: timed out loading ${this.documentId}`)
+      else throw error
+    }
+    // Return the document
+    return this.#doc
+  }
+
+  async loadAttemptedValue() {
+    return this.value([READY, REQUESTING])
+  }
+
+  /** `load` is called by the repo when the document is found in storage */
+  load(binary: Uint8Array) {
+    if (binary.length) {
+      this.#machine.send(LOAD, { payload: { binary } })
+    }
+  }
+
+  /** `update` is called by the repo when we receive changes from the network */
+  update(callback: (doc: A.Doc<T>) => A.Doc<T>) {
+    this.#machine.send(UPDATE, { payload: { callback } })
+  }
+
+  /** `change` is called by the repo when the document is changed locally  */
+  change(callback: A.ChangeFn<T>, options: A.ChangeOptions<T> = {}) {
+    if (!this.isReady()) {
+      throw new Error(
+        `DocHandle#${this.documentId} is not ready. Check \`handle.isReady()\` before accessing the document.`
+      )
+    }
+    this.#machine.send(UPDATE, {
+      payload: {
+        callback: (doc: A.Doc<T>) => {
+          return A.change(doc, options, callback)
+        },
+      },
+    })
+  }
+
+  changeAt(heads: A.Heads, callback: A.ChangeFn<T>, options: A.ChangeOptions<T> = {}) {
+    if (!this.isReady()) {
+      throw new Error(
+        `DocHandle#${this.documentId} is not ready. Check \`handle.isReady()\` before accessing the document.`
+      )
+    }
+    this.#machine.send(UPDATE, {
+      payload: {
+        callback: (doc: A.Doc<T>) => {
+          return A.changeAt(doc, heads, options, callback)
+        },
+      },
+    })
+  }
+
+  /** `request` is called by the repo when the document is not found in storage */
+  request() {
+    if (this.#state === LOADING) this.#machine.send(REQUEST)
+  }
+
+  /** `delete` is called by the repo when the document is deleted */
+  delete() {
+    this.#machine.send(DELETE)
+  }
+}
+
+// WRAPPER CLASS TYPES
+
+interface DocHandleOptions {
+  isNew?: boolean
+  timeoutDelay?: number
+}
+
+export interface DocHandleMessagePayload {
+  destinationId: PeerId
+  channelId: ChannelId
+  data: Uint8Array
+}
+
+export interface DocHandleChangePayload<T> {
+  handle: DocHandle<T>
+  doc: A.Doc<T>
+}
+
+export interface DocHandleDeletePayload<T> {
+  handle: DocHandle<T>
+}
+
+export interface DocHandlePatchPayload<T> {
+  handle: DocHandle<T>
+  patches: A.Patch[]
+  before: A.Doc<T>
+  after: A.Doc<T>
+}
+
+export interface DocHandleEvents<T> {
+  change: (payload: DocHandleChangePayload<T>) => void
+  patch: (payload: DocHandlePatchPayload<T>) => void
+  delete: (payload: DocHandleDeletePayload<T>) => void
+}
+
+// STATE MACHINE TYPES
+
+// state
+
+export const HandleState = {
+  IDLE: "idle",
+  LOADING: "loading",
+  REQUESTING: "requesting",
+  READY: "ready",
+  ERROR: "error",
+  DELETED: "deleted",
+} as const
+export type HandleState = (typeof HandleState)[keyof typeof HandleState]
+
+type DocHandleMachineState = {
+  states: Record<
+    (typeof HandleState)[keyof typeof HandleState],
+    StateSchema<HandleState>
+  >
+}
+
+// context
+
+interface DocHandleContext<T> {
+  documentId: string
+  doc: A.Doc<T>
+}
+
+// events
+
+export const Event = {
+  CREATE: "CREATE",
+  LOAD: "LOAD",
+  FIND: "FIND",
+  REQUEST: "REQUEST",
+  REQUEST_COMPLETE: "REQUEST_COMPLETE",
+  UPDATE: "UPDATE",
+  TIMEOUT: "TIMEOUT",
+  DELETE: "DELETE",
+} as const
+type Event = (typeof Event)[keyof typeof Event]
+
+type CreateEvent = { type: typeof CREATE; payload: { documentId: string } }
+type LoadEvent = { type: typeof LOAD; payload: { binary: Uint8Array } }
+type FindEvent = { type: typeof FIND; payload: { documentId: string } }
+type RequestEvent = { type: typeof REQUEST }
+type RequestCompleteEvent = { type: typeof REQUEST_COMPLETE }
+type DeleteEvent = { type: typeof DELETE }
+type UpdateEvent<T> = {
+  type: typeof UPDATE
+  payload: { callback: (doc: A.Doc<T>) => A.Doc<T> }
+}
+type TimeoutEvent = { type: typeof TIMEOUT }
+
+type DocHandleEvent<T> =
+  | CreateEvent
+  | LoadEvent
+  | FindEvent
+  | RequestEvent
+  | RequestCompleteEvent
+  | UpdateEvent<T>
+  | TimeoutEvent
+  | DeleteEvent
+
+type DocHandleXstateMachine<T> = Interpreter<
+  DocHandleContext<T>,
+  DocHandleMachineState,
+  DocHandleEvent<T>,
+  {
+    value: StateValue // Should this be unknown or T?
+    context: DocHandleContext<T>
+  },
+  ResolveTypegenMeta<
+    TypegenDisabled,
+    DocHandleEvent<T>,
+    BaseActionObject,
+    ServiceMap
+  >
+>
+
+// CONSTANTS
+
+const { IDLE, LOADING, REQUESTING, READY, ERROR, DELETED } = HandleState
+const {
+  CREATE,
+  LOAD,
+  FIND,
+  REQUEST,
+  UPDATE,
+  TIMEOUT,
+  DELETE,
+  REQUEST_COMPLETE,
+} = Event

package/src/EphemeralData.ts

@@ -0,0 +1,46 @@
+import { decode, encode } from "cbor-x"
+import EventEmitter from "eventemitter3"
+import { ChannelId, PeerId } from "./index.js"
+import { MessagePayload } from "./network/NetworkAdapter.js"
+
+/**
+ * EphemeralData provides a mechanism to broadcast short-lived data — cursor positions, presence,
+ * heartbeats, etc. — that is useful in the moment but not worth persisting.
+ */
+export class EphemeralData extends EventEmitter<EphemeralDataMessageEvents> {
+  /** Broadcast an ephemeral message */
+  broadcast(channelId: ChannelId, message: unknown) {
+    const messageBytes = encode(message)
+
+    this.emit("message", {
+      targetId: "*" as PeerId, // TODO: we don't really need a targetId for broadcast
+      channelId: ("m/" + channelId) as ChannelId,
+      message: messageBytes,
+      broadcast: true,
+    })
+  }
+
+  /** Receive an ephemeral message */
+  receive(senderId: PeerId, grossChannelId: ChannelId, message: Uint8Array) {
+    const data = decode(message)
+    const channelId = grossChannelId.slice(2) as ChannelId
+    this.emit("data", {
+      peerId: senderId,
+      channelId,
+      data,
+    })
+  }
+}
+
+// types
+
+export interface EphemeralDataPayload {
+  channelId: ChannelId
+  peerId: PeerId
+  data: { peerId: PeerId; channelId: ChannelId; data: unknown }
+}
+
+export type EphemeralDataMessageEvents = {
+  message: (event: MessagePayload) => void
+  data: (event: EphemeralDataPayload) => void
+}

package/src/Repo.ts

@@ -0,0 +1,155 @@
+import { DocCollection } from "./DocCollection.js"
+import { EphemeralData } from "./EphemeralData.js"
+import { NetworkAdapter } from "./network/NetworkAdapter.js"
+import { NetworkSubsystem } from "./network/NetworkSubsystem.js"
+import { StorageAdapter } from "./storage/StorageAdapter.js"
+import { StorageSubsystem } from "./storage/StorageSubsystem.js"
+import { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
+import { ChannelId, DocumentId, PeerId } from "./types.js"
+
+import debug from "debug"
+
+const SYNC_CHANNEL = "sync_channel" as ChannelId
+
+/** A Repo is a DocCollection with networking, syncing, and storage capabilities. */
+export class Repo extends DocCollection {
+  #log: debug.Debugger
+
+  networkSubsystem: NetworkSubsystem
+  storageSubsystem?: StorageSubsystem
+  ephemeralData: EphemeralData
+
+  constructor({ storage, network, peerId, sharePolicy }: RepoConfig) {
+    super()
+    this.#log = debug(`automerge-repo:repo`)
+    this.sharePolicy = sharePolicy ?? this.sharePolicy
+
+    // DOC COLLECTION
+
+    // The `document` event is fired by the DocCollection any time we create a new document or look
+    // up a document by ID. We listen for it in order to wire up storage and network synchronization.
+    this.on("document", async ({ handle }) => {
+      if (storageSubsystem) {
+        // Save when the document changes
+        handle.on("change", async ({ handle }) => {
+          const doc = await handle.value()
+          storageSubsystem.save(handle.documentId, doc)
+        })
+
+        // Try to load from disk
+        const binary = await storageSubsystem.loadBinary(handle.documentId)
+        handle.load(binary)
+      }
+
+      handle.request()
+
+      // Register the document with the synchronizer. This advertises our interest in the document.
+      synchronizer.addDocument(handle.documentId)
+    })
+
+    this.on("delete-document", ({ documentId }) => {
+      // TODO Pass the delete on to the network
+      // synchronizer.removeDocument(documentId)
+
+      if (storageSubsystem) {
+        storageSubsystem.remove(documentId)
+      }
+    })
+
+    // SYNCHRONIZER
+    // The synchronizer uses the network subsystem to keep documents in sync with peers.
+
+    const synchronizer = new CollectionSynchronizer(this)
+
+    // When the synchronizer emits sync messages, send them to peers
+    synchronizer.on(
+      "message",
+      ({ targetId, channelId, message, broadcast }) => {
+        this.#log(`sending sync message to ${targetId}`)
+        networkSubsystem.sendMessage(targetId, channelId, message, broadcast)
+      }
+    )
+
+    // STORAGE
+    // The storage subsystem has access to some form of persistence, and deals with save and loading documents.
+
+    const storageSubsystem = storage ? new StorageSubsystem(storage) : undefined
+    this.storageSubsystem = storageSubsystem
+
+    // NETWORK
+    // The network subsystem deals with sending and receiving messages to and from peers.
+
+    const networkSubsystem = new NetworkSubsystem(network, peerId)
+    this.networkSubsystem = networkSubsystem
+
+    // When we get a new peer, register it with the synchronizer
+    networkSubsystem.on("peer", async ({ peerId }) => {
+      this.#log("peer connected", { peerId })
+      synchronizer.addPeer(peerId)
+    })
+
+    // When a peer disconnects, remove it from the synchronizer
+    networkSubsystem.on("peer-disconnected", ({ peerId }) => {
+      synchronizer.removePeer(peerId)
+    })
+
+    // Handle incoming messages
+    networkSubsystem.on("message", async msg => {
+      const { senderId, channelId, message } = msg
+
+      // TODO: this demands a more principled way of associating channels with recipients
+
+      // Ephemeral channel ids start with "m/"
+      if (channelId.startsWith("m/")) {
+        // Ephemeral message
+        this.#log(`receiving ephemeral message from ${senderId}`)
+        ephemeralData.receive(senderId, channelId, message)
+      } else {
+        // Sync message
+        this.#log(`receiving sync message from ${senderId}`)
+        await synchronizer.receiveSyncMessage(senderId, channelId, message)
+      }
+    })
+
+    // We establish a special channel for sync messages
+    networkSubsystem.join(SYNC_CHANNEL)
+
+    // EPHEMERAL DATA
+    // The ephemeral data subsystem uses the network to send and receive messages that are not
+    // persisted to storage, e.g. cursor position, presence, etc.
+
+    const ephemeralData = new EphemeralData()
+    this.ephemeralData = ephemeralData
+
+    // Send ephemeral messages to peers
+    ephemeralData.on(
+      "message",
+      ({ targetId, channelId, message, broadcast }) => {
+        this.#log(`sending ephemeral message to ${targetId}`)
+        networkSubsystem.sendMessage(targetId, channelId, message, broadcast)
+      }
+    )
+  }
+}
+
+export interface RepoConfig {
+  /** Our unique identifier */
+  peerId?: PeerId
+
+  /** A storage adapter can be provided, or not */
+  storage?: StorageAdapter
+
+  /** One or more network adapters must be provided */
+  network: NetworkAdapter[]
+
+  /**
+   * Normal peers typically share generously with everyone (meaning we sync all our documents with
+   * all peers). A server only syncs documents that a peer explicitly requests by ID.
+   */
+  sharePolicy?: SharePolicy
+}
+
+export type SharePolicy = (
+  peerId: PeerId,
+  documentId?: DocumentId
+) => Promise<boolean>

package/src/helpers/arraysAreEqual.ts

@@ -0,0 +1,2 @@
+export const arraysAreEqual = <T>(a: T[], b: T[]) =>
+  a.length === b.length && a.every((element, index) => element === b[index])

package/src/helpers/eventPromise.ts

@@ -0,0 +1,10 @@
+import EventEmitter from "eventemitter3"
+
+/** Returns a promise that resolves when the given event is emitted on the given emitter. */
+export const eventPromise = (emitter: EventEmitter, event: string) =>
+  new Promise<any>(resolve => emitter.once(event, d => resolve(d)))
+
+export const eventPromises = (emitters: EventEmitter[], event: string) => {
+  const promises = emitters.map(emitter => eventPromise(emitter, event))
+  return Promise.all(promises)
+}

package/src/helpers/headsAreSame.ts

@@ -0,0 +1,8 @@
+import * as A from "@automerge/automerge"
+import { arraysAreEqual } from "./arraysAreEqual.js"
+
+export const headsAreSame = <T>(a: A.Doc<T>, b: A.Doc<T>) => {
+  const aHeads = A.getHeads(a)
+  const bHeads = A.getHeads(b)
+  return arraysAreEqual(aHeads, bHeads)
+}

package/src/helpers/mergeArrays.ts

@@ -0,0 +1,17 @@
+export function mergeArrays(myArrays: Uint8Array[]) {
+  // Get the total length of all arrays.
+  let length = 0
+  myArrays.forEach(item => {
+    length += item.length
+  })
+
+  // Create a new array with total length and merge all source arrays.
+  const mergedArray = new Uint8Array(length)
+  let offset = 0
+  myArrays.forEach(item => {
+    mergedArray.set(item, offset)
+    offset += item.length
+  })
+
+  return mergedArray
+}

package/src/helpers/pause.ts

@@ -0,0 +1,9 @@
+export const pause = (t = 0) =>
+  new Promise<void>(resolve => setTimeout(() => resolve(), t))
+
+export function rejectOnTimeout<T>(promise: Promise<T>, millis: number): Promise<T> {
+  return Promise.race([
+    promise,
+    pause(millis).then(() => { throw new Error("timeout exceeded") }),
+  ])
+}

package/src/helpers/withTimeout.ts

@@ -0,0 +1,28 @@
+/**
+ * If `promise` is resolved before `t` ms elapse, the timeout is cleared and the result of the
+ * promise is returned. If the timeout ends first, a `TimeoutError` is thrown.
+ */
+export const withTimeout = async <T>(
+  promise: Promise<T>,
+  t: number
+): Promise<T> => {
+  let timeoutId: ReturnType<typeof setTimeout>
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeoutId = setTimeout(
+      () => reject(new TimeoutError(`withTimeout: timed out after ${t}ms`)),
+      t
+    )
+  })
+  try {
+    return await Promise.race([promise, timeoutPromise])
+  } finally {
+    clearTimeout(timeoutId!)
+  }
+}
+
+export class TimeoutError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "TimeoutError"
+  }
+}

package/src/index.ts

@@ -0,0 +1,22 @@
+export { DocCollection } from "./DocCollection.js"
+export { DocHandle, HandleState } from "./DocHandle.js"
+export type {
+  DocHandleChangePayload,
+  DocHandleMessagePayload,
+  DocHandlePatchPayload,
+} from "./DocHandle.js"
+export { NetworkAdapter } from "./network/NetworkAdapter.js"
+export type {
+  InboundMessagePayload,
+  MessagePayload,
+  OpenPayload,
+  PeerCandidatePayload,
+  PeerDisconnectedPayload,
+} from "./network/NetworkAdapter.js"
+export { NetworkSubsystem } from "./network/NetworkSubsystem.js"
+export { Repo, type SharePolicy } from "./Repo.js"
+export { StorageAdapter } from "./storage/StorageAdapter.js"
+export { StorageSubsystem } from "./storage/StorageSubsystem.js"
+export { CollectionSynchronizer } from "./synchronizer/CollectionSynchronizer.js"
+export * from "./types.js"
+export * from "./test-utilities/adapter-tests.js"