@automerge/automerge-repo 1.0.19 → 1.1.0-alpha.13

package/src/helpers/debounce.ts

@@ -19,7 +19,7 @@ export const throttle = <F extends (...args: Parameters<F>) => ReturnType<F>>(
   return function (...args: Parameters<F>) {
     clearTimeout(timeout)
     timeout = setTimeout(() => {
-      fn.apply(null, args)
+      fn(...args)
     }, rate)
   }
 }

package/src/helpers/pause.ts

@@ -1,3 +1,5 @@
+/* c8 ignore start */
+
 export const pause = (t = 0) =>
   new Promise<void>(resolve => setTimeout(() => resolve(), t))
 
@@ -12,3 +14,5 @@ export function rejectOnTimeout<T>(
     }),
   ])
 }
+
+/* c8 ignore end */

package/src/helpers/throttle.ts

@@ -36,7 +36,7 @@ export const throttle = <F extends (...args: Parameters<F>) => ReturnType<F>>(
     wait = lastCall + delay - Date.now()
     clearTimeout(timeout)
     timeout = setTimeout(() => {
-      fn.apply(null, args)
+      fn(...args)
       lastCall = Date.now()
     }, wait)
   }

package/src/helpers/withTimeout.ts

@@ -1,3 +1,4 @@
+/* c8 ignore start */
 /**
  * If `promise` is resolved before `t` ms elapse, the timeout is cleared and the result of the
  * promise is returned. If the timeout ends first, a `TimeoutError` is thrown.
@@ -26,3 +27,4 @@ export class TimeoutError extends Error {
     this.name = "TimeoutError"
   }
 }
+/* c8 ignore end */

package/src/index.ts

@@ -34,7 +34,7 @@ export {
 } from "./AutomergeUrl.js"
 export { Repo } from "./Repo.js"
 export { NetworkAdapter } from "./network/NetworkAdapter.js"
-export { isValidRepoMessage } from "./network/messages.js"
+export { isRepoMessage } from "./network/messages.js"
 export { StorageAdapter } from "./storage/StorageAdapter.js"
 
 /** @hidden **/
@@ -67,6 +67,7 @@ export type {
   OpenPayload,
   PeerCandidatePayload,
   PeerDisconnectedPayload,
+  PeerMetadata,
 } from "./network/NetworkAdapter.js"
 
 export type {
@@ -83,6 +84,7 @@ export type {
   ChunkInfo,
   ChunkType,
   StorageKey,
+  StorageId,
 } from "./storage/types.js"
 
 export * from "./types.js"

package/src/network/NetworkAdapter.ts

@@ -1,6 +1,20 @@
+/* c8 ignore start */
+
 import { EventEmitter } from "eventemitter3"
 import { PeerId } from "../types.js"
 import { Message } from "./messages.js"
+import { StorageId } from "../storage/types.js"
+
+/**
+ * Describes a peer intent to the system
+ * storageId: the key for syncState to decide what the other peer already has
+ * isEphemeral: to decide if we bother recording this peer's sync state
+ *
+ */
+export interface PeerMetadata {
+  storageId?: StorageId
+  isEphemeral?: boolean
+}
 
 /** An interface representing some way to connect to other peers
  *
@@ -10,13 +24,15 @@ import { Message } from "./messages.js"
  * until the adapter emits a `ready` event before it starts trying to use it
  */
 export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
-  peerId?: PeerId // hmmm, maybe not
+  peerId?: PeerId
+  peerMetadata?: PeerMetadata
 
   /** Called by the {@link Repo} to start the connection process
    *
    * @argument peerId - the peerId of this repo
+   * @argument peerMetadata - how this adapter should present itself to other peers
    */
-  abstract connect(peerId: PeerId): void
+  abstract connect(peerId: PeerId, peerMetadata?: PeerMetadata): void
 
   /** Called by the {@link Repo} to send a message to a peer
    *
@@ -53,6 +69,7 @@ export interface OpenPayload {
 
 export interface PeerCandidatePayload {
   peerId: PeerId
+  peerMetadata: PeerMetadata
 }
 
 export interface PeerDisconnectedPayload {

package/src/network/NetworkSubsystem.ts

@@ -1,13 +1,17 @@
 import debug from "debug"
 import { EventEmitter } from "eventemitter3"
 import { PeerId, SessionId } from "../types.js"
-import { NetworkAdapter, PeerDisconnectedPayload } from "./NetworkAdapter.js"
+import type {
+  NetworkAdapter,
+  PeerDisconnectedPayload,
+  PeerMetadata,
+} from "./NetworkAdapter.js"
 import {
   EphemeralMessage,
   MessageContents,
   RepoMessage,
   isEphemeralMessage,
-  isValidRepoMessage,
+  isRepoMessage,
 } from "./messages.js"
 
 type EphemeralMessageSource = `${PeerId}:${SessionId}`
@@ -25,7 +29,11 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   #readyAdapterCount = 0
   #adapters: NetworkAdapter[] = []
 
-  constructor(adapters: NetworkAdapter[], public peerId = randomPeerId()) {
+  constructor(
+    adapters: NetworkAdapter[],
+    public peerId = randomPeerId(),
+    private peerMetadata: Promise<PeerMetadata>
+  ) {
     super()
     this.#log = debug(`automerge-repo:network:${this.peerId}`)
     adapters.forEach(a => this.addNetworkAdapter(a))
@@ -46,9 +54,8 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       }
     })
 
-    networkAdapter.on("peer-candidate", ({ peerId }) => {
+    networkAdapter.on("peer-candidate", ({ peerId, peerMetadata }) => {
       this.#log(`peer candidate: ${peerId} `)
-
       // TODO: This is where authentication would happen
 
       if (!this.#adaptersByPeer[peerId]) {
@@ -56,7 +63,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
         this.#adaptersByPeer[peerId] = networkAdapter
       }
 
-      this.emit("peer", { peerId })
+      this.emit("peer", { peerId, peerMetadata })
     })
 
     networkAdapter.on("peer-disconnected", ({ peerId }) => {
@@ -66,7 +73,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     })
 
     networkAdapter.on("message", msg => {
-      if (!isValidRepoMessage(msg)) {
+      if (!isRepoMessage(msg)) {
         this.#log(`invalid message: ${JSON.stringify(msg)}`)
         return
       }
@@ -98,7 +105,11 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       })
     })
 
-    networkAdapter.connect(this.peerId)
+    this.peerMetadata.then(peerMetadata => {
+      networkAdapter.connect(this.peerId, peerMetadata)
+    }).catch(err => {
+      this.#log("error connecting to network", err)
+    })
   }
 
   send(message: MessageContents) {
@@ -135,7 +146,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     }
 
     const outbound = prepareMessage(message)
-    this.#log("sending message", outbound)
+    this.#log("sending message %o", outbound)
     peer.send(outbound as RepoMessage)
   }
 
@@ -171,4 +182,5 @@ export interface NetworkSubsystemEvents {
 
 export interface PeerPayload {
   peerId: PeerId
+  peerMetadata: PeerMetadata
 }

package/src/network/messages.ts

@@ -1,16 +1,27 @@
 import { SyncState } from "@automerge/automerge"
+import { StorageId } from "../storage/types.js"
 import { DocumentId, PeerId, SessionId } from "../types.js"
 
+export type Message = {
+  type: string
+
+  /** The peer ID of the sender of this message */
+  senderId: PeerId
+
+  /** The peer ID of the recipient of this message */
+  targetId: PeerId
+
+  data?: Uint8Array
+
+  documentId?: DocumentId
+}
+
 /**
  * A sync message for a particular document
  */
 export type SyncMessage = {
   type: "sync"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
   /** The automerge sync message */
@@ -20,53 +31,50 @@ export type SyncMessage = {
   documentId: DocumentId
 }
 
-/** An ephemeral message
+/**
+ * An ephemeral message.
  *
  * @remarks
- * Ephemeral messages are not persisted anywhere and have no particular
- * structure. `automerge-repo` will gossip them around, in order to avoid
- * eternal loops of ephemeral messages every message has a session ID, which
- * is a random number generated by the sender at startup time, and a sequence
- * number. The combination of these two things allows us to discard messages
- * we have already seen.
+ * Ephemeral messages are not persisted anywhere. The data property can be used by the application
+ * as needed. The repo gossips these around.
+ *
+ * In order to avoid infinite loops of ephemeral messages, every message has (a) a session ID, which
+ * is a random number generated by the sender at startup time; and (b) a sequence number. The
+ * combination of these two things allows us to discard messages we have already seen.
  * */
 export type EphemeralMessage = {
   type: "ephemeral"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** A sequence number which must be incremented for each message sent by this peer */
+  /** A sequence number which must be incremented for each message sent by this peer. */
   count: number
 
-  /** The ID of the session this message is part of. The sequence number for a given session always increases */
+  /** The ID of the session this message is part of. The sequence number for a given session always increases. */
   sessionId: SessionId
 
-  /** The document ID this message pertains to */
+  /** The document ID this message pertains to. */
   documentId: DocumentId
 
-  /** The actual data of the message */
+  /** The actual data of the message. */
   data: Uint8Array
 }
 
-/** Sent by a {@link Repo} to indicate that it does not have the document and none of it's connected peers do either */
+/**
+ * Sent by a {@link Repo} to indicate that it does not have the document and none of its connected
+ * peers do either.
+ */
 export type DocumentUnavailableMessage = {
   type: "doc-unavailable"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
   /** The document which the peer claims it doesn't have */
   documentId: DocumentId
 }
 
-/** Sent by a {@link Repo} to request a document from a peer
+/**
+ * Sent by a {@link Repo} to request a document from a peer.
  *
  * @remarks
  * This is identical to a {@link SyncMessage} except that it is sent by a {@link Repo}
@@ -74,32 +82,44 @@ export type DocumentUnavailableMessage = {
  * */
 export type RequestMessage = {
   type: "request"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** The initial automerge sync message */
+  /** The automerge sync message */
   data: Uint8Array
 
-  /** The document ID this message requests */
+  /** The document ID of the document this message is for */
   documentId: DocumentId
 }
 
-/** (anticipating work in progress) */
-export type AuthMessage<TPayload = any> = {
-  type: "auth"
-
-  /** The peer ID of the sender of this message */
+/**
+ * Sent by a {@link Repo} to add or remove storage IDs from a remote peer's subscription.
+ */
+export type RemoteSubscriptionControlMessage = {
+  type: "remote-subscription-change"
   senderId: PeerId
+  targetId: PeerId
 
-  /** The peer ID of the recipient of this message */
+  /** The storage IDs to add to the subscription */
+  add?: StorageId[]
+
+  /** The storage IDs to remove from the subscription */
+  remove?: StorageId[]
+}
+
+/**
+ * Sent by a {@link Repo} to indicate that the heads of a document have changed on a remote peer.
+ */
+export type RemoteHeadsChanged = {
+  type: "remote-heads-changed"
+  senderId: PeerId
   targetId: PeerId
 
-  /** The payload of the auth message (up to the specific auth provider) */
-  payload: TPayload
+  /** The document ID of the document that has changed */
+  documentId: DocumentId
+
+  /** The document's new heads */
+  newHeads: { [key: StorageId]: { heads: string[]; timestamp: number } }
 }
 
 /** These are message types that a {@link NetworkAdapter} surfaces to a {@link Repo}. */
@@ -108,14 +128,20 @@ export type RepoMessage =
   | EphemeralMessage
   | RequestMessage
   | DocumentUnavailableMessage
+  | RemoteSubscriptionControlMessage
+  | RemoteHeadsChanged
 
-/** These are all the message types that a {@link NetworkAdapter} might see. */
-export type Message = RepoMessage | AuthMessage
+/** These are message types that are handled by the {@link CollectionSynchronizer}.*/
+export type DocMessage =
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
 
 /**
  * The contents of a message, without the sender ID or other properties added by the {@link NetworkSubsystem})
  */
-export type MessageContents<T extends Message = Message> =
+export type MessageContents<T extends Message = RepoMessage> =
   T extends EphemeralMessage
     ? Omit<T, "senderId" | "count" | "sessionId">
     : Omit<T, "senderId">
@@ -127,16 +153,21 @@ export interface SyncStateMessage {
   syncState: SyncState
 }
 
+/** Notify the repo that a peer started syncing with a doc */
+export interface OpenDocMessage {
+  peerId: PeerId
+  documentId: DocumentId
+}
+
 // TYPE GUARDS
 
-export const isValidRepoMessage = (message: Message): message is RepoMessage =>
-  typeof message === "object" &&
-  typeof message.type === "string" &&
-  typeof message.senderId === "string" &&
-  (isSyncMessage(message) ||
-    isEphemeralMessage(message) ||
-    isRequestMessage(message) ||
-    isDocumentUnavailableMessage(message))
+export const isRepoMessage = (message: Message): message is RepoMessage =>
+  isSyncMessage(message) ||
+  isEphemeralMessage(message) ||
+  isRequestMessage(message) ||
+  isDocumentUnavailableMessage(message) ||
+  isRemoteSubscriptionControlMessage(message) ||
+  isRemoteHeadsChanged(message)
 
 // prettier-ignore
 export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage => 
@@ -150,3 +181,10 @@ export const isSyncMessage = (msg: Message): msg is SyncMessage =>
 
 export const isEphemeralMessage = (msg: Message): msg is EphemeralMessage =>
   msg.type === "ephemeral"
+
+// prettier-ignore
+export const isRemoteSubscriptionControlMessage = (msg: Message): msg is RemoteSubscriptionControlMessage =>
+  msg.type === "remote-subscription-change"
+
+export const isRemoteHeadsChanged = (msg: Message): msg is RemoteHeadsChanged =>
+  msg.type === "remote-heads-changed"

package/src/storage/StorageSubsystem.ts

@@ -2,11 +2,12 @@ import * as A from "@automerge/automerge/next"
 import debug from "debug"
 import { headsAreSame } from "../helpers/headsAreSame.js"
 import { mergeArrays } from "../helpers/mergeArrays.js"
-import { PeerId, type DocumentId } from "../types.js"
+import { type DocumentId } from "../types.js"
 import { StorageAdapter } from "./StorageAdapter.js"
-import { ChunkInfo, StorageKey } from "./types.js"
+import { ChunkInfo, StorageKey, StorageId } from "./types.js"
 import { keyHash, headsHash } from "./keyHash.js"
 import { chunkTypeFromKey } from "./chunkTypeFromKey.js"
+import * as Uuid from "uuid"
 
 /**
  * The storage subsystem is responsible for saving and loading Automerge documents to and from
@@ -31,6 +32,23 @@ export class StorageSubsystem {
     this.#storageAdapter = storageAdapter
   }
 
+  async id(): Promise<StorageId> {
+    const storedId = await this.#storageAdapter.load(["storage-adapter-id"])
+
+    let id: StorageId
+    if (storedId) {
+      id = new TextDecoder().decode(storedId) as StorageId
+    } else {
+      id = Uuid.v4() as StorageId
+      await this.#storageAdapter.save(
+        ["storage-adapter-id"],
+        new TextEncoder().encode(id)
+      )
+    }
+
+    return id
+  }
+
   // ARBITRARY KEY/VALUE STORAGE
 
   // The `load`, `save`, and `remove` methods are for generic key/value storage, as opposed to
@@ -211,19 +229,19 @@ export class StorageSubsystem {
 
   async loadSyncState(
     documentId: DocumentId,
-    peerId: PeerId
+    storageId: StorageId
   ): Promise<A.SyncState | undefined> {
-    const key = [documentId, "sync-state", peerId]
+    const key = [documentId, "sync-state", storageId]
     const loaded = await this.#storageAdapter.load(key)
     return loaded ? A.decodeSyncState(loaded) : undefined
   }
 
   async saveSyncState(
     documentId: DocumentId,
-    peerId: PeerId,
+    storageId: StorageId,
     syncState: A.SyncState
   ): Promise<void> {
-    const key = [documentId, "sync-state", peerId]
+    const key = [documentId, "sync-state", storageId]
     await this.#storageAdapter.save(key, A.encodeSyncState(syncState))
   }
 
@@ -261,6 +279,11 @@ export class StorageSubsystem {
         incrementalSize += chunk.size
       }
     }
-    return incrementalSize >= snapshotSize
+    // if the file is currently small, don't worry, just compact
+    // this might seem a bit arbitrary (1k is arbitrary) but is designed to ensure compaction
+    // for documents with only a single large change on top of an empty (or nearly empty) document
+    // for example: imported NPM modules, images, etc.
+    // if we have even more incrementals (so far) than the snapshot, compact
+    return snapshotSize < 1024 || incrementalSize >= snapshotSize
   }
 }

package/src/storage/keyHash.ts

@@ -7,11 +7,13 @@ export function keyHash(binary: Uint8Array) {
   const hash = sha256.hash(binary)
   return bufferToHexString(hash)
 }
+
 export function headsHash(heads: A.Heads): string {
   const encoder = new TextEncoder()
   const headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
   return keyHash(headsbinary)
 }
+
 function bufferToHexString(data: Uint8Array) {
   return Array.from(data, byte => byte.toString(16).padStart(2, "0")).join("")
 }

package/src/storage/types.ts

@@ -37,3 +37,6 @@ export type ChunkType = "snapshot" | "incremental"
  * should not assume any particular structure.
  **/
 export type StorageKey = string[]
+
+/** A branded type for storage IDs */
+export type StorageId = string & { __storageId: true }

package/src/synchronizer/CollectionSynchronizer.ts

@@ -2,7 +2,7 @@ import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
 import { stringifyAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
-import { RepoMessage } from "../network/messages.js"
+import { DocMessage } from "../network/messages.js"
 import { DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
@@ -37,18 +37,25 @@ export class CollectionSynchronizer extends Synchronizer {
   #initDocSynchronizer(handle: DocHandle<unknown>): DocSynchronizer {
     const docSynchronizer = new DocSynchronizer({
       handle,
-      onLoadSyncState: peerId => {
+      onLoadSyncState: async peerId => {
         if (!this.repo.storageSubsystem) {
-          return Promise.resolve(undefined)
+          return
+        }
+
+        const { storageId, isEphemeral } =
+          this.repo.peerMetadataByPeerId[peerId] || {}
+        if (!storageId || isEphemeral) {
+          return
         }
 
         return this.repo.storageSubsystem.loadSyncState(
           handle.documentId,
-          peerId
+          storageId
         )
       },
     })
     docSynchronizer.on("message", event => this.emit("message", event))
+    docSynchronizer.on("open-doc", event => this.emit("open-doc", event))
     docSynchronizer.on("sync-state", event => this.emit("sync-state", event))
     return docSynchronizer
   }
@@ -70,7 +77,7 @@ export class CollectionSynchronizer extends Synchronizer {
    * When we receive a sync message for a document we haven't got in memory, we
    * register it with the repo and start synchronizing
    */
-  async receiveMessage(message: RepoMessage) {
+  async receiveMessage(message: DocMessage) {
     log(
       `onSyncMessage: ${message.senderId}, ${message.documentId}, ${
         "data" in message ? message.data.byteLength + "bytes" : ""
@@ -110,6 +117,7 @@ export class CollectionSynchronizer extends Synchronizer {
   }
 
   // TODO: implement this
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   removeDocument(documentId: DocumentId) {
     throw new Error("not implemented")
   }