@automerge/automerge-repo 1.1.0-alpha.1 → 1.1.0-alpha.13

package/src/network/NetworkAdapter.ts

@@ -1,8 +1,21 @@
+/* c8 ignore start */
+
 import { EventEmitter } from "eventemitter3"
 import { PeerId } from "../types.js"
 import { Message } from "./messages.js"
 import { StorageId } from "../storage/types.js"
 
+/**
+ * Describes a peer intent to the system
+ * storageId: the key for syncState to decide what the other peer already has
+ * isEphemeral: to decide if we bother recording this peer's sync state
+ *
+ */
+export interface PeerMetadata {
+  storageId?: StorageId
+  isEphemeral?: boolean
+}
+
 /** An interface representing some way to connect to other peers
  *
  * @remarks
@@ -11,21 +24,15 @@ import { StorageId } from "../storage/types.js"
  * until the adapter emits a `ready` event before it starts trying to use it
  */
 export abstract class NetworkAdapter extends EventEmitter<NetworkAdapterEvents> {
-  peerId?: PeerId // hmmm, maybe not
-  storageId?: StorageId
-  isEphemeral = true
+  peerId?: PeerId
+  peerMetadata?: PeerMetadata
 
   /** Called by the {@link Repo} to start the connection process
    *
    * @argument peerId - the peerId of this repo
-   * @argument storageId - the storage id of the peer
-   * @argument isEphemeral - weather or not the other end should persist our sync state
+   * @argument peerMetadata - how this adapter should present itself to other peers
    */
-  abstract connect(
-    peerId: PeerId,
-    storageId: StorageId | undefined,
-    isEphemeral: boolean
-  ): void
+  abstract connect(peerId: PeerId, peerMetadata?: PeerMetadata): void
 
   /** Called by the {@link Repo} to send a message to a peer
    *
@@ -62,8 +69,7 @@ export interface OpenPayload {
 
 export interface PeerCandidatePayload {
   peerId: PeerId
-  storageId?: StorageId
-  isEphemeral: boolean
+  peerMetadata: PeerMetadata
 }
 
 export interface PeerDisconnectedPayload {

package/src/network/NetworkSubsystem.ts

@@ -1,15 +1,18 @@
 import debug from "debug"
 import { EventEmitter } from "eventemitter3"
 import { PeerId, SessionId } from "../types.js"
-import { NetworkAdapter, PeerDisconnectedPayload } from "./NetworkAdapter.js"
+import type {
+  NetworkAdapter,
+  PeerDisconnectedPayload,
+  PeerMetadata,
+} from "./NetworkAdapter.js"
 import {
   EphemeralMessage,
   MessageContents,
   RepoMessage,
   isEphemeralMessage,
-  isValidRepoMessage,
+  isRepoMessage,
 } from "./messages.js"
-import { StorageId } from "../storage/types.js"
 
 type EphemeralMessageSource = `${PeerId}:${SessionId}`
 
@@ -29,8 +32,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
   constructor(
     adapters: NetworkAdapter[],
     public peerId = randomPeerId(),
-    private storageId: Promise<StorageId | undefined>, // todo: we shouldn't pass a promise here
-    private isEphemeral: boolean
+    private peerMetadata: Promise<PeerMetadata>
   ) {
     super()
     this.#log = debug(`automerge-repo:network:${this.peerId}`)
@@ -52,21 +54,17 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       }
     })
 
-    networkAdapter.on(
-      "peer-candidate",
-      ({ peerId, storageId, isEphemeral }) => {
-        this.#log(`peer candidate: ${peerId} `)
+    networkAdapter.on("peer-candidate", ({ peerId, peerMetadata }) => {
+      this.#log(`peer candidate: ${peerId} `)
+      // TODO: This is where authentication would happen
 
-        // TODO: This is where authentication would happen
-
-        if (!this.#adaptersByPeer[peerId]) {
-          // TODO: handle losing a server here
-          this.#adaptersByPeer[peerId] = networkAdapter
-        }
-
-        this.emit("peer", { peerId, storageId, isEphemeral })
+      if (!this.#adaptersByPeer[peerId]) {
+        // TODO: handle losing a server here
+        this.#adaptersByPeer[peerId] = networkAdapter
       }
-    )
+
+      this.emit("peer", { peerId, peerMetadata })
+    })
 
     networkAdapter.on("peer-disconnected", ({ peerId }) => {
       this.#log(`peer disconnected: ${peerId} `)
@@ -75,7 +73,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     })
 
     networkAdapter.on("message", msg => {
-      if (!isValidRepoMessage(msg)) {
+      if (!isRepoMessage(msg)) {
         this.#log(`invalid message: ${JSON.stringify(msg)}`)
         return
       }
@@ -107,8 +105,10 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
       })
     })
 
-    this.storageId.then(storageId => {
-      networkAdapter.connect(this.peerId, storageId, this.isEphemeral)
+    this.peerMetadata.then(peerMetadata => {
+      networkAdapter.connect(this.peerId, peerMetadata)
+    }).catch(err => {
+      this.#log("error connecting to network", err)
     })
   }
 
@@ -146,7 +146,7 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     }
 
     const outbound = prepareMessage(message)
-    this.#log("sending message", outbound)
+    this.#log("sending message %o", outbound)
     peer.send(outbound as RepoMessage)
   }
 
@@ -182,6 +182,5 @@ export interface NetworkSubsystemEvents {
 
 export interface PeerPayload {
   peerId: PeerId
-  storageId?: StorageId
-  isEphemeral: boolean
+  peerMetadata: PeerMetadata
 }

package/src/network/messages.ts

@@ -1,17 +1,27 @@
 import { SyncState } from "@automerge/automerge"
-import { DocumentId, PeerId, SessionId } from "../types.js"
 import { StorageId } from "../storage/types.js"
+import { DocumentId, PeerId, SessionId } from "../types.js"
+
+export type Message = {
+  type: string
+
+  /** The peer ID of the sender of this message */
+  senderId: PeerId
+
+  /** The peer ID of the recipient of this message */
+  targetId: PeerId
+
+  data?: Uint8Array
+
+  documentId?: DocumentId
+}
 
 /**
  * A sync message for a particular document
  */
 export type SyncMessage = {
   type: "sync"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
   /** The automerge sync message */
@@ -21,53 +31,50 @@ export type SyncMessage = {
   documentId: DocumentId
 }
 
-/** An ephemeral message
+/**
+ * An ephemeral message.
  *
  * @remarks
- * Ephemeral messages are not persisted anywhere and have no particular
- * structure. `automerge-repo` will gossip them around, in order to avoid
- * eternal loops of ephemeral messages every message has a session ID, which
- * is a random number generated by the sender at startup time, and a sequence
- * number. The combination of these two things allows us to discard messages
- * we have already seen.
+ * Ephemeral messages are not persisted anywhere. The data property can be used by the application
+ * as needed. The repo gossips these around.
+ *
+ * In order to avoid infinite loops of ephemeral messages, every message has (a) a session ID, which
+ * is a random number generated by the sender at startup time; and (b) a sequence number. The
+ * combination of these two things allows us to discard messages we have already seen.
  * */
 export type EphemeralMessage = {
   type: "ephemeral"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** A sequence number which must be incremented for each message sent by this peer */
+  /** A sequence number which must be incremented for each message sent by this peer. */
   count: number
 
-  /** The ID of the session this message is part of. The sequence number for a given session always increases */
+  /** The ID of the session this message is part of. The sequence number for a given session always increases. */
   sessionId: SessionId
 
-  /** The document ID this message pertains to */
+  /** The document ID this message pertains to. */
   documentId: DocumentId
 
-  /** The actual data of the message */
+  /** The actual data of the message. */
   data: Uint8Array
 }
 
-/** Sent by a {@link Repo} to indicate that it does not have the document and none of it's connected peers do either */
+/**
+ * Sent by a {@link Repo} to indicate that it does not have the document and none of its connected
+ * peers do either.
+ */
 export type DocumentUnavailableMessage = {
   type: "doc-unavailable"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
   /** The document which the peer claims it doesn't have */
   documentId: DocumentId
 }
 
-/** Sent by a {@link Repo} to request a document from a peer
+/**
+ * Sent by a {@link Repo} to request a document from a peer.
  *
  * @remarks
  * This is identical to a {@link SyncMessage} except that it is sent by a {@link Repo}
@@ -75,48 +82,44 @@ export type DocumentUnavailableMessage = {
  * */
 export type RequestMessage = {
   type: "request"
-
-  /** The peer ID of the sender of this message */
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** The initial automerge sync message */
+  /** The automerge sync message */
   data: Uint8Array
 
-  /** The document ID this message requests */
+  /** The document ID of the document this message is for */
   documentId: DocumentId
 }
 
-/** (anticipating work in progress) */
-export type AuthMessage<TPayload = any> = {
-  type: "auth"
-
-  /** The peer ID of the sender of this message */
+/**
+ * Sent by a {@link Repo} to add or remove storage IDs from a remote peer's subscription.
+ */
+export type RemoteSubscriptionControlMessage = {
+  type: "remote-subscription-change"
   senderId: PeerId
-
-  /** The peer ID of the recipient of this message */
   targetId: PeerId
 
-  /** The payload of the auth message (up to the specific auth provider) */
-  payload: TPayload
-}
+  /** The storage IDs to add to the subscription */
+  add?: StorageId[]
 
-export type RemoteSubscriptionControlMessage = {
-  type: "remote-subscription-change",
-  senderId: PeerId,
-  targetId: PeerId,
-  add?: StorageId[],
-  remove?: StorageId[],
+  /** The storage IDs to remove from the subscription */
+  remove?: StorageId[]
 }
 
+/**
+ * Sent by a {@link Repo} to indicate that the heads of a document have changed on a remote peer.
+ */
 export type RemoteHeadsChanged = {
-  type: "remote-heads-changed",
-  senderId: PeerId,
-  targetId: PeerId,
-  documentId: DocumentId,
-  newHeads: {[key: StorageId]: {heads: string[], timestamp: number}},
+  type: "remote-heads-changed"
+  senderId: PeerId
+  targetId: PeerId
+
+  /** The document ID of the document that has changed */
+  documentId: DocumentId
+
+  /** The document's new heads */
+  newHeads: { [key: StorageId]: { heads: string[]; timestamp: number } }
 }
 
 /** These are message types that a {@link NetworkAdapter} surfaces to a {@link Repo}. */
@@ -128,15 +131,17 @@ export type RepoMessage =
   | RemoteSubscriptionControlMessage
   | RemoteHeadsChanged
 
-export type DocMessage = SyncMessage | EphemeralMessage | RequestMessage | DocumentUnavailableMessage
-
-/** These are all the message types that a {@link NetworkAdapter} might see. */
-export type Message = RepoMessage | AuthMessage
+/** These are message types that are handled by the {@link CollectionSynchronizer}.*/
+export type DocMessage =
+  | SyncMessage
+  | EphemeralMessage
+  | RequestMessage
+  | DocumentUnavailableMessage
 
 /**
  * The contents of a message, without the sender ID or other properties added by the {@link NetworkSubsystem})
  */
-export type MessageContents<T extends Message = Message> =
+export type MessageContents<T extends Message = RepoMessage> =
   T extends EphemeralMessage
     ? Omit<T, "senderId" | "count" | "sessionId">
     : Omit<T, "senderId">
@@ -148,18 +153,21 @@ export interface SyncStateMessage {
   syncState: SyncState
 }
 
+/** Notify the repo that a peer started syncing with a doc */
+export interface OpenDocMessage {
+  peerId: PeerId
+  documentId: DocumentId
+}
+
 // TYPE GUARDS
 
-export const isValidRepoMessage = (message: Message): message is RepoMessage =>
-  typeof message === "object" &&
-  typeof message.type === "string" &&
-  typeof message.senderId === "string" &&
-  (isSyncMessage(message) ||
-    isEphemeralMessage(message) ||
-    isRequestMessage(message) ||
-    isDocumentUnavailableMessage(message) ||
-    isRemoteSubscriptionControlMessage(message) ||
-    isRemoteHeadsChanged(message))
+export const isRepoMessage = (message: Message): message is RepoMessage =>
+  isSyncMessage(message) ||
+  isEphemeralMessage(message) ||
+  isRequestMessage(message) ||
+  isDocumentUnavailableMessage(message) ||
+  isRemoteSubscriptionControlMessage(message) ||
+  isRemoteHeadsChanged(message)
 
 // prettier-ignore
 export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage => 
@@ -174,6 +182,7 @@ export const isSyncMessage = (msg: Message): msg is SyncMessage =>
 export const isEphemeralMessage = (msg: Message): msg is EphemeralMessage =>
   msg.type === "ephemeral"
 
+// prettier-ignore
 export const isRemoteSubscriptionControlMessage = (msg: Message): msg is RemoteSubscriptionControlMessage =>
   msg.type === "remote-subscription-change"
 

package/src/storage/StorageSubsystem.ts

@@ -33,7 +33,7 @@ export class StorageSubsystem {
   }
 
   async id(): Promise<StorageId> {
-    let storedId = await this.#storageAdapter.load(["storage-adapter-id"])
+    const storedId = await this.#storageAdapter.load(["storage-adapter-id"])
 
     let id: StorageId
     if (storedId) {
@@ -279,6 +279,11 @@ export class StorageSubsystem {
         incrementalSize += chunk.size
       }
     }
-    return incrementalSize >= snapshotSize
+    // if the file is currently small, don't worry, just compact
+    // this might seem a bit arbitrary (1k is arbitrary) but is designed to ensure compaction
+    // for documents with only a single large change on top of an empty (or nearly empty) document
+    // for example: imported NPM modules, images, etc.
+    // if we have even more incrementals (so far) than the snapshot, compact
+    return snapshotSize < 1024 || incrementalSize >= snapshotSize
   }
 }

package/src/storage/keyHash.ts

@@ -7,11 +7,13 @@ export function keyHash(binary: Uint8Array) {
   const hash = sha256.hash(binary)
   return bufferToHexString(hash)
 }
+
 export function headsHash(heads: A.Heads): string {
   const encoder = new TextEncoder()
   const headsbinary = mergeArrays(heads.map((h: string) => encoder.encode(h)))
   return keyHash(headsbinary)
 }
+
 function bufferToHexString(data: Uint8Array) {
   return Array.from(data, byte => byte.toString(16).padStart(2, "0")).join("")
 }

package/src/synchronizer/CollectionSynchronizer.ts

@@ -2,7 +2,7 @@ import debug from "debug"
 import { DocHandle } from "../DocHandle.js"
 import { stringifyAutomergeUrl } from "../AutomergeUrl.js"
 import { Repo } from "../Repo.js"
-import { DocMessage, RepoMessage } from "../network/messages.js"
+import { DocMessage } from "../network/messages.js"
 import { DocumentId, PeerId } from "../types.js"
 import { DocSynchronizer } from "./DocSynchronizer.js"
 import { Synchronizer } from "./Synchronizer.js"
@@ -42,18 +42,20 @@ export class CollectionSynchronizer extends Synchronizer {
           return
         }
 
-        const persistanceInfo = this.repo.persistanceInfoByPeerId[peerId]
-        if (!persistanceInfo || persistanceInfo.isEphemeral) {
+        const { storageId, isEphemeral } =
+          this.repo.peerMetadataByPeerId[peerId] || {}
+        if (!storageId || isEphemeral) {
           return
         }
 
         return this.repo.storageSubsystem.loadSyncState(
           handle.documentId,
-          persistanceInfo.storageId
+          storageId
         )
       },
     })
     docSynchronizer.on("message", event => this.emit("message", event))
+    docSynchronizer.on("open-doc", event => this.emit("open-doc", event))
     docSynchronizer.on("sync-state", event => this.emit("sync-state", event))
     return docSynchronizer
   }
@@ -115,6 +117,7 @@ export class CollectionSynchronizer extends Synchronizer {
   }
 
   // TODO: implement this
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   removeDocument(documentId: DocumentId) {
     throw new Error("not implemented")
   }

package/src/synchronizer/DocSynchronizer.ts

@@ -20,7 +20,6 @@ import {
 import { PeerId } from "../types.js"
 import { Synchronizer } from "./Synchronizer.js"
 import { throttle } from "../helpers/throttle.js"
-import { headsAreSame } from "../helpers/headsAreSame.js"
 
 type PeerDocumentStatus = "unknown" | "has" | "unavailable" | "wants"
 
@@ -124,9 +123,7 @@ export class DocSynchronizer extends Synchronizer {
   }
 
   #withSyncState(peerId: PeerId, callback: (syncState: A.SyncState) => void) {
-    if (!this.#peers.includes(peerId)) {
-      this.#peers.push(peerId)
-    }
+    this.#addPeer(peerId)
 
     if (!(peerId in this.#peerDocumentStatuses)) {
       this.#peerDocumentStatuses[peerId] = "unknown"
@@ -140,15 +137,26 @@ export class DocSynchronizer extends Synchronizer {
 
     let pendingCallbacks = this.#pendingSyncStateCallbacks[peerId]
     if (!pendingCallbacks) {
-      this.#onLoadSyncState(peerId).then(syncState => {
-        this.#initSyncState(peerId, syncState ?? A.initSyncState())
-      })
+      this.#onLoadSyncState(peerId)
+        .then(syncState => {
+          this.#initSyncState(peerId, syncState ?? A.initSyncState())
+        })
+        .catch(err => {
+          this.#log(`Error loading sync state for ${peerId}: ${err}`)
+        })
       pendingCallbacks = this.#pendingSyncStateCallbacks[peerId] = []
     }
 
     pendingCallbacks.push(callback)
   }
 
+  #addPeer(peerId: PeerId) {
+    if (!this.#peers.includes(peerId)) {
+      this.#peers.push(peerId)
+      this.emit("open-doc", { documentId: this.documentId, peerId })
+    }
+  }
+
   #initSyncState(peerId: PeerId, syncState: A.SyncState) {
     const pendingCallbacks = this.#pendingSyncStateCallbacks[peerId]
     if (pendingCallbacks) {
@@ -256,11 +264,15 @@ export class DocSynchronizer extends Synchronizer {
         )
         this.#setSyncState(peerId, reparsedSyncState)
 
-        docPromise.then(doc => {
-          if (doc) {
-            this.#sendSyncMessage(peerId, doc)
-          }
-        })
+        docPromise
+          .then(doc => {
+            if (doc) {
+              this.#sendSyncMessage(peerId, doc)
+            }
+          })
+          .catch(err => {
+            this.#log(`Error loading doc for ${peerId}: ${err}`)
+          })
       })
     })
   }
@@ -322,10 +334,10 @@ export class DocSynchronizer extends Synchronizer {
     }
 
     this.#processAllPendingSyncMessages()
-    this.#processSyncMessage(message, new Date())
+    this.#processSyncMessage(message)
   }
 
-  #processSyncMessage(message: SyncMessage | RequestMessage, received: Date) {
+  #processSyncMessage(message: SyncMessage | RequestMessage) {
     if (isRequestMessage(message)) {
       this.#peerDocumentStatuses[message.senderId] = "wants"
     }
@@ -384,7 +396,7 @@ export class DocSynchronizer extends Synchronizer {
 
   #processAllPendingSyncMessages() {
     for (const message of this.#pendingSyncMessages) {
-      this.#processSyncMessage(message.message, message.received)
+      this.#processSyncMessage(message.message)
     }
 
     this.#pendingSyncMessages = []

package/src/synchronizer/Synchronizer.ts

@@ -1,15 +1,25 @@
 import { EventEmitter } from "eventemitter3"
 import {
   MessageContents,
+  OpenDocMessage,
   RepoMessage,
-  SyncStateMessage,
 } from "../network/messages.js"
+import { SyncState } from "@automerge/automerge"
+import { PeerId, DocumentId } from "../types.js"
 
 export abstract class Synchronizer extends EventEmitter<SynchronizerEvents> {
   abstract receiveMessage(message: RepoMessage): void
 }
 
 export interface SynchronizerEvents {
-  message: (arg: MessageContents) => void
-  "sync-state": (arg: SyncStateMessage) => void
+  message: (payload: MessageContents) => void
+  "sync-state": (payload: SyncStatePayload) => void
+  "open-doc": (arg: OpenDocMessage) => void
+}
+
+/** Notify the repo that the sync state has changed  */
+export interface SyncStatePayload {
+  peerId: PeerId
+  documentId: DocumentId
+  syncState: SyncState
 }