@comapeo/core 2.3.1 → 3.0.0-0

package/src/blob-store/index.js

@@ -6,12 +6,26 @@ import { FilterEntriesStream } from './utils.js'
 import { noop } from '../utils.js'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import { HyperdriveIndexImpl as HyperdriveIndex } from './hyperdrive-index.js'
+import { Logger } from '../logger.js'
+import { getErrorCode, getErrorMessage } from '../lib/error.js'
 
 /** @import Hyperdrive from 'hyperdrive' */
 /** @import { JsonObject } from 'type-fest' */
 /** @import { Readable as NodeReadable } from 'node:stream' */
 /** @import { Readable as StreamxReadable, Writable } from 'streamx' */
-/** @import { BlobFilter, BlobId, BlobStoreEntriesStream } from '../types.js' */
+/** @import { GenericBlobFilter, BlobFilter, BlobId, BlobStoreEntriesStream } from '../types.js' */
+
+/**
+ * @typedef {object} BlobStoreEvents
+ * @prop {(peerId: string, blobFilter: GenericBlobFilter | null) => void} blob-filter
+ * @prop {(opts: {
+ *   peerId: string
+ *   start: number
+ *   length: number
+ *   blobCoreId: string
+ * }) => void} want-blob-range
+ * @prop {(error: Error) => void} error
+ */
 
 /**
  * @internal
@@ -31,6 +45,13 @@ const SUPPORTED_BLOB_VARIANTS = /** @type {const} */ ({
 // the type with JSDoc
 export { SUPPORTED_BLOB_VARIANTS }
 
+/** @type {import('../types.js').BlobFilter} */
+const NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER = {
+  photo: ['preview', 'thumbnail'],
+  // Don't download any audio of video files, since previews and
+  // thumbnails aren't supported yet.
+}
+
 class ErrNotFound extends Error {
   constructor(message = 'NotFound') {
     super(message)
@@ -38,24 +59,117 @@ class ErrNotFound extends Error {
   }
 }
 
-/** @extends {TypedEmitter<{ error: (error: Error) => void }>} */
+/** @extends {TypedEmitter<BlobStoreEvents>} */
 export class BlobStore extends TypedEmitter {
   #driveIndex
   /** @type {Downloader} */
   #downloader
+  /** @type {Map<string, GenericBlobFilter | null>} */
+  #blobFilters = new Map()
+  #l
+  /** @type {Map<string, BlobStoreEntriesStream>} */
+  #entriesStreams = new Map()
+  #isArchiveDevice
+  #deviceId
+
+  /**
+   * Bound function for handling download intents for both peers and self
+   * @param {GenericBlobFilter | null} filter
+   * @param {string} peerId
+   */
+  #handleDownloadIntent = async (filter, peerId) => {
+    this.#l.log('Download intent %o for peer %S', filter, peerId)
+    try {
+      this.#entriesStreams.get(peerId)?.destroy()
+      this.emit('blob-filter', peerId, filter)
+      this.#blobFilters.set(peerId, filter)
+
+      if (filter === null) return
+
+      const entriesReadStream = this.createEntriesReadStream({
+        live: true,
+        filter,
+      })
+      this.#entriesStreams.set(peerId, entriesReadStream)
+
+      entriesReadStream.once('close', () => {
+        if (this.#entriesStreams.get(peerId) === entriesReadStream) {
+          this.#entriesStreams.delete(peerId)
+        }
+      })
+
+      for await (const {
+        blobCoreId,
+        value: { blob },
+      } of entriesReadStream) {
+        const { blockOffset: start, blockLength: length } = blob
+        this.emit('want-blob-range', {
+          peerId,
+          start,
+          length,
+          blobCoreId,
+        })
+      }
+    } catch (err) {
+      if (getErrorCode(err) === 'ERR_STREAM_PREMATURE_CLOSE') return
+      this.#l.log(
+        'Error getting blob entries stream for peer %h: %s',
+        peerId,
+        getErrorMessage(err)
+      )
+    }
+  }
+
+  /**
+   * Bound to `this`
+   * This will be called whenever a peer is successfully added to the creatorcore
+   * @param {import('../types.js').HypercorePeer & { protomux: import('protomux')<import('../lib/noise-secret-stream-helpers.js').OpenedNoiseStream> }} peer
+   */
+  #handlePeerAdd = (peer) => {
+    const downloadFilter = getBlobDownloadFilter(this.#isArchiveDevice)
+    this.#coreManager.sendDownloadIntents(downloadFilter, peer)
+  }
+
+  /**
+   * Bound to `this`
+   * @param {import('../types.js').HypercorePeer & { protomux: import('protomux')<import('../lib/noise-secret-stream-helpers.js').OpenedNoiseStream> }} peer
+   */
+  #handlePeerRemove = (peer) => {
+    const peerKey = peer.protomux.stream.remotePublicKey
+    const peerId = peerKey.toString('hex')
+    this.#entriesStreams.get(peerId)?.destroy()
+    this.#entriesStreams.delete(peerId)
+  }
+
+  #coreManager
+  #logger
 
   /**
    * @param {object} options
    * @param {import('../core-manager/index.js').CoreManager} options.coreManager
-   * @param {BlobFilter | null} options.downloadFilter - Filter blob types and/or variants to download. Set to `null` to download all blobs.
+   * @param {boolean} [options.isArchiveDevice] Set to `true` if this is an archive device which should download all blobs, or just a selection of blobs
+   * @param {import('../logger.js').Logger} [options.logger]
    */
-  constructor({ coreManager, downloadFilter }) {
+  constructor({ coreManager, isArchiveDevice = true, logger }) {
     super()
+    this.#logger = logger
+    this.#l = Logger.create('blobStore', logger)
+    this.#isArchiveDevice = isArchiveDevice
     this.#driveIndex = new HyperdriveIndex(coreManager)
+    this.#coreManager = coreManager
+    this.#deviceId = coreManager.deviceId
+    const downloadFilter = getBlobDownloadFilter(isArchiveDevice)
+    if (downloadFilter) {
+      this.#handleDownloadIntent(downloadFilter, this.#deviceId)
+    }
     this.#downloader = new Downloader(this.#driveIndex, {
       filter: downloadFilter,
     })
     this.#downloader.on('error', (error) => this.emit('error', error))
+
+    coreManager.on('peer-download-intent', this.#handleDownloadIntent)
+    coreManager.creatorCore.on('peer-add', this.#handlePeerAdd)
+    coreManager.creatorCore.on('peer-remove', this.#handlePeerRemove)
   }
 
   /**
@@ -65,6 +179,38 @@ export class BlobStore extends TypedEmitter {
     return getDiscoveryId(this.#driveIndex.writerKey)
   }
 
+  get isArchiveDevice() {
+    return this.#isArchiveDevice
+  }
+
+  /**
+   * @param {string} peerId
+   * @returns {GenericBlobFilter | null}
+   */
+  getBlobFilter(peerId) {
+    return this.#blobFilters.get(peerId) ?? null
+  }
+
+  /** @param {boolean} isArchiveDevice */
+  async setIsArchiveDevice(isArchiveDevice) {
+    this.#l.log('Setting isArchiveDevice to %s', isArchiveDevice)
+    if (this.#isArchiveDevice === isArchiveDevice) return
+    this.#isArchiveDevice = isArchiveDevice
+    const blobDownloadFilter = getBlobDownloadFilter(isArchiveDevice)
+    this.#downloader.removeAllListeners()
+    this.#downloader.destroy()
+    this.#downloader = new Downloader(this.#driveIndex, {
+      filter: blobDownloadFilter,
+    })
+    this.#downloader.on('error', (error) => this.emit('error', error))
+    // Even if blobFilter is null, e.g. we plan to download everything, we still
+    // need to inform connected peers of the change.
+    for (const peer of this.#coreManager.creatorCore.peers) {
+      this.#coreManager.sendDownloadIntents(blobDownloadFilter, peer)
+    }
+    this.#handleDownloadIntent(blobDownloadFilter, this.#deviceId)
+  }
+
   /**
    * @param {string} driveId hex-encoded discovery key
    * @returns {Hyperdrive}
@@ -90,21 +236,6 @@ export class BlobStore extends TypedEmitter {
     return blob
   }
 
-  /**
-   * Set the filter for downloading blobs.
-   *
-   * @param {import('../types.js').BlobFilter | null} filter Filter blob types and/or variants to download. Filter is { [BlobType]: BlobVariants[] }. At least one blob variant must be specified for each blob type.
-   * @returns {void}
-   */
-  setDownloadFilter(filter) {
-    this.#downloader.removeAllListeners()
-    this.#downloader.destroy()
-    this.#downloader = new Downloader(this.#driveIndex, {
-      filter,
-    })
-    this.#downloader.on('error', (error) => this.emit('error', error))
-  }
-
   /**
    * @param {BlobId} blobId
    * @param {object} [options]
@@ -245,6 +376,9 @@ export class BlobStore extends TypedEmitter {
   close() {
     this.#downloader.removeAllListeners()
     this.#downloader.destroy()
+    this.#coreManager.off('peer-download-intent', this.#handleDownloadIntent)
+    this.#coreManager.creatorCore.off('peer-add', this.#handlePeerAdd)
+    this.#coreManager.creatorCore.off('peer-remove', this.#handlePeerRemove)
   }
 }
 
@@ -280,3 +414,11 @@ function makePath({ type, variant, name }) {
 function getDiscoveryId(key) {
   return discoveryKey(key).toString('hex')
 }
+
+/**
+ * @param {boolean} isArchiveDevice
+ * @returns {null | BlobFilter}
+ */
+function getBlobDownloadFilter(isArchiveDevice) {
+  return isArchiveDevice ? null : NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER
+}

package/src/core-manager/index.js

@@ -30,7 +30,7 @@ export const kCoreManagerReplicate = Symbol('replicate core manager')
  * @typedef {Object} Events
  * @property {(coreRecord: CoreRecord) => void} add-core
  * @property {(namespace: Namespace, msg: { coreDiscoveryId: string, peerId: string, start: number, bitfield: Uint32Array }) => void} peer-have
- * @property {(blobFilter: GenericBlobFilter, peerId: string) => void} peer-download-intent
+ * @property {(blobFilter: GenericBlobFilter | null, peerId: string) => void} peer-download-intent
  */
 
 /**
@@ -412,7 +412,7 @@ export class CoreManager extends TypedEmitter {
   }
 
   /**
-   * @param {GenericBlobFilter} blobFilter
+   * @param {GenericBlobFilter | null} blobFilter
    * @param {HypercorePeer} peer
    */
   #handleDownloadIntentMessage(blobFilter, peer) {
@@ -421,7 +421,7 @@ export class CoreManager extends TypedEmitter {
   }
 
   /**
-   * @param {BlobFilter} blobFilter
+   * @param {BlobFilter | null} blobFilter
    * @param {HypercorePeer} peer
    */
   sendDownloadIntents(blobFilter, peer) {
@@ -540,20 +540,27 @@ const HaveExtensionCodec = {
 }
 
 const DownloadIntentCodec = {
-  /** @param {BlobFilter} filter */
+  /** @param {BlobFilter | null} filter */
   encode(filter) {
-    const downloadIntents = mapObject(filter, (key, value) => [
+    const downloadIntents = mapObject(filter || {}, (key, value) => [
       key,
       { variants: value || [] },
     ])
-    return DownloadIntentExtension.encode({ downloadIntents }).finish()
+    // If filter is null, we want to download everything
+    const everything = !filter
+    return DownloadIntentExtension.encode({
+      downloadIntents,
+      everything,
+    }).finish()
   },
   /**
    * @param {Buffer | Uint8Array} buf
-   * @returns {GenericBlobFilter}
+   * @returns {GenericBlobFilter | null}
    */
   decode(buf) {
     const msg = DownloadIntentExtension.decode(buf)
+    // If everything is true, we ignore the downloadIntents and return null, which means download everything
+    if (msg.everything) return null
     return mapObject(msg.downloadIntents, (key, value) => [
       key + '', // keep TS happy
       value.variants,

package/src/errors.js

@@ -1,6 +1,39 @@
 export class NotFoundError extends Error {
   constructor(message = 'Not found') {
     super(message)
+    this.name = 'NotFoundError'
+  }
+}
+
+export class AlreadyJoinedError extends Error {
+  /** @param {string} [message] */
+  constructor(message = 'AlreadyJoinedError') {
+    super(message)
+    this.name = 'AlreadyJoinedError'
+  }
+}
+
+export class InviteSendError extends Error {
+  /** @param {string} [message] */
+  constructor(message = 'Invite Send Error') {
+    super(message)
+    this.name = 'InviteSendError'
+  }
+}
+
+export class InviteAbortedError extends Error {
+  /** @param {string} [message] */
+  constructor(message = 'Invite Aborted') {
+    super(message)
+    this.name = 'InviteAbortedError'
+  }
+}
+
+export class TimeoutError extends Error {
+  /** @param {string} [message] */
+  constructor(message = 'TimeoutError') {
+    super(message)
+    this.name = 'TimeoutError'
   }
 }
 

package/src/generated/extensions.d.ts

@@ -24,6 +24,13 @@ export interface DownloadIntentExtension {
     downloadIntents: {
         [key: string]: DownloadIntentExtension_DownloadIntent;
     };
+    /**
+     * If true, the peer intends to download all blobs - this is the default
+     * assumption when a peer has not sent a download intent, but if a peer
+     * changes their intent while connected, we need to send the new intent to
+     * download everything.
+     */
+    everything: boolean;
 }
 export interface DownloadIntentExtension_DownloadIntent {
     variants: string[];

package/src/generated/extensions.js

@@ -170,7 +170,7 @@ export var HaveExtension = {
     },
 };
 function createBaseDownloadIntentExtension() {
-    return { downloadIntents: {} };
+    return { downloadIntents: {}, everything: false };
 }
 export var DownloadIntentExtension = {
     encode: function (message, writer) {
@@ -180,6 +180,9 @@ export var DownloadIntentExtension = {
             DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key, value: value }, writer.uint32(10).fork())
                 .ldelim();
         });
+        if (message.everything === true) {
+            writer.uint32(16).bool(message.everything);
+        }
         return writer;
     },
     decode: function (input, length) {
@@ -198,6 +201,12 @@ export var DownloadIntentExtension = {
                         message.downloadIntents[entry1.key] = entry1.value;
                     }
                     continue;
+                case 2:
+                    if (tag !== 16) {
+                        break;
+                    }
+                    message.everything = reader.bool();
+                    continue;
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -210,7 +219,7 @@ export var DownloadIntentExtension = {
         return DownloadIntentExtension.fromPartial(base !== null && base !== void 0 ? base : {});
     },
     fromPartial: function (object) {
-        var _a;
+        var _a, _b;
         var message = createBaseDownloadIntentExtension();
         message.downloadIntents = Object.entries((_a = object.downloadIntents) !== null && _a !== void 0 ? _a : {}).reduce(function (acc, _a) {
             var key = _a[0], value = _a[1];
@@ -219,6 +228,7 @@ export var DownloadIntentExtension = {
             }
             return acc;
         }, {});
+        message.everything = (_b = object.everything) !== null && _b !== void 0 ? _b : false;
         return message;
     },
 };

package/src/generated/extensions.ts

@@ -69,6 +69,13 @@ export function haveExtension_NamespaceToNumber(object: HaveExtension_Namespace)
 /** A map of blob types and variants that a peer intends to download */
 export interface DownloadIntentExtension {
   downloadIntents: { [key: string]: DownloadIntentExtension_DownloadIntent };
+  /**
+   * If true, the peer intends to download all blobs - this is the default
+   * assumption when a peer has not sent a download intent, but if a peer
+   * changes their intent while connected, we need to send the new intent to
+   * download everything.
+   */
+  everything: boolean;
 }
 
 export interface DownloadIntentExtension_DownloadIntent {
@@ -209,7 +216,7 @@ export const HaveExtension = {
 };
 
 function createBaseDownloadIntentExtension(): DownloadIntentExtension {
-  return { downloadIntents: {} };
+  return { downloadIntents: {}, everything: false };
 }
 
 export const DownloadIntentExtension = {
@@ -218,6 +225,9 @@ export const DownloadIntentExtension = {
       DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key as any, value }, writer.uint32(10).fork())
         .ldelim();
     });
+    if (message.everything === true) {
+      writer.uint32(16).bool(message.everything);
+    }
     return writer;
   },
 
@@ -238,6 +248,13 @@ export const DownloadIntentExtension = {
             message.downloadIntents[entry1.key] = entry1.value;
           }
           continue;
+        case 2:
+          if (tag !== 16) {
+            break;
+          }
+
+          message.everything = reader.bool();
+          continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -260,6 +277,7 @@ export const DownloadIntentExtension = {
       }
       return acc;
     }, {});
+    message.everything = object.everything ?? false;
     return message;
   },
 };

package/src/invite/StateDiagram.md

@@ -0,0 +1,47 @@
+```mermaid
+---
+title: Invite Receive State Diagram
+---
+stateDiagram-v2
+  state "invite" as invite {
+    [*] --> invite.pending
+    invite.pending --> invite.canceled : CANCEL_INVITE
+    invite.pending --> invite.responding.accept : ACCEPT_INVITE
+    invite.pending --> invite.responding.already : ALREADY_IN_PROJECT
+    invite.pending --> invite.responding.reject : REJECT_INVITE
+    invite.responding.default --> invite.error
+    invite.responding.accept --> invite.joining.addingProject : RECEIVE_PROJECT_DETAILS
+    invite.responding.accept --> invite.joining : Responded Accept
+    invite.responding.accept --> invite.error : Error responding
+    invite.responding.reject --> invite.rejected : Responded Reject
+    invite.responding.reject --> invite.error : Error responding
+    invite.responding.already --> invite.respondedAlready : Responded Already
+    invite.responding.already --> invite.error : Error responding
+    invite.responding --> invite.canceled : CANCEL_INVITE
+    invite.joining.awaitingDetails --> invite.canceled : CANCEL_INVITE
+    invite.joining.awaitingDetails --> invite.error : projectDetailsTimeout
+    invite.joining.addingProject --> invite.error : addProject Timeout
+    invite.joining.addingProject --> invite.joined :  Project Added
+    invite.joining.addingProject --> invite.error : addProject Error
+    state "Pending invite awaiting response" as invite.pending
+        state "Responding to invite" as invite.responding {
+            [*] --> invite.responding.default
+            state "default" as invite.responding.default
+            state "accept" as invite.responding.accept
+            state "reject" as invite.responding.reject
+            state "already" as invite.responding.already
+        }
+        state "Joining project from invite" as invite.joining {
+            [*] --> invite.joining.awaitingDetails
+            invite.joining.awaitingDetails --> invite.joining.addingProject : RECEIVE_PROJECT_DETAILS
+            state "Awaiting project details" as invite.joining.awaitingDetails
+          state "Adding project" as invite.joining.addingProject
+        }
+    state "Invite Canceled" as invite.canceled
+    state "Invite Rejected" as invite.rejected
+    state "Responded already in project" as invite.respondedAlready
+    state "Joined project" as invite.joined
+    state "Invite Error" as invite.error
+  }
+
+```