@comapeo/core 2.3.1 → 2.3.2

package/src/blob-store/entries-stream.js

@@ -2,6 +2,7 @@ import SubEncoder from 'sub-encoder'
 import mergeStreams from '@sindresorhus/merge-streams'
 import { Transform, pipeline } from 'node:stream'
 import { noop } from '../utils.js'
+import ensureError from 'ensure-error'
 
 /** @import Hyperdrive from 'hyperdrive' */
 /** @import { BlobStoreEntriesStream } from '../types.js' */
@@ -18,7 +19,7 @@ const keyEncoding = new SubEncoder('files', 'utf-8')
  */
 export function createEntriesStream(driveIndex, { live = false } = {}) {
   const mergedEntriesStreams = mergeStreams(
-    [...driveIndex].map((drive) => getHistoryStream(drive.db, { live }))
+    [...driveIndex].map((drive) => getHistoryStream(drive, { live }))
   )
   driveIndex.on('add-drive', addDrive)
   // Close is always emitted, so we can use it to remove the listener
@@ -29,53 +30,77 @@ export function createEntriesStream(driveIndex, { live = false } = {}) {
 
   /** @param {Hyperdrive} drive */
   function addDrive(drive) {
-    mergedEntriesStreams.add(getHistoryStream(drive.db, { live }))
+    mergedEntriesStreams.add(getHistoryStream(drive, { live }))
   }
 }
 
 /**
  *
- * @param {import('hyperbee')} bee
+ * @param {Hyperdrive} drive
  * @param {object} opts
  * @param {boolean} opts.live
  */
-function getHistoryStream(bee, { live }) {
+function getHistoryStream(drive, { live }) {
   // This will also include old versions of files, but it is the only way to
   // get a live stream from a Hyperbee, however we currently do not support
   // edits of blobs, so this should not be an issue, and the consequence is
   // that old versions are downloaded too, which is acceptable.
-  const historyStream = bee.createHistoryStream({
+  const historyStream = drive.db.createHistoryStream({
     live,
     // `keyEncoding` is necessary because hyperdrive stores file index data
     // under the `files` sub-encoding key
     keyEncoding,
   })
-  return pipeline(historyStream, new AddDriveIds(bee.core), noop)
+  return pipeline(historyStream, new AddDriveIds(drive), noop)
 }
 
 class AddDriveIds extends Transform {
-  #core
+  #drive
+  /** @type {string | undefined} */
   #cachedDriveId
+  /** @type {string | undefined} */
+  #cachedBlobCoreId
 
-  /** @param {import('hypercore')} core */
-  constructor(core) {
+  /** @param {Hyperdrive} drive */
+  constructor(drive) {
     super({ objectMode: true })
-    this.#core = core
-    this.#cachedDriveId = core.discoveryKey?.toString('hex')
+    this.#drive = drive
+    this.#cachedDriveId = drive.db.core.discoveryKey?.toString('hex')
   }
 
-  /** @type {Transform['_transform']} */
-  _transform(entry, _, callback) {
+  get #driveId() {
     // Minimal performance optimization to only call toString() once.
     // core.discoveryKey will always be defined by the time it starts
     // streaming, but could be null when the instance is first created.
-    let driveId
     if (this.#cachedDriveId) {
-      driveId = this.#cachedDriveId
+      return this.#cachedDriveId
     } else {
-      driveId = this.#core.discoveryKey?.toString('hex')
-      this.#cachedDriveId = driveId
+      this.#cachedDriveId = this.#drive.db.core.discoveryKey?.toString('hex')
+      return this.#cachedDriveId
+    }
+  }
+
+  async #getBlobCoreId() {
+    if (this.#cachedBlobCoreId) return this.#cachedBlobCoreId
+    const blobs = await this.#drive.getBlobs()
+    this.#cachedBlobCoreId = blobs.core.discoveryKey?.toString('hex')
+    return this.#cachedBlobCoreId
+  }
+
+  /** @type {Transform['_transform']} */
+  _transform(entry, _, callback) {
+    if (!this.#driveId) {
+      return callback(new Error('Drive discovery key unexpectedly missing'))
     }
-    callback(null, { ...entry, driveId })
+    this.#getBlobCoreId()
+      .then((blobCoreId) => {
+        if (!blobCoreId) {
+          return callback(
+            new Error('Blob core discovery key unexpectedly missing')
+          )
+        }
+        callback(null, { ...entry, driveId: this.#driveId, blobCoreId })
+      })
+      .catch((reason) => callback(ensureError(reason)))
   }
 }

package/src/blob-store/index.js

@@ -6,12 +6,26 @@ import { FilterEntriesStream } from './utils.js'
 import { noop } from '../utils.js'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import { HyperdriveIndexImpl as HyperdriveIndex } from './hyperdrive-index.js'
+import { Logger } from '../logger.js'
+import { getErrorCode, getErrorMessage } from '../lib/error.js'
 
 /** @import Hyperdrive from 'hyperdrive' */
 /** @import { JsonObject } from 'type-fest' */
 /** @import { Readable as NodeReadable } from 'node:stream' */
 /** @import { Readable as StreamxReadable, Writable } from 'streamx' */
-/** @import { BlobFilter, BlobId, BlobStoreEntriesStream } from '../types.js' */
+/** @import { GenericBlobFilter, BlobFilter, BlobId, BlobStoreEntriesStream } from '../types.js' */
+
+/**
+ * @typedef {object} BlobStoreEvents
+ * @prop {(peerId: string, blobFilter: GenericBlobFilter | null) => void} blob-filter
+ * @prop {(opts: {
+ *   peerId: string
+ *   start: number
+ *   length: number
+ *   blobCoreId: string
+ * }) => void} want-blob-range
+ * @prop {(error: Error) => void} error
+ */
 
 /**
  * @internal
@@ -31,6 +45,13 @@ const SUPPORTED_BLOB_VARIANTS = /** @type {const} */ ({
 // the type with JSDoc
 export { SUPPORTED_BLOB_VARIANTS }
 
+/** @type {import('../types.js').BlobFilter} */
+const NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER = {
+  photo: ['preview', 'thumbnail'],
+  // Don't download any audio of video files, since previews and
+  // thumbnails aren't supported yet.
+}
+
 class ErrNotFound extends Error {
   constructor(message = 'NotFound') {
     super(message)
@@ -38,24 +59,117 @@ class ErrNotFound extends Error {
   }
 }
 
-/** @extends {TypedEmitter<{ error: (error: Error) => void }>} */
+/** @extends {TypedEmitter<BlobStoreEvents>} */
 export class BlobStore extends TypedEmitter {
   #driveIndex
   /** @type {Downloader} */
   #downloader
+  /** @type {Map<string, GenericBlobFilter | null>} */
+  #blobFilters = new Map()
+  #l
+  /** @type {Map<string, BlobStoreEntriesStream>} */
+  #entriesStreams = new Map()
+  #isArchiveDevice
+  #deviceId
+
+  /**
+   * Bound function for handling download intents for both peers and self
+   * @param {GenericBlobFilter | null} filter
+   * @param {string} peerId
+   */
+  #handleDownloadIntent = async (filter, peerId) => {
+    this.#l.log('Download intent %o for peer %S', filter, peerId)
+    try {
+      this.#entriesStreams.get(peerId)?.destroy()
+      this.emit('blob-filter', peerId, filter)
+      this.#blobFilters.set(peerId, filter)
+
+      if (filter === null) return
+
+      const entriesReadStream = this.createEntriesReadStream({
+        live: true,
+        filter,
+      })
+      this.#entriesStreams.set(peerId, entriesReadStream)
+
+      entriesReadStream.once('close', () => {
+        if (this.#entriesStreams.get(peerId) === entriesReadStream) {
+          this.#entriesStreams.delete(peerId)
+        }
+      })
+
+      for await (const {
+        blobCoreId,
+        value: { blob },
+      } of entriesReadStream) {
+        const { blockOffset: start, blockLength: length } = blob
+        this.emit('want-blob-range', {
+          peerId,
+          start,
+          length,
+          blobCoreId,
+        })
+      }
+    } catch (err) {
+      if (getErrorCode(err) === 'ERR_STREAM_PREMATURE_CLOSE') return
+      this.#l.log(
+        'Error getting blob entries stream for peer %h: %s',
+        peerId,
+        getErrorMessage(err)
+      )
+    }
+  }
+
+  /**
+   * Bound to `this`
+   * This will be called whenever a peer is successfully added to the creatorcore
+   * @param {import('../types.js').HypercorePeer & { protomux: import('protomux')<import('../lib/noise-secret-stream-helpers.js').OpenedNoiseStream> }} peer
+   */
+  #handlePeerAdd = (peer) => {
+    const downloadFilter = getBlobDownloadFilter(this.#isArchiveDevice)
+    this.#coreManager.sendDownloadIntents(downloadFilter, peer)
+  }
+
+  /**
+   * Bound to `this`
+   * @param {import('../types.js').HypercorePeer & { protomux: import('protomux')<import('../lib/noise-secret-stream-helpers.js').OpenedNoiseStream> }} peer
+   */
+  #handlePeerRemove = (peer) => {
+    const peerKey = peer.protomux.stream.remotePublicKey
+    const peerId = peerKey.toString('hex')
+    this.#entriesStreams.get(peerId)?.destroy()
+    this.#entriesStreams.delete(peerId)
+  }
+
+  #coreManager
+  #logger
 
   /**
    * @param {object} options
    * @param {import('../core-manager/index.js').CoreManager} options.coreManager
-   * @param {BlobFilter | null} options.downloadFilter - Filter blob types and/or variants to download. Set to `null` to download all blobs.
+   * @param {boolean} [options.isArchiveDevice] Set to `true` if this is an archive device which should download all blobs, or just a selection of blobs
+   * @param {import('../logger.js').Logger} [options.logger]
    */
-  constructor({ coreManager, downloadFilter }) {
+  constructor({ coreManager, isArchiveDevice = true, logger }) {
     super()
+    this.#logger = logger
+    this.#l = Logger.create('blobStore', logger)
+    this.#isArchiveDevice = isArchiveDevice
     this.#driveIndex = new HyperdriveIndex(coreManager)
+    this.#coreManager = coreManager
+    this.#deviceId = coreManager.deviceId
+    const downloadFilter = getBlobDownloadFilter(isArchiveDevice)
+    if (downloadFilter) {
+      this.#handleDownloadIntent(downloadFilter, this.#deviceId)
+    }
     this.#downloader = new Downloader(this.#driveIndex, {
       filter: downloadFilter,
     })
     this.#downloader.on('error', (error) => this.emit('error', error))
+
+    coreManager.on('peer-download-intent', this.#handleDownloadIntent)
+    coreManager.creatorCore.on('peer-add', this.#handlePeerAdd)
+    coreManager.creatorCore.on('peer-remove', this.#handlePeerRemove)
   }
 
   /**
@@ -65,6 +179,38 @@ export class BlobStore extends TypedEmitter {
     return getDiscoveryId(this.#driveIndex.writerKey)
   }
 
+  get isArchiveDevice() {
+    return this.#isArchiveDevice
+  }
+
+  /**
+   * @param {string} peerId
+   * @returns {GenericBlobFilter | null}
+   */
+  getBlobFilter(peerId) {
+    return this.#blobFilters.get(peerId) ?? null
+  }
+
+  /** @param {boolean} isArchiveDevice */
+  async setIsArchiveDevice(isArchiveDevice) {
+    this.#l.log('Setting isArchiveDevice to %s', isArchiveDevice)
+    if (this.#isArchiveDevice === isArchiveDevice) return
+    this.#isArchiveDevice = isArchiveDevice
+    const blobDownloadFilter = getBlobDownloadFilter(isArchiveDevice)
+    this.#downloader.removeAllListeners()
+    this.#downloader.destroy()
+    this.#downloader = new Downloader(this.#driveIndex, {
+      filter: blobDownloadFilter,
+    })
+    this.#downloader.on('error', (error) => this.emit('error', error))
+    // Even if blobFilter is null, e.g. we plan to download everything, we still
+    // need to inform connected peers of the change.
+    for (const peer of this.#coreManager.creatorCore.peers) {
+      this.#coreManager.sendDownloadIntents(blobDownloadFilter, peer)
+    }
+    this.#handleDownloadIntent(blobDownloadFilter, this.#deviceId)
+  }
+
   /**
    * @param {string} driveId hex-encoded discovery key
    * @returns {Hyperdrive}
@@ -90,21 +236,6 @@ export class BlobStore extends TypedEmitter {
     return blob
   }
 
-  /**
-   * Set the filter for downloading blobs.
-   *
-   * @param {import('../types.js').BlobFilter | null} filter Filter blob types and/or variants to download. Filter is { [BlobType]: BlobVariants[] }. At least one blob variant must be specified for each blob type.
-   * @returns {void}
-   */
-  setDownloadFilter(filter) {
-    this.#downloader.removeAllListeners()
-    this.#downloader.destroy()
-    this.#downloader = new Downloader(this.#driveIndex, {
-      filter,
-    })
-    this.#downloader.on('error', (error) => this.emit('error', error))
-  }
-
   /**
    * @param {BlobId} blobId
    * @param {object} [options]
@@ -245,6 +376,9 @@ export class BlobStore extends TypedEmitter {
   close() {
     this.#downloader.removeAllListeners()
     this.#downloader.destroy()
+    this.#coreManager.off('peer-download-intent', this.#handleDownloadIntent)
+    this.#coreManager.creatorCore.off('peer-add', this.#handlePeerAdd)
+    this.#coreManager.creatorCore.off('peer-remove', this.#handlePeerRemove)
   }
 }
 
@@ -280,3 +414,11 @@ function makePath({ type, variant, name }) {
 function getDiscoveryId(key) {
   return discoveryKey(key).toString('hex')
 }
+
+/**
+ * @param {boolean} isArchiveDevice
+ * @returns {null | BlobFilter}
+ */
+function getBlobDownloadFilter(isArchiveDevice) {
+  return isArchiveDevice ? null : NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER
+}

package/src/core-manager/index.js

@@ -30,7 +30,7 @@ export const kCoreManagerReplicate = Symbol('replicate core manager')
  * @typedef {Object} Events
  * @property {(coreRecord: CoreRecord) => void} add-core
  * @property {(namespace: Namespace, msg: { coreDiscoveryId: string, peerId: string, start: number, bitfield: Uint32Array }) => void} peer-have
- * @property {(blobFilter: GenericBlobFilter, peerId: string) => void} peer-download-intent
+ * @property {(blobFilter: GenericBlobFilter | null, peerId: string) => void} peer-download-intent
  */
 
 /**
@@ -412,7 +412,7 @@ export class CoreManager extends TypedEmitter {
   }
 
   /**
-   * @param {GenericBlobFilter} blobFilter
+   * @param {GenericBlobFilter | null} blobFilter
    * @param {HypercorePeer} peer
    */
   #handleDownloadIntentMessage(blobFilter, peer) {
@@ -421,7 +421,7 @@ export class CoreManager extends TypedEmitter {
   }
 
   /**
-   * @param {BlobFilter} blobFilter
+   * @param {BlobFilter | null} blobFilter
    * @param {HypercorePeer} peer
    */
   sendDownloadIntents(blobFilter, peer) {
@@ -540,20 +540,27 @@ const HaveExtensionCodec = {
 }
 
 const DownloadIntentCodec = {
-  /** @param {BlobFilter} filter */
+  /** @param {BlobFilter | null} filter */
   encode(filter) {
-    const downloadIntents = mapObject(filter, (key, value) => [
+    const downloadIntents = mapObject(filter || {}, (key, value) => [
       key,
       { variants: value || [] },
     ])
-    return DownloadIntentExtension.encode({ downloadIntents }).finish()
+    // If filter is null, we want to download everything
+    const everything = !filter
+    return DownloadIntentExtension.encode({
+      downloadIntents,
+      everything,
+    }).finish()
   },
   /**
    * @param {Buffer | Uint8Array} buf
-   * @returns {GenericBlobFilter}
+   * @returns {GenericBlobFilter | null}
    */
   decode(buf) {
     const msg = DownloadIntentExtension.decode(buf)
+    // If everything is true, we ignore the downloadIntents and return null, which means download everything
+    if (msg.everything) return null
     return mapObject(msg.downloadIntents, (key, value) => [
       key + '', // keep TS happy
       value.variants,

package/src/generated/extensions.d.ts

@@ -24,6 +24,13 @@ export interface DownloadIntentExtension {
     downloadIntents: {
         [key: string]: DownloadIntentExtension_DownloadIntent;
     };
+    /**
+     * If true, the peer intends to download all blobs - this is the default
+     * assumption when a peer has not sent a download intent, but if a peer
+     * changes their intent while connected, we need to send the new intent to
+     * download everything.
+     */
+    everything: boolean;
 }
 export interface DownloadIntentExtension_DownloadIntent {
     variants: string[];

package/src/generated/extensions.js

@@ -170,7 +170,7 @@ export var HaveExtension = {
     },
 };
 function createBaseDownloadIntentExtension() {
-    return { downloadIntents: {} };
+    return { downloadIntents: {}, everything: false };
 }
 export var DownloadIntentExtension = {
     encode: function (message, writer) {
@@ -180,6 +180,9 @@ export var DownloadIntentExtension = {
             DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key, value: value }, writer.uint32(10).fork())
                 .ldelim();
         });
+        if (message.everything === true) {
+            writer.uint32(16).bool(message.everything);
+        }
         return writer;
     },
     decode: function (input, length) {
@@ -198,6 +201,12 @@ export var DownloadIntentExtension = {
                         message.downloadIntents[entry1.key] = entry1.value;
                     }
                     continue;
+                case 2:
+                    if (tag !== 16) {
+                        break;
+                    }
+                    message.everything = reader.bool();
+                    continue;
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -210,7 +219,7 @@ export var DownloadIntentExtension = {
         return DownloadIntentExtension.fromPartial(base !== null && base !== void 0 ? base : {});
     },
     fromPartial: function (object) {
-        var _a;
+        var _a, _b;
         var message = createBaseDownloadIntentExtension();
         message.downloadIntents = Object.entries((_a = object.downloadIntents) !== null && _a !== void 0 ? _a : {}).reduce(function (acc, _a) {
             var key = _a[0], value = _a[1];
@@ -219,6 +228,7 @@ export var DownloadIntentExtension = {
             }
             return acc;
         }, {});
+        message.everything = (_b = object.everything) !== null && _b !== void 0 ? _b : false;
         return message;
     },
 };

package/src/generated/extensions.ts

@@ -69,6 +69,13 @@ export function haveExtension_NamespaceToNumber(object: HaveExtension_Namespace)
 /** A map of blob types and variants that a peer intends to download */
 export interface DownloadIntentExtension {
   downloadIntents: { [key: string]: DownloadIntentExtension_DownloadIntent };
+  /**
+   * If true, the peer intends to download all blobs - this is the default
+   * assumption when a peer has not sent a download intent, but if a peer
+   * changes their intent while connected, we need to send the new intent to
+   * download everything.
+   */
+  everything: boolean;
 }
 
 export interface DownloadIntentExtension_DownloadIntent {
@@ -209,7 +216,7 @@ export const HaveExtension = {
 };
 
 function createBaseDownloadIntentExtension(): DownloadIntentExtension {
-  return { downloadIntents: {} };
+  return { downloadIntents: {}, everything: false };
 }
 
 export const DownloadIntentExtension = {
@@ -218,6 +225,9 @@ export const DownloadIntentExtension = {
       DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key as any, value }, writer.uint32(10).fork())
         .ldelim();
     });
+    if (message.everything === true) {
+      writer.uint32(16).bool(message.everything);
+    }
     return writer;
   },
 
@@ -238,6 +248,13 @@ export const DownloadIntentExtension = {
             message.downloadIntents[entry1.key] = entry1.value;
           }
           continue;
+        case 2:
+          if (tag !== 16) {
+            break;
+          }
+
+          message.everything = reader.bool();
+          continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -260,6 +277,7 @@ export const DownloadIntentExtension = {
       }
       return acc;
     }, {});
+    message.everything = object.everything ?? false;
     return message;
   },
 };