@comapeo/core 2.3.1 → 2.3.2

package/src/mapeo-project.js

@@ -49,10 +49,7 @@ import { omit } from './lib/omit.js'
 import { MemberApi } from './member-api.js'
 import {
   SyncApi,
-  kAddBlobWantRange,
-  kClearBlobWantRanges,
   kHandleDiscoveryKey,
-  kSetBlobDownloadFilter,
   kWaitForInitialSyncWithPeer,
 } from './sync/sync-api.js'
 import { Logger } from './logger.js'
@@ -60,9 +57,8 @@ import { IconApi } from './icon-api.js'
 import { readConfig } from './config-import.js'
 import TranslationApi from './translation-api.js'
 import { NotFoundError, nullIfNotFound } from './errors.js'
-import { getErrorCode, getErrorMessage } from './lib/error.js'
 /** @import { ProjectSettingsValue } from '@comapeo/schema' */
-/** @import { CoreStorage, BlobFilter, BlobStoreEntriesStream, KeyPair, Namespace, ReplicationStream } from './types.js' */
+/** @import { CoreStorage, BlobFilter, BlobStoreEntriesStream, KeyPair, Namespace, ReplicationStream, GenericBlobFilter } from './types.js' */
 
 /** @typedef {Omit<ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
 /** @typedef {ProjectSettingsValue['configMetadata']} ConfigMetadata */
@@ -82,13 +78,6 @@ export const kIsArchiveDevice = Symbol('isArchiveDevice (temp - test only)')
 
 const EMPTY_PROJECT_SETTINGS = Object.freeze({})
 
-/** @type {import('./types.js').BlobFilter} */
-const NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER = {
-  photo: ['preview', 'thumbnail'],
-  // Don't download any audio of video files, since previews and
-  // thumbnails aren't supported yet.
-}
-
 /**
  * @extends {TypedEmitter<{ close: () => void }>}
  */
@@ -112,7 +101,6 @@ export class MapeoProject extends TypedEmitter {
   #l
   /** @type {Boolean} this avoids loading multiple configs in parallel */
   #loadingConfig
-  #isArchiveDevice
 
   static EMPTY_PROJECT_SETTINGS = EMPTY_PROJECT_SETTINGS
 
@@ -154,12 +142,9 @@ export class MapeoProject extends TypedEmitter {
     this.#deviceId = getDeviceId(keyManager)
     this.#projectKey = projectKey
     this.#loadingConfig = false
-    this.#isArchiveDevice = isArchiveDevice
 
     const getReplicationStream = this[kProjectReplicate].bind(this, true)
 
-    const blobDownloadFilter = getBlobDownloadFilter(isArchiveDevice)
-
     ///////// 1. Setup database
 
     this.#sqlite = new Database(dbPath)
@@ -376,7 +361,8 @@ export class MapeoProject extends TypedEmitter {
 
     this.#blobStore = new BlobStore({
       coreManager: this.#coreManager,
-      downloadFilter: blobDownloadFilter,
+      isArchiveDevice: isArchiveDevice,
+      logger: this.#l,
     })
 
     this.#blobStore.on('error', (err) => {
@@ -412,7 +398,7 @@ export class MapeoProject extends TypedEmitter {
       coreManager: this.#coreManager,
       coreOwnership: this.#coreOwnership,
       roles: this.#roles,
-      blobDownloadFilter,
+      blobStore: this.#blobStore,
       logger: this.#l,
       getServerWebsocketUrls: async () => {
         const members = await this.#memberApi.getMany()
@@ -434,48 +420,6 @@ export class MapeoProject extends TypedEmitter {
       getReplicationStream,
     })
 
-    /** @type {Map<string, BlobStoreEntriesStream>} */
-    const entriesReadStreams = new Map()
-
-    this.#coreManager.on('peer-download-intent', async (filter, peerId) => {
-      entriesReadStreams.get(peerId)?.destroy()
-
-      const entriesReadStream = this.#blobStore.createEntriesReadStream({
-        live: true,
-        filter,
-      })
-      entriesReadStreams.set(peerId, entriesReadStream)
-
-      entriesReadStream.once('close', () => {
-        if (entriesReadStreams.get(peerId) === entriesReadStream) {
-          entriesReadStreams.delete(peerId)
-        }
-      })
-
-      this.#syncApi[kClearBlobWantRanges](peerId)
-
-      try {
-        for await (const entry of entriesReadStream) {
-          const { blockOffset, blockLength } = entry.value.blob
-          this.#syncApi[kAddBlobWantRange](peerId, blockOffset, blockLength)
-        }
-      } catch (err) {
-        if (getErrorCode(err) === 'ERR_STREAM_PREMATURE_CLOSE') return
-        this.#l.log(
-          'Error getting blob entries stream for peer %h: %s',
-          peerId,
-          getErrorMessage(err)
-        )
-      }
-    })
-
-    this.#coreManager.creatorCore.on('peer-remove', (peer) => {
-      const peerKey = peer.protomux.stream.remotePublicKey
-      const peerId = peerKey.toString('hex')
-      entriesReadStreams.get(peerId)?.destroy()
-      entriesReadStreams.delete(peerId)
-    })
-
     this.#translationApi = new TranslationApi({
       dataType: this.#dataTypes.translation,
     })
@@ -516,7 +460,7 @@ export class MapeoProject extends TypedEmitter {
       localPeers.off('discovery-key', onDiscoverykey)
     })
 
-    this.#l.log('Created project instance %h', projectKey)
+    this.#l.log('Created project instance %h, %s', projectKey, isArchiveDevice)
   }
 
   /**
@@ -785,16 +729,12 @@ export class MapeoProject extends TypedEmitter {
 
   /** @param {boolean} isArchiveDevice */
   async [kSetIsArchiveDevice](isArchiveDevice) {
-    if (this.#isArchiveDevice === isArchiveDevice) return
-    const blobDownloadFilter = getBlobDownloadFilter(isArchiveDevice)
-    this.#blobStore.setDownloadFilter(blobDownloadFilter)
-    this.#syncApi[kSetBlobDownloadFilter](blobDownloadFilter)
-    this.#isArchiveDevice = isArchiveDevice
+    this.#blobStore.setIsArchiveDevice(isArchiveDevice)
   }
 
   /** @returns {boolean} */
   get [kIsArchiveDevice]() {
-    return this.#isArchiveDevice
+    return this.#blobStore.isArchiveDevice
   }
 
   /**
@@ -1054,15 +994,6 @@ export class MapeoProject extends TypedEmitter {
     }
   }
 }
-
-/**
- * @param {boolean} isArchiveDevice
- * @returns {null | BlobFilter}
- */
-function getBlobDownloadFilter(isArchiveDevice) {
-  return isArchiveDevice ? null : NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER
-}
-
 /**
  * @param {import("@comapeo/schema").ProjectSettings & { forks: string[] }} projectDoc
  * @returns {EditableProjectSettings}

package/src/sync/core-sync-state.js

@@ -67,25 +67,41 @@ export class CoreSyncState {
   /** @type {InternalState['remoteStates']} */
   #remoteStates = new Map()
   /** @type {InternalState['localState']} */
-  #localState = new PeerState()
+  #localState
   #preHavesLength = 0
   #update
   #peerSyncControllers
   #namespace
+  #deviceId
   #l
+  #hasDownloadFilter
 
   /**
    * @param {object} opts
    * @param {() => void} opts.onUpdate Called when a state update is available (via getState())
    * @param {Map<string, import('./peer-sync-controller.js').PeerSyncController>} opts.peerSyncControllers
    * @param {Namespace} opts.namespace
+   * @param {string} opts.deviceId
+   * @param {(peerId: string) => boolean} opts.hasDownloadFilter
    * @param {Logger} [opts.logger]
    */
-  constructor({ onUpdate, peerSyncControllers, namespace, logger }) {
+  constructor({
+    onUpdate,
+    peerSyncControllers,
+    namespace,
+    deviceId,
+    hasDownloadFilter,
+    logger,
+  }) {
     // The logger parameter is already namespaced by NamespaceSyncState
     this.#l = logger || Logger.create('css')
     this.#peerSyncControllers = peerSyncControllers
     this.#namespace = namespace
+    this.#deviceId = deviceId
+    this.#hasDownloadFilter = hasDownloadFilter
+    this.#localState = new PeerState({
+      wantsEverything: !hasDownloadFilter(deviceId),
+    })
     // Called whenever the state changes, so we clear the cache because next
     // call to getState() will need to re-derive the state
     this.#update = () => {
@@ -187,18 +203,21 @@ export class CoreSyncState {
    * @returns {void}
    */
   addWantRange(peerId, start, length) {
+    this.#l.log('Peer %S wants range %d-%d', peerId, start, start + length)
     const peerState = this.#getOrCreatePeerState(peerId)
     peerState.addWantRange(start, length)
     this.#update()
   }
 
   /**
+   * Set whether a peer wants everything or only blocks specified by addWantRange()
    * @param {PeerId} peerId
-   * @returns {void}
+   * @param {boolean} wantsEverything
    */
-  clearWantRanges(peerId) {
+  setWantsEverything(peerId, wantsEverything) {
+    this.#l.log('Peer %S wants everything: %s', peerId, wantsEverything)
     const peerState = this.#getOrCreatePeerState(peerId)
-    peerState.clearWantRanges()
+    peerState.setWantsEverything(wantsEverything)
     this.#update()
   }
 
@@ -206,8 +225,7 @@ export class CoreSyncState {
    * @param {PeerId} peerId
    */
   addPeer(peerId) {
-    if (this.#remoteStates.has(peerId)) return
-    this.#remoteStates.set(peerId, new PeerState())
+    this.#getOrCreatePeerState(peerId)
     this.#update()
   }
 
@@ -225,9 +243,12 @@ export class CoreSyncState {
    * @param {PeerId} peerId
    */
   #getOrCreatePeerState(peerId) {
+    if (peerId === this.#deviceId) return this.#localState
     let peerState = this.#remoteStates.get(peerId)
     if (!peerState) {
-      peerState = new PeerState()
+      peerState = new PeerState({
+        wantsEverything: !this.#hasDownloadFilter(peerId),
+      })
       this.#remoteStates.set(peerId, peerState)
     }
     return peerState
@@ -305,9 +326,14 @@ export class PeerState {
    * What blocks do we want? If `null`, we want everything.
    * @type {null | Bitfield}
    */
-  #wants = null
+  #wants
   /** @type {PeerNamespaceState['status']} */
   status = 'stopped'
+
+  constructor({ wantsEverything = true } = {}) {
+    this.#wants = wantsEverything ? null : new RemoteBitfield()
+  }
+
   get preHavesBitfield() {
     return this.#preHaves
   }
@@ -340,11 +366,11 @@ export class PeerState {
     this.#wants.setRange(start, length, true)
   }
   /**
-   * Set the range of blocks that this peer wants to the empty set. In other
-   * words, this peer wants nothing from this core.
+   * Set whether this peer wants everything or only blocks specified by addWantRange()
+   * @param {boolean} wantsEverything
    */
-  clearWantRanges() {
-    this.#wants = new RemoteBitfield()
+  setWantsEverything(wantsEverything) {
+    this.#wants = wantsEverything ? null : new RemoteBitfield()
   }
   /**
    * Returns whether the peer has the block at `index`. If a pre-have bitfield
@@ -427,6 +453,7 @@ export function deriveState(coreState) {
     const truncate = 2 ** Math.min(32, length - i) - 1
 
     const localHaves = coreState.localState.haveWord(i) & truncate
+    const localWants = coreState.localState.wantWord(i) & truncate
     localState.have += bitCount32(localHaves)
 
     let someoneElseWantsFromMe = 0
@@ -440,7 +467,7 @@ export function deriveState(coreState) {
       remoteStates[peerId].want += bitCount32(theyWantFromMe)
       someoneElseWantsFromMe |= theyWantFromMe
 
-      const iWantFromThem = peerHaves & ~localHaves
+      const iWantFromThem = peerHaves & ~localHaves & localWants
       remoteStates[peerId].wanted += bitCount32(iWantFromThem)
       iWantFromSomeoneElse |= iWantFromThem
     }

package/src/sync/namespace-sync-state.js

@@ -20,6 +20,8 @@ export class NamespaceSyncState {
   #cachedState = null
   #peerSyncControllers
   #logger
+  #deviceId
+  #blobStore
 
   /**
    * @param {object} opts
@@ -27,6 +29,7 @@ export class NamespaceSyncState {
    * @param {import('../core-manager/index.js').CoreManager} opts.coreManager
    * @param {() => void} opts.onUpdate Called when a state update is available (via getState())
    * @param {Map<string, import('./peer-sync-controller.js').PeerSyncController>} opts.peerSyncControllers
+   * @param {import('../blob-store/index.js').BlobStore} opts.blobStore
    * @param {Logger} [opts.logger]
    */
   constructor({
@@ -34,12 +37,15 @@ export class NamespaceSyncState {
     coreManager,
     onUpdate,
     peerSyncControllers,
+    blobStore,
     logger,
   }) {
     // Currently we don't create a logger for this class, just pass it down
     this.#logger = logger
     this.#namespace = namespace
     this.#peerSyncControllers = peerSyncControllers
+    this.#blobStore = blobStore
+    this.#deviceId = coreManager.deviceId
     // Called whenever the state changes, so we clear the cache because next
     // call to getState() will need to re-derive the state
     this.#handleUpdate = () => {
@@ -60,6 +66,18 @@ export class NamespaceSyncState {
       if (namespace !== this.#namespace) return
       this.#insertPreHaves(msg)
     })
+
+    if (this.#namespace !== 'blob') return
+    blobStore.on('blob-filter', (peerId, filter) => {
+      const wantsEverything = !filter
+      for (const css of this.#coreStates.values()) {
+        css.setWantsEverything(peerId, wantsEverything)
+      }
+    })
+    blobStore.on('want-blob-range', ({ blobCoreId, peerId, start, length }) => {
+      const coreState = this.#getCoreState(blobCoreId)
+      coreState.addWantRange(peerId, start, length)
+    })
   }
 
   get namespace() {
@@ -136,28 +154,6 @@ export class NamespaceSyncState {
     this.#getCoreState(coreDiscoveryId).insertPreHaves(peerId, start, bitfield)
   }
 
-  /**
-   * @param {string} peerId
-   * @param {number} start
-   * @param {number} length
-   * @returns {void}
-   */
-  addWantRange(peerId, start, length) {
-    for (const coreState of this.#coreStates.values()) {
-      coreState.addWantRange(peerId, start, length)
-    }
-  }
-
-  /**
-   * @param {string} peerId
-   * @returns {void}
-   */
-  clearWantRanges(peerId) {
-    for (const coreState of this.#coreStates.values()) {
-      coreState.clearWantRanges(peerId)
-    }
-  }
-
   /**
    * @param {string} discoveryId
    */
@@ -169,6 +165,13 @@ export class NamespaceSyncState {
         onUpdate: this.#handleUpdate,
         peerSyncControllers: this.#peerSyncControllers,
         namespace: this.#namespace,
+        deviceId: this.#deviceId,
+        hasDownloadFilter: (peerId) => {
+          return (
+            this.#namespace === 'blob' &&
+            !!this.#blobStore.getBlobFilter(peerId)
+          )
+        },
         logger: Logger.create('css:' + this.#namespace, this.#logger, {
           prefix: logPrefix,
         }),

package/src/sync/sync-api.js

@@ -25,9 +25,6 @@ export const kRescindFullStopRequest = Symbol('foreground')
 export const kWaitForInitialSyncWithPeer = Symbol(
   'wait for initial sync with peer'
 )
-export const kSetBlobDownloadFilter = Symbol('set isArchiveDevice')
-export const kAddBlobWantRange = Symbol('add blob want range')
-export const kClearBlobWantRanges = Symbol('clear blob want ranges')
 
 /**
  * @typedef {'initial' | 'full'} SyncType
@@ -37,6 +34,15 @@ export const kClearBlobWantRanges = Symbol('clear blob want ranges')
  * @typedef {'none' | 'presync' | 'all'} SyncEnabledState
  */
 
+/**
+ * @internal
+ * @typedef {object} BlobWantRange
+ * @property {number} start
+ * @property {number} length
+ * @property {string} blobCoreId
+ * @property {string} peerId
+ */
+
 /**
  * @internal
  * @typedef {object} RemoteDeviceNamespaceGroupSyncState
@@ -93,8 +99,6 @@ export class SyncApi extends TypedEmitter {
   #getReplicationStream
   /** @type {Map<string, WebSocket>} */
   #serverWebsockets = new Map()
-  /** @type {null | BlobFilter} */
-  #blobDownloadFilter = null
 
   /**
    * @param {object} opts
@@ -103,7 +107,7 @@ export class SyncApi extends TypedEmitter {
    * @param {import('../roles.js').Roles} opts.roles
    * @param {() => Promise<Iterable<string>>} opts.getServerWebsocketUrls
    * @param {() => ReplicationStream} opts.getReplicationStream
-   * @param {null | BlobFilter} opts.blobDownloadFilter
+   * @param {import('../blob-store/index.js').BlobStore} opts.blobStore
    * @param {number} [opts.throttleMs]
    * @param {Logger} [opts.logger]
    */
@@ -115,7 +119,7 @@ export class SyncApi extends TypedEmitter {
     getReplicationStream,
     logger,
     coreOwnership,
-    blobDownloadFilter,
+    blobStore,
   }) {
     super()
     this.#l = Logger.create('syncApi', logger)
@@ -128,6 +132,7 @@ export class SyncApi extends TypedEmitter {
       coreManager,
       throttleMs,
       peerSyncControllers: this.#pscByPeerId,
+      blobStore,
       logger,
     })
     this[kSyncState].setMaxListeners(0)
@@ -135,8 +140,6 @@ export class SyncApi extends TypedEmitter {
       this.#updateState(namespaceSyncState)
     })
 
-    this[kSetBlobDownloadFilter](blobDownloadFilter)
-
     this.#coreManager.creatorCore.on('peer-add', this.#handlePeerAdd)
     this.#coreManager.creatorCore.on('peer-remove', this.#handlePeerDisconnect)
 
@@ -156,37 +159,6 @@ export class SyncApi extends TypedEmitter {
       .catch(noop)
   }
 
-  /** @param {import('../types.js').BlobFilter | null} blobDownloadFilter */
-  [kSetBlobDownloadFilter](blobDownloadFilter) {
-    this.#blobDownloadFilter = blobDownloadFilter
-    if (!blobDownloadFilter) return // No download intents = intend to download everything
-    for (const peer of this.#coreManager.creatorCore.peers) {
-      this.#coreManager.sendDownloadIntents(blobDownloadFilter, peer)
-    }
-  }
-
-  /**
-   * Add some blob blocks this peer wants.
-   *
-   * @param {string} peerId
-   * @param {number} start
-   * @param {number} length
-   * @returns {void}
-   */
-  [kAddBlobWantRange](peerId, start, length) {
-    this[kSyncState].addBlobWantRange(peerId, start, length)
-  }
-
-  /**
-   * Clear the blob blocks this peer wants.
-   *
-   * @param {string} peerId
-   * @returns {void}
-   */
-  [kClearBlobWantRanges](peerId) {
-    this[kSyncState].clearBlobWantRanges(peerId)
-  }
-
   /** @type {import('../local-peers.js').LocalPeersEvents['discovery-key']} */
   [kHandleDiscoveryKey](discoveryKey, protomux) {
     const peerSyncController = this.#peerSyncControllers.get(protomux)
@@ -540,9 +512,6 @@ export class SyncApi extends TypedEmitter {
       )
       return
     }
-    if (this.#blobDownloadFilter) {
-      this.#coreManager.sendDownloadIntents(this.#blobDownloadFilter, peer)
-    }
     const peerSyncController = new PeerSyncController({
       protomux,
       coreManager: this.#coreManager,

package/src/sync/sync-state.js

@@ -23,10 +23,17 @@ export class SyncState extends TypedEmitter {
    * @param {object} opts
    * @param {import('../core-manager/index.js').CoreManager} opts.coreManager
    * @param {Map<string, import('./peer-sync-controller.js').PeerSyncController>} opts.peerSyncControllers
+   * @param {import('../blob-store/index.js').BlobStore} opts.blobStore
    * @param {number} [opts.throttleMs]
    * @param {import('../logger.js').Logger} [opts.logger]
    */
-  constructor({ coreManager, peerSyncControllers, throttleMs = 200, logger }) {
+  constructor({
+    coreManager,
+    peerSyncControllers,
+    blobStore,
+    throttleMs = 200,
+    logger,
+  }) {
     super()
     const throttledHandleUpdate = throttle(throttleMs, this.#handleUpdate)
     for (const namespace of NAMESPACES) {
@@ -35,6 +42,7 @@ export class SyncState extends TypedEmitter {
         coreManager,
         onUpdate: throttledHandleUpdate,
         peerSyncControllers,
+        blobStore,
         logger,
       })
     }
@@ -68,24 +76,6 @@ export class SyncState extends TypedEmitter {
     ])
   }
 
-  /**
-   * @param {string} peerId
-   * @param {number} start
-   * @param {number} length
-   * @returns {void}
-   */
-  addBlobWantRange(peerId, start, length) {
-    this.#syncStates.blob.addWantRange(peerId, start, length)
-  }
-
-  /**
-   * @param {string} peerId
-   * @returns {void}
-   */
-  clearBlobWantRanges(peerId) {
-    this.#syncStates.blob.clearWantRanges(peerId)
-  }
-
   #handleUpdate = () => {
     this.emit('state', this.getState())
   }

package/src/types.ts

@@ -152,6 +152,6 @@ export type DefaultEmitterEvents<
 
 export type BlobStoreEntriesStream = Readable & {
   [Symbol.asyncIterator](): AsyncIterableIterator<
-    HyperdriveEntry & { driveId: string }
+    HyperdriveEntry & { driveId: string; blobCoreId: string }
   >
 }