@comapeo/core 2.1.0 → 2.2.0

package/src/lib/error.js

@@ -21,6 +21,30 @@ export class ErrorWithCode extends Error {
   }
 }
 
+/**
+ * If the argument is an `Error` instance, return its `code` property if it is a string.
+ * Otherwise, returns `undefined`.
+ *
+ * @param {unknown} maybeError
+ * @returns {undefined | string}
+ * @example
+ * try {
+ *   // do something
+ * } catch (err) {
+ *   console.error(getErrorCode(err))
+ * }
+ */
+export function getErrorCode(maybeError) {
+  if (
+    maybeError instanceof Error &&
+    'code' in maybeError &&
+    typeof maybeError.code === 'string'
+  ) {
+    return maybeError.code
+  }
+  return undefined
+}
+
 /**
  * Get the error message from an object if possible.
  * Otherwise, stringify the argument.

package/src/mapeo-manager.js

@@ -51,6 +51,7 @@ import {
   kRequestFullStop,
   kRescindFullStopRequest,
 } from './sync/sync-api.js'
+import { NotFoundError } from './errors.js'
 /** @import { ProjectSettingsValue as ProjectValue } from '@comapeo/schema' */
 /** @import NoiseSecretStream from '@hyperswarm/secret-stream' */
 /** @import { SetNonNullable } from 'type-fest' */
@@ -456,7 +457,7 @@ export class MapeoManager extends TypedEmitter {
       .get()
 
     if (!projectKeysTableResult) {
-      throw new Error(`NotFound: project ID ${projectPublicId} not found`)
+      throw new NotFoundError(`Project ID ${projectPublicId} not found`)
     }
 
     const { projectId } = projectKeysTableResult
@@ -896,7 +897,7 @@ export class MapeoManager extends TypedEmitter {
       .get()
 
     if (!row) {
-      throw new Error(`NotFound: project ID ${projectPublicId} not found`)
+      throw new NotFoundError(`Project ID ${projectPublicId} not found`)
     }
 
     const { keysCipher, projectId, projectInfo } = row

package/src/mapeo-project.js

@@ -49,15 +49,20 @@ import { omit } from './lib/omit.js'
 import { MemberApi } from './member-api.js'
 import {
   SyncApi,
+  kAddBlobWantRange,
+  kClearBlobWantRanges,
   kHandleDiscoveryKey,
+  kSetBlobDownloadFilter,
   kWaitForInitialSyncWithPeer,
 } from './sync/sync-api.js'
 import { Logger } from './logger.js'
 import { IconApi } from './icon-api.js'
 import { readConfig } from './config-import.js'
 import TranslationApi from './translation-api.js'
+import { NotFoundError, nullIfNotFound } from './errors.js'
+import { getErrorCode, getErrorMessage } from './lib/error.js'
 /** @import { ProjectSettingsValue } from '@comapeo/schema' */
-/** @import { CoreStorage, KeyPair, Namespace, ReplicationStream } from './types.js' */
+/** @import { CoreStorage, BlobFilter, BlobStoreEntriesStream, KeyPair, Namespace, ReplicationStream } from './types.js' */
 
 /** @typedef {Omit<ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
 /** @typedef {ProjectSettingsValue['configMetadata']} ConfigMetadata */
@@ -77,6 +82,13 @@ export const kIsArchiveDevice = Symbol('isArchiveDevice (temp - test only)')
 
 const EMPTY_PROJECT_SETTINGS = Object.freeze({})
 
+/** @type {import('./types.js').BlobFilter} */
+const NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER = {
+  photo: ['preview', 'thumbnail'],
+  // Don't download any audio of video files, since previews and
+  // thumbnails aren't supported yet.
+}
+
 /**
  * @extends {TypedEmitter<{ close: () => void }>}
  */
@@ -146,6 +158,8 @@ export class MapeoProject extends TypedEmitter {
 
     const getReplicationStream = this[kProjectReplicate].bind(this, true)
 
+    const blobDownloadFilter = getBlobDownloadFilter(isArchiveDevice)
+
     ///////// 1. Setup database
 
     this.#sqlite = new Database(dbPath)
@@ -362,6 +376,13 @@ export class MapeoProject extends TypedEmitter {
 
     this.#blobStore = new BlobStore({
       coreManager: this.#coreManager,
+      downloadFilter: blobDownloadFilter,
+    })
+
+    this.#blobStore.on('error', (err) => {
+      // TODO: Handle this error in some way - this error will come from an
+      // unexpected error with background blob downloads
+      console.error('BlobStore error', err)
     })
 
     this.$blobs = new BlobApi({
@@ -391,7 +412,7 @@ export class MapeoProject extends TypedEmitter {
       coreManager: this.#coreManager,
       coreOwnership: this.#coreOwnership,
       roles: this.#roles,
-      blobDownloadFilter: null,
+      blobDownloadFilter,
       logger: this.#l,
       getServerWebsocketUrls: async () => {
         const members = await this.#memberApi.getMany()
@@ -413,6 +434,48 @@ export class MapeoProject extends TypedEmitter {
       getReplicationStream,
     })
 
+    /** @type {Map<string, BlobStoreEntriesStream>} */
+    const entriesReadStreams = new Map()
+
+    this.#coreManager.on('peer-download-intent', async (filter, peerId) => {
+      entriesReadStreams.get(peerId)?.destroy()
+
+      const entriesReadStream = this.#blobStore.createEntriesReadStream({
+        live: true,
+        filter,
+      })
+      entriesReadStreams.set(peerId, entriesReadStream)
+
+      entriesReadStream.once('close', () => {
+        if (entriesReadStreams.get(peerId) === entriesReadStream) {
+          entriesReadStreams.delete(peerId)
+        }
+      })
+
+      this.#syncApi[kClearBlobWantRanges](peerId)
+
+      try {
+        for await (const entry of entriesReadStream) {
+          const { blockOffset, blockLength } = entry.value.blob
+          this.#syncApi[kAddBlobWantRange](peerId, blockOffset, blockLength)
+        }
+      } catch (err) {
+        if (getErrorCode(err) === 'ERR_STREAM_PREMATURE_CLOSE') return
+        this.#l.log(
+          'Error getting blob entries stream for peer %h: %s',
+          peerId,
+          getErrorMessage(err)
+        )
+      }
+    })
+
+    this.#coreManager.creatorCore.on('peer-remove', (peer) => {
+      const peerKey = peer.protomux.stream.remotePublicKey
+      const peerId = peerKey.toString('hex')
+      entriesReadStreams.get(peerId)?.destroy()
+      entriesReadStreams.delete(peerId)
+    })
+
     this.#translationApi = new TranslationApi({
       dataType: this.#dataTypes.translation,
     })
@@ -506,6 +569,7 @@ export class MapeoProject extends TypedEmitter {
    */
   async close() {
     this.#l.log('closing project %h', this.#projectId)
+    this.#blobStore.close()
     const dataStorePromises = []
     for (const dataStore of Object.values(this.#dataStores)) {
       dataStorePromises.push(dataStore.close())
@@ -599,14 +663,9 @@ export class MapeoProject extends TypedEmitter {
   async $setProjectSettings(settings) {
     const { projectSettings } = this.#dataTypes
 
-    // We only want to catch the error to the getByDocId call
-    // Using try/catch for this is a little verbose when dealing with TS types
     const existing = await projectSettings
       .getByDocId(this.#projectId)
-      .catch(() => {
-        // project does not exist so return null
-        return null
-      })
+      .catch(nullIfNotFound)
 
     if (existing) {
       return extractEditableProjectSettings(
@@ -659,7 +718,7 @@ export class MapeoProject extends TypedEmitter {
     const coreId = this.#coreManager
       .getCoreByDiscoveryKey(coreDiscoveryKey)
       ?.key.toString('hex')
-    if (!coreId) throw new Error('NotFound')
+    if (!coreId) throw new NotFoundError()
     return this.#coreOwnership.getOwner(coreId)
   }
 
@@ -714,20 +773,23 @@ export class MapeoProject extends TypedEmitter {
       schemaName: /** @type {const} */ ('deviceInfo'),
     }
 
-    let existingDoc
-    try {
-      existingDoc = await deviceInfo.getByDocId(configCoreId)
-    } catch (err) {
+    const existingDoc = await deviceInfo
+      .getByDocId(configCoreId)
+      .catch(nullIfNotFound)
+    if (existingDoc) {
+      return await deviceInfo.update(existingDoc.versionId, doc)
+    } else {
       return await deviceInfo[kCreateWithDocId](configCoreId, doc)
     }
-
-    return deviceInfo.update(existingDoc.versionId, doc)
   }
 
   /** @param {boolean} isArchiveDevice */
   async [kSetIsArchiveDevice](isArchiveDevice) {
+    if (this.#isArchiveDevice === isArchiveDevice) return
+    const blobDownloadFilter = getBlobDownloadFilter(isArchiveDevice)
+    this.#blobStore.setDownloadFilter(blobDownloadFilter)
+    this.#syncApi[kSetBlobDownloadFilter](blobDownloadFilter)
     this.#isArchiveDevice = isArchiveDevice
-    // TODO: call this.#syncApi[kSetBlobDownloadFilter]()
   }
 
   /** @returns {boolean} */
@@ -879,7 +941,7 @@ export class MapeoProject extends TypedEmitter {
         const fieldRefs = fieldNames.map((fieldName) => {
           const fieldRef = fieldNameToRef.get(fieldName)
           if (!fieldRef) {
-            throw new Error(
+            throw new NotFoundError(
               `field ${fieldName} not found (referenced by preset ${value.name})})`
             )
           }
@@ -891,7 +953,7 @@ export class MapeoProject extends TypedEmitter {
         }
         const iconRef = iconNameToRef.get(iconName)
         if (!iconRef) {
-          throw new Error(
+          throw new NotFoundError(
             `icon ${iconName} not found (referenced by preset ${value.name})`
           )
         }
@@ -938,7 +1000,7 @@ export class MapeoProject extends TypedEmitter {
             })
           )
         } else {
-          throw new Error(
+          throw new NotFoundError(
             `docRef for ${value.docRefType} with name ${name} not found`
           )
         }
@@ -993,6 +1055,14 @@ export class MapeoProject extends TypedEmitter {
   }
 }
 
+/**
+ * @param {boolean} isArchiveDevice
+ * @returns {null | BlobFilter}
+ */
+function getBlobDownloadFilter(isArchiveDevice) {
+  return isArchiveDevice ? null : NON_ARCHIVE_DEVICE_DOWNLOAD_FILTER
+}
+
 /**
  * @param {import("@comapeo/schema").ProjectSettings & { forks: string[] }} projectDoc
  * @returns {EditableProjectSettings}

package/src/member-api.js

@@ -277,6 +277,10 @@ export class MemberApi extends TypedEmitter {
    *   peer. For example, the project must have a name.
    * - `NETWORK_ERROR`: there was an issue connecting to the server. Is the
    *   device online? Is the server online?
+   * - `SERVER_HAS_TOO_MANY_PROJECTS`: the server limits the number of projects
+   *   it can have, and it's at the limit.
+   * - `PROJECT_NOT_IN_SERVER_ALLOWLIST`: the server only allows specific
+   *   projects to be added and ours wasn't one of them.
    * - `INVALID_SERVER_RESPONSE`: we connected to the server but it returned
    *   an unexpected response. Is the server running a compatible version of
    *   CoMapeo Cloud?
@@ -351,32 +355,7 @@ export class MemberApi extends TypedEmitter {
       )
     }
 
-    if (response.status !== 200 && response.status !== 201) {
-      throw new ErrorWithCode(
-        'INVALID_SERVER_RESPONSE',
-        `Failed to add server peer due to HTTP status code ${response.status}`
-      )
-    }
-
-    try {
-      const responseBody = await response.json()
-      assert(
-        responseBody &&
-          typeof responseBody === 'object' &&
-          'data' in responseBody &&
-          responseBody.data &&
-          typeof responseBody.data === 'object' &&
-          'deviceId' in responseBody.data &&
-          typeof responseBody.data.deviceId === 'string',
-        'Response body is valid'
-      )
-      return { serverDeviceId: responseBody.data.deviceId }
-    } catch (err) {
-      throw new ErrorWithCode(
-        'INVALID_SERVER_RESPONSE',
-        "Failed to add server peer because we couldn't parse the response"
-      )
-    }
+    return await parseAddServerResponse(response)
   }
 
   /**
@@ -575,3 +554,66 @@ function isValidServerBaseUrl(
 function encodeBufferForServer(buffer) {
   return buffer ? b4a.toString(buffer, 'hex') : undefined
 }
+
+/**
+ * @param {Response} response
+ * @returns {Promise<{ serverDeviceId: string }>}
+ */
+async function parseAddServerResponse(response) {
+  if (response.status === 200) {
+    try {
+      const responseBody = await response.json()
+      assert(
+        responseBody &&
+          typeof responseBody === 'object' &&
+          'data' in responseBody &&
+          responseBody.data &&
+          typeof responseBody.data === 'object' &&
+          'deviceId' in responseBody.data &&
+          typeof responseBody.data.deviceId === 'string',
+        'Response body is valid'
+      )
+      return { serverDeviceId: responseBody.data.deviceId }
+    } catch (err) {
+      throw new ErrorWithCode(
+        'INVALID_SERVER_RESPONSE',
+        "Failed to add server peer because we couldn't parse the response"
+      )
+    }
+  }
+
+  let responseBody
+  try {
+    responseBody = await response.json()
+  } catch (_) {
+    responseBody = null
+  }
+  if (
+    responseBody &&
+    typeof responseBody === 'object' &&
+    'error' in responseBody &&
+    responseBody.error &&
+    typeof responseBody.error === 'object' &&
+    'code' in responseBody.error
+  ) {
+    switch (responseBody.error.code) {
+      case 'PROJECT_NOT_IN_ALLOWLIST':
+        throw new ErrorWithCode(
+          'PROJECT_NOT_IN_SERVER_ALLOWLIST',
+          "The server only allows specific projects to be added, and this isn't one of them"
+        )
+      case 'TOO_MANY_PROJECTS':
+        throw new ErrorWithCode(
+          'SERVER_HAS_TOO_MANY_PROJECTS',
+          "The server limits the number of projects it can have and it's at the limit"
+        )
+      default:
+        break
+    }
+  }
+
+  throw new ErrorWithCode(
+    'INVALID_SERVER_RESPONSE',
+    `Failed to add server peer due to HTTP status code ${response.status}`
+  )
+}

package/src/roles.js

@@ -2,6 +2,7 @@ import { currentSchemaVersions } from '@comapeo/schema'
 import mapObject from 'map-obj'
 import { kCreateWithDocId, kDataStore } from './datatype/index.js'
 import { assert, setHas } from './utils.js'
+import { nullIfNotFound } from './errors.js'
 import { TypedEmitter } from 'tiny-typed-emitter'
 /** @import { Namespace } from './types.js' */
 
@@ -98,6 +99,33 @@ export const CREATOR_ROLE = {
   },
 }
 
+/**
+ * @type {Role<typeof BLOCKED_ROLE_ID>}
+ */
+const BLOCKED_ROLE = {
+  roleId: BLOCKED_ROLE_ID,
+  name: 'Blocked',
+  docs: mapObject(currentSchemaVersions, (key) => {
+    return [
+      key,
+      {
+        readOwn: false,
+        writeOwn: false,
+        readOthers: false,
+        writeOthers: false,
+      },
+    ]
+  }),
+  roleAssignment: [],
+  sync: {
+    auth: 'blocked',
+    config: 'blocked',
+    data: 'blocked',
+    blobIndex: 'blocked',
+    blob: 'blocked',
+  },
+}
+
 /**
  * This is the role assumed for a device when no role record can be found. This
  * can happen when an invited device did not manage to sync with the device that
@@ -166,29 +194,7 @@ export const ROLES = {
       blob: 'allowed',
     },
   },
-  [BLOCKED_ROLE_ID]: {
-    roleId: BLOCKED_ROLE_ID,
-    name: 'Blocked',
-    docs: mapObject(currentSchemaVersions, (key) => {
-      return [
-        key,
-        {
-          readOwn: false,
-          writeOwn: false,
-          readOthers: false,
-          writeOthers: false,
-        },
-      ]
-    }),
-    roleAssignment: [],
-    sync: {
-      auth: 'blocked',
-      config: 'blocked',
-      data: 'blocked',
-      blobIndex: 'blocked',
-      blob: 'blocked',
-    },
-  },
+  [BLOCKED_ROLE_ID]: BLOCKED_ROLE,
   [LEFT_ROLE_ID]: {
     roleId: LEFT_ROLE_ID,
     name: 'Left',
@@ -264,12 +270,10 @@ export class Roles extends TypedEmitter {
    * @returns {Promise<Role>}
    */
   async getRole(deviceId) {
-    /** @type {string} */
-    let roleId
-    try {
-      const roleAssignment = await this.#dataType.getByDocId(deviceId)
-      roleId = roleAssignment.roleId
-    } catch (e) {
+    const roleRecord = await this.#dataType
+      .getByDocId(deviceId)
+      .catch(nullIfNotFound)
+    if (!roleRecord) {
       // The project creator will have the creator role
       const authCoreId = await this.#coreOwnership.getCoreId(deviceId, 'auth')
       if (authCoreId === this.#projectCreatorAuthCoreId) {
@@ -280,8 +284,10 @@ export class Roles extends TypedEmitter {
         return NO_ROLE
       }
     }
+
+    const { roleId } = roleRecord
     if (!isRoleId(roleId)) {
-      return ROLES[BLOCKED_ROLE_ID]
+      return BLOCKED_ROLE
     }
     return ROLES[roleId]
   }
@@ -383,7 +389,7 @@ export class Roles extends TypedEmitter {
 
     const existingRoleDoc = await this.#dataType
       .getByDocId(deviceId)
-      .catch(() => null)
+      .catch(nullIfNotFound)
 
     if (existingRoleDoc) {
       await this.#dataType.update(
@@ -403,7 +409,7 @@ export class Roles extends TypedEmitter {
     }
   }
 
-  async #isProjectCreator() {
+  #isProjectCreator() {
     const ownAuthCoreId = this.#coreManager
       .getWriterCore('auth')
       .key.toString('hex')

package/src/sync/core-sync-state.js

@@ -182,13 +182,23 @@ export class CoreSyncState {
    * blocks/ranges that are added here
    *
    * @param {PeerId} peerId
-   * @param {Array<{ start: number, length: number }>} ranges
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
    */
-  setPeerWants(peerId, ranges) {
+  addWantRange(peerId, start, length) {
     const peerState = this.#getOrCreatePeerState(peerId)
-    for (const { start, length } of ranges) {
-      peerState.setWantRange({ start, length })
-    }
+    peerState.addWantRange(start, length)
+    this.#update()
+  }
+
+  /**
+   * @param {PeerId} peerId
+   * @returns {void}
+   */
+  clearWantRanges(peerId) {
+    const peerState = this.#getOrCreatePeerState(peerId)
+    peerState.clearWantRanges()
     this.#update()
   }
 
@@ -291,14 +301,13 @@ export class PeerState {
   #preHaves = new RemoteBitfield()
   /** @type {HypercoreRemoteBitfield | undefined} */
   #haves
-  /** @type {Bitfield} */
-  #wants = new RemoteBitfield()
+  /**
+   * What blocks do we want? If `null`, we want everything.
+   * @type {null | Bitfield}
+   */
+  #wants = null
   /** @type {PeerNamespaceState['status']} */
   status = 'stopped'
-  #wantAll
-  constructor({ wantAll = true } = {}) {
-    this.#wantAll = wantAll
-  }
   get preHavesBitfield() {
     return this.#preHaves
   }
@@ -316,18 +325,27 @@ export class PeerState {
     this.#haves = bitfield
   }
   /**
-   * Set a range of blocks that a peer wants. This is not part of the Hypercore
+   * Add a range of blocks that a peer wants. This is not part of the Hypercore
    * protocol, so we need our own extension messages that a peer can use to
    * inform us which blocks they are interested in. For most cores peers always
-   * want all blocks, but for blob cores often peers only want preview or
+   * want all blocks, but for blob cores peers may only want preview or
    * thumbnail versions of media
    *
-   * @param {{ start: number, length: number }} range
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
    */
-  setWantRange({ start, length }) {
-    this.#wantAll = false
+  addWantRange(start, length) {
+    this.#wants ??= new RemoteBitfield()
     this.#wants.setRange(start, length, true)
   }
+  /**
+   * Set the range of blocks that this peer wants to the empty set. In other
+   * words, this peer wants nothing from this core.
+   */
+  clearWantRanges() {
+    this.#wants = new RemoteBitfield()
+  }
   /**
    * Returns whether the peer has the block at `index`. If a pre-have bitfield
    * has been passed, this is used if no connected peer bitfield is available.
@@ -355,8 +373,7 @@ export class PeerState {
    * @param {number} index
    */
   want(index) {
-    if (this.#wantAll) return true
-    return this.#wants.get(index)
+    return this.#wants ? this.#wants.get(index) : true
   }
   /**
    * Return the "wants" for the 32 blocks from `index`, as a 32-bit integer
@@ -366,11 +383,10 @@ export class PeerState {
    * the 32 blocks from `index`
    */
   wantWord(index) {
-    if (this.#wantAll) {
-      // This is a 32-bit number with all bits set
-      return 2 ** 32 - 1
-    }
-    return getBitfieldWord(this.#wants, index)
+    return this.#wants
+      ? getBitfieldWord(this.#wants, index)
+      : // This is a 32-bit number with all bits set
+        2 ** 32 - 1
   }
 }
 

package/src/sync/namespace-sync-state.js

@@ -136,6 +136,28 @@ export class NamespaceSyncState {
     this.#getCoreState(coreDiscoveryId).insertPreHaves(peerId, start, bitfield)
   }
 
+  /**
+   * @param {string} peerId
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
+   */
+  addWantRange(peerId, start, length) {
+    for (const coreState of this.#coreStates.values()) {
+      coreState.addWantRange(peerId, start, length)
+    }
+  }
+
+  /**
+   * @param {string} peerId
+   * @returns {void}
+   */
+  clearWantRanges(peerId) {
+    for (const coreState of this.#coreStates.values()) {
+      coreState.clearWantRanges(peerId)
+    }
+  }
+
   /**
    * @param {string} discoveryId
    */