@comapeo/core 2.1.0 → 2.3.0

package/src/sync/sync-api.js

@@ -16,7 +16,7 @@ import { NO_ROLE_ID } from '../roles.js'
 /** @import * as http from 'node:http' */
 /** @import { CoreOwnership } from '../core-ownership.js' */
 /** @import { OpenedNoiseStream } from '../lib/noise-secret-stream-helpers.js' */
-/** @import { ReplicationStream } from '../types.js' */
+/** @import { BlobFilter, ReplicationStream } from '../types.js' */
 
 export const kHandleDiscoveryKey = Symbol('handle discovery key')
 export const kSyncState = Symbol('sync state')
@@ -26,6 +26,8 @@ export const kWaitForInitialSyncWithPeer = Symbol(
   'wait for initial sync with peer'
 )
 export const kSetBlobDownloadFilter = Symbol('set isArchiveDevice')
+export const kAddBlobWantRange = Symbol('add blob want range')
+export const kClearBlobWantRanges = Symbol('clear blob want ranges')
 
 /**
  * @typedef {'initial' | 'full'} SyncType
@@ -91,7 +93,8 @@ export class SyncApi extends TypedEmitter {
   #getReplicationStream
   /** @type {Map<string, WebSocket>} */
   #serverWebsockets = new Map()
-  #blobDownloadFilter
+  /** @type {null | BlobFilter} */
+  #blobDownloadFilter = null
 
   /**
    * @param {object} opts
@@ -100,7 +103,7 @@ export class SyncApi extends TypedEmitter {
    * @param {import('../roles.js').Roles} opts.roles
    * @param {() => Promise<Iterable<string>>} opts.getServerWebsocketUrls
    * @param {() => ReplicationStream} opts.getReplicationStream
-   * @param {import('../types.js').BlobFilter | null} opts.blobDownloadFilter
+   * @param {null | BlobFilter} opts.blobDownloadFilter
    * @param {number} [opts.throttleMs]
    * @param {Logger} [opts.logger]
    */
@@ -116,7 +119,6 @@ export class SyncApi extends TypedEmitter {
   }) {
     super()
     this.#l = Logger.create('syncApi', logger)
-    this.#blobDownloadFilter = blobDownloadFilter
     this.#coreManager = coreManager
     this.#coreOwnership = coreOwnership
     this.#roles = roles
@@ -133,6 +135,8 @@ export class SyncApi extends TypedEmitter {
       this.#updateState(namespaceSyncState)
     })
 
+    this[kSetBlobDownloadFilter](blobDownloadFilter)
+
     this.#coreManager.creatorCore.on('peer-add', this.#handlePeerAdd)
     this.#coreManager.creatorCore.on('peer-remove', this.#handlePeerDisconnect)
 
@@ -161,6 +165,28 @@ export class SyncApi extends TypedEmitter {
     }
   }
 
+  /**
+   * Add some blob blocks this peer wants.
+   *
+   * @param {string} peerId
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
+   */
+  [kAddBlobWantRange](peerId, start, length) {
+    this[kSyncState].addBlobWantRange(peerId, start, length)
+  }
+
+  /**
+   * Clear the blob blocks this peer wants.
+   *
+   * @param {string} peerId
+   * @returns {void}
+   */
+  [kClearBlobWantRanges](peerId) {
+    this[kSyncState].clearBlobWantRanges(peerId)
+  }
+
   /** @type {import('../local-peers.js').LocalPeersEvents['discovery-key']} */
   [kHandleDiscoveryKey](discoveryKey, protomux) {
     const peerSyncController = this.#peerSyncControllers.get(protomux)

package/src/sync/sync-state.js

@@ -68,6 +68,24 @@ export class SyncState extends TypedEmitter {
     ])
   }
 
+  /**
+   * @param {string} peerId
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
+   */
+  addBlobWantRange(peerId, start, length) {
+    this.#syncStates.blob.addWantRange(peerId, start, length)
+  }
+
+  /**
+   * @param {string} peerId
+   * @returns {void}
+   */
+  clearBlobWantRanges(peerId) {
+    this.#syncStates.blob.clearWantRanges(peerId)
+  }
+
   #handleUpdate = () => {
     this.emit('state', this.getState())
   }

package/src/translation-api.js

@@ -1,7 +1,7 @@
 import { and, sql } from 'drizzle-orm'
 import { kCreateWithDocId, kSelect } from './datatype/index.js'
 import { hashObject } from './utils.js'
-import { NotFoundError } from './errors.js'
+import { nullIfNotFound } from './errors.js'
 import { omit } from './lib/omit.js'
 /** @import { Translation, TranslationValue } from '@comapeo/schema' */
 /** @import { SetOptional } from 'type-fest' */
@@ -50,15 +50,11 @@ export default class TranslationApi {
   async put(value) {
     const identifiers = omit(value, ['message'])
     const docId = hashObject(identifiers)
-    try {
-      const doc = await this.#dataType.getByDocId(docId)
+    const doc = await this.#dataType.getByDocId(docId).catch(nullIfNotFound)
+    if (doc) {
       return await this.#dataType.update(doc.versionId, value)
-    } catch (e) {
-      if (e instanceof NotFoundError) {
-        return await this.#dataType[kCreateWithDocId](docId, value)
-      } else {
-        throw new Error(`Error on translation ${e}`)
-      }
+    } else {
+      return await this.#dataType[kCreateWithDocId](docId, value)
     }
   }
 

package/src/types.ts

@@ -14,6 +14,8 @@ import { Duplex } from 'streamx'
 import RandomAccessStorage from 'random-access-storage'
 import { DefaultListener, ListenerSignature } from 'tiny-typed-emitter'
 import type { NAMESPACES } from './constants.js'
+import type { Readable } from 'stream'
+import type { HyperdriveEntry } from 'hyperdrive'
 
 export type Namespace = (typeof NAMESPACES)[number]
 
@@ -147,3 +149,9 @@ export type DefaultEmitterEvents<
   newListener: (event: keyof L, listener: L[keyof L]) => void
   removeListener: (event: keyof L, listener: L[keyof L]) => void
 }
+
+export type BlobStoreEntriesStream = Readable & {
+  [Symbol.asyncIterator](): AsyncIterableIterator<
+    HyperdriveEntry & { driveId: string }
+  >
+}

package/dist/blob-store/live-download.d.ts

@@ -1,107 +0,0 @@
-/**
- * Reduce multiple states into one. Factored out for unit testing because I
- * don't trust my coding. Probably a smarter way to do this, but this works.
- *
- * @param {Iterable<{ state: BlobDownloadState | BlobDownloadStateError }>} liveDownloads
- * @param {{ signal?: AbortSignal }} options
- * @returns {BlobDownloadState | BlobDownloadStateError}
- */
-export function combineStates(liveDownloads: Iterable<{
-    state: BlobDownloadState | BlobDownloadStateError;
-}>, { signal }?: {
-    signal?: AbortSignal;
-}): BlobDownloadState | BlobDownloadStateError;
-/**
- * @typedef {object} BlobDownloadState
- * @property {number} haveCount The number of files already downloaded
- * @property {number} haveBytes The bytes already downloaded
- * @property {number} wantCount The number of files pending download
- * @property {number} wantBytes The bytes pending download
- * @property {null} error If status = 'error' then this will be an Error object
- * @property {'checking' | 'downloading' | 'downloaded' | 'aborted'} status
- */
-/** @typedef {Omit<BlobDownloadState, 'error' | 'status'> & { status: 'error', error: Error }} BlobDownloadStateError */
-/**
- * @typedef {object} BlobDownloadEvents
- * @property {(state: BlobDownloadState | BlobDownloadStateError ) => void} state Emitted with the current download state whenever it changes (not emitted during initial 'checking' status)
- */
-/**
- * LiveDownload class
- * @extends {TypedEmitter<BlobDownloadEvents>}
- */
-export class LiveDownload extends TypedEmitter<BlobDownloadEvents> {
-    /**
-     * Like drive.download() but 'live', and for multiple drives
-     * @param {Iterable<import('hyperdrive')>} drives
-     * @param {import('./index.js').InternalDriveEmitter} emitter
-     * @param {object} options
-     * @param {import('../types.js').BlobFilter} [options.filter] Filter blobs of specific types and/or sizes to download
-     * @param {AbortSignal} [options.signal]
-     */
-    constructor(drives: Iterable<import("hyperdrive")>, emitter: import("./index.js").InternalDriveEmitter, { filter, signal }: {
-        filter?: import("../types.js").BlobFilter | undefined;
-        signal?: AbortSignal | undefined;
-    });
-    /**
-     * @returns {BlobDownloadState | BlobDownloadStateError}
-     */
-    get state(): BlobDownloadState | BlobDownloadStateError;
-    #private;
-}
-/**
- * LiveDownload class
- * @extends {TypedEmitter<BlobDownloadEvents>}
- */
-export class DriveLiveDownload extends TypedEmitter<BlobDownloadEvents> {
-    /**
-     * Like drive.download() but 'live',
-     * @param {import('hyperdrive')} drive
-     * @param {object} options
-     * @param {import('../types.js').BlobFilter} [options.filter] Filter blobs of specific types and/or sizes to download
-     * @param {AbortSignal} [options.signal]
-     */
-    constructor(drive: import("hyperdrive"), { filter, signal }?: {
-        filter?: import("../types.js").BlobFilter | undefined;
-        signal?: AbortSignal | undefined;
-    });
-    /**
-     * @returns {BlobDownloadState | BlobDownloadStateError}
-     */
-    get state(): BlobDownloadState | BlobDownloadStateError;
-    #private;
-}
-export type BlobDownloadState = {
-    /**
-     * The number of files already downloaded
-     */
-    haveCount: number;
-    /**
-     * The bytes already downloaded
-     */
-    haveBytes: number;
-    /**
-     * The number of files pending download
-     */
-    wantCount: number;
-    /**
-     * The bytes pending download
-     */
-    wantBytes: number;
-    /**
-     * If status = 'error' then this will be an Error object
-     */
-    error: null;
-    status: "checking" | "downloading" | "downloaded" | "aborted";
-};
-export type BlobDownloadStateError = Omit<BlobDownloadState, "error" | "status"> & {
-    status: "error";
-    error: Error;
-};
-export type BlobDownloadEvents = {
-    /**
-     * Emitted with the current download state whenever it changes (not emitted during initial 'checking' status)
-     */
-    state: (state: BlobDownloadState | BlobDownloadStateError) => void;
-};
-import { TypedEmitter } from 'tiny-typed-emitter';
-//# sourceMappingURL=live-download.d.ts.map

package/dist/blob-store/live-download.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"live-download.d.ts","sourceRoot":"","sources":["../../src/blob-store/live-download.js"],"names":[],"mappings":"AA+RA;;;;;;;GAOG;AACH,6CAJW,QAAQ,CAAC;IAAE,KAAK,EAAE,iBAAiB,GAAG,sBAAsB,CAAA;CAAE,CAAC,eAC/D;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACtB,iBAAiB,GAAG,sBAAsB,CAqCtD;AApUD;;;;;;;;GAQG;AAEH,wHAAwH;AAExH;;;GAGG;AAEH;;;GAGG;AACH;IAKE;;;;;;;OAOG;IACH,oBANW,QAAQ,CAAC,OAAO,YAAY,CAAC,CAAC,WAC9B,OAAO,YAAY,EAAE,oBAAoB,sBAEjD;QAAmD,MAAM;QAC3B,MAAM;KAAC,EAiCvC;IAED;;OAEG;IACH,wDAEC;;CACF;AAED;;;GAGG;AACH;IAaE;;;;;;OAMG;IACH,mBALW,OAAO,YAAY,CAAC,uBAE5B;QAAmD,MAAM;QAC3B,MAAM;KAAC,EAmBvC;IAED;;OAEG;IACH,wDAyBC;;CAqIF;;;;;eArRa,MAAM;;;;eACN,MAAM;;;;eACN,MAAM;;;;eACN,MAAM;;;;WACN,IAAI;YACJ,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS;;qCAGrD,IAAI,CAAC,iBAAiB,EAAE,OAAO,GAAG,QAAQ,CAAC,GAAG;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE;;;;;WAI/E,CAAC,KAAK,EAAE,iBAAiB,GAAG,sBAAsB,KAAM,IAAI;;6BApB7C,oBAAoB"}

package/src/blob-store/live-download.js

@@ -1,373 +0,0 @@
-import { TypedEmitter } from 'tiny-typed-emitter'
-import { once } from 'node:events'
-import SubEncoder from 'sub-encoder'
-
-const keyEncoding = new SubEncoder('files', 'utf-8')
-
-/**
- * @typedef {object} BlobDownloadState
- * @property {number} haveCount The number of files already downloaded
- * @property {number} haveBytes The bytes already downloaded
- * @property {number} wantCount The number of files pending download
- * @property {number} wantBytes The bytes pending download
- * @property {null} error If status = 'error' then this will be an Error object
- * @property {'checking' | 'downloading' | 'downloaded' | 'aborted'} status
- */
-
-/** @typedef {Omit<BlobDownloadState, 'error' | 'status'> & { status: 'error', error: Error }} BlobDownloadStateError */
-
-/**
- * @typedef {object} BlobDownloadEvents
- * @property {(state: BlobDownloadState | BlobDownloadStateError ) => void} state Emitted with the current download state whenever it changes (not emitted during initial 'checking' status)
- */
-
-/**
- * LiveDownload class
- * @extends {TypedEmitter<BlobDownloadEvents>}
- */
-export class LiveDownload extends TypedEmitter {
-  /** @type {Set<DriveLiveDownload>} */
-  #driveLiveDownloads = new Set()
-  #signal
-
-  /**
-   * Like drive.download() but 'live', and for multiple drives
-   * @param {Iterable<import('hyperdrive')>} drives
-   * @param {import('./index.js').InternalDriveEmitter} emitter
-   * @param {object} options
-   * @param {import('../types.js').BlobFilter} [options.filter] Filter blobs of specific types and/or sizes to download
-   * @param {AbortSignal} [options.signal]
-   */
-  constructor(drives, emitter, { filter, signal }) {
-    super()
-    this.#signal = signal
-
-    const emitState = () => {
-      this.emit('state', this.state)
-    }
-
-    /** @param {import('hyperdrive')} drive */
-    const addDrive = (drive) => {
-      const download = new DriveLiveDownload(drive, {
-        filter,
-        signal,
-      })
-      this.#driveLiveDownloads.add(download)
-      download.on('state', emitState)
-    }
-
-    for (const drive of drives) addDrive(drive)
-    emitter.on('add-drive', addDrive)
-
-    signal?.addEventListener(
-      'abort',
-      () => {
-        emitter.off('add-drive', addDrive)
-        for (const download of this.#driveLiveDownloads) {
-          download.off('state', emitState)
-        }
-      },
-      { once: true }
-    )
-  }
-
-  /**
-   * @returns {BlobDownloadState | BlobDownloadStateError}
-   */
-  get state() {
-    return combineStates(this.#driveLiveDownloads, { signal: this.#signal })
-  }
-}
-
-/**
- * LiveDownload class
- * @extends {TypedEmitter<BlobDownloadEvents>}
- */
-export class DriveLiveDownload extends TypedEmitter {
-  #haveCount = 0
-  #haveBytes = 0
-  #wantBytes = 0
-  #initialCheck = true
-  #drive
-  #folders
-  /** @type {Set<{ done(): Promise<void>, destroy(): void }>} */
-  #downloads = new Set()
-  /** @type {Error | null} */
-  #error = null
-  #signal
-
-  /**
-   * Like drive.download() but 'live',
-   * @param {import('hyperdrive')} drive
-   * @param {object} options
-   * @param {import('../types.js').BlobFilter} [options.filter] Filter blobs of specific types and/or sizes to download
-   * @param {AbortSignal} [options.signal]
-   */
-  constructor(drive, { filter, signal } = {}) {
-    super()
-    this.#drive = drive
-    this.#folders = filterToFolders(filter)
-    this.#signal = signal
-    if (signal && !signal.aborted) {
-      signal.addEventListener(
-        'abort',
-        () => {
-          for (const download of this.#downloads) download.destroy()
-          this.#downloads.clear()
-          this.emit('state', this.state)
-        },
-        { once: true }
-      )
-    }
-    this.#start().catch(this.#handleError.bind(this))
-  }
-
-  /**
-   * @returns {BlobDownloadState | BlobDownloadStateError}
-   */
-  get state() {
-    if (this.#error) {
-      return {
-        haveCount: this.#haveCount,
-        haveBytes: this.#haveBytes,
-        wantCount: this.#downloads.size,
-        wantBytes: this.#wantBytes,
-        error: this.#error,
-        status: 'error',
-      }
-    }
-    return {
-      haveCount: this.#haveCount,
-      haveBytes: this.#haveBytes,
-      wantCount: this.#downloads.size,
-      wantBytes: this.#wantBytes,
-      error: null,
-      status: this.#signal?.aborted
-        ? 'aborted'
-        : this.#initialCheck
-        ? 'checking'
-        : this.#downloads.size > 0
-        ? 'downloading'
-        : 'downloaded',
-    }
-  }
-
-  async #start() {
-    const blobsCore = await this.#getBlobsCore()
-    /* c8 ignore next */
-    if (this.#signal?.aborted || !blobsCore) return // Can't get here in tests
-    let seq = 0
-
-    for (const folder of this.#folders) {
-      // Don't emit state during initial iteration of existing data, since this is
-      // likely fast and not useful UX feedback
-      const entryStream = this.#drive.list(folder, { recursive: true })
-      if (this.#signal) {
-        this.#signal.addEventListener('abort', () => entryStream.destroy(), {
-          once: true,
-        })
-      }
-      for await (const entry of entryStream) {
-        if (this.#signal?.aborted) return
-        seq = Math.max(seq, entry.seq)
-        const { blob } = entry.value
-        if (!blob) continue
-        await this.#processEntry(blobsCore, blob)
-      }
-      if (this.#signal?.aborted) return
-    }
-
-    this.#initialCheck = false
-    this.emit('state', this.state)
-
-    const bee = this.#drive.db
-    // This will also download old versions of files, but it is the only way to
-    // get a live stream from a Hyperbee, however we currently do not support
-    // edits of blobs, so this should not be an issue. `keyEncoding` is
-    // necessary because hyperdrive stores file index data under the `files`
-    // sub-encoding key
-    const historyStream = bee.createHistoryStream({
-      live: true,
-      gt: seq,
-      keyEncoding,
-    })
-    if (this.#signal) {
-      this.#signal.addEventListener('abort', () => historyStream.destroy(), {
-        once: true,
-      })
-    }
-    for await (const entry of historyStream) {
-      if (this.#signal?.aborted) return
-      const { blob } = entry.value
-      if (!blob) continue
-      if (!matchesFolder(entry.key, this.#folders)) continue
-      // TODO: consider cancelling downloads when a delete entry is found?
-      // Probably not worth the extra work.
-      if (entry.type !== 'put') continue
-      const wasDownloaded = this.state.status === 'downloaded'
-      await this.#processEntry(blobsCore, blob)
-      if (wasDownloaded && this.state.status === 'downloading') {
-        // State has changed, so emit
-        this.emit('state', this.state)
-      }
-    }
-    /* c8 ignore next 2 */
-    // Could possibly reach here if aborted after check in loop, hard to test
-    this.emit('state', this.state)
-  }
-
-  /**
-   * If a Hyperdrive has been added by its key and has never replicated, then
-   * drive.getBlobs() will not resolve until replication starts. Since we do not
-   * want the downloader to remain in the "checking" state forever, we catch
-   * this case and update the state before waiting for the hyperdrive hyperblobs
-   * instance. This also makes waiting for the blobs instance cancellable.
-   *
-   * @returns {Promise<import('hypercore') | undefined>}
-   */
-  async #getBlobsCore() {
-    if (this.#drive.blobs) return this.#drive.blobs.core
-    await this.#drive.ready()
-    await this.#drive.core.update({ wait: true })
-
-    // If no peers at this stage, we are not going to be able to get the blobs
-    // until a peer appears, so consider this state "downloaded", because
-    // otherwise this will just hang as "checking"
-    if (!this.#drive.core.peers.length) {
-      this.#initialCheck = false
-      this.emit('state', this.state)
-    }
-    try {
-      const [blobs] = await once(this.#drive, 'blobs', { signal: this.#signal })
-      return blobs.core
-    } catch (e) {
-      if (e instanceof Error && e.name === 'AbortError') return
-      throw e
-    }
-  }
-
-  /** @param {Error} e */
-  #handleError(e) {
-    this.#error = e
-    this.emit('state', this.state)
-  }
-
-  /**
-   * Update state and queue missing entries for download
-   *
-   * @param {import('hypercore')} core
-   * @param {{ blockOffset: number, blockLength: number, byteLength: number }} blob
-   */
-  async #processEntry(
-    core,
-    { blockOffset: start, blockLength: length, byteLength }
-  ) {
-    const end = start + length
-    const have = await core.has(start, end)
-    if (have) {
-      this.#haveCount++
-      this.#haveBytes += byteLength
-    } else {
-      this.#wantBytes += byteLength
-      const download = core.download({ start, end })
-      this.#downloads.add(download)
-      download
-        .done()
-        .then(() => {
-          this.#downloads.delete(download)
-          this.#haveCount++
-          this.#haveBytes += byteLength
-          this.#wantBytes -= byteLength
-          this.emit('state', this.state)
-        })
-        .catch(this.#handleError.bind(this))
-    }
-  }
-}
-
-/**
- * Reduce multiple states into one. Factored out for unit testing because I
- * don't trust my coding. Probably a smarter way to do this, but this works.
- *
- * @param {Iterable<{ state: BlobDownloadState | BlobDownloadStateError }>} liveDownloads
- * @param {{ signal?: AbortSignal }} options
- * @returns {BlobDownloadState | BlobDownloadStateError}
- */
-export function combineStates(liveDownloads, { signal } = {}) {
-  /** @type {BlobDownloadState | BlobDownloadStateError} */
-  let combinedState = {
-    haveCount: 0,
-    haveBytes: 0,
-    wantCount: 0,
-    wantBytes: 0,
-    error: null,
-    status: 'downloaded',
-  }
-  for (const { state } of liveDownloads) {
-    combinedState.haveCount += state.haveCount
-    combinedState.haveBytes += state.haveBytes
-    combinedState.wantCount += state.wantCount
-    combinedState.wantBytes += state.wantBytes
-    if (state.status === combinedState.status) continue
-    if (state.status === 'error') {
-      combinedState = { ...combinedState, error: state.error, status: 'error' }
-    } else if (
-      state.status === 'downloading' &&
-      combinedState.status === 'downloaded'
-    ) {
-      combinedState = { ...combinedState, status: 'downloading' }
-    } else if (
-      state.status === 'checking' &&
-      (combinedState.status === 'downloaded' ||
-        combinedState.status === 'downloading')
-    ) {
-      combinedState = { ...combinedState, status: 'checking' }
-    }
-  }
-  if (signal?.aborted) {
-    combinedState.status = 'aborted'
-  }
-  return combinedState
-}
-
-/**
- * Convert a filter to an array of folders that need to be downloaded
- *
- * @param {import('../types.js').BlobFilter} [filter]
- * @returns {string[]} array of folders that match the filter
- */
-function filterToFolders(filter) {
-  if (!filter) return ['/']
-  const folders = []
-  for (const [
-    type,
-    variants,
-  ] of /** @type {import('type-fest').Entries<typeof filter>} */ (
-    Object.entries(filter)
-  )) {
-    // De-dupe variants array
-    for (const variant of new Set(variants)) {
-      folders.push(makePath({ type, variant }))
-    }
-  }
-  return folders
-}
-
-/**
- * Returns true if the path is within one of the given folders
- *
- * @param {string} path
- * @param {string[]} folders
- * @returns {boolean}
- */
-function matchesFolder(path, folders) {
-  for (const folder of folders) {
-    if (path.startsWith(folder)) return true
-  }
-  return false
-}
-
-/** @param {Pick<import('../types.js').BlobId, 'type' | 'variant'>} opts */
-function makePath({ type, variant }) {
-  return `/${type}/${variant}`
-}