@helia/utils 2.5.2 → 3.0.0-3a1dd561

package/src/routing.ts

@@ -1,376 +0,0 @@
-import { NoRoutersAvailableError } from '@helia/interface'
-import { NotFoundError, start, stop } from '@libp2p/interface'
-import { PeerQueue } from '@libp2p/utils'
-import merge from 'it-merge'
-import { CustomProgressEvent } from 'progress-events'
-import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { GetFailedError } from './errors.ts'
-import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents } from '@helia/interface'
-import type { ComponentLogger, Logger, Metrics, PeerId, PeerInfo, Startable } from '@libp2p/interface'
-import type { CID } from 'multiformats/cid'
-
-const DEFAULT_PROVIDER_LOOKUP_CONCURRENCY = 5
-
-export interface RoutingInit {
-  routers: Array<Partial<RoutingInterface>>
-  providerLookupConcurrency?: number
-}
-
-export interface RoutingComponents {
-  logger: ComponentLogger
-  metrics?: Metrics
-}
-
-export class Routing implements RoutingInterface, Startable {
-  public name: string
-
-  private readonly log: Logger
-  private readonly routers: Array<Partial<RoutingInterface>>
-  private readonly providerLookupConcurrency: number
-
-  constructor (components: RoutingComponents, init: RoutingInit) {
-    this.name = 'helia'
-    this.log = components.logger.forComponent('helia:routing')
-    this.routers = init.routers ?? []
-    this.providerLookupConcurrency = init.providerLookupConcurrency ?? DEFAULT_PROVIDER_LOOKUP_CONCURRENCY
-
-    this.findProviders = components.metrics?.traceFunction('helia.routing.findProviders', this.findProviders.bind(this), {
-      optionsIndex: 1
-    }) ?? this.findProviders
-    this.provide = components.metrics?.traceFunction('helia.routing.provide', this.provide.bind(this), {
-      optionsIndex: 1
-    }) ?? this.provide
-    this.cancelReprovide = components.metrics?.traceFunction('helia.routing.cancelReprovide', this.cancelReprovide.bind(this), {
-      optionsIndex: 1
-    }) ?? this.cancelReprovide
-    this.put = components.metrics?.traceFunction('helia.routing.put', this.put.bind(this), {
-      optionsIndex: 2
-    }) ?? this.put
-    this.get = components.metrics?.traceFunction('helia.routing.get', this.get.bind(this), {
-      optionsIndex: 1
-    }) ?? this.get
-    this.findPeer = components.metrics?.traceFunction('helia.routing.findPeer', this.findPeer.bind(this), {
-      optionsIndex: 1
-    }) ?? this.findPeer
-    this.getClosestPeers = components.metrics?.traceFunction('helia.routing.getClosestPeers', this.getClosestPeers.bind(this), {
-      optionsIndex: 1
-    }) ?? this.getClosestPeers
-  }
-
-  async start (): Promise<void> {
-    await start(...this.routers)
-  }
-
-  async stop (): Promise<void> {
-    await stop(...this.routers)
-  }
-
-  /**
-   * Iterates over all content routers in parallel to find providers of the
-   * given key
-   */
-  async * findProviders (key: CID, options: RoutingOptions<RoutingFindProvidersProgressEvents> = {}): AsyncIterable<Provider> {
-    if (this.routers.length === 0) {
-      throw new NoRoutersAvailableError('No content routers available')
-    }
-
-    // provider multiaddrs are only cached for a limited time, so they can come
-    // back as an empty array - when this happens we have to do a FIND_PEER
-    // query to get updated addresses, but we shouldn't block on this so use a
-    // separate bounded queue to perform this lookup
-    const queue = new PeerQueue<Provider | null>({
-      concurrency: this.providerLookupConcurrency
-    })
-
-    let foundProviders = 0
-    const errors: Error[] = []
-    const self = this
-    let routersFinished = 0
-
-    this.log('findProviders for %c start using routers %s', key, this.routers.map(r => r.toString()).join(', '))
-
-    const routers = supports(this.routers, 'findProviders')
-      .map(async function * (router) {
-        let foundProviders = 0
-
-        options?.onProgress?.(new CustomProgressEvent('helia:routing:find-providers:start', {
-          routing: router.name,
-          cid: key
-        }))
-
-        try {
-          for await (const prov of router.findProviders(key, options)) {
-            foundProviders++
-
-            // @ts-expect-error router.name is a string, needs to be specific
-            options?.onProgress?.(new CustomProgressEvent('helia:routing:find-providers:provider', {
-              routing: router.name,
-              cid: key,
-              provider: prov
-            }))
-
-            yield prov
-          }
-        } catch (err: any) {
-          errors.push(err)
-        } finally {
-          self.log('router %s found %d providers for %c', router, foundProviders, key)
-
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:find-providers:end', {
-            routing: router.name,
-            cid: key,
-            found: foundProviders
-          }))
-
-          routersFinished++
-
-          // if all routers have finished and there are no jobs to find updated
-          // peer multiaddres running or queued, cause the generator to exit
-          if (routersFinished === routers.length && queue.size === 0) {
-            queue.emitIdle()
-          }
-        }
-      })
-
-    for await (const peer of merge(
-      queue.toGenerator(),
-      ...routers)
-    ) {
-      // the peer was yielded by a content router without multiaddrs and we
-      // failed to load them
-      if (peer == null) {
-        continue
-      }
-
-      // have to refresh peer info for this peer to get updated multiaddrs
-      if (peer.multiaddrs.length === 0) {
-        // already looking this peer up
-        if (queue.find(peer.id) != null) {
-          continue
-        }
-
-        queue.add(async () => {
-          try {
-            const provider = await this.findPeer(peer.id, options)
-
-            if (provider.multiaddrs.length === 0) {
-              return null
-            }
-
-            return {
-              ...provider,
-              protocols: peer.protocols,
-              routing: peer.routing
-            }
-          } catch (err) {
-            this.log.error('could not load multiaddrs for peer %p - %e', peer.id, err)
-            return null
-          }
-        }, {
-          peerId: peer.id,
-          signal: options.signal
-        })
-          .catch(err => {
-            this.log.error('could not load multiaddrs for peer %p - %e', peer.id, err)
-          })
-
-        continue
-      }
-
-      foundProviders++
-      yield peer
-    }
-
-    this.log('findProviders finished, found %d providers for %c', foundProviders, key)
-  }
-
-  /**
-   * Iterates over all content routers in parallel to notify it is
-   * a provider of the given key
-   */
-  async provide (key: CID, options: RoutingOptions<RoutingProvideProgressEvents> = {}): Promise<void> {
-    if (this.routers.length === 0) {
-      throw new NoRoutersAvailableError('No content routers available')
-    }
-
-    await Promise.all(
-      supports(this.routers, 'provide')
-        .map(async (router) => {
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:provide:start', {
-            routing: router.name,
-            cid: key
-          }))
-
-          await router.provide(key, options)
-
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:provide:end', {
-            routing: router.name,
-            cid: key
-          }))
-        })
-    )
-  }
-
-  async cancelReprovide (key: CID, options: RoutingOptions<RoutingCancelReprovideProgressEvents> = {}): Promise<void> {
-    await Promise.all(
-      supports(this.routers, 'cancelReprovide')
-        .map(async (router) => {
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:cancel-reprovide:start', {
-            routing: router.name,
-            cid: key
-          }))
-
-          await router.cancelReprovide(key, options)
-
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:cancel-reprovide:end', {
-            routing: router.name,
-            cid: key
-          }))
-        })
-    )
-  }
-
-  /**
-   * Store the given key/value pair in the available content routings
-   */
-  async put (key: Uint8Array, value: Uint8Array, options?: RoutingOptions<RoutingPutProgressEvents>): Promise<void> {
-    await Promise.all(
-      supports(this.routers, 'put')
-        .map(async (router) => {
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:put:start', {
-            routing: router.name,
-            key,
-            value
-          }))
-
-          await router.put(key, value, options)
-
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:put:end', {
-            routing: router.name,
-            key,
-            value
-          }))
-        })
-    )
-  }
-
-  /**
-   * Get the value to the given key. The first value offered by any configured
-   * router will be returned.
-   */
-  async get (key: Uint8Array, options?: RoutingOptions<RoutingGetProgressEvents>): Promise<Uint8Array> {
-    const errors: Error[] = []
-    let result: Uint8Array | undefined
-
-    try {
-      result = await Promise.any(
-        supports(this.routers, 'get')
-          .map(async (router) => {
-            options?.onProgress?.(new CustomProgressEvent('helia:routing:get:start', {
-              routing: router.name,
-              key
-            }))
-
-            try {
-              return await router.get(key, options)
-            } catch (err: any) {
-              this.log('router %s failed with %e', router, err)
-              errors.push(err)
-            } finally {
-              options?.onProgress?.(new CustomProgressEvent('helia:routing:get:end', {
-                routing: router.name,
-                key
-              }))
-            }
-          })
-      )
-    } catch {
-      // ignore AggregateError as we will throw a better-named one
-    }
-
-    if (result == null) {
-      throw new GetFailedError(errors, `Failed to get value key ${uint8ArrayToString(key, 'base58btc')}`)
-    }
-
-    return result
-  }
-
-  /**
-   * Iterates over all peer routers in parallel to find the given peer
-   */
-  async findPeer (id: PeerId, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<PeerInfo> {
-    if (this.routers.length === 0) {
-      throw new NoRoutersAvailableError('No peer routers available')
-    }
-
-    const self = this
-    const source = merge(
-      ...supports(this.routers, 'findPeer')
-        .map(router => (async function * () {
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:find-peer:start', {
-            routing: router.name,
-            peerId: id
-          }))
-
-          try {
-            yield await router.findPeer(id, options)
-          } catch (err) {
-            self.log.error(err)
-          } finally {
-            options?.onProgress?.(new CustomProgressEvent('helia:routing:find-peer:end', {
-              routing: router.name,
-              peerId: id
-            }))
-          }
-        })())
-    )
-
-    for await (const peer of source) {
-      if (peer == null) {
-        continue
-      }
-
-      return peer
-    }
-
-    throw new NotFoundError('Could not find peer in routing')
-  }
-
-  /**
-   * Attempt to find the closest peers on the network to the given key
-   */
-  async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<PeerInfo> {
-    if (this.routers.length === 0) {
-      throw new NoRoutersAvailableError('No peer routers available')
-    }
-
-    for await (const peer of merge(
-      ...supports(this.routers, 'getClosestPeers')
-        .map(async function * (router) {
-          options?.onProgress?.(new CustomProgressEvent('helia:routing:get-closest-peers:start', {
-            routing: router.name,
-            key
-          }))
-
-          try {
-            yield * router.getClosestPeers(key, options)
-          } finally {
-            options?.onProgress?.(new CustomProgressEvent('helia:routing:get-closest-peers:end', {
-              routing: router.name,
-              key
-            }))
-          }
-        })
-    )) {
-      if (peer == null) {
-        continue
-      }
-
-      yield peer
-    }
-  }
-}
-
-function supports <Operation extends keyof Routing> (routers: any[], key: Operation): Array<Pick<Routing, Operation | 'name'>> {
-  return routers.filter(router => router[key] != null)
-}

package/src/storage.ts

@@ -1,195 +0,0 @@
-import { start, stop } from '@libp2p/interface'
-import createMortice from 'mortice'
-import { BlockPinnedError } from './errors.ts'
-import type { Routing } from '@helia/interface'
-import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore } from '@helia/interface/blocks'
-import type { Pins } from '@helia/interface/pins'
-import type { AbortOptions, Startable } from '@libp2p/interface'
-import type { Blockstore, InputPair } from 'interface-blockstore'
-import type { AwaitIterable } from 'interface-store'
-import type { Mortice } from 'mortice'
-import type { CID } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
-
-export interface BlockStorageInit {
-  holdGcLock?: boolean
-}
-
-export interface GetOptions extends AbortOptions {
-  progress?(evt: Event): void
-}
-
-/**
- * BlockStorage is a hybrid blockstore that puts/gets blocks from a configured
- * blockstore (that may be on disk, s3, or something else). If the blocks are
- * not present Bitswap will be used to fetch them from network peers.
- */
-export class BlockStorage implements Blocks, Startable {
-  public lock: Mortice
-  private readonly child: Blocks
-  private readonly pins: Pins
-  private readonly routing: Routing
-  private started: boolean
-
-  /**
-   * Create a new BlockStorage
-   */
-  constructor (blockstore: Blocks, pins: Pins, routing: Routing, options: BlockStorageInit = {}) {
-    this.child = blockstore
-    this.pins = pins
-    this.routing = routing
-    this.lock = createMortice({
-      singleProcess: options.holdGcLock
-    })
-    this.started = false
-  }
-
-  isStarted (): boolean {
-    return this.started
-  }
-
-  async start (): Promise<void> {
-    await start(this.child)
-    this.started = true
-  }
-
-  async stop (): Promise<void> {
-    await stop(this.child)
-    this.started = false
-  }
-
-  unwrap (): Blockstore {
-    return this.child
-  }
-
-  /**
-   * Put a block to the underlying datastore
-   */
-  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      return await this.child.put(cid, block, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Put a multiple blocks to the underlying datastore
-   */
-  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.putMany(blocks, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Get a block by cid
-   */
-  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.get(cid, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Get multiple blocks back from an (async) iterable of cids
-   */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.getMany(cids, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Delete a block from the blockstore
-   */
-  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.writeLock()
-
-    try {
-      if (await this.pins.isPinned(cid)) {
-        throw new BlockPinnedError('Block was pinned - please unpin and try again')
-      }
-
-      // stop re-providing this CID if necessary
-      await this.routing.cancelReprovide(cid, options)
-
-      await this.child.delete(cid, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Delete multiple blocks from the blockstore
-   */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.writeLock()
-
-    try {
-      const storage = this
-
-      yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
-        for await (const cid of cids) {
-          if (await storage.pins.isPinned(cid)) {
-            throw new BlockPinnedError('Block was pinned - please unpin and try again')
-          }
-
-          // stop re-providing this CID if necessary
-          await storage.routing.cancelReprovide(cid, options)
-
-          yield cid
-        }
-      }()), options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      return await this.child.has(cid, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
-    options?.signal?.throwIfAborted()
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.getAll(options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  createSession (root: CID, options?: AbortOptions): SessionBlockstore {
-    options?.signal?.throwIfAborted()
-    return this.child.createSession(root, options)
-  }
-}

package/src/utils/datastore-version.ts

@@ -1,25 +0,0 @@
-import { Key } from 'interface-datastore'
-import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
-import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { InvalidDatastoreVersionError } from '../errors.ts'
-import type { Datastore } from 'interface-datastore'
-
-const DS_VERSION_KEY = new Key('/version')
-const CURRENT_VERSION = 1
-
-export async function assertDatastoreVersionIsCurrent (datastore: Datastore): Promise<void> {
-  if (!(await datastore.has(DS_VERSION_KEY))) {
-    await datastore.put(DS_VERSION_KEY, uint8ArrayFromString(`${CURRENT_VERSION}`))
-
-    return
-  }
-
-  const buf = await datastore.get(DS_VERSION_KEY)
-  const str = uint8ArrayToString(buf)
-  const version = parseInt(str, 10)
-
-  if (version !== CURRENT_VERSION) {
-    // TODO: write migrations when we break compatibility - for an example, see https://github.com/ipfs/js-ipfs-repo/tree/master/packages/ipfs-repo-migrations
-    throw new InvalidDatastoreVersionError('Invalid datastore version, a datastore migration may be required')
-  }
-}

package/src/utils/get-codec.ts

@@ -1,47 +0,0 @@
-/* eslint max-depth: ["error", 7] */
-
-import { UnknownCodecError } from '@helia/interface'
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagJson from '@ipld/dag-json'
-import * as dagPb from '@ipld/dag-pb'
-import * as json from 'multiformats/codecs/json'
-import * as raw from 'multiformats/codecs/raw'
-import { isPromise } from './is-promise.ts'
-import type { Await } from '@helia/interface'
-import type { BlockCodec } from 'multiformats/codecs/interface'
-
-export function getCodec <T = any, Code extends number = any> (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: (code: number) => Await<BlockCodec<any, any>>): (code: Code) => Await<BlockCodec<Code, T>> {
-  const codecs: Record<number, BlockCodec<any, any>> = {
-    [dagPb.code]: dagPb,
-    [raw.code]: raw,
-    [dagCbor.code]: dagCbor,
-    [dagJson.code]: dagJson,
-    [json.code]: json
-  }
-
-  initialCodecs.forEach(codec => {
-    codecs[codec.code] = codec
-  })
-
-  return async (code) => {
-    let codec = codecs[code]
-
-    if (codec == null && loadCodec != null) {
-      const res = loadCodec(code)
-
-      if (isPromise(res)) {
-        codec = await res
-      } else {
-        codec = res
-      }
-
-      codecs[codec.code] = codec
-    }
-
-    if (codec != null) {
-      return codec
-    }
-
-    throw new UnknownCodecError(`Could not load codec for ${code}`)
-  }
-}

package/src/utils/get-hasher.ts

@@ -1,40 +0,0 @@
-import { UnknownHashAlgorithmError } from '@helia/interface'
-import { identity } from 'multiformats/hashes/identity'
-import { sha256, sha512 } from 'multiformats/hashes/sha2'
-import { isPromise } from './is-promise.ts'
-import type { Await } from '@helia/interface'
-import type { MultihashHasher } from 'multiformats/hashes/interface'
-
-export function getHasher (initialHashers: MultihashHasher[] = [], loadHasher?: (code: number) => Await<MultihashHasher>): (code: number) => Await<MultihashHasher> {
-  const hashers: Record<number, MultihashHasher> = {
-    [sha256.code]: sha256,
-    [sha512.code]: sha512,
-    [identity.code]: identity
-  }
-
-  initialHashers.forEach(hasher => {
-    hashers[hasher.code] = hasher
-  })
-
-  return async (code) => {
-    let hasher = hashers[code]
-
-    if (hasher == null && loadHasher != null) {
-      const res = loadHasher(code)
-
-      if (isPromise(res)) {
-        hasher = await res
-      } else {
-        hasher = res
-      }
-
-      hashers[hasher.code] = hasher
-    }
-
-    if (hasher != null) {
-      return hasher
-    }
-
-    throw new UnknownHashAlgorithmError(`No hasher configured for multihash code 0x${code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`)
-  }
-}