helia 6.1.4 → 7.0.0

package/src/pins.ts

@@ -0,0 +1,208 @@
+import { AlreadyPinnedError, depthFirstWalker } from '@helia/utils'
+import { InvalidParametersError } from '@libp2p/interface'
+import * as cborg from 'cborg'
+import { Key } from 'interface-datastore'
+import { base36 } from 'multiformats/bases/base36'
+import { CID } from 'multiformats/cid'
+import { CustomProgressEvent } from 'progress-events'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import type { CodecLoader } from '@helia/interface'
+import type { AddOptions, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { Datastore } from 'interface-datastore'
+import type { Version } from 'multiformats/cid'
+
+interface DatastorePin {
+  /**
+   * 0 for a direct pin or an arbitrary (+ve, whole) number or Infinity
+   */
+  depth: number
+
+  /**
+   * User-specific metadata for the pin
+   */
+  metadata: Record<string, string | number | boolean>
+}
+
+interface DatastorePinnedBlock {
+  pinCount: number
+  pinnedBy: Uint8Array[]
+}
+
+/**
+ * Callback for updating a {@link DatastorePinnedBlock}'s properties when
+ * calling `#updatePinnedBlock`
+ *
+ * The callback should return `false` to prevent any pinning modifications to
+ * the block, and true in all other cases.
+ */
+interface WithPinnedBlockCallback {
+  (pinnedBlock: DatastorePinnedBlock): boolean
+}
+
+const DATASTORE_PIN_PREFIX = '/pin/'
+const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
+const DATASTORE_ENCODING = base36
+
+function toDSKey (cid: CID): Key {
+  if (cid.version === 0) {
+    cid = cid.toV1()
+  }
+
+  return new Key(`${DATASTORE_PIN_PREFIX}${cid.toString(DATASTORE_ENCODING)}`)
+}
+
+export class PinsImpl implements Pins {
+  private readonly datastore: Datastore
+  private readonly blockstore: Blockstore
+  private readonly getCodec: CodecLoader
+
+  constructor (datastore: Datastore, blockstore: Blockstore, getCodec: CodecLoader) {
+    this.datastore = datastore
+    this.blockstore = blockstore
+    this.getCodec = getCodec
+  }
+
+  async * add (cid: CID<unknown, number, number, Version>, options: AddOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const pinKey = toDSKey(cid)
+
+    if (await this.datastore.has(pinKey)) {
+      throw new AlreadyPinnedError('Already pinned')
+    }
+
+    const depth = Math.round(options.depth ?? Infinity)
+
+    if (depth < 0) {
+      throw new InvalidParametersError('Depth must be greater than or equal to 0')
+    }
+
+    const walker = (options.walker ?? depthFirstWalker())({
+      blockstore: this.blockstore,
+      getCodec: this.getCodec
+    })
+
+    for await (const node of walker.walk(cid, { ...options, depth })) {
+      const childCid = node.block.cid
+      await this.#updatePinnedBlock(childCid, (pinnedBlock: DatastorePinnedBlock) => {
+        // do not update pinned block if this block is already pinned by this CID
+        if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
+          return false
+        }
+
+        pinnedBlock.pinCount++
+        pinnedBlock.pinnedBy.push(cid.bytes)
+        return true
+      }, options)
+
+      yield childCid
+    }
+
+    const pin: DatastorePin = {
+      depth,
+      metadata: options.metadata ?? {}
+    }
+
+    await this.datastore.put(pinKey, cborg.encode(pin), options)
+  }
+
+  /**
+   * Update the pin count for the CID
+   */
+  async #updatePinnedBlock (cid: CID, withPinnedBlock: WithPinnedBlockCallback, options: AddOptions): Promise<void> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    let pinnedBlock: DatastorePinnedBlock = {
+      pinCount: 0,
+      pinnedBy: []
+    }
+
+    try {
+      pinnedBlock = cborg.decode(await this.datastore.get(blockKey, options))
+    } catch (err: any) {
+      if (err.name !== 'NotFoundError') {
+        throw err
+      }
+    }
+
+    const shouldContinue = withPinnedBlock(pinnedBlock)
+
+    if (!shouldContinue) {
+      return
+    }
+
+    if (pinnedBlock.pinCount === 0) {
+      if (await this.datastore.has(blockKey)) {
+        await this.datastore.delete(blockKey)
+        return
+      }
+    }
+
+    await this.datastore.put(blockKey, cborg.encode(pinnedBlock), options)
+    options.onProgress?.(new CustomProgressEvent<CID>('helia:pin:add', cid))
+  }
+
+  async * rm (cid: CID<unknown, number, number, Version>, options: RmOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const pinKey = toDSKey(cid)
+    const buf = await this.datastore.get(pinKey, options)
+    const pin = cborg.decode(buf)
+
+    await this.datastore.delete(pinKey, options)
+
+    const walker = (options.walker ?? depthFirstWalker())({
+      blockstore: this.blockstore,
+      getCodec: this.getCodec
+    })
+
+    for await (const node of walker.walk(cid, { ...options, depth: pin.depth })) {
+      const childCid = node.block.cid
+      await this.#updatePinnedBlock(childCid, (pinnedBlock): boolean => {
+        pinnedBlock.pinCount--
+        pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
+        return true
+      }, {
+        ...options,
+        depth: pin.depth
+      })
+
+      yield childCid
+    }
+  }
+
+  async * ls (options: LsOptions = {}): AsyncGenerator<Pin, void, undefined> {
+    for await (const { key, value } of this.datastore.query({
+      prefix: DATASTORE_PIN_PREFIX + (options.cid != null ? `${options.cid.toString(base36)}` : '')
+    }, options)) {
+      const cid = CID.parse(key.toString().substring(5), base36)
+      const pin = cborg.decode(value)
+
+      yield {
+        cid,
+        ...pin
+      }
+    }
+  }
+
+  async isPinned (cid: CID, options: IsPinnedOptions = {}): Promise<boolean> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    return this.datastore.has(blockKey, options)
+  }
+
+  async get (cid: CID, options?: AbortOptions): Promise<Pin> {
+    const pinKey = toDSKey(cid)
+    const buf = await this.datastore.get(pinKey, options)
+
+    return cborg.decode(buf)
+  }
+
+  async setMetadata (cid: CID, metadata: Record<string, string | number | boolean> | undefined, options?: AbortOptions): Promise<void> {
+    const pinKey = toDSKey(cid)
+    const buf = await this.datastore.get(pinKey, options)
+    const pin: DatastorePin = cborg.decode(buf)
+
+    pin.metadata = metadata ?? {}
+
+    await this.datastore.put(pinKey, cborg.encode(pin), options)
+  }
+}

package/src/routing.ts

@@ -0,0 +1,389 @@
+import { NoRoutersAvailableError } from '@helia/interface'
+import { GetFailedError } from '@helia/utils'
+import { NotFoundError, start, stop } from '@libp2p/interface'
+import { Queue } from '@libp2p/utils'
+import merge from 'it-merge'
+import { CustomProgressEvent } from 'progress-events'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents, Router, Peer } from '@helia/interface'
+import type { ComponentLogger, Logger, Metrics, Startable } from '@libp2p/interface'
+import type { AbortOptions } from 'abort-error'
+import type { CID } from 'multiformats/cid'
+
+const DEFAULT_PROVIDER_LOOKUP_CONCURRENCY = 5
+
+export interface RoutingInit {
+  routers: Router[]
+  providerLookupConcurrency?: number
+}
+
+export interface RoutingComponents {
+  logger: ComponentLogger
+  metrics?: Metrics
+}
+
+interface PeerQueueOptions extends AbortOptions {
+  peer: CID
+}
+
+export class Routing implements RoutingInterface, Startable {
+  public name: string
+
+  private readonly log: Logger
+  private readonly routers: Router[]
+  private readonly providerLookupConcurrency: number
+
+  constructor (components: RoutingComponents, init: RoutingInit) {
+    this.name = 'helia'
+    this.log = components.logger.forComponent('helia:routing')
+    this.routers = init.routers ?? []
+    this.providerLookupConcurrency = init.providerLookupConcurrency ?? DEFAULT_PROVIDER_LOOKUP_CONCURRENCY
+
+    this.findProviders = components.metrics?.traceFunction('helia.routing.findProviders', this.findProviders.bind(this), {
+      optionsIndex: 1
+    }) ?? this.findProviders
+    this.provide = components.metrics?.traceFunction('helia.routing.provide', this.provide.bind(this), {
+      optionsIndex: 1
+    }) ?? this.provide
+    this.cancelReprovide = components.metrics?.traceFunction('helia.routing.cancelReprovide', this.cancelReprovide.bind(this), {
+      optionsIndex: 1
+    }) ?? this.cancelReprovide
+    this.put = components.metrics?.traceFunction('helia.routing.put', this.put.bind(this), {
+      optionsIndex: 2
+    }) ?? this.put
+    this.get = components.metrics?.traceFunction('helia.routing.get', this.get.bind(this), {
+      optionsIndex: 1
+    }) ?? this.get
+    this.findPeer = components.metrics?.traceFunction('helia.routing.findPeer', this.findPeer.bind(this), {
+      optionsIndex: 1
+    }) ?? this.findPeer
+    this.getClosestPeers = components.metrics?.traceFunction('helia.routing.getClosestPeers', this.getClosestPeers.bind(this), {
+      optionsIndex: 1
+    }) ?? this.getClosestPeers
+  }
+
+  async start (): Promise<void> {
+    await start(...this.routers)
+  }
+
+  async stop (): Promise<void> {
+    await stop(...this.routers)
+  }
+
+  hasRouter (name: string): boolean {
+    return this.routers.findIndex(r => r.name === name) !== -1
+  }
+
+  addRouter (router: Router): void {
+    this.routers.push(router)
+  }
+
+  /**
+   * Iterates over all content routers in parallel to find providers of the
+   * given key
+   */
+  async * findProviders (key: CID, options: RoutingOptions<RoutingFindProvidersProgressEvents> = {}): AsyncIterable<Provider> {
+    if (this.routers.length === 0) {
+      throw new NoRoutersAvailableError('No content routers available')
+    }
+
+    // provider multiaddrs are only cached for a limited time, so they can come
+    // back as an empty array - when this happens we have to do a FIND_PEER
+    // query to get updated addresses, but we shouldn't block on this so use a
+    // separate bounded queue to perform this lookup
+    const queue = new Queue<Provider | null, PeerQueueOptions>({
+      concurrency: this.providerLookupConcurrency
+    })
+
+    let foundProviders = 0
+    const errors: Error[] = []
+    const self = this
+    let routersFinished = 0
+
+    this.log('findProviders for %c start using routers %s', key, this.routers.map(r => r.toString()).join(', '))
+
+    const routers = supports(this.routers, 'findProviders')
+      .map(async function * (router) {
+        let foundProviders = 0
+
+        options?.onProgress?.(new CustomProgressEvent('helia:routing:find-providers:start', {
+          routing: router.name,
+          cid: key
+        }))
+
+        try {
+          for await (const prov of router.findProviders(key, options)) {
+            foundProviders++
+
+            // @ts-expect-error router.name is a string, needs to be specific
+            options?.onProgress?.(new CustomProgressEvent('helia:routing:find-providers:provider', {
+              routing: router.name,
+              cid: key,
+              provider: prov
+            }))
+
+            yield prov
+          }
+        } catch (err: any) {
+          errors.push(err)
+        } finally {
+          self.log('router %s found %d providers for %c', router, foundProviders, key)
+
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:find-providers:end', {
+            routing: router.name,
+            cid: key,
+            found: foundProviders
+          }))
+
+          routersFinished++
+
+          // if all routers have finished and there are no jobs to find updated
+          // peer multiaddres running or queued, cause the generator to exit
+          if (routersFinished === routers.length && queue.size === 0) {
+            queue.emitIdle()
+          }
+        }
+      })
+
+    for await (const peer of merge(
+      queue.toGenerator(),
+      ...routers)
+    ) {
+      // the peer was yielded by a content router without multiaddrs and we
+      // failed to load them
+      if (peer == null) {
+        continue
+      }
+
+      // have to refresh peer info for this peer to get updated multiaddrs
+      if (peer.multiaddrs.length === 0) {
+        // already looking this peer up
+        if (queue.queue.find(job => job.options.peer.equals(peer.id)) != null) {
+          continue
+        }
+
+        queue.add(async () => {
+          try {
+            const provider = await this.findPeer(peer.id, options)
+
+            if (provider.multiaddrs.length === 0) {
+              return null
+            }
+
+            return {
+              ...provider,
+              protocols: peer.protocols,
+              routing: peer.routing
+            }
+          } catch (err) {
+            this.log.error('could not load multiaddrs for peer %p - %e', peer.id, err)
+            return null
+          }
+        }, {
+          peer: peer.id,
+          signal: options.signal
+        })
+          .catch(err => {
+            this.log.error('could not load multiaddrs for peer %p - %e', peer.id, err)
+          })
+
+        continue
+      }
+
+      foundProviders++
+      yield peer
+    }
+
+    this.log('findProviders finished, found %d providers for %c', foundProviders, key)
+  }
+
+  /**
+   * Iterates over all content routers in parallel to notify it is
+   * a provider of the given key
+   */
+  async provide (key: CID, options: RoutingOptions<RoutingProvideProgressEvents> = {}): Promise<void> {
+    if (this.routers.length === 0) {
+      throw new NoRoutersAvailableError('No content routers available')
+    }
+
+    await Promise.all(
+      supports(this.routers, 'provide')
+        .map(async (router) => {
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:provide:start', {
+            routing: router.name,
+            cid: key
+          }))
+
+          await router.provide(key, options)
+
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:provide:end', {
+            routing: router.name,
+            cid: key
+          }))
+        })
+    )
+  }
+
+  async cancelReprovide (key: CID, options: RoutingOptions<RoutingCancelReprovideProgressEvents> = {}): Promise<void> {
+    await Promise.all(
+      supports(this.routers, 'cancelReprovide')
+        .map(async (router) => {
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:cancel-reprovide:start', {
+            routing: router.name,
+            cid: key
+          }))
+
+          await router.cancelReprovide(key, options)
+
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:cancel-reprovide:end', {
+            routing: router.name,
+            cid: key
+          }))
+        })
+    )
+  }
+
+  /**
+   * Store the given key/value pair in the available content routings
+   */
+  async put (key: Uint8Array, value: Uint8Array, options?: RoutingOptions<RoutingPutProgressEvents>): Promise<void> {
+    await Promise.all(
+      supports(this.routers, 'put')
+        .map(async (router) => {
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:put:start', {
+            routing: router.name,
+            key,
+            value
+          }))
+
+          await router.put(key, value, options)
+
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:put:end', {
+            routing: router.name,
+            key,
+            value
+          }))
+        })
+    )
+  }
+
+  /**
+   * Get the value to the given key. The first value offered by any configured
+   * router will be returned.
+   */
+  async get (key: Uint8Array, options?: RoutingOptions<RoutingGetProgressEvents>): Promise<Uint8Array> {
+    const errors: Error[] = []
+    let result: Uint8Array | undefined
+
+    try {
+      result = await Promise.any(
+        supports(this.routers, 'get')
+          .map(async (router) => {
+            options?.onProgress?.(new CustomProgressEvent('helia:routing:get:start', {
+              routing: router.name,
+              key
+            }))
+
+            try {
+              return await router.get(key, options)
+            } catch (err: any) {
+              this.log('router %s failed with %e', router, err)
+              errors.push(err)
+            } finally {
+              options?.onProgress?.(new CustomProgressEvent('helia:routing:get:end', {
+                routing: router.name,
+                key
+              }))
+            }
+          })
+      )
+    } catch {
+      // ignore AggregateError as we will throw a better-named one
+    }
+
+    if (result == null) {
+      throw new GetFailedError(errors, `Failed to get value key ${uint8ArrayToString(key, 'base58btc')}`)
+    }
+
+    return result
+  }
+
+  /**
+   * Iterates over all peer routers in parallel to find the given peer
+   */
+  async findPeer (id: CID, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<Peer> {
+    if (this.routers.length === 0) {
+      throw new NoRoutersAvailableError('No peer routers available')
+    }
+
+    const self = this
+    const source = merge(
+      ...supports(this.routers, 'findPeer')
+        .map(router => (async function * () {
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:find-peer:start', {
+            routing: router.name,
+            peerId: id
+          }))
+
+          try {
+            yield await router.findPeer(id, options)
+          } catch (err) {
+            self.log.error(err)
+          } finally {
+            options?.onProgress?.(new CustomProgressEvent('helia:routing:find-peer:end', {
+              routing: router.name,
+              peerId: id
+            }))
+          }
+        })())
+    )
+
+    for await (const peer of source) {
+      if (peer == null) {
+        continue
+      }
+
+      return peer
+    }
+
+    throw new NotFoundError('Could not find peer in routing')
+  }
+
+  /**
+   * Attempt to find the closest peers on the network to the given key
+   */
+  async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<Peer> {
+    if (this.routers.length === 0) {
+      throw new NoRoutersAvailableError('No peer routers available')
+    }
+
+    for await (const peer of merge(
+      ...supports(this.routers, 'getClosestPeers')
+        .map(async function * (router) {
+          options?.onProgress?.(new CustomProgressEvent('helia:routing:get-closest-peers:start', {
+            routing: router.name,
+            key
+          }))
+
+          try {
+            yield * router.getClosestPeers(key, options)
+          } finally {
+            options?.onProgress?.(new CustomProgressEvent('helia:routing:get-closest-peers:end', {
+              routing: router.name,
+              key
+            }))
+          }
+        })
+    )) {
+      if (peer == null) {
+        continue
+      }
+
+      yield peer
+    }
+  }
+}
+
+function supports <Operation extends keyof Routing> (routers: any[], key: Operation): Array<Pick<Routing, Operation | 'name'>> {
+  return routers.filter(router => router[key] != null)
+}