@helia/utils 0.0.0-031519c

package/src/index.ts

@@ -0,0 +1,225 @@
+/**
+ * @packageDocumentation
+ *
+ * Exports a `Helia` class that implements the {@link HeliaInterface} API.
+ *
+ * In general you should use the `helia` or `@helia/http` modules instead which
+ * pre-configure Helia for certain use-cases (p2p or pure-HTTP).
+ *
+ * @example
+ *
+ * ```typescript
+ * import { Helia } from '@helia/utils'
+ *
+ * const node = new Helia({
+ *   // ...options
+ * })
+ * ```
+ */
+
+import { contentRoutingSymbol, peerRoutingSymbol, start, stop } from '@libp2p/interface'
+import { defaultLogger } from '@libp2p/logger'
+import drain from 'it-drain'
+import { CustomProgressEvent } from 'progress-events'
+import { PinsImpl } from './pins.js'
+import { Routing as RoutingClass } from './routing.js'
+import { BlockStorage } from './storage.js'
+import { defaultDagWalkers } from './utils/dag-walkers.js'
+import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.js'
+import { defaultHashers } from './utils/default-hashers.js'
+import { NetworkedStorage } from './utils/networked-storage.js'
+import type { DAGWalker, GCOptions, Helia as HeliaInterface, Routing } from '@helia/interface'
+import type { BlockBroker } from '@helia/interface/blocks'
+import type { Pins } from '@helia/interface/pins'
+import type { ComponentLogger, Logger } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { Datastore } from 'interface-datastore'
+import type { CID } from 'multiformats/cid'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+/**
+ * Options used to create a Helia node.
+ */
+export interface HeliaInit {
+  /**
+   * The blockstore is where blocks are stored
+   */
+  blockstore: Blockstore
+
+  /**
+   * The datastore is where data is stored
+   */
+  datastore: Datastore
+
+  /**
+   * By default sha256, sha512 and identity hashes are supported for
+   * bitswap operations. To bitswap blocks with CIDs using other hashes
+   * pass appropriate MultihashHashers here.
+   */
+  hashers?: MultihashHasher[]
+
+  /**
+   * In order to pin CIDs that correspond to a DAG, it's necessary to know
+   * how to traverse that DAG.  DAGWalkers take a block and yield any CIDs
+   * encoded within that block.
+   */
+  dagWalkers?: DAGWalker[]
+
+  /**
+   * A list of strategies used to fetch blocks when they are not present in
+   * the local blockstore
+   */
+  blockBrokers: Array<(components: any) => BlockBroker>
+
+  /**
+   * Garbage collection requires preventing blockstore writes during searches
+   * for unpinned blocks as DAGs are typically pinned after they've been
+   * imported - without locking this could lead to the deletion of blocks while
+   * they are being added to the blockstore.
+   *
+   * By default this lock is held on the current process and other processes
+   * will contact this process for access.
+   *
+   * If Helia is being run in multiple processes, one process must hold the GC
+   * lock so use this option to control which process that is.
+   *
+   * @default true
+   */
+  holdGcLock?: boolean
+
+  /**
+   * An optional logging component to pass to libp2p. If not specified the
+   * default implementation from libp2p will be used.
+   */
+  logger?: ComponentLogger
+
+  /**
+   * Routers perform operations such as looking up content providers,
+   * information about network peers or getting/putting records.
+   */
+  routers?: Array<Partial<Routing>>
+
+  /**
+   * Components used by subclasses
+   */
+  components?: Record<string, any>
+}
+
+interface Components {
+  blockstore: Blockstore
+  datastore: Datastore
+  hashers: Record<number, MultihashHasher>
+  dagWalkers: Record<number, DAGWalker>
+  logger: ComponentLogger
+  blockBrokers: BlockBroker[]
+}
+
+export class Helia implements HeliaInterface {
+  public blockstore: BlockStorage
+  public datastore: Datastore
+  public pins: Pins
+  public logger: ComponentLogger
+  public routing: Routing
+  public dagWalkers: Record<number, DAGWalker>
+  public hashers: Record<number, MultihashHasher>
+  private readonly log: Logger
+
+  constructor (init: HeliaInit) {
+    this.logger = init.logger ?? defaultLogger()
+    this.log = this.logger.forComponent('helia')
+    this.hashers = defaultHashers(init.hashers)
+    this.dagWalkers = defaultDagWalkers(init.dagWalkers)
+
+    const components: Components = {
+      blockstore: init.blockstore,
+      datastore: init.datastore,
+      hashers: this.hashers,
+      dagWalkers: this.dagWalkers,
+      logger: this.logger,
+      blockBrokers: [],
+      ...(init.components ?? {})
+    }
+
+    components.blockBrokers = init.blockBrokers.map((fn) => {
+      return fn(components)
+    })
+
+    const networkedStorage = new NetworkedStorage(components)
+
+    this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers)
+
+    this.blockstore = new BlockStorage(networkedStorage, this.pins, {
+      holdGcLock: init.holdGcLock ?? true
+    })
+    this.datastore = init.datastore
+    this.routing = new RoutingClass(components, {
+      routers: (init.routers ?? []).flatMap((router: any) => {
+        // if the router itself is a router
+        const routers = [
+          router
+        ]
+
+        // if the router provides a libp2p-style ContentRouter
+        if (router[contentRoutingSymbol] != null) {
+          routers.push(router[contentRoutingSymbol])
+        }
+
+        // if the router provides a libp2p-style PeerRouter
+        if (router[peerRoutingSymbol] != null) {
+          routers.push(router[peerRoutingSymbol])
+        }
+
+        return routers
+      })
+    })
+  }
+
+  async start (): Promise<void> {
+    await assertDatastoreVersionIsCurrent(this.datastore)
+    await start(
+      this.blockstore,
+      this.datastore,
+      this.routing
+    )
+  }
+
+  async stop (): Promise<void> {
+    await stop(
+      this.blockstore,
+      this.datastore,
+      this.routing
+    )
+  }
+
+  async gc (options: GCOptions = {}): Promise<void> {
+    const releaseLock = await this.blockstore.lock.writeLock()
+
+    try {
+      const helia = this
+      const blockstore = this.blockstore.unwrap()
+
+      this.log('gc start')
+
+      await drain(blockstore.deleteMany((async function * (): AsyncGenerator<CID> {
+        for await (const { cid } of blockstore.getAll()) {
+          try {
+            if (await helia.pins.isPinned(cid, options)) {
+              continue
+            }
+
+            yield cid
+
+            options.onProgress?.(new CustomProgressEvent<CID>('helia:gc:deleted', cid))
+          } catch (err) {
+            helia.log.error('Error during gc', err)
+            options.onProgress?.(new CustomProgressEvent<Error>('helia:gc:error', err))
+          }
+        }
+      }())))
+    } finally {
+      releaseLock()
+    }
+
+    this.log('gc finished')
+  }
+}

package/src/pins.ts

@@ -0,0 +1,227 @@
+import { Queue } from '@libp2p/utils/queue'
+import * as cborg from 'cborg'
+import { type Datastore, Key } from 'interface-datastore'
+import { base36 } from 'multiformats/bases/base36'
+import { CID, type Version } from 'multiformats/cid'
+import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import type { DAGWalker } from '@helia/interface'
+import type { GetBlockProgressEvents } from '@helia/interface/blocks'
+import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+
+interface DatastorePin {
+  /**
+   * 0 for a direct pin or an arbitrary (+ve, whole) number or Infinity
+   */
+  depth: number
+
+  /**
+   * User-specific metadata for the pin
+   */
+  metadata: Record<string, string | number | boolean>
+}
+
+interface DatastorePinnedBlock {
+  pinCount: number
+  pinnedBy: Uint8Array[]
+}
+
+/**
+ * Callback for updating a {@link DatastorePinnedBlock}'s properties when
+ * calling `#updatePinnedBlock`
+ *
+ * The callback should return `false` to prevent any pinning modifications to
+ * the block, and true in all other cases.
+ */
+interface WithPinnedBlockCallback {
+  (pinnedBlock: DatastorePinnedBlock): boolean
+}
+
+const DATASTORE_PIN_PREFIX = '/pin/'
+const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
+const DATASTORE_ENCODING = base36
+const DAG_WALK_QUEUE_CONCURRENCY = 1
+
+interface WalkDagOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | AddPinEvents> {
+  depth: number
+}
+
+function toDSKey (cid: CID): Key {
+  if (cid.version === 0) {
+    cid = cid.toV1()
+  }
+
+  return new Key(`${DATASTORE_PIN_PREFIX}${cid.toString(DATASTORE_ENCODING)}`)
+}
+
+export class PinsImpl implements Pins {
+  private readonly datastore: Datastore
+  private readonly blockstore: Blockstore
+  private readonly dagWalkers: Record<number, DAGWalker>
+
+  constructor (datastore: Datastore, blockstore: Blockstore, dagWalkers: Record<number, DAGWalker>) {
+    this.datastore = datastore
+    this.blockstore = blockstore
+    this.dagWalkers = dagWalkers
+  }
+
+  async * add (cid: CID<unknown, number, number, Version>, options: AddOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const pinKey = toDSKey(cid)
+
+    if (await this.datastore.has(pinKey)) {
+      throw new Error('Already pinned')
+    }
+
+    const depth = Math.round(options.depth ?? Infinity)
+
+    if (depth < 0) {
+      throw new Error('Depth must be greater than or equal to 0')
+    }
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
+    const queue = new Queue<AsyncGenerator<CID>>({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+
+    for await (const childCid of this.#walkDag(cid, queue, {
+      ...options,
+      depth
+    })) {
+      await this.#updatePinnedBlock(childCid, (pinnedBlock: DatastorePinnedBlock) => {
+        // do not update pinned block if this block is already pinned by this CID
+        if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
+          return false
+        }
+
+        pinnedBlock.pinCount++
+        pinnedBlock.pinnedBy.push(cid.bytes)
+        return true
+      }, options)
+
+      yield childCid
+    }
+
+    const pin: DatastorePin = {
+      depth,
+      metadata: options.metadata ?? {}
+    }
+
+    await this.datastore.put(pinKey, cborg.encode(pin), options)
+  }
+
+  /**
+   * Walk a DAG in an iterable fashion
+   */
+  async * #walkDag (cid: CID, queue: Queue<AsyncGenerator<CID>>, options: WalkDagOptions): AsyncGenerator<CID> {
+    if (options.depth === -1) {
+      return
+    }
+
+    const dagWalker = this.dagWalkers[cid.code]
+
+    if (dagWalker == null) {
+      throw new Error(`No dag walker found for cid codec ${cid.code}`)
+    }
+
+    const block = await this.blockstore.get(cid, options)
+
+    yield cid
+
+    // walk dag, ensure all blocks are present
+    for await (const cid of dagWalker.walk(block)) {
+      yield * await queue.add(async () => {
+        return this.#walkDag(cid, queue, {
+          ...options,
+          depth: options.depth - 1
+        })
+      })
+    }
+  }
+
+  /**
+   * Update the pin count for the CID
+   */
+  async #updatePinnedBlock (cid: CID, withPinnedBlock: WithPinnedBlockCallback, options: AddOptions): Promise<void> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    let pinnedBlock: DatastorePinnedBlock = {
+      pinCount: 0,
+      pinnedBy: []
+    }
+
+    try {
+      pinnedBlock = cborg.decode(await this.datastore.get(blockKey, options))
+    } catch (err: any) {
+      if (err.code !== 'ERR_NOT_FOUND') {
+        throw err
+      }
+    }
+
+    const shouldContinue = withPinnedBlock(pinnedBlock)
+
+    if (!shouldContinue) {
+      return
+    }
+
+    if (pinnedBlock.pinCount === 0) {
+      if (await this.datastore.has(blockKey)) {
+        await this.datastore.delete(blockKey)
+        return
+      }
+    }
+
+    await this.datastore.put(blockKey, cborg.encode(pinnedBlock), options)
+    options.onProgress?.(new CustomProgressEvent<CID>('helia:pin:add', cid))
+  }
+
+  async * rm (cid: CID<unknown, number, number, Version>, options: RmOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const pinKey = toDSKey(cid)
+    const buf = await this.datastore.get(pinKey, options)
+    const pin = cborg.decode(buf)
+
+    await this.datastore.delete(pinKey, options)
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
+    const queue = new Queue<AsyncGenerator<CID>>({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+
+    for await (const childCid of this.#walkDag(cid, queue, {
+      ...options,
+      depth: pin.depth
+    })) {
+      await this.#updatePinnedBlock(childCid, (pinnedBlock): boolean => {
+        pinnedBlock.pinCount--
+        pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
+        return true
+      }, {
+        ...options,
+        depth: pin.depth
+      })
+
+      yield childCid
+    }
+  }
+
+  async * ls (options: LsOptions = {}): AsyncGenerator<Pin, void, undefined> {
+    for await (const { key, value } of this.datastore.query({
+      prefix: DATASTORE_PIN_PREFIX + (options.cid != null ? `${options.cid.toString(base36)}` : '')
+    }, options)) {
+      const cid = CID.parse(key.toString().substring(5), base36)
+      const pin = cborg.decode(value)
+
+      yield {
+        cid,
+        ...pin
+      }
+    }
+  }
+
+  async isPinned (cid: CID, options: IsPinnedOptions = {}): Promise<boolean> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    return this.datastore.has(blockKey, options)
+  }
+}

package/src/routing.ts

@@ -0,0 +1,169 @@
+import { CodeError, start, stop } from '@libp2p/interface'
+import { PeerSet } from '@libp2p/peer-collections'
+import merge from 'it-merge'
+import type { Routing as RoutingInterface, Provider, RoutingOptions } from '@helia/interface'
+import type { AbortOptions, ComponentLogger, Logger, PeerId, PeerInfo, Startable } from '@libp2p/interface'
+import type { CID } from 'multiformats/cid'
+
+export interface RoutingInit {
+  routers: Array<Partial<RoutingInterface>>
+}
+
+export interface RoutingComponents {
+  logger: ComponentLogger
+}
+
+export class Routing implements RoutingInterface, Startable {
+  private readonly log: Logger
+  private readonly routers: Array<Partial<RoutingInterface>>
+
+  constructor (components: RoutingComponents, init: RoutingInit) {
+    this.log = components.logger.forComponent('helia:routing')
+    this.routers = init.routers ?? []
+  }
+
+  async start (): Promise<void> {
+    await start(...this.routers)
+  }
+
+  async stop (): Promise<void> {
+    await stop(...this.routers)
+  }
+
+  /**
+   * Iterates over all content routers in parallel to find providers of the given key
+   */
+  async * findProviders (key: CID, options: RoutingOptions = {}): AsyncIterable<Provider> {
+    if (this.routers.length === 0) {
+      throw new CodeError('No content routers available', 'ERR_NO_ROUTERS_AVAILABLE')
+    }
+
+    const seen = new PeerSet()
+
+    for await (const peer of merge(
+      ...supports(this.routers, 'findProviders')
+        .map(router => router.findProviders(key, options))
+    )) {
+      // the peer was yielded by a content router without multiaddrs and we
+      // failed to load them
+      if (peer == null) {
+        continue
+      }
+
+      // deduplicate peers
+      if (seen.has(peer.id)) {
+        continue
+      }
+
+      seen.add(peer.id)
+
+      yield peer
+    }
+  }
+
+  /**
+   * Iterates over all content routers in parallel to notify it is
+   * a provider of the given key
+   */
+  async provide (key: CID, options: AbortOptions = {}): Promise<void> {
+    if (this.routers.length === 0) {
+      throw new CodeError('No content routers available', 'ERR_NO_ROUTERS_AVAILABLE')
+    }
+
+    await Promise.all(
+      supports(this.routers, 'provide')
+        .map(async (router) => {
+          await router.provide(key, options)
+        })
+    )
+  }
+
+  /**
+   * Store the given key/value pair in the available content routings
+   */
+  async put (key: Uint8Array, value: Uint8Array, options?: AbortOptions): Promise<void> {
+    await Promise.all(
+      supports(this.routers, 'put')
+        .map(async (router) => {
+          await router.put(key, value, options)
+        })
+    )
+  }
+
+  /**
+   * Get the value to the given key.
+   * Times out after 1 minute by default.
+   */
+  async get (key: Uint8Array, options?: AbortOptions): Promise<Uint8Array> {
+    return Promise.any(
+      supports(this.routers, 'get')
+        .map(async (router) => {
+          return router.get(key, options)
+        })
+    )
+  }
+
+  /**
+   * Iterates over all peer routers in parallel to find the given peer
+   */
+  async findPeer (id: PeerId, options?: RoutingOptions): Promise<PeerInfo> {
+    if (this.routers.length === 0) {
+      throw new CodeError('No peer routers available', 'ERR_NO_ROUTERS_AVAILABLE')
+    }
+
+    const self = this
+    const source = merge(
+      ...supports(this.routers, 'findPeer')
+        .map(router => (async function * () {
+          try {
+            yield await router.findPeer(id, options)
+          } catch (err) {
+            self.log.error(err)
+          }
+        })())
+    )
+
+    for await (const peer of source) {
+      if (peer == null) {
+        continue
+      }
+
+      return peer
+    }
+
+    throw new CodeError('Could not find peer in routing', 'ERR_NOT_FOUND')
+  }
+
+  /**
+   * Attempt to find the closest peers on the network to the given key
+   */
+  async * getClosestPeers (key: Uint8Array, options: RoutingOptions = {}): AsyncIterable<PeerInfo> {
+    if (this.routers.length === 0) {
+      throw new CodeError('No peer routers available', 'ERR_NO_ROUTERS_AVAILABLE')
+    }
+
+    const seen = new PeerSet()
+
+    for await (const peer of merge(
+      ...supports(this.routers, 'getClosestPeers')
+        .map(router => router.getClosestPeers(key, options))
+    )) {
+      if (peer == null) {
+        continue
+      }
+
+      // deduplicate peers
+      if (seen.has(peer.id)) {
+        continue
+      }
+
+      seen.add(peer.id)
+
+      yield peer
+    }
+  }
+}
+
+function supports <Operation extends keyof Routing> (routers: any[], key: Operation): Array<Pick<Routing, Operation>> {
+  return routers.filter(router => router[key] != null)
+}