@helia/utils 2.5.2-73a28eda → 2.5.2-9114743f

package/src/index.ts

@@ -6,8 +6,11 @@
  */
 
 import { keychain } from '@ipshipyard/keychain'
-import { contentRoutingSymbol, peerRoutingSymbol, start, stop, TypedEventEmitter } from '@libp2p/interface'
+import { start, stop, TypedEventEmitter } from '@libp2p/interface'
 import { dns } from '@multiformats/dns'
+import { defaultLogger } from 'birnam'
+import { MemoryBlockstore } from 'blockstore-core'
+import { MemoryDatastore } from 'datastore-core'
 import drain from 'it-drain'
 import { CustomProgressEvent } from 'progress-events'
 import { PinsImpl } from './pins.ts'
@@ -19,15 +22,14 @@ import { getCrypto } from './utils/get-crypto.ts'
 import { getHasher } from './utils/get-hasher.ts'
 import { NetworkedStorage } from './utils/networked-storage.ts'
 import type { BlockStorageInit } from './storage.ts'
-import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing, CryptoLoader, Crypto } from '@helia/interface'
+import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing, CryptoLoader, Crypto, NodeInfo, Router, HeliaMixin } from '@helia/interface'
 import type { BlockBroker } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
 import type { Keychain, KeychainInit } from '@ipshipyard/keychain'
-import type { ComponentLogger, ContentRouting, Libp2p, Logger, Metrics, PeerRouting } from '@libp2p/interface'
+import type { ComponentLogger, Logger, Metrics } from '@libp2p/interface'
 import type { DNS } from '@multiformats/dns'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
-import type { Libp2pOptions } from 'libp2p'
 import type { BlockCodec } from 'multiformats'
 import type { CID } from 'multiformats/cid'
 import type { MultihashHasher } from 'multiformats/hashes/interface'
@@ -35,6 +37,8 @@ import type { MultihashHasher } from 'multiformats/hashes/interface'
 export { AbstractSession } from './abstract-session.ts'
 export type { AbstractCreateSessionOptions, BlockstoreSessionEvents, AbstractSessionComponents } from './abstract-session.ts'
 
+export { isCID } from './utils/is-cid.ts'
+
 export type { BlockStorage, BlockStorageInit }
 
 export { breadthFirstWalker, depthFirstWalker, naturalOrderWalker } from './graph-walker.ts'
@@ -43,26 +47,7 @@ export type { GraphWalkerComponents, GraphWalkerInit, GraphNode, GraphWalker } f
 /**
  * Options used to create a Helia node.
  */
-export interface HeliaInit<T extends Libp2p = Libp2p> {
-  /**
-   * A libp2p node is required to perform network operations. Either a
-   * pre-configured node or options to configure a node can be passed
-   * here.
-   *
-   * If node options are passed, they will be merged with the default
-   * config for the current platform. In this case all passed config
-   * keys will replace those from the default config.
-   *
-   * The libp2p `start` option is not supported, instead please pass `start` in
-   * the root of the HeliaInit object.
-   */
-  libp2p: T | Omit<Libp2pOptions<any>, 'start'>
-
-  /**
-   * Pass `false` to not start the Helia node
-   */
-  start?: boolean
-
+export interface HeliaInit {
   /**
    * By default Helia stores the node's PeerId in an encrypted form in a
    * libp2p keystore. These options control how that keystore is configured.
@@ -72,12 +57,12 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
   /**
    * The blockstore is where blocks are stored
    */
-  blockstore: Blockstore
+  blockstore?: Blockstore
 
   /**
    * The datastore is where data is stored
    */
-  datastore: Datastore
+  datastore?: Datastore
 
   /**
    * By default sha256, sha512 and identity hashes are supported for
@@ -105,12 +90,6 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    */
   loadCodec?(code: number): BlockCodec<any, any> | Promise<BlockCodec<any, any>>
 
-  /**
-   * A list of strategies used to fetch blocks when they are not present in
-   * the local blockstore
-   */
-  blockBrokers: Array<(components: any) => BlockBroker>
-
   /**
    * A list of pre-supported public/private key implementations
    */
@@ -143,11 +122,17 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    */
   logger?: ComponentLogger
 
+  /**
+   * A list of strategies used to fetch blocks when they are not present in
+   * the local blockstore
+   */
+  blockBrokers?: Array<BlockBroker | ((components: any) => BlockBroker)>
+
   /**
    * Routers perform operations such as looking up content providers,
    * information about network peers or getting/putting records.
    */
-  routers?: Array<Partial<Routing> | ((components: any) => Partial<Routing>)>
+  routers?: Array<Router | ((components: any) => Router)>
 
   /**
    * During provider lookups, peers can be returned from routing implementations
@@ -192,7 +177,6 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
 }
 
 interface Components {
-  libp2p: Libp2p
   blockstore: Blockstore
   datastore: Datastore
   logger: ComponentLogger
@@ -206,39 +190,46 @@ interface Components {
   getCrypto: CryptoLoader
 }
 
-export class Helia<T extends Libp2p> implements HeliaInterface<T> {
-  public libp2p: T
+export class Helia implements HeliaInterface {
+  public info: NodeInfo
   public blockstore: BlockStorage
   public datastore: Datastore
-  public events: TypedEventEmitter<HeliaEvents<T>>
+  public events: TypedEventEmitter<HeliaEvents<this>>
   public pins: Pins
   public logger: ComponentLogger
-  public routing: Routing
+  public routing: RoutingClass
   public getCodec: CodecLoader
   public getHasher: HasherLoader
   public getCrypto: CryptoLoader
   public dns: DNS
   public keychain: Keychain
   public metrics?: Metrics
+  public status: 'stopped' | 'stopping' | 'starting' | 'started'
   private readonly log: Logger
+  private readonly blockBrokers: BlockBroker[]
+  private readonly mixins: HeliaMixin[]
 
-  constructor (init: Omit<HeliaInit, 'start' | 'libp2p'> & { libp2p: T }) {
-    this.logger = init.logger ?? init.libp2p.logger
+  constructor (init: HeliaInit & { name: string, version: string }) {
+    this.info = {
+      name: init.name,
+      version: init.version
+    }
+    this.logger = init.logger ?? defaultLogger()
     this.log = this.logger.forComponent('helia')
     this.getHasher = getHasher(init.hashers, init.loadHasher)
     this.getCodec = getCodec(init.codecs, init.loadCodec)
     this.getCrypto = getCrypto(init.cryptos, init.loadCrypto)
     this.dns = init.dns ?? dns()
     this.metrics = init.metrics
-    this.libp2p = init.libp2p
-    this.events = new TypedEventEmitter<HeliaEvents<T>>()
+    this.events = new TypedEventEmitter<HeliaEvents<typeof this>>()
+    this.status = 'stopped'
+    this.mixins = []
 
     // @ts-expect-error routing and keychain are not set
     const components: Components = {
-      blockstore: init.blockstore,
-      datastore: init.datastore,
+      blockstore: init.blockstore ?? new MemoryBlockstore(),
+      datastore: init.datastore ?? new MemoryDatastore(),
       logger: this.logger,
-      libp2p: this.libp2p,
       blockBrokers: [],
       getHasher: this.getHasher,
       getCodec: this.getCodec,
@@ -251,7 +242,7 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
     this.keychain = components.keychain = keychain()(components)
 
     this.routing = components.routing = new RoutingClass(components, {
-      routers: (init.routers ?? []).flatMap((router: Partial<Routing> | ((components: any) => Partial<Routing>)) => {
+      routers: (init.routers ?? []).flatMap((router: Router | ((components: any) => Router)) => {
         if (typeof router === 'function') {
           router = router(components)
         }
@@ -261,54 +252,86 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
           router
         ]
 
-        // if the router provides a libp2p-style ContentRouter
-        const contentRouting = asContentRouting(router)
-        if (contentRouting != null) {
-          routers.push(contentRouting)
-        }
-
-        // if the router provides a libp2p-style PeerRouter
-        const peerRouting = asPeerRouting(router)
-        if (peerRouting != null) {
-          routers.push(peerRouting)
-        }
-
         return routers
       }),
       providerLookupConcurrency: init.providerLookupConcurrency
     })
 
-    components.blockBrokers = init.blockBrokers.map((fn) => {
-      return fn(components)
+    this.blockBrokers = components.blockBrokers = (init.blockBrokers ?? []).map((broker) => {
+      if (typeof broker === 'function') {
+        broker = broker(components)
+      }
+
+      return broker
     })
 
     const networkedStorage = new NetworkedStorage(components, init)
-    this.pins = new PinsImpl(init.datastore, networkedStorage, this.getCodec)
+    this.pins = new PinsImpl(components.datastore, networkedStorage, this.getCodec)
     this.blockstore = new BlockStorage(networkedStorage, this.pins, this.routing, {
       holdGcLock: init.holdGcLock ?? true
     })
-    this.datastore = init.datastore
+    this.datastore = components.datastore
+  }
+
+  hasRouter (name: string): boolean {
+    return this.routing.hasRouter(name)
+  }
+
+  addRouter (router: Router): void {
+    this.routing.addRouter(router)
+  }
+
+  hasBlockBroker (name: string): boolean {
+    return this.blockBrokers.findIndex(b => b.name === name) !== -1
   }
 
-  async start (): Promise<void> {
+  addBlockBroker (blockBroker: BlockBroker): void {
+    this.blockBrokers.push(blockBroker)
+  }
+
+  addMixin (mixin: HeliaMixin): void {
+    this.mixins.push(mixin)
+  }
+
+  async start (): Promise<this> {
+    this.status = 'starting'
+
     await assertDatastoreVersionIsCurrent(this.datastore)
     await start(
       this.blockstore,
       this.datastore,
       this.routing,
-      this.libp2p
+      ...this.blockBrokers
     )
+
+    for (const mixin of this.mixins) {
+      await mixin.start?.(this)
+    }
+
+    this.status = 'started'
     this.events.dispatchEvent(new CustomEvent('start', { detail: this }))
+
+    return this
   }
 
-  async stop (): Promise<void> {
+  async stop (): Promise<this> {
+    this.status = 'stopping'
+
+    for (const mixin of this.mixins) {
+      await mixin.stop?.(this)
+    }
+
     await stop(
       this.blockstore,
       this.datastore,
       this.routing,
-      this.libp2p
+      ...this.blockBrokers
     )
+
+    this.status = 'stopped'
     this.events.dispatchEvent(new CustomEvent('stop', { detail: this }))
+
+    return this
   }
 
   async gc (options: GCOptions = {}): Promise<void> {
@@ -343,11 +366,3 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
     this.log('gc finished')
   }
 }
-
-function asContentRouting (obj?: any): ContentRouting | undefined {
-  return obj?.[contentRoutingSymbol]
-}
-
-function asPeerRouting (obj?: any): PeerRouting | undefined {
-  return obj?.[peerRoutingSymbol]
-}

package/src/pins.ts

@@ -1,22 +1,18 @@
 import { InvalidParametersError } from '@libp2p/interface'
-import { Queue } from '@libp2p/utils'
 import * as cborg from 'cborg'
 import { Key } from 'interface-datastore'
-import toBuffer from 'it-to-buffer'
 import { base36 } from 'multiformats/bases/base36'
-import { createUnsafe } from 'multiformats/block'
 import { CID } from 'multiformats/cid'
 import { CustomProgressEvent } from 'progress-events'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
 import { AlreadyPinnedError } from './errors.ts'
+import { depthFirstWalker } from './graph-walker.ts'
 import type { CodecLoader } from '@helia/interface'
-import type { GetBlockProgressEvents } from '@helia/interface/blocks'
-import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
+import type { AddOptions, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
 import type { AbortOptions } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
 import type { Version } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
 
 interface DatastorePin {
   /**
@@ -49,11 +45,6 @@ interface WithPinnedBlockCallback {
 const DATASTORE_PIN_PREFIX = '/pin/'
 const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
 const DATASTORE_ENCODING = base36
-const DAG_WALK_QUEUE_CONCURRENCY = 1
-
-interface WalkDagOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | AddPinEvents> {
-  depth: number
-}
 
 function toDSKey (cid: CID): Key {
   if (cid.version === 0) {
@@ -87,15 +78,13 @@ export class PinsImpl implements Pins {
       throw new InvalidParametersError('Depth must be greater than or equal to 0')
     }
 
-    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
-    const queue = new Queue<AsyncGenerator<CID>>({
-      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    const walker = (options.walker ?? depthFirstWalker())({
+      blockstore: this.blockstore,
+      getCodec: this.getCodec
     })
 
-    for await (const childCid of this.#walkDag(cid, queue, {
-      ...options,
-      depth
-    })) {
+    for await (const node of walker.walk(cid, { ...options, depth })) {
+      const childCid = node.block.cid
       await this.#updatePinnedBlock(childCid, (pinnedBlock: DatastorePinnedBlock) => {
         // do not update pinned block if this block is already pinned by this CID
         if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
@@ -118,31 +107,6 @@ export class PinsImpl implements Pins {
     await this.datastore.put(pinKey, cborg.encode(pin), options)
   }
 
-  /**
-   * Walk a DAG in an iterable fashion
-   */
-  async * #walkDag (cid: CID, queue: Queue<AsyncGenerator<CID>>, options: WalkDagOptions): AsyncGenerator<CID> {
-    if (options.depth === -1) {
-      return
-    }
-
-    const codec = await this.getCodec(cid.code)
-    const bytes = await toBuffer(this.blockstore.get(cid, options))
-    const block = createUnsafe({ bytes, cid, codec })
-
-    yield cid
-
-    // walk dag, ensure all blocks are present
-    for (const [,cid] of block.links()) {
-      yield * await queue.add(async () => {
-        return this.#walkDag(cid, queue, {
-          ...options,
-          depth: options.depth - 1
-        })
-      })
-    }
-  }
-
   /**
    * Update the pin count for the CID
    */
@@ -186,15 +150,13 @@ export class PinsImpl implements Pins {
 
     await this.datastore.delete(pinKey, options)
 
-    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
-    const queue = new Queue<AsyncGenerator<CID>>({
-      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    const walker = (options.walker ?? depthFirstWalker())({
+      blockstore: this.blockstore,
+      getCodec: this.getCodec
     })
 
-    for await (const childCid of this.#walkDag(cid, queue, {
-      ...options,
-      depth: pin.depth
-    })) {
+    for await (const node of walker.walk(cid, { ...options, depth: pin.depth })) {
+      const childCid = node.block.cid
       await this.#updatePinnedBlock(childCid, (pinnedBlock): boolean => {
         pinnedBlock.pinCount--
         pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))

package/src/routing.ts

@@ -1,18 +1,19 @@
 import { NoRoutersAvailableError } from '@helia/interface'
 import { NotFoundError, start, stop } from '@libp2p/interface'
-import { PeerQueue } from '@libp2p/utils'
+import { Queue } from '@libp2p/utils'
 import merge from 'it-merge'
 import { CustomProgressEvent } from 'progress-events'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { GetFailedError } from './errors.ts'
-import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents } from '@helia/interface'
-import type { ComponentLogger, Logger, Metrics, PeerId, PeerInfo, Startable } from '@libp2p/interface'
+import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents, Router, Peer } from '@helia/interface'
+import type { ComponentLogger, Logger, Metrics, Startable } from '@libp2p/interface'
+import type { AbortOptions } from 'abort-error'
 import type { CID } from 'multiformats/cid'
 
 const DEFAULT_PROVIDER_LOOKUP_CONCURRENCY = 5
 
 export interface RoutingInit {
-  routers: Array<Partial<RoutingInterface>>
+  routers: Router[]
   providerLookupConcurrency?: number
 }
 
@@ -21,11 +22,15 @@ export interface RoutingComponents {
   metrics?: Metrics
 }
 
+interface PeerQueueOptions extends AbortOptions {
+  peer: CID
+}
+
 export class Routing implements RoutingInterface, Startable {
   public name: string
 
   private readonly log: Logger
-  private readonly routers: Array<Partial<RoutingInterface>>
+  private readonly routers: Router[]
   private readonly providerLookupConcurrency: number
 
   constructor (components: RoutingComponents, init: RoutingInit) {
@@ -65,6 +70,14 @@ export class Routing implements RoutingInterface, Startable {
     await stop(...this.routers)
   }
 
+  hasRouter (name: string): boolean {
+    return this.routers.findIndex(r => r.name === name) !== -1
+  }
+
+  addRouter (router: Router): void {
+    this.routers.push(router)
+  }
+
   /**
    * Iterates over all content routers in parallel to find providers of the
    * given key
@@ -78,7 +91,7 @@ export class Routing implements RoutingInterface, Startable {
     // back as an empty array - when this happens we have to do a FIND_PEER
     // query to get updated addresses, but we shouldn't block on this so use a
     // separate bounded queue to perform this lookup
-    const queue = new PeerQueue<Provider | null>({
+    const queue = new Queue<Provider | null, PeerQueueOptions>({
       concurrency: this.providerLookupConcurrency
     })
 
@@ -145,7 +158,7 @@ export class Routing implements RoutingInterface, Startable {
       // have to refresh peer info for this peer to get updated multiaddrs
       if (peer.multiaddrs.length === 0) {
         // already looking this peer up
-        if (queue.find(peer.id) != null) {
+        if (queue.queue.find(job => job.options.peer.equals(peer.id)) != null) {
           continue
         }
 
@@ -167,7 +180,7 @@ export class Routing implements RoutingInterface, Startable {
             return null
           }
         }, {
-          peerId: peer.id,
+          peer: peer.id,
           signal: options.signal
         })
           .catch(err => {
@@ -298,7 +311,7 @@ export class Routing implements RoutingInterface, Startable {
   /**
    * Iterates over all peer routers in parallel to find the given peer
    */
-  async findPeer (id: PeerId, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<PeerInfo> {
+  async findPeer (id: CID, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<Peer> {
     if (this.routers.length === 0) {
       throw new NoRoutersAvailableError('No peer routers available')
     }
@@ -339,7 +352,7 @@ export class Routing implements RoutingInterface, Startable {
   /**
    * Attempt to find the closest peers on the network to the given key
    */
-  async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<PeerInfo> {
+  async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<Peer> {
     if (this.routers.length === 0) {
       throw new NoRoutersAvailableError('No peer routers available')
     }

package/src/utils/is-cid.ts

@@ -0,0 +1,9 @@
+import { CID } from 'multiformats/cid'
+
+export function isCID (obj?: any): obj is CID {
+  if (obj == null) {
+    return false
+  }
+
+  return CID.asCID(obj) != null
+}

package/src/utils/networked-storage.ts

@@ -4,7 +4,8 @@ import { SessionStorage } from './session-storage.ts'
 import { Storage } from './storage.ts'
 import type { StorageComponents, StorageInit } from './storage.ts'
 import type { BlockBroker, Blocks, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
-import type { AbortOptions, Startable } from '@libp2p/interface'
+import type { Startable } from '@libp2p/interface'
+import type { AbortOptions } from 'abort-error'
 import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 

package/src/utils/session-storage.ts

@@ -3,7 +3,7 @@ import { anySignal } from 'any-signal'
 import { Storage } from './storage.ts'
 import type { StorageComponents } from './storage.ts'
 import type { Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore, SessionBlockBroker } from '@helia/interface/blocks'
-import type { AbortOptions, PeerId } from '@libp2p/interface'
+import type { AbortOptions } from '@libp2p/interface'
 import type { Multiaddr } from '@multiformats/multiaddr'
 import type { InputPair } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
@@ -35,7 +35,7 @@ export class SessionStorage extends Storage<SessionBlockBroker> implements Sessi
     this.closeController.abort()
   }
 
-  async addPeer (peer: PeerId | Multiaddr | Multiaddr[], options?: AbortOptions): Promise<void> {
+  async addPeer (peer: CID | Multiaddr | Multiaddr[], options?: AbortOptions): Promise<void> {
     await Promise.all(
       this.blockBrokers
         .map(broker => broker.addPeer(peer, options))