helia 6.1.4 → 7.0.0

package/src/get-crypto.ts

@@ -0,0 +1,44 @@
+import { UnknownCryptoError } from '@helia/interface'
+import { isPromise } from '@helia/utils'
+import { ecdsaCrypto, ed25519Crypto, rsaCrypto } from '@ipshipyard/crypto'
+import type { CryptoLoader } from '@helia/interface'
+import type { Crypto } from '@ipshipyard/crypto'
+
+export function getCrypto (initialCryptos: Array<Crypto> = [], loadCrypto?: CryptoLoader): CryptoLoader {
+  const cryptos: Record<string | number, Crypto> = {}
+
+  initialCryptos = [
+    ecdsaCrypto(),
+    ed25519Crypto(),
+    rsaCrypto(),
+    ...initialCryptos
+  ]
+
+  initialCryptos.forEach(crypto => {
+    cryptos[crypto.type] = crypto
+    cryptos[crypto.code] = crypto
+  })
+
+  return async (nameOrCode) => {
+    let crypto = cryptos[nameOrCode]
+
+    if (crypto == null && loadCrypto != null) {
+      const res = loadCrypto(nameOrCode)
+
+      if (isPromise(res)) {
+        crypto = await res
+      } else {
+        crypto = res
+      }
+
+      cryptos[crypto.type] = crypto
+      cryptos[crypto.code] = crypto
+    }
+
+    if (crypto != null) {
+      return crypto
+    }
+
+    throw new UnknownCryptoError(`Could not load crypto for ${crypto}`)
+  }
+}

package/src/get-hasher.ts

@@ -0,0 +1,39 @@
+import { UnknownHashAlgorithmError } from '@helia/interface'
+import { isPromise } from '@helia/utils'
+import { identity } from 'multiformats/hashes/identity'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { HasherLoader } from '@helia/interface'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export function getHasher (initialHashers: MultihashHasher[] = [], loadHasher?: HasherLoader): HasherLoader {
+  const hashers: Record<number, MultihashHasher> = {
+    [sha256.code]: sha256,
+    [identity.code]: identity
+  }
+
+  initialHashers.forEach(hasher => {
+    hashers[hasher.code] = hasher
+  })
+
+  return async (code) => {
+    let hasher = hashers[code]
+
+    if (hasher == null && loadHasher != null) {
+      const res = loadHasher(code)
+
+      if (isPromise(res)) {
+        hasher = await res
+      } else {
+        hasher = res
+      }
+
+      hashers[hasher.code] = hasher
+    }
+
+    if (hasher != null) {
+      return hasher
+    }
+
+    throw new UnknownHashAlgorithmError(`No hasher configured for multihash code 0x${code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`)
+  }
+}

package/src/helia.ts

@@ -0,0 +1,364 @@
+/**
+ * @packageDocumentation
+ *
+ * This module contains utility code that is shared between various Helia
+ * modules such as `helia`, `@helia/http`, etc.
+ */
+
+import { InvalidConfigurationError } from '@helia/utils'
+import { keychain } from '@ipshipyard/keychain'
+import { start, stop } from '@libp2p/interface'
+import { dns } from '@multiformats/dns'
+import { defaultLogger } from 'birnam'
+import { MemoryBlockstore } from 'blockstore-core'
+import { MemoryDatastore } from 'datastore-core'
+import drain from 'it-drain'
+import { TypedEventEmitter } from 'main-event'
+import { CustomProgressEvent } from 'progress-events'
+import { BlockStorage } from './block-storage.ts'
+import { assertDatastoreVersionIsCurrent } from './datastore-version.ts'
+import { getCodec } from './get-codec.ts'
+import { getCrypto } from './get-crypto.ts'
+import { getHasher } from './get-hasher.ts'
+import { NetworkedStorage } from './networked-storage.ts'
+import { PinsImpl } from './pins.ts'
+import { Routing as RoutingClass } from './routing.ts'
+import type { BlockStorageInit } from './block-storage.ts'
+import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing, CryptoLoader, Crypto, NodeInfo, Router, HeliaMixin } from '@helia/interface'
+import type { BlockBroker } from '@helia/interface'
+import type { Pins } from '@helia/interface'
+import type { Keychain, KeychainInit } from '@ipshipyard/keychain'
+import type { ComponentLogger, Logger, Metrics } from '@libp2p/interface'
+import type { DNS } from '@multiformats/dns'
+import type { Blockstore } from 'interface-blockstore'
+import type { Datastore } from 'interface-datastore'
+import type { BlockCodec } from 'multiformats'
+import type { CID } from 'multiformats/cid'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export type { BlockStorage, BlockStorageInit }
+
+/**
+ * Options used to create a Helia node.
+ */
+export interface HeliaInit {
+  /**
+   * By default Helia stores the node's PeerId in an encrypted form in a
+   * libp2p keystore. These options control how that keystore is configured.
+   */
+  keychain?: KeychainInit
+
+  /**
+   * The blockstore is where blocks are stored
+   */
+  blockstore?: Blockstore
+
+  /**
+   * The datastore is where data is stored
+   */
+  datastore?: Datastore
+
+  /**
+   * By default sha256, sha512 and identity hashes are supported for
+   * bitswap operations. To bitswap blocks with CIDs using other hashes
+   * pass appropriate MultihashHashers here.
+   */
+  hashers?: MultihashHasher[]
+
+  /**
+   * An optional function that can load a MultihashHasher on demand. May return
+   * a promise.
+   */
+  loadHasher?(code: number): MultihashHasher | Promise<MultihashHasher>
+
+  /**
+   * A BlockCodec allows converting a block to an object representation
+   */
+  codecs?: Array<BlockCodec<any, any>>
+
+  /**
+   * An optional function that can load a BlockCodec on demand. May return a
+   * promise.
+   */
+  loadCodec?(code: number): BlockCodec<any, any> | Promise<BlockCodec<any, any>>
+
+  /**
+   * A list of pre-supported public/private key implementations
+   */
+  cryptos?: Array<Crypto>
+
+  /**
+   * Dynamically load a cryptography implementation
+   */
+  loadCrypto?: CryptoLoader
+
+  /**
+   * Garbage collection requires preventing blockstore writes during searches
+   * for unpinned blocks as DAGs are typically pinned after they've been
+   * imported - without locking this could lead to the deletion of blocks while
+   * they are being added to the blockstore.
+   *
+   * By default this lock is held on the current process and other processes
+   * will contact this process for access.
+   *
+   * If Helia is being run in multiple processes, one process must hold the GC
+   * lock so use this option to control which process that is.
+   *
+   * @default true
+   */
+  holdGcLock?: boolean
+
+  /**
+   * An optional logging component to pass to libp2p. If not specified the
+   * default implementation from libp2p will be used.
+   */
+  logger?: ComponentLogger
+
+  /**
+   * A list of strategies used to fetch blocks when they are not present in
+   * the local blockstore
+   */
+  blockBrokers?: Array<BlockBroker | ((components: any) => BlockBroker)>
+
+  /**
+   * Routers perform operations such as looking up content providers,
+   * information about network peers or getting/putting records.
+   */
+  routers?: Array<Router | ((components: any) => Router)>
+
+  /**
+   * During provider lookups, peers can be returned from routing implementations
+   * with no multiaddrs.
+   *
+   * This can happen when they've been retrieved from network peers that only
+   * store multiaddrs for a limited amount of time.
+   *
+   * When this happens the peer's info has to be looked up with a further query.
+   *
+   * To not have this query block the yielding of other providers returned with
+   * multiaddrs, a separate queue is used to perform this lookup.
+   *
+   * This config value controls the concurrency of that queue.
+   *
+   * @default 5
+   */
+  providerLookupConcurrency?: number
+
+  /**
+   * Components used by subclasses
+   */
+  components?: Record<string, any>
+
+  /**
+   * An optional DNS implementation used to perform queries for DNS records.
+   */
+  dns?: DNS
+
+  /**
+   * A metrics object that can be used to collected arbitrary stats about node
+   * usage.
+   */
+  metrics?: Metrics
+
+  /**
+   * Limit the maximum supported size of identity hash digests to this value
+   *
+   * @default 128
+   */
+  maxIdentityHashDigestLength?: number
+}
+
+interface Components {
+  blockstore: Blockstore
+  datastore: Datastore
+  logger: ComponentLogger
+  blockBrokers: BlockBroker[]
+  routing: Routing
+  dns: DNS
+  keychain: Keychain
+  metrics?: Metrics
+  getCodec: CodecLoader
+  getHasher: HasherLoader
+  getCrypto: CryptoLoader
+}
+
+export class Helia implements HeliaInterface {
+  public info: NodeInfo
+  public blockstore: BlockStorage
+  public datastore: Datastore
+  public events: TypedEventEmitter<HeliaEvents<this>>
+  public pins: Pins
+  public logger: ComponentLogger
+  public routing: RoutingClass
+  public getCodec: CodecLoader
+  public getHasher: HasherLoader
+  public getCrypto: CryptoLoader
+  public dns: DNS
+  public keychain: Keychain
+  public metrics?: Metrics
+  public status: 'stopped' | 'stopping' | 'starting' | 'started'
+  private readonly log: Logger
+  private readonly blockBrokers: BlockBroker[]
+  private readonly mixins: HeliaMixin[]
+
+  constructor (init: HeliaInit & { name: string, version: string }) {
+    this.info = {
+      name: init.name,
+      version: init.version
+    }
+    this.logger = init.logger ?? defaultLogger()
+    this.log = this.logger.forComponent('helia')
+    this.getHasher = getHasher(init.hashers, init.loadHasher)
+    this.getCodec = getCodec(init.codecs, init.loadCodec)
+    this.getCrypto = getCrypto(init.cryptos, init.loadCrypto)
+    this.dns = init.dns ?? dns()
+    this.metrics = init.metrics
+    this.events = new TypedEventEmitter<HeliaEvents<typeof this>>()
+    this.status = 'stopped'
+    this.mixins = []
+
+    // @ts-expect-error routing and keychain are not set
+    const components: Components = {
+      blockstore: init.blockstore ?? new MemoryBlockstore(),
+      datastore: init.datastore ?? new MemoryDatastore(),
+      logger: this.logger,
+      blockBrokers: [],
+      getHasher: this.getHasher,
+      getCodec: this.getCodec,
+      getCrypto: this.getCrypto,
+      dns: this.dns,
+      metrics: this.metrics,
+      ...(init.components ?? {})
+    }
+
+    this.keychain = components.keychain = keychain()(components)
+
+    this.routing = components.routing = new RoutingClass(components, {
+      routers: (init.routers ?? []).flatMap((router: Router | ((components: any) => Router)) => {
+        if (typeof router === 'function') {
+          router = router(components)
+        }
+
+        // if the router itself is a router
+        const routers = [
+          router
+        ]
+
+        return routers
+      }),
+      providerLookupConcurrency: init.providerLookupConcurrency
+    })
+
+    this.blockBrokers = components.blockBrokers = (init.blockBrokers ?? []).map((broker) => {
+      if (typeof broker === 'function') {
+        broker = broker(components)
+      }
+
+      return broker
+    })
+
+    const networkedStorage = new NetworkedStorage(components, init)
+    this.pins = new PinsImpl(components.datastore, networkedStorage, this.getCodec)
+    this.blockstore = new BlockStorage(networkedStorage, this.pins, this.routing, {
+      holdGcLock: init.holdGcLock ?? true
+    })
+    this.datastore = components.datastore
+  }
+
+  hasRouter (name: string): boolean {
+    return this.routing.hasRouter(name)
+  }
+
+  addRouter (router: Router): void {
+    this.routing.addRouter(router)
+  }
+
+  hasBlockBroker (name: string): boolean {
+    return this.blockBrokers.findIndex(b => b.name === name) !== -1
+  }
+
+  addBlockBroker (blockBroker: BlockBroker): void {
+    this.blockBrokers.push(blockBroker)
+  }
+
+  addMixin (mixin: HeliaMixin): void {
+    if (this.mixins.find(m => m.name === mixin.name) != null) {
+      throw new InvalidConfigurationError(`A mixin with the name "${mixin.name} is already present`)
+    }
+
+    this.mixins.push(mixin)
+  }
+
+  async start (): Promise<this> {
+    this.status = 'starting'
+
+    await assertDatastoreVersionIsCurrent(this.datastore)
+    await start(
+      this.blockstore,
+      this.datastore,
+      this.routing,
+      ...this.blockBrokers
+    )
+
+    for (const mixin of this.mixins) {
+      await mixin.start?.(this)
+    }
+
+    this.status = 'started'
+    this.events.dispatchEvent(new CustomEvent('start', { detail: this }))
+
+    return this
+  }
+
+  async stop (): Promise<this> {
+    this.status = 'stopping'
+
+    for (const mixin of this.mixins) {
+      await mixin.stop?.(this)
+    }
+
+    await stop(
+      this.blockstore,
+      this.datastore,
+      this.routing,
+      ...this.blockBrokers
+    )
+
+    this.status = 'stopped'
+    this.events.dispatchEvent(new CustomEvent('stop', { detail: this }))
+
+    return this
+  }
+
+  async gc (options: GCOptions = {}): Promise<void> {
+    const releaseLock = await this.blockstore.lock.writeLock()
+
+    try {
+      const helia = this
+      const blockstore = this.blockstore.unwrap()
+
+      this.log('gc start')
+
+      await drain(blockstore.deleteMany((async function * (): AsyncGenerator<CID> {
+        for await (const { cid } of blockstore.getAll()) {
+          try {
+            if (await helia.pins.isPinned(cid, options)) {
+              continue
+            }
+
+            yield cid
+
+            options.onProgress?.(new CustomProgressEvent<CID>('helia:gc:deleted', cid))
+          } catch (err: any) {
+            helia.log.error('error during gc - %e', err)
+            options.onProgress?.(new CustomProgressEvent<Error>('helia:gc:error', err))
+          }
+        }
+      }())))
+    } finally {
+      releaseLock()
+    }
+
+    this.log('gc finished')
+  }
+}

package/src/index.ts

@@ -19,15 +19,17 @@
  * ```
  */
 
-import { Helia as HeliaClass } from '@helia/utils'
-import { heliaDefaults } from './utils/helia-defaults.ts'
-import { libp2pDefaults } from './utils/libp2p-defaults.ts'
-import type { DefaultLibp2pServices } from './utils/libp2p-defaults.ts'
-import type { Libp2pDefaultsOptions } from './utils/libp2p.ts'
+import { withBitswap } from '@helia/bitswap'
+import { withHTTP } from '@helia/http'
+import { withLibp2p } from '@helia/libp2p'
+import * as dagCbor from '@ipld/dag-cbor'
+import * as dagJson from '@ipld/dag-json'
+import * as json from 'multiformats/codecs/json'
+import { sha512 } from 'multiformats/hashes/sha2'
+import { Helia as HeliaClass } from './helia.ts'
+import { name, version } from './version.ts'
+import type { HeliaInit } from './helia.ts'
 import type { Helia } from '@helia/interface'
-import type { HeliaInit } from '@helia/utils'
-import type { Libp2p } from '@libp2p/interface'
-import type { CID } from 'multiformats/cid'
 
 // re-export interface types so people don't have to depend on @helia/interface
 // if they don't want to
@@ -35,20 +37,6 @@ export * from '@helia/interface'
 
 export type { HeliaInit }
 
-export type { DefaultLibp2pServices, Libp2pDefaultsOptions }
-
-// allow amending the default config
-export { libp2pDefaults }
-export { heliaDefaults }
-
-/**
- * DAGWalkers take a block and yield CIDs encoded in that block
- */
-export interface DAGWalker {
-  codec: number
-  walk(block: Uint8Array): Generator<CID, void, undefined>
-}
-
 /**
  * Create and return a Helia node
  *
@@ -70,15 +58,55 @@ export interface DAGWalker {
  * }
  * ```
  */
-export async function createHelia <T extends Libp2p> (init: Partial<HeliaInit<T>>): Promise<Helia<T>>
-export async function createHelia (init?: Partial<HeliaInit<Libp2p<DefaultLibp2pServices>>>): Promise<Helia<Libp2p<DefaultLibp2pServices>>>
-export async function createHelia (init: Partial<HeliaInit> = {}): Promise<Helia> {
-  const options = await heliaDefaults(init)
-  const helia = new HeliaClass(options)
+export function createHelia (init: HeliaInit = {}): Helia {
+  return withBitswap(withLibp2p(withHTTP(createHeliaLight({
+    ...init,
+    codecs: [
+      dagCbor,
+      dagJson,
+      json,
+      ...(init.codecs ?? [])
+    ],
+    hashers: [
+      sha512,
+      ...(init.hashers ?? [])
+    ]
+  }))))
+}
 
-  if (options.start !== false) {
-    await helia.start()
-  }
+/**
+ * Create and return a Helia node without and routing or block broker config.
+ *
+ * The only supported codecs are `dag-pb` and `raw`, and the only supported
+ * hashes are `sha2-256` and `identity`.
+ *
+ * This allows more flexible customization and the smallest possible bundle size
+ * in web browsers.
+ *
+ * @example Creating a Helia node
+ *
+ * ```ts
+ * import { createHeliaLight } from 'helia'
+ * import { unixfs } from '@helia/unixfs'
+ * import { CID } from 'multiformats/cid'
+ *
+ * const helia = await createHeliaLight()
+ * const fs = unixfs(helia)
+ * const cid = CID.parse('QmFoo...')
+ *
+ * for await (const buf of fs.cat(cid, {
+ *   signal: AbortSignal.timeout(5_000)
+ * })) {
+ *   console.info(buf)
+ * }
+ * ```
+ */
+export function createHeliaLight (init: HeliaInit = {}): Helia {
+  const helia = new HeliaClass({
+    name,
+    version,
+    ...init
+  })
 
   return helia
 }

package/src/networked-storage.ts

@@ -0,0 +1,74 @@
+import { InvalidConfigurationError } from '@helia/utils'
+import { start, stop } from '@libp2p/interface'
+import { SessionStorage } from './session-storage.ts'
+import { Storage } from './storage.ts'
+import type { StorageComponents, StorageInit } from './storage.ts'
+import type { BlockBroker, Blocks, CreateSessionOptions, SessionBlockstore } from '@helia/interface'
+import type { Startable } from '@libp2p/interface'
+import type { AbortOptions } from 'abort-error'
+import type { Blockstore } from 'interface-blockstore'
+import type { CID } from 'multiformats/cid'
+
+export interface GetOptions extends AbortOptions {
+  progress?(evt: Event): void
+}
+
+export type NetworkedStorageComponents = StorageComponents<BlockBroker>
+
+/**
+ * Networked storage wraps a regular blockstore - when getting blocks if the
+ * blocks are not present, the configured BlockBrokers will be used to fetch them.
+ */
+export class NetworkedStorage extends Storage<BlockBroker> implements Blocks, Startable {
+  private started: boolean
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (components: NetworkedStorageComponents, init: StorageInit = {}) {
+    super(components, init)
+
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child, ...this.blockBrokers)
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child, ...this.blockBrokers)
+    this.started = false
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  createSession (root: CID, options?: CreateSessionOptions): SessionBlockstore {
+    if (this.blockBrokers.length === 0) {
+      throw new InvalidConfigurationError('No block brokers configured')
+    }
+
+    const blockBrokers = this.blockBrokers
+      .map(broker => broker.createSession?.(options))
+      .filter(broker => broker != null)
+
+    if (blockBrokers.length === 0) {
+      throw new InvalidConfigurationError(`No configured block brokers support sessions - tried ${this.blockBrokers.map(b => b.name).join(', ')}`)
+    }
+
+    return new SessionStorage({
+      blockstore: this.child,
+      blockBrokers,
+      getHasher: this.getHasher,
+      logger: this.logger
+    }, {
+      root
+    })
+  }
+}