helia 0.0.0-031ca73

package/src/helia.ts

@@ -0,0 +1,117 @@
+import type { GCOptions, Helia } from '@helia/interface'
+import type { Libp2p } from '@libp2p/interface-libp2p'
+import type { Datastore } from 'interface-datastore'
+import { identity } from 'multiformats/hashes/identity'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+import type { HeliaInit } from '.'
+import { Bitswap, createBitswap } from 'ipfs-bitswap'
+import { BlockStorage } from './storage.js'
+import type { Pins } from '@helia/interface/pins'
+import { PinsImpl } from './pins.js'
+import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.js'
+import drain from 'it-drain'
+import { CustomProgressEvent } from 'progress-events'
+import { MemoryDatastore } from 'datastore-core'
+import { MemoryBlockstore } from 'blockstore-core'
+
+export class HeliaImpl implements Helia {
+  public libp2p: Libp2p
+  public blockstore: BlockStorage
+  public datastore: Datastore
+  public pins: Pins
+
+  #bitswap?: Bitswap
+
+  constructor (init: HeliaInit) {
+    const hashers: MultihashHasher[] = [
+      sha256,
+      sha512,
+      identity,
+      ...(init.hashers ?? [])
+    ]
+
+    const datastore = init.datastore ?? new MemoryDatastore()
+    const blockstore = init.blockstore ?? new MemoryBlockstore()
+
+    // @ts-expect-error incomplete libp2p implementation
+    const libp2p = init.libp2p ?? new Proxy<Libp2p>({}, {
+      get (_, prop) {
+        const noop = (): void => {}
+        const noops = ['start', 'stop']
+
+        if (noops.includes(prop.toString())) {
+          return noop
+        }
+
+        if (prop === 'isProxy') {
+          return true
+        }
+
+        throw new Error('Please configure Helia with a libp2p instance')
+      },
+      set () {
+        throw new Error('Please configure Helia with a libp2p instance')
+      }
+    })
+
+    this.pins = new PinsImpl(datastore, blockstore, init.dagWalkers ?? [])
+
+    if (init.libp2p != null) {
+      this.#bitswap = createBitswap(libp2p, blockstore, {
+        hashLoader: {
+          getHasher: async (codecOrName: string | number) => {
+            const hasher = hashers.find(hasher => {
+              return hasher.code === codecOrName || hasher.name === codecOrName
+            })
+
+            if (hasher != null) {
+              return await Promise.resolve(hasher)
+            }
+
+            throw new Error(`Could not load hasher for code/name "${codecOrName}"`)
+          }
+        }
+      })
+    }
+
+    this.libp2p = libp2p
+    this.blockstore = new BlockStorage(blockstore, this.pins, this.#bitswap)
+    this.datastore = datastore
+  }
+
+  async start (): Promise<void> {
+    await assertDatastoreVersionIsCurrent(this.datastore)
+
+    this.#bitswap?.start()
+    await this.libp2p.start()
+  }
+
+  async stop (): Promise<void> {
+    this.#bitswap?.stop()
+    await this.libp2p.stop()
+  }
+
+  async gc (options: GCOptions = {}): Promise<void> {
+    const releaseLock = await this.blockstore.lock.writeLock()
+
+    try {
+      const helia = this
+      const blockstore = this.blockstore.unwrap()
+
+      await drain(blockstore.deleteMany((async function * () {
+        for await (const cid of blockstore.queryKeys({})) {
+          if (await helia.pins.isPinned(cid, options)) {
+            continue
+          }
+
+          yield cid
+
+          options.onProgress?.(new CustomProgressEvent('helia:gc:deleted', cid))
+        }
+      }())))
+    } finally {
+      releaseLock()
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,94 @@
+/**
+ * @packageDocumentation
+ *
+ * Create a Helia node.
+ *
+ * @example
+ *
+ * ```typescript
+ * import { createLibp2p } from 'libp2p'
+ * import { MemoryDatastore } from 'datastore-core'
+ * import { MemoryBlockstore } from 'blockstore-core'
+ * import { createHelia } from 'helia'
+ * import { unixfs } from '@helia/unixfs'
+ * import { CID } from 'multiformats/cid'
+ *
+ * const node = await createHelia({
+ *   blockstore: new MemoryBlockstore(),
+ *   datastore: new MemoryDatastore(),
+ *   libp2p: await createLibp2p({
+ *     //... libp2p options
+ *   })
+ * })
+ * const fs = unixfs(node)
+ * fs.cat(CID.parse('bafyFoo'))
+ * ```
+ */
+
+import type { Helia } from '@helia/interface'
+import type { Libp2p } from '@libp2p/interface-libp2p'
+import type { Blockstore } from 'interface-blockstore'
+import type { Datastore } from 'interface-datastore'
+import type { CID } from 'multiformats/cid'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+import { HeliaImpl } from './helia.js'
+
+/**
+ * DAGWalkers take a block and yield CIDs encoded in that block
+ */
+export interface DAGWalker {
+  codec: number
+  walk: (block: Uint8Array) => AsyncGenerator<CID, void, undefined>
+}
+
+/**
+ * Options used to create a Helia node.
+ */
+export interface HeliaInit {
+  /**
+   * A libp2p node is required to perform network operations
+   */
+  libp2p?: Libp2p
+
+  /**
+   * The blockstore is where blocks are stored
+   */
+  blockstore?: Blockstore
+
+  /**
+   * The datastore is where data is stored
+   */
+  datastore?: Datastore
+
+  /**
+   * By default sha256, sha512 and identity hashes are supported for
+   * bitswap operations. To bitswap blocks with CIDs using other hashes
+   * pass appropriate MultihashHashers here.
+   */
+  hashers?: MultihashHasher[]
+
+  /**
+   * In order to pin CIDs that correspond to a DAG, it's necessary to know
+   * how to traverse that DAG.  DAGWalkers take a block and yield any CIDs
+   * encoded within that block.
+   */
+  dagWalkers?: DAGWalker[]
+
+  /**
+   * Pass `false` to not start the helia node
+   */
+  start?: boolean
+}
+
+/**
+ * Create and return a Helia node
+ */
+export async function createHelia (init: HeliaInit = {}): Promise<Helia> {
+  const helia = new HeliaImpl(init)
+
+  if (init.start !== false) {
+    await helia.start()
+  }
+
+  return helia
+}

package/src/pins.ts

@@ -0,0 +1,238 @@
+import type { AddOptions, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
+import { Datastore, Key } from 'interface-datastore'
+import { CID, Version } from 'multiformats/cid'
+import * as cborg from 'cborg'
+import { base36 } from 'multiformats/bases/base36'
+import type { Blockstore } from 'interface-blockstore'
+import PQueue from 'p-queue'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import defer from 'p-defer'
+import type { DAGWalker } from './index.js'
+import { cborWalker, dagPbWalker, jsonWalker, rawWalker } from './utils/dag-walkers.js'
+
+const DEFAULT_DAG_WALKERS = [
+  rawWalker,
+  dagPbWalker,
+  cborWalker,
+  jsonWalker
+]
+
+interface DatastorePin {
+  /**
+   * 0 for a direct pin or an arbitrary (+ve, whole) number or Infinity
+   */
+  depth: number
+
+  /**
+   * User-specific metadata for the pin
+   */
+  metadata: Record<string, string | number | boolean>
+}
+
+interface DatastorePinnedBlock {
+  pinCount: number
+  pinnedBy: Uint8Array[]
+}
+
+const DATASTORE_PIN_PREFIX = '/pin/'
+const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
+const DATASTORE_ENCODING = base36
+// const DAG_WALK_MAX_QUEUE_LENGTH = 10
+const DAG_WALK_QUEUE_CONCURRENCY = 1
+
+interface WalkDagOptions extends AbortOptions {
+  depth: number
+}
+
+function toDSKey (cid: CID): Key {
+  if (cid.version === 0) {
+    cid = cid.toV1()
+  }
+
+  return new Key(`${DATASTORE_PIN_PREFIX}${cid.toString(DATASTORE_ENCODING)}`)
+}
+
+export class PinsImpl implements Pins {
+  private readonly datastore: Datastore
+  private readonly blockstore: Blockstore
+  private dagWalkers: Record<number, DAGWalker>
+
+  constructor (datastore: Datastore, blockstore: Blockstore, dagWalkers: DAGWalker[]) {
+    this.datastore = datastore
+    this.blockstore = blockstore
+    this.dagWalkers = {}
+
+    ;[...DEFAULT_DAG_WALKERS, ...dagWalkers].forEach(dagWalker => {
+      this.dagWalkers[dagWalker.codec] = dagWalker
+    })
+  }
+
+  async add (cid: CID<unknown, number, number, Version>, options: AddOptions = {}): Promise<Pin> {
+    const pinKey = toDSKey(cid)
+
+    if (await this.datastore.has(pinKey)) {
+      throw new Error('Already pinned')
+    }
+
+    const depth = Math.round(options.depth ?? Infinity)
+
+    if (depth < 0) {
+      throw new Error('Depth must be greater than or equal to 0')
+    }
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
+    const queue = new PQueue({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+    void queue.add(async () => {
+      await this.#walkDag(cid, queue, (pinnedBlock) => {
+        // do not update pinned block if this block is already pinned by this CID
+        if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
+          return
+        }
+
+        pinnedBlock.pinCount++
+        pinnedBlock.pinnedBy.push(cid.bytes)
+      }, {
+        ...options,
+        depth
+      })
+    })
+
+    // if a job in the queue errors, throw that error
+    const deferred = defer()
+
+    queue.on('error', (err) => {
+      queue.clear()
+      deferred.reject(err)
+    })
+
+    // wait for the queue to complete or error
+    await Promise.race([
+      queue.onIdle(),
+      deferred.promise
+    ])
+
+    const pin: DatastorePin = {
+      depth,
+      metadata: options.metadata ?? {}
+    }
+
+    await this.datastore.put(pinKey, cborg.encode(pin), options)
+
+    return {
+      cid,
+      ...pin
+    }
+  }
+
+  /**
+   * Walk the DAG behind the passed CID, ensure all blocks are present in the blockstore
+   * and update the pin count for them
+   */
+  async #walkDag (cid: CID, queue: PQueue, withPinnedBlock: (pinnedBlock: DatastorePinnedBlock) => void, options: WalkDagOptions): Promise<void> {
+    if (options.depth === -1) {
+      return
+    }
+
+    const dagWalker = this.dagWalkers[cid.code]
+
+    if (dagWalker == null) {
+      throw new Error(`No dag walker found for cid codec ${cid.code}`)
+    }
+
+    const block = await this.blockstore.get(cid)
+
+    await this.#updatePinnedBlock(cid, withPinnedBlock, options)
+
+    // walk dag, ensure all blocks are present
+    for await (const cid of dagWalker.walk(block)) {
+      void queue.add(async () => {
+        await this.#walkDag(cid, queue, withPinnedBlock, {
+          ...options,
+          depth: options.depth - 1
+        })
+      })
+    }
+  }
+
+  /**
+   * Update the pin count for the CID
+   */
+  async #updatePinnedBlock (cid: CID, withPinnedBlock: (pinnedBlock: DatastorePinnedBlock) => void, options: AbortOptions): Promise<void> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    let pinnedBlock: DatastorePinnedBlock = {
+      pinCount: 0,
+      pinnedBy: []
+    }
+
+    try {
+      pinnedBlock = cborg.decode(await this.datastore.get(blockKey, options))
+    } catch (err: any) {
+      if (err.code !== 'ERR_NOT_FOUND') {
+        throw err
+      }
+    }
+
+    withPinnedBlock(pinnedBlock)
+
+    if (pinnedBlock.pinCount === 0) {
+      if (await this.datastore.has(blockKey)) {
+        await this.datastore.delete(blockKey)
+        return
+      }
+    }
+
+    await this.datastore.put(blockKey, cborg.encode(pinnedBlock), options)
+  }
+
+  async rm (cid: CID<unknown, number, number, Version>, options: RmOptions = {}): Promise<Pin> {
+    const pinKey = toDSKey(cid)
+    const buf = await this.datastore.get(pinKey, options)
+    const pin = cborg.decode(buf)
+
+    await this.datastore.delete(pinKey, options)
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
+    const queue = new PQueue({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+    void queue.add(async () => {
+      await this.#walkDag(cid, queue, (pinnedBlock) => {
+        pinnedBlock.pinCount--
+        pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
+      }, {
+        ...options,
+        depth: pin.depth
+      })
+    })
+    await queue.onIdle()
+
+    return {
+      cid,
+      ...pin
+    }
+  }
+
+  async * ls (options: LsOptions = {}): AsyncGenerator<Pin, void, undefined> {
+    for await (const { key, value } of this.datastore.query({
+      prefix: DATASTORE_PIN_PREFIX + (options.cid != null ? `${options.cid.toString(base36)}` : '')
+    }, options)) {
+      const cid = CID.parse(key.toString().substring(5), base36)
+      const pin = cborg.decode(value)
+
+      yield {
+        cid,
+        ...pin
+      }
+    }
+  }
+
+  async isPinned (cid: CID, options: IsPinnedOptions = {}): Promise<boolean> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    return await this.datastore.has(blockKey, options)
+  }
+}

package/src/storage.ts

@@ -0,0 +1,214 @@
+import { BaseBlockstore } from 'blockstore-core'
+import merge from 'it-merge'
+import { pushable } from 'it-pushable'
+import filter from 'it-filter'
+import type { Blockstore, KeyQuery, Query } from 'interface-blockstore'
+import type { Bitswap } from 'ipfs-bitswap'
+import type { CID } from 'multiformats/cid'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { AwaitIterable } from 'interface-store'
+import type { Mortice } from 'mortice'
+import createMortice from 'mortice'
+import type { Pins } from '@helia/interface/pins'
+
+export interface BlockStorageOptions extends AbortOptions {
+  progress?: (evt: Event) => void
+}
+
+/**
+ * BlockStorage is a hybrid blockstore that puts/gets blocks from a configured
+ * blockstore (that may be on disk, s3, or something else). If the blocks are
+ * not present Bitswap will be used to fetch them from network peers.
+ */
+export class BlockStorage extends BaseBlockstore implements Blockstore {
+  public lock: Mortice
+  private readonly child: Blockstore
+  private readonly bitswap?: Bitswap
+  private readonly pins: Pins
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (blockstore: Blockstore, pins: Pins, bitswap?: Bitswap) {
+    super()
+
+    this.child = blockstore
+    this.bitswap = bitswap
+    this.pins = pins
+    this.lock = createMortice()
+  }
+
+  async open (): Promise<void> {
+    await this.child.open()
+  }
+
+  async close (): Promise<void> {
+    await this.child.close()
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions = {}): Promise<void> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      if (this.bitswap?.isStarted() === true) {
+        await this.bitswap.put(cid, block, options)
+      } else {
+        await this.child.put(cid, block, options)
+      }
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: AwaitIterable<{ key: CID, value: Uint8Array }>, options: AbortOptions = {}): AsyncGenerator<{ key: CID, value: Uint8Array }, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      const missingBlocks = filter(blocks, async ({ key }) => {
+        return !(await this.child.has(key))
+      })
+
+      const store = this.bitswap?.isStarted() === true ? this.bitswap : this.child
+
+      yield * store.putMany(missingBlocks, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async get (cid: CID, options: BlockStorageOptions = {}): Promise<Uint8Array> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      if (!(await this.has(cid)) && this.bitswap?.isStarted() === true) {
+        return await this.bitswap?.get(cid, options)
+      } else {
+        return await this.child.get(cid, options)
+      }
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get multiple blocks back from an array of cids
+   */
+  async * getMany (cids: AwaitIterable<CID>, options: BlockStorageOptions = {}): AsyncGenerator<Uint8Array, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      const getFromBitswap = pushable<CID>({ objectMode: true })
+      const getFromChild = pushable<CID>({ objectMode: true })
+
+      void Promise.resolve().then(async () => {
+        for await (const cid of cids) {
+          if (!(await this.has(cid)) && this.bitswap?.isStarted() === true) {
+            getFromBitswap.push(cid)
+          } else {
+            getFromChild.push(cid)
+          }
+        }
+
+        getFromBitswap.end()
+        getFromChild.end()
+      }).catch(err => {
+        getFromBitswap.throw(err)
+      })
+
+      const streams = [
+        this.child.getMany(getFromChild, options)
+      ]
+
+      if (this.bitswap?.isStarted() === true) {
+        streams.push(this.bitswap.getMany(getFromBitswap, options))
+      }
+
+      yield * merge(...streams)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions = {}): Promise<void> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      if (await this.pins.isPinned(cid)) {
+        throw new Error('CID was pinned')
+      }
+
+      await this.child.delete(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      const storage = this
+
+      yield * this.child.deleteMany((async function * () {
+        for await (const cid of cids) {
+          if (await storage.pins.isPinned(cid)) {
+            throw new Error('CID was pinned')
+          }
+
+          yield cid
+        }
+      }()), options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.has(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async * query (q: Query, options: AbortOptions = {}): AsyncGenerator<{ key: CID, value: Uint8Array }, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.query(q, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async * queryKeys (q: KeyQuery, options: AbortOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.queryKeys(q, options)
+    } finally {
+      releaseLock()
+    }
+  }
+}