@helia/utils 0.0.0-031519c

package/src/storage.ts

@@ -0,0 +1,172 @@
+import { start, stop } from '@libp2p/interface'
+import createMortice from 'mortice'
+import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks'
+import type { Pins } from '@helia/interface/pins'
+import type { AbortOptions, Startable } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { AwaitIterable } from 'interface-store'
+import type { Mortice } from 'mortice'
+import type { CID } from 'multiformats/cid'
+import type { ProgressOptions } from 'progress-events'
+
+export interface BlockStorageInit {
+  holdGcLock?: boolean
+}
+
+export interface GetOptions extends AbortOptions {
+  progress?(evt: Event): void
+}
+
+/**
+ * BlockStorage is a hybrid blockstore that puts/gets blocks from a configured
+ * blockstore (that may be on disk, s3, or something else). If the blocks are
+ * not present Bitswap will be used to fetch them from network peers.
+ */
+export class BlockStorage implements Blocks, Startable {
+  public lock: Mortice
+  private readonly child: Blockstore
+  private readonly pins: Pins
+  private started: boolean
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (blockstore: Blockstore, pins: Pins, options: BlockStorageInit = {}) {
+    this.child = blockstore
+    this.pins = pins
+    this.lock = createMortice({
+      singleProcess: options.holdGcLock
+    })
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child)
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child)
+    this.started = false
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.put(cid, block, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.putMany(blocks, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.get(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get multiple blocks back from an (async) iterable of cids
+   */
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.getMany(cids, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      if (await this.pins.isPinned(cid)) {
+        throw new Error('CID was pinned')
+      }
+
+      await this.child.delete(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      const storage = this
+
+      yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
+        for await (const cid of cids) {
+          if (await storage.pins.isPinned(cid)) {
+            throw new Error('CID was pinned')
+          }
+
+          yield cid
+        }
+      }()), options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.has(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.getAll(options)
+    } finally {
+      releaseLock()
+    }
+  }
+}

package/src/utils/dag-walkers.ts

@@ -0,0 +1,198 @@
+/* eslint max-depth: ["error", 7] */
+
+import * as dagCbor from '@ipld/dag-cbor'
+import * as dagJson from '@ipld/dag-json'
+import * as dagPb from '@ipld/dag-pb'
+import * as cborg from 'cborg'
+import { Type, Token } from 'cborg'
+import * as cborgJson from 'cborg/json'
+import { CID } from 'multiformats'
+import { base64 } from 'multiformats/bases/base64'
+import * as json from 'multiformats/codecs/json'
+import * as raw from 'multiformats/codecs/raw'
+import type { DAGWalker } from '@helia/interface'
+
+/**
+ * Dag walker for dag-pb CIDs
+ */
+export const dagPbWalker: DAGWalker = {
+  codec: dagPb.code,
+  * walk (block) {
+    const node = dagPb.decode(block)
+
+    yield * node.Links.map(l => l.Hash)
+  }
+}
+
+/**
+ * Dag walker for raw CIDs
+ */
+export const rawWalker: DAGWalker = {
+  codec: raw.code,
+  * walk () {
+    // no embedded CIDs in a raw block
+  }
+}
+
+// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692
+const CID_TAG = 42
+
+/**
+ * Dag walker for dag-cbor CIDs. Does not actually use dag-cbor since
+ * all we are interested in is extracting the the CIDs from the block
+ * so we can just use cborg for that.
+ */
+export const dagCborWalker: DAGWalker = {
+  codec: dagCbor.code,
+  * walk (block) {
+    const cids: CID[] = []
+    const tags: cborg.TagDecoder[] = []
+    tags[CID_TAG] = (bytes) => {
+      if (bytes[0] !== 0) {
+        throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00')
+      }
+
+      const cid = CID.decode(bytes.subarray(1)) // ignore leading 0x00
+
+      cids.push(cid)
+
+      return cid
+    }
+
+    cborg.decode(block, {
+      tags
+    })
+
+    yield * cids
+  }
+}
+
+/**
+ * Borrowed from @ipld/dag-json
+ */
+class DagJsonTokenizer extends cborgJson.Tokenizer {
+  private readonly tokenBuffer: cborg.Token[]
+
+  constructor (data: Uint8Array, options?: cborg.DecodeOptions) {
+    super(data, options)
+
+    this.tokenBuffer = []
+  }
+
+  done (): boolean {
+    return this.tokenBuffer.length === 0 && super.done()
+  }
+
+  _next (): cborg.Token {
+    if (this.tokenBuffer.length > 0) {
+      // @ts-expect-error https://github.com/Microsoft/TypeScript/issues/30406
+      return this.tokenBuffer.pop()
+    }
+    return super.next()
+  }
+
+  /**
+   * Implements rules outlined in https://github.com/ipld/specs/pull/356
+   */
+  next (): cborg.Token {
+    const token = this._next()
+
+    if (token.type === Type.map) {
+      const keyToken = this._next()
+      if (keyToken.type === Type.string && keyToken.value === '/') {
+        const valueToken = this._next()
+        if (valueToken.type === Type.string) { // *must* be a CID
+          const breakToken = this._next() // swallow the end-of-map token
+          if (breakToken.type !== Type.break) {
+            throw new Error('Invalid encoded CID form')
+          }
+          this.tokenBuffer.push(valueToken) // CID.parse will pick this up after our tag token
+          return new Token(Type.tag, 42, 0)
+        }
+        if (valueToken.type === Type.map) {
+          const innerKeyToken = this._next()
+          if (innerKeyToken.type === Type.string && innerKeyToken.value === 'bytes') {
+            const innerValueToken = this._next()
+            if (innerValueToken.type === Type.string) { // *must* be Bytes
+              for (let i = 0; i < 2; i++) {
+                const breakToken = this._next() // swallow two end-of-map tokens
+                if (breakToken.type !== Type.break) {
+                  throw new Error('Invalid encoded Bytes form')
+                }
+              }
+              const bytes = base64.decode(`m${innerValueToken.value}`)
+              return new Token(Type.bytes, bytes, innerValueToken.value.length)
+            }
+            this.tokenBuffer.push(innerValueToken) // bail
+          }
+          this.tokenBuffer.push(innerKeyToken) // bail
+        }
+        this.tokenBuffer.push(valueToken) // bail
+      }
+      this.tokenBuffer.push(keyToken) // bail
+    }
+    return token
+  }
+}
+
+/**
+ * Dag walker for dag-json CIDs. Does not actually use dag-json since
+ * all we are interested in is extracting the the CIDs from the block
+ * so we can just use cborg/json for that.
+ */
+export const dagJsonWalker: DAGWalker = {
+  codec: dagJson.code,
+  * walk (block) {
+    const cids: CID[] = []
+    const tags: cborg.TagDecoder[] = []
+    tags[CID_TAG] = (string) => {
+      const cid = CID.parse(string)
+
+      cids.push(cid)
+
+      return cid
+    }
+
+    cborgJson.decode(block, {
+      tags,
+      tokenizer: new DagJsonTokenizer(block, {
+        tags,
+        allowIndefinite: true,
+        allowUndefined: true,
+        allowNaN: true,
+        allowInfinity: true,
+        allowBigInt: true,
+        strict: false,
+        rejectDuplicateMapKeys: false
+      })
+    })
+
+    yield * cids
+  }
+}
+
+/**
+ * Dag walker for json CIDs. JSON has no facility for linking to
+ * external blocks so the walker is a no-op.
+ */
+export const jsonWalker: DAGWalker = {
+  codec: json.code,
+  * walk () {}
+}
+
+export function defaultDagWalkers (walkers: DAGWalker[] = []): Record<number, DAGWalker> {
+  const output: Record<number, DAGWalker> = {}
+
+  ;[
+    dagPbWalker,
+    rawWalker,
+    dagCborWalker,
+    dagJsonWalker,
+    jsonWalker,
+    ...walkers
+  ].forEach(dagWalker => {
+    output[dagWalker.codec] = dagWalker
+  })
+
+  return output
+}

package/src/utils/datastore-version.ts

@@ -0,0 +1,23 @@
+import { type Datastore, Key } from 'interface-datastore'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+
+const DS_VERSION_KEY = new Key('/version')
+const CURRENT_VERSION = 1
+
+export async function assertDatastoreVersionIsCurrent (datastore: Datastore): Promise<void> {
+  if (!(await datastore.has(DS_VERSION_KEY))) {
+    await datastore.put(DS_VERSION_KEY, uint8ArrayFromString(`${CURRENT_VERSION}`))
+
+    return
+  }
+
+  const buf = await datastore.get(DS_VERSION_KEY)
+  const str = uint8ArrayToString(buf)
+  const version = parseInt(str, 10)
+
+  if (version !== CURRENT_VERSION) {
+    // TODO: write migrations when we break compatibility - for an example, see https://github.com/ipfs/js-ipfs-repo/tree/master/packages/ipfs-repo-migrations
+    throw new Error('Unknown datastore version, a datastore migration may be required')
+  }
+}

package/src/utils/default-hashers.ts

@@ -0,0 +1,18 @@
+import { identity } from 'multiformats/hashes/identity'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export function defaultHashers (hashers: MultihashHasher[] = []): Record<number, MultihashHasher> {
+  const output: Record<number, MultihashHasher> = {}
+
+  ;[
+    sha256,
+    sha512,
+    identity,
+    ...hashers
+  ].forEach(hasher => {
+    output[hasher.code] = hasher
+  })
+
+  return output
+}

package/src/utils/networked-storage.ts

@@ -0,0 +1,261 @@
+import { CodeError, start, stop } from '@libp2p/interface'
+import { anySignal } from 'any-signal'
+import filter from 'it-filter'
+import forEach from 'it-foreach'
+import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetriever, BlockAnnouncer, BlockRetrievalOptions } from '@helia/interface/blocks'
+import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { AwaitIterable } from 'interface-store'
+import type { CID } from 'multiformats/cid'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+
+export interface GetOptions extends AbortOptions {
+  progress?(evt: Event): void
+}
+
+function isBlockRetriever (b: any): b is BlockRetriever {
+  return typeof b.retrieve === 'function'
+}
+
+function isBlockAnnouncer (b: any): b is BlockAnnouncer {
+  return typeof b.announce === 'function'
+}
+
+export interface NetworkedStorageComponents {
+  blockstore: Blockstore
+  logger: ComponentLogger
+  blockBrokers?: BlockBroker[]
+  hashers?: Record<number, MultihashHasher>
+}
+
+/**
+ * Networked storage wraps a regular blockstore - when getting blocks if the
+ * blocks are not present Bitswap will be used to fetch them from network peers.
+ */
+export class NetworkedStorage implements Blocks, Startable {
+  private readonly child: Blockstore
+  private readonly blockRetrievers: BlockRetriever[]
+  private readonly blockAnnouncers: BlockAnnouncer[]
+  private readonly hashers: Record<number, MultihashHasher>
+  private started: boolean
+  private readonly log: Logger
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (components: NetworkedStorageComponents) {
+    this.log = components.logger.forComponent('helia:networked-storage')
+    this.child = components.blockstore
+    this.blockRetrievers = (components.blockBrokers ?? []).filter(isBlockRetriever)
+    this.blockAnnouncers = (components.blockBrokers ?? []).filter(isBlockAnnouncer)
+    this.hashers = components.hashers ?? {}
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers]))
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers]))
+    this.started = false
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
+    if (await this.child.has(cid)) {
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:duplicate', cid))
+      return cid
+    }
+
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:providers:notify', cid))
+
+    this.blockAnnouncers.forEach(provider => {
+      provider.announce(cid, block, options)
+    })
+
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:blockstore:put', cid))
+
+    return this.child.put(cid, block, options)
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
+      const has = await this.child.has(cid)
+
+      if (has) {
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:duplicate', cid))
+      }
+
+      return !has
+    })
+
+    const notifyEach = forEach(missingBlocks, ({ cid, block }): void => {
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:providers:notify', cid))
+      this.blockAnnouncers.forEach(provider => {
+        provider.announce(cid, block, options)
+      })
+    })
+
+    options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'))
+    yield * this.child.putMany(notifyEach, options)
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+    if (options.offline !== true && !(await this.child.has(cid))) {
+      // we do not have the block locally, get it from a block provider
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:get', cid))
+      const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers[cid.multihash.code], {
+        ...options,
+        log: this.log
+      })
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:put', cid))
+      await this.child.put(cid, block, options)
+
+      // notify other block providers of the new block
+      options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:notify', cid))
+      this.blockAnnouncers.forEach(provider => {
+        provider.announce(cid, block, options)
+      })
+
+      return block
+    }
+
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:get', cid))
+
+    return this.child.get(cid, options)
+  }
+
+  /**
+   * Get multiple blocks back from an (async) iterable of cids
+   */
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
+
+    yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
+      if (options.offline !== true && !(await this.child.has(cid))) {
+        // we do not have the block locally, get it from a block provider
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:get', cid))
+        const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers[cid.multihash.code], {
+          ...options,
+          log: this.log
+        })
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:blockstore:put', cid))
+        await this.child.put(cid, block, options)
+
+        // notify other block providers of the new block
+        options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:notify', cid))
+        this.blockAnnouncers.forEach(provider => {
+          provider.announce(cid, block, options)
+        })
+      }
+    }))
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
+    options.onProgress?.(new CustomProgressEvent<CID>('blocks:delete:blockstore:delete', cid))
+
+    await this.child.delete(cid, options)
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'))
+    yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
+      for await (const cid of cids) {
+        yield cid
+      }
+    }()), options)
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    return this.child.has(cid, options)
+  }
+
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+    options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'))
+    yield * this.child.getAll(options)
+  }
+}
+
+export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher): Required<BlockRetrievalOptions>['validateFn'] => {
+  if (hasher == null) {
+    throw new CodeError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`, 'ERR_UNKNOWN_HASH_ALG')
+  }
+
+  return async (block: Uint8Array): Promise<void> => {
+    // verify block
+    const hash = await hasher.digest(block)
+
+    if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
+      // if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
+      throw new CodeError('Hash of downloaded block did not match multihash from passed CID', 'ERR_HASH_MISMATCH')
+    }
+  }
+}
+
+/**
+ * Race block providers cancelling any pending requests once the block has been
+ * found.
+ */
+async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hasher: MultihashHasher, options: AbortOptions & LoggerOptions): Promise<Uint8Array> {
+  const validateFn = getCidBlockVerifierFunction(cid, hasher)
+
+  const controller = new AbortController()
+  const signal = anySignal([controller.signal, options.signal])
+
+  try {
+    return await Promise.any(
+      providers.map(async provider => {
+        try {
+          let blocksWereValidated = false
+          const block = await provider.retrieve(cid, {
+            ...options,
+            signal,
+            validateFn: async (block: Uint8Array): Promise<void> => {
+              await validateFn(block)
+              blocksWereValidated = true
+            }
+          })
+
+          if (!blocksWereValidated) {
+            // the blockBroker either did not throw an error when attempting to validate the block
+            // or did not call the validateFn at all. We should validate the block ourselves
+            await validateFn(block)
+          }
+
+          return block
+        } catch (err) {
+          options.log.error('could not retrieve verified block for %c', cid, err)
+          throw err
+        }
+      })
+    )
+  } finally {
+    signal.clear()
+  }
+}