@helia/utils 2.1.0 → 2.1.1-3eb6f3a7

package/src/graph-walker.ts

@@ -1,4 +1,5 @@
 import { Queue } from '@libp2p/utils'
+import filter from 'it-filter'
 import toBuffer from 'it-to-buffer'
 import { createUnsafe } from 'multiformats/block'
 import type { CodecLoader } from '@helia/interface'
@@ -22,7 +23,7 @@ export interface GraphNode <T = unknown, C extends number = number, A extends nu
 }
 
 export interface GraphWalker {
-  walk <T = any> (cid: CID, options?: AbortOptions): AsyncGenerator<GraphNode<T>>
+  walk <T = any> (cid: CID, options?: WalkOptions<T>): AsyncGenerator<GraphNode<T>>
 }
 
 export function depthFirstWalker (components: GraphWalkerComponents, init: GraphWalkerInit = {}): GraphWalker {
@@ -39,57 +40,56 @@ interface JobOptions extends AbortOptions {
   path: CID[]
 }
 
-class DepthFirstGraphWalker {
+export interface WalkOptions<T> extends AbortOptions {
+  includeChild?(child: CID, parent: BlockView<T, number, number, 0 | 1>): boolean
+}
+
+abstract class AbstractGraphWalker {
   private readonly components: GraphWalkerComponents
 
-  constructor (components: GraphWalkerComponents, init: GraphWalkerInit = {}) {
+  constructor (components: GraphWalkerComponents, init: GraphWalkerInit) {
     this.components = components
   }
 
-  async * walk <T = any> (cid: CID, options: AbortOptions): AsyncGenerator<GraphNode<T>> {
-    const queue = new Queue<GraphNode<T>, JobOptions>({
-      concurrency: 1,
-      sort: (a, b) => {
-        if (a.options.depth === b.options.depth) {
-          return 0
-        }
+  async * walk <T = any> (cid: CID, options?: WalkOptions<T>): AsyncGenerator<GraphNode<T>> {
+    const queue = this.getQueue()
+    const gen = filter(queue.toGenerator(options), (node) => node != null) as AsyncGenerator<GraphNode<T>>
+    let finished = false
 
-        if (a.options.depth < b.options.depth) {
-          return 1
-        }
-
-        return -1
-      }
-    })
-
-    const gen = queue.toGenerator()
-
-    const job = async (options: JobOptions): Promise<GraphNode<T>> => {
-      const cid = options.cid
-      const bytes = await toBuffer(this.components.blockstore.get(cid, options))
-      const block = createUnsafe({
+    const job = async (opts: JobOptions): Promise<GraphNode<T> | undefined> => {
+      const cid = opts.cid
+      const bytes = await toBuffer(this.components.blockstore.get(cid, opts))
+      const block = createUnsafe<T, number, number, 0 | 1>({
         cid,
         bytes,
         codec: await this.components.getCodec(cid.code)
       })
 
       for (const [, linkedCid] of block.links()) {
+        if (options?.includeChild?.(linkedCid, block) === false) {
+          continue
+        }
+
         queue.add(job, {
-          ...options,
+          ...opts,
           cid: linkedCid,
-          depth: options.depth + 1,
-          path: [...options.path, linkedCid]
+          depth: opts.depth + 1,
+          path: [...opts.path, linkedCid]
         })
+          // eslint-disable-next-line no-loop-func
           .catch(err => {
-            gen.throw(err)
-            queue.abort()
+            // only throw if the generator is still yielding results, otherwise
+            // it can cause unhandled promise rejections
+            if (!finished) {
+              gen.throw(err)
+            }
           })
       }
 
       return {
         block,
-        depth: options.depth,
-        path: options.path
+        depth: opts.depth,
+        path: opts.path
       }
     }
 
@@ -100,23 +100,28 @@ class DepthFirstGraphWalker {
       path: [cid]
     })
       .catch(err => {
-        gen.throw(err)
-        queue.abort()
+        // only throw if the generator is still yielding results, otherwise it
+        // can cause unhandled promise rejections
+        if (!finished) {
+          gen.throw(err)
+        }
       })
 
-    yield * gen
+    try {
+      yield * gen
+    } finally {
+      finished = true
+      // abort any in-progress operations
+      queue.abort()
+    }
   }
-}
 
-class BreadthFirstGraphWalker {
-  private readonly components: GraphWalkerComponents
-
-  constructor (components: GraphWalkerComponents, init: GraphWalkerInit = {}) {
-    this.components = components
-  }
+  abstract getQueue <T> (): Queue<GraphNode<T> | undefined, JobOptions>
+}
 
-  async * walk <T = any> (cid: CID, options: AbortOptions): AsyncGenerator<GraphNode<T>> {
-    const queue = new Queue<GraphNode<T>, JobOptions>({
+class DepthFirstGraphWalker extends AbstractGraphWalker {
+  getQueue<T>(): Queue<GraphNode<T> | undefined, JobOptions> {
+    return new Queue<GraphNode<T> | undefined, JobOptions>({
       concurrency: 1,
       sort: (a, b) => {
         if (a.options.depth === b.options.depth) {
@@ -124,55 +129,30 @@ class BreadthFirstGraphWalker {
         }
 
         if (a.options.depth < b.options.depth) {
-          return -1
+          return 1
         }
 
-        return 1
+        return -1
       }
     })
+  }
+}
 
-    const gen = queue.toGenerator()
-
-    const job = async (options: JobOptions): Promise<GraphNode<T>> => {
-      const cid = options.cid
-      const bytes = await toBuffer(this.components.blockstore.get(cid, options))
-      const block = createUnsafe({
-        cid,
-        bytes,
-        codec: await this.components.getCodec(cid.code)
-      })
+class BreadthFirstGraphWalker extends AbstractGraphWalker {
+  getQueue<T>(): Queue<GraphNode<T> | undefined, JobOptions> {
+    return new Queue<GraphNode<T> | undefined, JobOptions>({
+      concurrency: 1,
+      sort: (a, b) => {
+        if (a.options.depth === b.options.depth) {
+          return 0
+        }
 
-      for (const [, linkedCid] of block.links()) {
-        queue.add(job, {
-          ...options,
-          cid: linkedCid,
-          depth: options.depth + 1,
-          path: [...options.path, linkedCid]
-        })
-          .catch(err => {
-            gen.throw(err)
-            queue.abort()
-          })
-      }
+        if (a.options.depth < b.options.depth) {
+          return -1
+        }
 
-      return {
-        block,
-        depth: options.depth,
-        path: options.path
+        return 1
       }
-    }
-
-    queue.add(job, {
-      ...options,
-      cid,
-      depth: 0,
-      path: [cid]
     })
-      .catch(err => {
-        gen.throw(err)
-        queue.abort()
-      })
-
-    yield * gen
   }
 }

package/src/utils/datastore-version.ts

@@ -1,7 +1,7 @@
 import { Key } from 'interface-datastore'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { InvalidDatastoreVersion } from '../errors.js'
+import { InvalidDatastoreVersionError } from '../errors.js'
 import type { Datastore } from 'interface-datastore'
 
 const DS_VERSION_KEY = new Key('/version')
@@ -20,6 +20,6 @@ export async function assertDatastoreVersionIsCurrent (datastore: Datastore): Pr
 
   if (version !== CURRENT_VERSION) {
     // TODO: write migrations when we break compatibility - for an example, see https://github.com/ipfs/js-ipfs-repo/tree/master/packages/ipfs-repo-migrations
-    throw new InvalidDatastoreVersion('Invalid datastore version, a datastore migration may be required')
+    throw new InvalidDatastoreVersionError('Invalid datastore version, a datastore migration may be required')
   }
 }

package/src/utils/networked-storage.ts

@@ -5,6 +5,7 @@ import filter from 'it-filter'
 import forEach from 'it-foreach'
 import { CustomProgressEvent } from 'progress-events'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import { InvalidConfigurationError } from '../errors.ts'
 import { isPromise } from './is-promise.js'
 import type { HasherLoader } from '@helia/interface'
 import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
@@ -446,6 +447,10 @@ async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hashe
     }
   }
 
+  if (retrievers.length === 0) {
+    throw new InvalidConfigurationError(`No block brokers capable of retrieving blocks are configured, the CID ${cid} cannot be fetched from the network`)
+  }
+
   try {
     return await Promise.any(
       retrievers
@@ -457,6 +462,7 @@ async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hashe
               signal,
               validateFn: async (block: Uint8Array): Promise<void> => {
                 await validateFn(block)
+                options.signal?.throwIfAborted()
                 blocksWereValidated = true
               }
             })
@@ -465,6 +471,7 @@ async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hashe
               // the blockBroker either did not throw an error when attempting to validate the block
               // or did not call the validateFn at all. We should validate the block ourselves
               await validateFn(block)
+              options.signal?.throwIfAborted()
             }
 
             return block

package/dist/typedoc-urls.json

@@ -1,18 +0,0 @@
-{
-  "AbstractSession": "https://ipfs.github.io/helia/classes/_helia_utils.AbstractSession.html",
-  "Helia": "https://ipfs.github.io/helia/classes/_helia_utils.Helia.html",
-  ".:Helia": "https://ipfs.github.io/helia/classes/_helia_utils.Helia.html",
-  "AbstractCreateSessionOptions": "https://ipfs.github.io/helia/interfaces/_helia_utils.AbstractCreateSessionOptions.html",
-  "AbstractSessionComponents": "https://ipfs.github.io/helia/interfaces/_helia_utils.AbstractSessionComponents.html",
-  "BlockStorage": "https://ipfs.github.io/helia/interfaces/_helia_utils.BlockStorage.html",
-  "BlockStorageInit": "https://ipfs.github.io/helia/interfaces/_helia_utils.BlockStorageInit.html",
-  "BlockstoreSessionEvents": "https://ipfs.github.io/helia/interfaces/_helia_utils.BlockstoreSessionEvents.html",
-  "GraphNode": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphNode.html",
-  "GraphWalker": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphWalker.html",
-  "GraphWalkerComponents": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphWalkerComponents.html",
-  "GraphWalkerInit": "https://ipfs.github.io/helia/interfaces/_helia_utils.GraphWalkerInit.html",
-  "HeliaInit": "https://ipfs.github.io/helia/interfaces/_helia_utils.HeliaInit.html",
-  ".:HeliaInit": "https://ipfs.github.io/helia/interfaces/helia.HeliaInit.html",
-  "breadthFirstWalker": "https://ipfs.github.io/helia/functions/_helia_utils.breadthFirstWalker.html",
-  "depthFirstWalker": "https://ipfs.github.io/helia/functions/_helia_utils.depthFirstWalker.html"
-}