@helia/car 5.0.1 → 5.1.0

package/src/car.ts

@@ -1,50 +1,22 @@
 import { CarWriter } from '@ipld/car'
-import { Queue } from '@libp2p/utils'
 import drain from 'it-drain'
 import map from 'it-map'
-import toBuffer from 'it-to-buffer'
-import { createUnsafe } from 'multiformats/block'
 import { raceSignal } from 'race-signal'
-import { DAG_WALK_QUEUE_CONCURRENCY } from './constants.js'
+import { DAG_PB_CODEC_CODE } from './constants.ts'
 import { SubgraphExporter } from './export-strategies/subgraph-exporter.js'
-import { GraphSearch } from './traversal-strategies/graph-search.js'
-import type { CarComponents, Car as CarInterface, ExportCarOptions, ExportStrategy, TraversalStrategy } from './index.js'
+import { UnixFSExporter } from './index.js'
+import type { CarComponents, Car as CarInterface, ExportCarOptions } from './index.js'
 import type { PutManyBlocksProgressEvents } from '@helia/interface/blocks'
 import type { CarReader } from '@ipld/car'
 import type { AbortOptions, Logger } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
 import type { ProgressOptions } from 'progress-events'
 
-/**
- * Context for the traversal process.
- */
-interface TraversalContext {
-  currentPath: CID[]
-  pathsToTarget: CID[][] | null // collect all target paths
-}
-
-interface WalkDagContext<Strategy> {
-  cid: CID
-  queue: Queue
-  strategy: Strategy
-  options?: ExportCarOptions
-}
-
-interface ExportWalkDagContext extends WalkDagContext<ExportStrategy> {
-  writer: Pick<CarWriter, 'put'>
-  recursive?: boolean
-}
-
-interface TraversalWalkDagContext extends WalkDagContext<TraversalStrategy> {
-  traversalContext: TraversalContext
-  parentPath: CID[]
-}
-
 export class Car implements CarInterface {
   private readonly components: CarComponents
   private readonly log: Logger
 
-  constructor (components: CarComponents, init: any) {
+  constructor (components: CarComponents) {
     this.components = components
     this.log = components.logger.forComponent('helia:car')
   }
@@ -91,148 +63,67 @@ export class Car implements CarInterface {
   }
 
   private async _export (root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void> {
-    const deferred = Promise.withResolvers<Error | void>()
     const roots = Array.isArray(root) ? root : [root]
-
-    // Create traversal-specific context
-    const traversalContext: TraversalContext = {
-      currentPath: [],
-      pathsToTarget: null
-    }
-
     const traversalStrategy = options?.traversal
-    const exportStrategy = options?.exporter ?? new SubgraphExporter()
-
-    // use a queue to walk the DAG instead of recursion so we can traverse very
-    // large DAGs
-    const queue = new Queue({
-      concurrency: DAG_WALK_QUEUE_CONCURRENCY
-    })
-
-    let startedExport = false
-    queue.addEventListener('idle', () => {
-      if (startedExport) {
-        // idle event was called, and started exporting, so we are done.
-        deferred.resolve()
-      } else if (!startedExport && traversalContext.pathsToTarget?.length === roots.length) {
-        // queue is idle, we haven't started exporting yet, and we have path(s)
-        // to the target(s), so we can start the export process.
-        this.log.trace('starting export of blocks to the car file')
-        startedExport = true
-
-        for (const path of traversalContext.pathsToTarget) {
-          const targetIndex = path.length - 1
-          const targetCid = path[targetIndex]
-
-          // Process all verification blocks in the path except the target
-          path.slice(0, -1).forEach(cid => {
-            queue.add(async () => {
-              await this.#exportDagNode({ cid, queue, writer, strategy: exportStrategy, options, recursive: false })
-            })
-              .catch((err) => {
-                this.log.error('error during queue operation - %e', err)
-              })
-          })
-
-          // Process the target block (which will recursively export its DAG)
-          queue.add(async () => {
-            await this.#exportDagNode({ cid: targetCid, queue, writer, strategy: exportStrategy, options })
-          })
-            .catch((err) => {
-              this.log.error('error during queue operation - %e', err)
+
+    for (const root of roots) {
+      const exportStrategy = options?.exporter ?? (root.code === DAG_PB_CODEC_CODE ? new UnixFSExporter() : new SubgraphExporter())
+      let current = root
+      let underRoot = false
+
+      if (traversalStrategy != null) {
+        for await (const { cid, bytes } of traversalStrategy.traverse(current, this.components.blockstore, this.components.getCodec, options)) {
+          this.log.trace('next CID on path to %c is %c', root, cid)
+          current = cid
+
+          // the traversal is under the root we are exporting
+          if (root.equals(cid)) {
+            underRoot = true
+          }
+
+          // include the traversal block if we are under the root CID or it has
+          // been explicitly requested
+          if (underRoot || options?.includeTraversalBlocks === true) {
+            // eslint-disable-next-line max-depth
+            if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
+              continue
+            }
+
+            // mark as processed
+            options?.blockFilter?.add(cid.multihash.bytes)
+
+            // store block on path to target
+            await writer.put({
+              cid,
+              bytes
             })
+          }
         }
-      } else {
-        // queue is idle, we haven't started exporting yet, and we don't have
-        // path(s) to the target(s), so we can't start the export process.
-        // this should not happen without a separate error during traversal, but
-        // we'll handle it here anyway.
-        this.log.trace('no paths to target, skipping export')
-        deferred.reject(new Error('Could not traverse to target CID(s)'))
       }
-    })
-    queue.addEventListener('failure', (evt) => {
-      queue.clear()
-      deferred.reject(evt.detail.error)
-    })
-
-    for (const root of roots) {
-      void queue.add(async () => {
-        this.log.trace('traversing dag from %c', root)
-        await this.#traverseDagNode({ cid: root, queue, strategy: traversalStrategy ?? new GraphSearch(root), traversalContext, parentPath: [], options })
-      })
-        .catch((err) => {
-          this.log.error('error during queue operation - %e', err)
-        })
-    }
 
-    // wait for the writer to end
-    try {
-      await deferred.promise
-    } finally {
-      await writer.close()
-    }
-  }
-
-  /**
-   * Traverse a DAG and stop when we reach the target node
-   */
-  async #traverseDagNode ({ cid, queue, strategy, traversalContext, parentPath, options }: TraversalWalkDagContext): Promise<void> {
-    // if we are traversing, we need to gather path(s) to the target(s)
-    const currentPath = [...parentPath, cid]
-
-    if (strategy.isTarget(cid)) {
-      traversalContext.pathsToTarget = traversalContext.pathsToTarget ?? []
-      traversalContext.pathsToTarget.push([...currentPath])
-      this.log.trace('found path to target %c', cid)
-      return
-    }
-
-    const codec = await this.components.getCodec(cid.code)
-    const bytes = await toBuffer(this.components.blockstore.get(cid, options))
-
-    // we are recursively traversing the dag
-    const decodedBlock = createUnsafe({ bytes, cid, codec })
-
-    for await (const nextCid of strategy.traverse(cid, decodedBlock)) {
-      void queue.add(async () => {
-        await this.#traverseDagNode({ cid: nextCid, queue, strategy, traversalContext, parentPath: currentPath ?? [], options })
-      })
-        .catch((err) => {
-          this.log.error('error during traversal queue operation - %e', err)
-        })
-    }
-  }
-
-  /**
-   * Use an ExportStrategy to export part of all of a DAG
-   */
-  async #exportDagNode ({ cid, queue, writer, strategy, options, recursive = true }: ExportWalkDagContext): Promise<void> {
-    if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
-      return
-    }
-
-    const codec = await this.components.getCodec(cid.code)
-    const bytes = await toBuffer(this.components.blockstore.get(cid, options))
-
-    // Mark as processed
-    options?.blockFilter?.add(cid.multihash.bytes)
+      for await (const { cid, bytes } of exportStrategy.export(current, this.components.blockstore, this.components.getCodec, options)) {
+        if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
+          continue
+        }
 
-    // Write to CAR
-    await writer.put({ cid, bytes })
+        // skip the export target block if we have already included it during
+        // traversal
+        if (underRoot && cid.equals(current)) {
+          continue
+        }
 
-    if (recursive) {
-      // we are recursively traversing the dag
-      const decodedBlock = createUnsafe({ bytes, cid, codec })
+        // mark as processed
+        options?.blockFilter?.add(cid.multihash.bytes)
 
-      for await (const nextCid of strategy.export(cid, decodedBlock)) {
-        void queue.add(async () => {
-          await this.#exportDagNode({ cid: nextCid, queue, writer, strategy, options })
+        // write to CAR
+        await writer.put({
+          cid,
+          bytes
         })
-          .catch((err) => {
-            this.log.error('error during export queue operation - %e', err)
-          })
       }
     }
+
+    // wait for the writer to end
+    await writer.close()
   }
 }

package/src/errors.ts

@@ -6,3 +6,13 @@ export class NotUnixFSError extends Error {
   message = 'Not a UnixFS node'
   name = 'NotUnixFSError'
 }
+
+export class NotDescendantError extends Error {
+  static name = 'NotDescendantError'
+  name = 'NotDescendantError'
+}
+
+export class InvalidTraversalError extends Error {
+  static name = 'InvalidTraversalError'
+  name = 'InvalidTraversalError'
+}

package/src/export-strategies/block-exporter.ts

@@ -1,13 +1,22 @@
+import toBuffer from 'it-to-buffer'
+import { createUnsafe } from 'multiformats/block'
 import type { ExportStrategy } from '../index.js'
-import type { BlockView } from 'multiformats/block/interface'
+import type { CodecLoader } from '@helia/interface'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { BlockView } from 'multiformats'
 import type { CID } from 'multiformats/cid'
 
 /**
  * Yields the first block from the first CID and stops
  */
 export class BlockExporter implements ExportStrategy {
-  async * export (cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
-    // don't yield the block, index.ts will add it to the car file and then
-    // we're done
+  async * export (cid: CID, blockstore: Blockstore, getCodec: CodecLoader, options?: AbortOptions): AsyncGenerator<BlockView<unknown, number, number, 0 | 1>, void, undefined> {
+    const bytes = await toBuffer(blockstore.get(cid, options))
+    yield createUnsafe({
+      cid,
+      bytes,
+      codec: await getCodec(cid.code)
+    })
   }
 }

package/src/export-strategies/subgraph-exporter.ts

@@ -1,5 +1,9 @@
+import { breadthFirstWalker } from '@helia/utils'
 import type { ExportStrategy } from '../index.js'
-import type { BlockView } from 'multiformats/block/interface'
+import type { CodecLoader } from '@helia/interface'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { BlockView } from 'multiformats'
 import type { CID } from 'multiformats/cid'
 
 /**
@@ -10,9 +14,14 @@ import type { CID } from 'multiformats/cid'
  * the helia config.
  */
 export class SubgraphExporter implements ExportStrategy {
-  async * export (_cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
-    for (const [, linkedCid] of block.links()) {
-      yield linkedCid
+  async * export (cid: CID, blockstore: Blockstore, getCodec: CodecLoader, options?: AbortOptions): AsyncGenerator<BlockView<unknown, number, number, 0 | 1>, void, undefined> {
+    const walker = breadthFirstWalker({
+      blockstore,
+      getCodec
+    })
+
+    for await (const node of walker.walk(cid, options)) {
+      yield node.block
     }
   }
 }

package/src/export-strategies/unixfs-exporter.ts

@@ -1,22 +1,33 @@
-import { DAG_PB_CODEC_CODE, RAW_PB_CODEC_CODE } from '../constants.js'
-import { NotUnixFSError } from '../errors.js'
+import { depthFirstWalker } from '@helia/utils'
+import { DAG_PB_CODEC_CODE, RAW_PB_CODEC_CODE } from '../constants.ts'
+import { NotUnixFSError } from '../errors.ts'
 import type { ExportStrategy } from '../index.js'
-import type { BlockView } from 'multiformats/block/interface'
+import type { CodecLoader } from '@helia/interface'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { BlockView } from 'multiformats'
 import type { CID } from 'multiformats/cid'
 
 /**
- * This exporter is used when you want to generate a car file that contains a
- * single UnixFS file or directory
+ * Traverses the DAG depth-first starting at the target CID and yields all
+ * encountered blocks.
+ *
+ * Blocks linked to from the target block are traversed using codecs defined in
+ * the helia config.
  */
 export class UnixFSExporter implements ExportStrategy {
-  async * export (cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+  async * export (cid: CID, blockstore: Blockstore, getCodec: CodecLoader, options?: AbortOptions): AsyncGenerator<BlockView<unknown, number, number, 0 | 1>, void, undefined> {
     if (cid.code !== DAG_PB_CODEC_CODE && cid.code !== RAW_PB_CODEC_CODE) {
       throw new NotUnixFSError('Target CID was not UnixFS - use the SubGraphExporter to export arbitrary graphs')
     }
 
-    // yield all the blocks that make up the file or directory
-    for (const [, linkedCid] of block.links()) {
-      yield linkedCid
+    const walker = depthFirstWalker({
+      blockstore,
+      getCodec
+    })
+
+    for await (const node of walker.walk(cid, options)) {
+      yield node.block
     }
   }
 }

package/src/index.ts

@@ -23,7 +23,9 @@
  * const c = car(helia)
  * const out = nodeFs.createWriteStream('example.car')
  *
- * for await (const buf of c.export(cid)) {
+ * for await (const buf of c.export(cid, {
+ *   signal: AbortSignal.timeout(5_000)
+ * })) {
  *   out.write(buf)
  * }
  *
@@ -32,6 +34,14 @@
  *
  * @example Exporting a part of a UnixFS DAG as a CAR file
  *
+ * Here the graph traversal will start at `root` and include the blocks for
+ * `root`, `/foo`, `/bar`, and all the blocks that make up `baz.txt`.
+ *
+ * If there are other files/directories in the UnixFS DAG under `root`, they
+ * will not be included.
+ *
+ * `root` will be the only entry in the CAR file roots.
+ *
  * ```typescript
  * import { createHelia } from 'helia'
  * import { car, UnixFSPath } from '@helia/car'
@@ -39,12 +49,13 @@
  * import nodeFs from 'node:fs'
  *
  * const helia = await createHelia()
- * const cid = CID.parse('QmFoo...')
+ * const root = CID.parse('QmFoo...')
  *
  * const c = car(helia)
  * const out = nodeFs.createWriteStream('example.car')
  *
- * for await (const buf of c.export(cid, {
+ * for await (const buf of c.export(root, {
+ *   signal: AbortSignal.timeout(5_000),
  *   traversal: new UnixFSPath('/foo/bar/baz.txt')
  * })) {
  *   out.write(buf)
@@ -53,6 +64,43 @@
  * out.end()
  * ```
  *
+ * @example Including traversal path above the root in a CAR
+ *
+ * The `includeTraversalBlocks` option will include the traversal blocks in the
+ * CAR when they would otherwise be excluded (for example when the traversal
+ * starts in a parent of the export root).
+ *
+ * Here `baz` is the CID for `baz.txt`.
+ *
+ * The CAR file will include the blocks for `parent`, `/foo`, `/bar`, and
+ * `/baz.txt`.
+ *
+ * `baz` will be the only entry in the CAR file roots.
+ *
+ * ```typescript
+ * import { createHelia } from 'helia'
+ * import { car, UnixFSPath } from '@helia/car'
+ * import { CID } from 'multiformats/cid'
+ * import nodeFs from 'node:fs'
+ *
+ * const helia = await createHelia()
+ * const parent = CID.parse('QmFoo...')
+ * const baz = CID.parse('QmBar...')
+ *
+ * const c = car(helia)
+ * const out = nodeFs.createWriteStream('example.car')
+ *
+ * for await (const buf of c.export(baz, {
+ *   signal: AbortSignal.timeout(5_000),
+ *   traversal: new UnixFSPath(parent, '/foo/bar/baz.txt'),
+ *   includeTraversalBlocks: true
+ * })) {
+ *   out.write(buf)
+ * }
+ *
+ * out.end()
+ * ```
+ *
  * @example Importing all blocks from a CAR file
  *
  * ```typescript
@@ -72,7 +120,9 @@
  * const reader = await CarReader.fromIterable(inStream)
  *
  * const c = car(helia)
- * await c.import(reader)
+ * await c.import(reader, {
+ *   signal: AbortSignal.timeout(5_000)
+ * })
  * ```
  */
 
@@ -102,13 +152,7 @@ export interface TraversalStrategy {
   /**
    * Traverse the DAG and yield the next CID to traverse
    */
-  traverse<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>
-
-  /**
-   * Returns true if the current CID is the target and we should switch to the
-   * export strategy
-   */
-  isTarget(cid: CID): boolean
+  traverse(root: CID, blockstore: Blockstore, getCodec: CodecLoader, options?: AbortOptions): AsyncGenerator<BlockView<unknown, number, number, 0 | 1>, void, undefined>
 }
 
 /**
@@ -121,14 +165,13 @@ export interface ExportStrategy {
   /**
    * Export the DAG and yield the next CID to traverse
    */
-  export<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>
+  export(cid: CID, blockstore: Blockstore, getCodec: CodecLoader, options?: AbortOptions): AsyncGenerator<BlockView<unknown, number, number, 0 | 1>, void, undefined>
 }
 
 export * from './export-strategies/index.js'
 export * from './traversal-strategies/index.js'
 
 export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents>, ProviderOptions {
-
   /**
    * If true, the blockstore will not do any network requests.
    *
@@ -153,6 +196,14 @@ export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlock
    * blocks included in the exported car file. (e.g. https://specs.ipfs.tech/http-gateways/trustless-gateway/#dag-scope-request-query-parameter)
    */
   exporter?: ExportStrategy
+
+  /**
+   * If `true`, and the traversal strategy starts above the root, include the
+   * traversed blocks in the CAR file before the root and subsequent blocks
+   *
+   * @default false
+   */
+  includeTraversalBlocks?: boolean
 }
 
 /**
@@ -208,6 +259,6 @@ export interface Car {
 /**
  * Create a {@link Car} instance for use with {@link https://github.com/ipfs/helia Helia}
  */
-export function car (helia: CarComponents, init: any = {}): Car {
-  return new CarClass(helia, init)
+export function car (helia: CarComponents): Car {
+  return new CarClass(helia)
 }

package/src/traversal-strategies/cid-path.ts

@@ -1,29 +1,59 @@
+import toBuffer from 'it-to-buffer'
+import { createUnsafe } from 'multiformats/block'
+import { InvalidTraversalError, NotDescendantError } from '../errors.js'
 import type { TraversalStrategy } from '../index.js'
-import type { BlockView } from 'multiformats/block/interface'
+import type { CodecLoader } from '@helia/interface'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { BlockView } from 'multiformats'
 import type { CID } from 'multiformats/cid'
 
 /**
  * Simple strategy that traverses a known path to a target CID.
  *
  * All this strategy does is yield the next CID in the known path.
+ *
+ * The path should end with the CID to be exported
  */
 export class CIDPath implements TraversalStrategy {
-  private readonly pathToTarget: CID[]
-  private readonly target: CID
+  private readonly path: CID[]
 
-  constructor (pathToTarget: CID[]) {
-    this.pathToTarget = pathToTarget
-    this.target = pathToTarget[pathToTarget.length - 1]
+  constructor (path: CID[]) {
+    this.path = path
   }
 
-  isTarget (cid: CID): boolean {
-    return this.target.equals(cid)
-  }
+  async * traverse (root: CID, blockstore: Blockstore, getCodec: CodecLoader, options?: AbortOptions): AsyncGenerator<BlockView<unknown, number, number, 0 | 1>, void, undefined> {
+    if (!this.path.some(c => c.equals(root))) {
+      throw new InvalidTraversalError(`CIDPath traversal must include ${root}`)
+    }
+
+    let parentBlock: BlockView<unknown, number, number, 0 | 1> | undefined
+
+    for (const cid of this.path) {
+      if (parentBlock != null) {
+        let isChild = false
+
+        for (const [, child] of parentBlock.links()) {
+          if (child.equals(cid)) {
+            isChild = true
+            break
+          }
+        }
+
+        if (!isChild) {
+          throw new NotDescendantError(`${cid} is not a child of ${parentBlock.cid}`)
+        }
+      }
 
-  async * traverse <T extends BlockView<any, any, any, 0 | 1>>(cid: CID, _block?: T): AsyncGenerator<CID, void, undefined> {
-    const givenCidIndex = this.pathToTarget.indexOf(cid)
-    const nextCid = this.pathToTarget[givenCidIndex + 1]
+      const bytes = await toBuffer(blockstore.get(cid, options))
+      const block = createUnsafe({
+        cid,
+        bytes,
+        codec: await getCodec(cid.code)
+      })
 
-    yield nextCid
+      parentBlock = block
+      yield block
+    }
   }
 }