@helia/car 4.0.3 → 4.0.4-bb2ab74

package/src/export-strategies/subgraph-exporter.ts

@@ -0,0 +1,18 @@
+import { type CID } from 'multiformats/cid'
+import { type ExportStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+
+/**
+ * Traverses the DAG breadth-first starting at the target CID and yields all
+ * encountered blocks.
+ *
+ * Blocks linked to from the target block are traversed using codecs defined in
+ * the helia config.
+ */
+export class SubgraphExporter implements ExportStrategy {
+  async * export (_cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+    for await (const [, linkedCid] of block.links()) {
+      yield linkedCid
+    }
+  }
+}

package/src/export-strategies/unixfs-exporter.ts

@@ -0,0 +1,22 @@
+import { DAG_PB_CODEC_CODE, RAW_PB_CODEC_CODE } from '../constants.js'
+import { NotUnixFSError } from '../errors.js'
+import type { ExportStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * This exporter is used when you want to generate a car file that contains a
+ * single UnixFS file or directory
+ */
+export class UnixFSExporter implements ExportStrategy {
+  async * export (cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+    if (cid.code !== DAG_PB_CODEC_CODE && cid.code !== RAW_PB_CODEC_CODE) {
+      throw new NotUnixFSError('Target CID was not UnixFS - use the SubGraphExporter to export arbitrary graphs')
+    }
+
+    // yield all the blocks that make up the file or directory
+    for await (const [, linkedCid] of block.links()) {
+      yield linkedCid
+    }
+  }
+}

package/src/index.ts

@@ -1,39 +1,56 @@
 /**
  * @packageDocumentation
  *
- * `@helia/car` provides `import` and `export` methods to read/write Car files to {@link https://github.com/ipfs/helia Helia}'s blockstore.
+ * `@helia/car` provides `import` and `export` methods to read/write Car files
+ * to {@link https://github.com/ipfs/helia Helia}'s blockstore.
  *
  * See the {@link Car} interface for all available operations.
  *
- * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more esoteric DAG walkers can be passed as an init option.
+ * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more
+ * esoteric DAG walkers can be passed as an init option.
  *
  * @example Exporting a DAG as a CAR file
  *
  * ```typescript
  * import { createHelia } from 'helia'
- * import { unixfs } from '@helia/unixfs'
  * import { car } from '@helia/car'
- * import { CarWriter } from '@ipld/car'
- * import { Readable } from 'node:stream'
+ * import { CID } from 'multiformats/cid'
  * import nodeFs from 'node:fs'
  *
- * const helia = await createHelia({
- *   // ... helia config
- * })
- * const fs = unixfs(helia)
+ * const helia = await createHelia()
+ * const cid = CID.parse('QmFoo...')
  *
- * // add some UnixFS data
- * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4]))
+ * const c = car(helia)
+ * const out = nodeFs.createWriteStream('example.car')
+ *
+ * for await (const buf of c.stream(cid)) {
+ *   out.write(buf)
+ * }
+ *
+ * out.end()
+ * ```
+ *
+ * @example Exporting a part of a UnixFS DAG as a CAR file
+ *
+ * ```typescript
+ * import { createHelia } from 'helia'
+ * import { car, UnixFSPath } from '@helia/car'
+ * import { CID } from 'multiformats/cid'
+ * import nodeFs from 'node:fs'
+ *
+ * const helia = await createHelia()
+ * const cid = CID.parse('QmFoo...')
  *
- * // export it as a Car
  * const c = car(helia)
- * const { writer, out } = await CarWriter.create(cid)
+ * const out = nodeFs.createWriteStream('example.car')
  *
- * // `out` needs to be directed somewhere, see the @ipld/car docs for more information
- * Readable.from(out).pipe(nodeFs.createWriteStream('example.car'))
+ * for await (const buf of c.stream(cid, {
+ *   traversal: new UnixFSPath('/foo/bar/baz.txt')
+ * })) {
+ *   out.write(buf)
+ * }
  *
- * // write the DAG behind `cid` into the writer
- * await c.export(cid, writer)
+ * out.end()
  * ```
  *
  * @example Importing all blocks from a CAR file
@@ -59,31 +76,83 @@
  * ```
  */
 
-import { CarWriter } from '@ipld/car'
-import drain from 'it-drain'
-import map from 'it-map'
-import { createUnsafe } from 'multiformats/block'
-import defer from 'p-defer'
-import PQueue from 'p-queue'
+import { Car as CarClass } from './car.js'
 import type { CodecLoader } from '@helia/interface'
-import type { GetBlockProgressEvents, PutManyBlocksProgressEvents } from '@helia/interface/blocks'
-import type { CarReader } from '@ipld/car'
-import type { AbortOptions } from '@libp2p/interface'
+import type { PutManyBlocksProgressEvents, GetBlockProgressEvents } from '@helia/interface/blocks'
+import type { CarWriter, CarReader } from '@ipld/car'
+import type { AbortOptions, ComponentLogger } from '@libp2p/interface'
 import type { Filter } from '@libp2p/utils/filters'
 import type { Blockstore } from 'interface-blockstore'
+import type { BlockView } from 'multiformats/block/interface'
 import type { CID } from 'multiformats/cid'
 import type { ProgressOptions } from 'progress-events'
 
 export interface CarComponents {
+  logger: ComponentLogger
   blockstore: Blockstore
   getCodec: CodecLoader
 }
 
+/**
+ * Interface for different traversal strategies.
+ *
+ * While traversing the DAG, it will yield blocks that it has traversed.
+ */
+export interface TraversalStrategy {
+  /**
+   * Traverse the DAG and yield the next CID to traverse
+   */
+  traverse<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>
+
+  /**
+   * Returns true if the current CID is the target and we should switch to the
+   * export strategy
+   */
+  isTarget(cid: CID): boolean
+}
+
+/**
+ * Interface for different export strategies.
+ *
+ * When traversal has ended the export begins starting at the target CID, and
+ * the export strategy may do further traversal and writing to the car file.
+ */
+export interface ExportStrategy {
+  /**
+   * Export the DAG and yield the next CID to traverse
+   */
+  export<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>
+}
+
+export * from './export-strategies/index.js'
+export * from './traversal-strategies/index.js'
+
 export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents> {
+
+  /**
+   * If true, the blockstore will not do any network requests.
+   *
+   * @default false
+   */
+  offline?: boolean
+
   /**
-   * If a filter is passed it will be used to deduplicate blocks exported in the car file
+   * If a filter is passed it will be used to deduplicate blocks exported in the
+   * car file
    */
   blockFilter?: Filter
+
+  /**
+   * The traversal strategy to use for the export. This determines how the dag
+   * is traversed: either depth first, breadth first, or a custom strategy.
+   */
+  traversal?: TraversalStrategy
+
+  /**
+   * Export strategy to use for the export. This should be used to change the
+   * blocks included in the exported car file. (e.g. https://specs.ipfs.tech/http-gateways/trustless-gateway/#dag-scope-request-query-parameter)
+   */
+  exporter?: ExportStrategy
 }
 
 /**
@@ -152,7 +221,7 @@ export interface Car {
    *
    * ```typescript
    * import { createHelia } from 'helia'
-   * import { car } from '@helia/car
+   * import { car } from '@helia/car'
    * import { CID } from 'multiformats/cid'
    *
    * const helia = await createHelia()
@@ -165,101 +234,12 @@ export interface Car {
    * }
    * ```
    */
-  stream(root: CID | CID[], options?: AbortOptions & ProgressOptions<GetBlockProgressEvents>): AsyncGenerator<Uint8Array>
-}
-
-const DAG_WALK_QUEUE_CONCURRENCY = 1
-
-class DefaultCar implements Car {
-  private readonly components: CarComponents
-
-  constructor (components: CarComponents, init: any) {
-    this.components = components
-  }
-
-  async import (reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void> {
-    await drain(this.components.blockstore.putMany(
-      map(reader.blocks(), ({ cid, bytes }) => ({ cid, block: bytes })),
-      options
-    ))
-  }
-
-  async export (root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void> {
-    const deferred = defer<Error | undefined>()
-    const roots = Array.isArray(root) ? root : [root]
-
-    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
-    const queue = new PQueue({
-      concurrency: DAG_WALK_QUEUE_CONCURRENCY
-    })
-    queue.on('idle', () => {
-      deferred.resolve()
-    })
-    queue.on('error', (err) => {
-      queue.clear()
-      deferred.reject(err)
-    })
-
-    for (const root of roots) {
-      void queue.add(async () => {
-        await this.#walkDag(root, queue, async (cid, bytes) => {
-          // if a filter has been passed, skip blocks that have already been written
-          if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
-            return
-          }
-
-          options?.blockFilter?.add(cid.multihash.bytes)
-          await writer.put({ cid, bytes })
-        }, options)
-      })
-        .catch(() => {})
-    }
-
-    // wait for the writer to end
-    try {
-      await deferred.promise
-    } finally {
-      await writer.close()
-    }
-  }
-
-  async * stream (root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined> {
-    const { writer, out } = CarWriter.create(root)
-
-    // has to be done async so we write to `writer` and read from `out` at the
-    // same time
-    this.export(root, writer, options)
-      .catch(() => {})
-
-    for await (const buf of out) {
-      yield buf
-    }
-  }
-
-  /**
-   * Walk the DAG behind the passed CID, ensure all blocks are present in the blockstore
-   * and update the pin count for them
-   */
-  async #walkDag (cid: CID, queue: PQueue, withBlock: (cid: CID, block: Uint8Array) => Promise<void>, options?: AbortOptions & ProgressOptions<GetBlockProgressEvents>): Promise<void> {
-    const codec = await this.components.getCodec(cid.code)
-    const bytes = await this.components.blockstore.get(cid, options)
-
-    await withBlock(cid, bytes)
-
-    const block = createUnsafe({ bytes, cid, codec })
-
-    // walk dag, ensure all blocks are present
-    for await (const [,cid] of block.links()) {
-      void queue.add(async () => {
-        await this.#walkDag(cid, queue, withBlock, options)
-      })
-    }
-  }
+  stream(root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined>
 }
 
 /**
  * Create a {@link Car} instance for use with {@link https://github.com/ipfs/helia Helia}
  */
 export function car (helia: CarComponents, init: any = {}): Car {
-  return new DefaultCar(helia, init)
+  return new CarClass(helia, init)
 }

package/src/traversal-strategies/cid-path.ts

@@ -0,0 +1,29 @@
+import type { TraversalStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * Simple strategy that traverses a known path to a target CID.
+ *
+ * All this strategy does is yield the next CID in the known path.
+ */
+export class CIDPath implements TraversalStrategy {
+  private readonly pathToTarget: CID[]
+  private readonly target: CID
+
+  constructor (pathToTarget: CID[]) {
+    this.pathToTarget = pathToTarget
+    this.target = pathToTarget[pathToTarget.length - 1]
+  }
+
+  isTarget (cid: CID): boolean {
+    return this.target.equals(cid)
+  }
+
+  async * traverse <T extends BlockView<any, any, any, 0 | 1>>(cid: CID, _block?: T): AsyncGenerator<CID, void, undefined> {
+    const givenCidIndex = this.pathToTarget.indexOf(cid)
+    const nextCid = this.pathToTarget[givenCidIndex + 1]
+
+    yield nextCid
+  }
+}

package/src/traversal-strategies/graph-search.ts

@@ -0,0 +1,25 @@
+import type { TraversalStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * A traversal strategy that performs a breadth-first search (so as to not load blocks unnecessarily) looking for a
+ * target CID. Traversal stops when we reach the target CID or run out of nodes.
+ */
+export class GraphSearch implements TraversalStrategy {
+  private readonly target: CID
+
+  constructor (target: CID) {
+    this.target = target
+  }
+
+  isTarget (cid: CID): boolean {
+    return this.target.equals(cid)
+  }
+
+  async * traverse <T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined> {
+    for await (const [, linkedCid] of block.links()) {
+      yield linkedCid
+    }
+  }
+}

package/src/traversal-strategies/index.ts

@@ -0,0 +1,3 @@
+export * from './graph-search.js'
+export * from './cid-path.js'
+export * from './unixfs-path.js'

package/src/traversal-strategies/unixfs-path.ts

@@ -0,0 +1,58 @@
+import { decode } from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import { DAG_PB_CODEC_CODE } from '../constants.js'
+import { NotUnixFSError } from '../errors.js'
+import type { TraversalStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * Traverses a DAG containing UnixFS directories
+ */
+export class UnixFSPath implements TraversalStrategy {
+  private readonly path: string[]
+
+  constructor (path: string) {
+    // "/foo/bar/baz.txt" -> ['foo', 'bar', 'baz.txt']
+    this.path = path.replace(/^\//, '').split('/')
+  }
+
+  isTarget (): boolean {
+    return this.path.length === 0
+  }
+
+  async * traverse <T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined> {
+    if (cid.code !== DAG_PB_CODEC_CODE) {
+      throw new NotUnixFSError('Target CID is not UnixFS')
+    }
+
+    const segment = this.path.shift()
+
+    if (segment == null) {
+      return
+    }
+
+    const pb = decode(block.bytes)
+
+    if (pb.Data == null) {
+      throw new NotUnixFSError('Target CID has no UnixFS data in decoded bytes')
+    }
+
+    const unixfs = UnixFS.unmarshal(pb.Data)
+
+    if (unixfs.type === 'directory') {
+      const link = pb.Links.filter(link => link.Name === segment).pop()
+
+      if (link == null) {
+        throw new NotUnixFSError(`Target CID directory has no link with name ${segment}`)
+      }
+
+      yield link.Hash
+      return
+    }
+
+    // TODO: HAMT support
+
+    throw new NotUnixFSError('Target CID is not a UnixFS directory')
+  }
+}

package/dist/typedoc-urls.json

@@ -1,10 +0,0 @@
-{
-  "Car": "https://ipfs.github.io/helia/interfaces/_helia_car.Car.html",
-  ".:Car": "https://ipfs.github.io/helia/interfaces/_helia_car.Car.html",
-  "CarComponents": "https://ipfs.github.io/helia/interfaces/_helia_car.CarComponents.html",
-  ".:CarComponents": "https://ipfs.github.io/helia/interfaces/_helia_car.CarComponents.html",
-  "ExportCarOptions": "https://ipfs.github.io/helia/interfaces/_helia_car.ExportCarOptions.html",
-  ".:ExportCarOptions": "https://ipfs.github.io/helia/interfaces/_helia_car.ExportCarOptions.html",
-  "car": "https://ipfs.github.io/helia/functions/_helia_car.car-1.html",
-  ".:car": "https://ipfs.github.io/helia/functions/_helia_car.car-1.html"
-}