@helia/car 4.0.3-c0bf36e → 4.0.4-313e2c1

package/dist/src/traversal-strategies/graph-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-search.js","sourceRoot":"","sources":["../../../src/traversal-strategies/graph-search.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,MAAM,OAAO,WAAW;IACL,MAAM,CAAK;IAE5B,YAAa,MAAW;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,QAAQ,CAAE,GAAQ;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,CAAE,QAAQ,CAA6C,GAAQ,EAAE,KAAQ;QAC7E,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChD,MAAM,SAAS,CAAA;QACjB,CAAC;IACH,CAAC;CACF"}

package/dist/src/traversal-strategies/index.d.ts

@@ -0,0 +1,4 @@
+export * from './graph-search.js';
+export * from './cid-path.js';
+export * from './unixfs-path.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/traversal-strategies/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/traversal-strategies/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA"}

package/dist/src/traversal-strategies/index.js

@@ -0,0 +1,4 @@
+export * from './graph-search.js';
+export * from './cid-path.js';
+export * from './unixfs-path.js';
+//# sourceMappingURL=index.js.map

package/dist/src/traversal-strategies/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/traversal-strategies/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA"}

package/dist/src/traversal-strategies/unixfs-path.d.ts

@@ -0,0 +1,13 @@
+import type { TraversalStrategy } from '../index.js';
+import type { BlockView } from 'multiformats/block/interface';
+import type { CID } from 'multiformats/cid';
+/**
+ * Traverses a DAG containing UnixFS directories
+ */
+export declare class UnixFSPath implements TraversalStrategy {
+    private readonly path;
+    constructor(path: string);
+    isTarget(): boolean;
+    traverse<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>;
+}
+//# sourceMappingURL=unixfs-path.d.ts.map

package/dist/src/traversal-strategies/unixfs-path.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unixfs-path.d.ts","sourceRoot":"","sources":["../../../src/traversal-strategies/unixfs-path.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAE3C;;GAEG;AACH,qBAAa,UAAW,YAAW,iBAAiB;IAClD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAU;gBAElB,IAAI,EAAE,MAAM;IAKzB,QAAQ,IAAK,OAAO;IAIZ,QAAQ,CAAE,CAAC,SAAS,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC;CAkCvH"}

package/dist/src/traversal-strategies/unixfs-path.js

@@ -0,0 +1,42 @@
+import { decode } from '@ipld/dag-pb';
+import { UnixFS } from 'ipfs-unixfs';
+import { DAG_PB_CODEC_CODE } from '../constants.js';
+import { NotUnixFSError } from '../errors.js';
+/**
+ * Traverses a DAG containing UnixFS directories
+ */
+export class UnixFSPath {
+    path;
+    constructor(path) {
+        // "/foo/bar/baz.txt" -> ['foo', 'bar', 'baz.txt']
+        this.path = path.replace(/^\//, '').split('/');
+    }
+    isTarget() {
+        return this.path.length === 0;
+    }
+    async *traverse(cid, block) {
+        if (cid.code !== DAG_PB_CODEC_CODE) {
+            throw new NotUnixFSError('Target CID is not UnixFS');
+        }
+        const segment = this.path.shift();
+        if (segment == null) {
+            return;
+        }
+        const pb = decode(block.bytes);
+        if (pb.Data == null) {
+            throw new NotUnixFSError('Target CID has no UnixFS data in decoded bytes');
+        }
+        const unixfs = UnixFS.unmarshal(pb.Data);
+        if (unixfs.type === 'directory') {
+            const link = pb.Links.filter(link => link.Name === segment).pop();
+            if (link == null) {
+                throw new NotUnixFSError(`Target CID directory has no link with name ${segment}`);
+            }
+            yield link.Hash;
+            return;
+        }
+        // TODO: HAMT support
+        throw new NotUnixFSError('Target CID is not a UnixFS directory');
+    }
+}
+//# sourceMappingURL=unixfs-path.js.map

package/dist/src/traversal-strategies/unixfs-path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unixfs-path.js","sourceRoot":"","sources":["../../../src/traversal-strategies/unixfs-path.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAK7C;;GAEG;AACH,MAAM,OAAO,UAAU;IACJ,IAAI,CAAU;IAE/B,YAAa,IAAY;QACvB,kDAAkD;QAClD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,CAAE,QAAQ,CAA6C,GAAQ,EAAE,KAAQ;QAC7E,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,cAAc,CAAC,0BAA0B,CAAC,CAAA;QACtD,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;QAEjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAM;QACR,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,EAAE,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,IAAI,cAAc,CAAC,gDAAgD,CAAC,CAAA;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAExC,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,GAAG,EAAE,CAAA;YAEjE,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,cAAc,CAAC,8CAA8C,OAAO,EAAE,CAAC,CAAA;YACnF,CAAC;YAED,MAAM,IAAI,CAAC,IAAI,CAAA;YACf,OAAM;QACR,CAAC;QAED,qBAAqB;QAErB,MAAM,IAAI,cAAc,CAAC,sCAAsC,CAAC,CAAA;IAClE,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/car",
-  "version": "4.0.3-c0bf36e",
+  "version": "4.0.4-313e2c1",
   "description": "Import/export car files from Helia",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/car#readme",
@@ -54,11 +54,13 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/interface": "5.2.1-c0bf36e",
+    "@helia/interface": "5.2.1-313e2c1",
     "@ipld/car": "^5.3.3",
+    "@ipld/dag-pb": "^4.1.3",
     "@libp2p/interface": "^2.2.1",
     "@libp2p/utils": "^6.2.1",
     "interface-blockstore": "^5.3.1",
+    "ipfs-unixfs": "^11.2.1",
     "it-drain": "^3.0.7",
     "it-map": "^3.1.1",
     "multiformats": "^13.3.1",
@@ -67,14 +69,18 @@
     "progress-events": "^1.0.1"
   },
   "devDependencies": {
-    "@helia/mfs": "4.0.3-c0bf36e",
-    "@helia/unixfs": "4.0.3-c0bf36e",
-    "@ipld/dag-pb": "^4.1.3",
+    "@helia/mfs": "5.0.0-313e2c1",
+    "@helia/unixfs": "5.0.0-313e2c1",
+    "@ipld/dag-cbor": "^9.2.2",
+    "@libp2p/logger": "^5.1.15",
     "aegir": "^45.1.1",
     "blockstore-core": "^5.0.2",
     "datastore-core": "^10.0.2",
     "ipfs-unixfs-importer": "^15.3.1",
-    "it-to-buffer": "^4.0.7"
+    "it-foreach": "^2.1.2",
+    "it-length": "^3.0.7",
+    "it-to-buffer": "^4.0.7",
+    "sinon": "^19.0.2"
   },
   "sideEffects": false
 }

package/src/car.ts

@@ -0,0 +1,218 @@
+import { CarWriter } from '@ipld/car'
+import drain from 'it-drain'
+import map from 'it-map'
+import { createUnsafe } from 'multiformats/block'
+import { type CID } from 'multiformats/cid'
+import defer from 'p-defer'
+import PQueue from 'p-queue'
+import { DAG_WALK_QUEUE_CONCURRENCY } from './constants.js'
+import { SubgraphExporter } from './export-strategies/subgraph-exporter.js'
+import { GraphSearch } from './traversal-strategies/graph-search.js'
+import type { CarComponents, Car as CarInterface, ExportCarOptions, ExportStrategy, TraversalStrategy } from './index.js'
+import type { PutManyBlocksProgressEvents } from '@helia/interface/blocks'
+import type { CarReader } from '@ipld/car'
+import type { AbortOptions, Logger } from '@libp2p/interface'
+import type { ProgressOptions } from 'progress-events'
+
+/**
+ * Context for the traversal process.
+ */
+interface TraversalContext {
+  currentPath: CID[]
+  pathsToTarget: CID[][] | null // collect all target paths
+}
+
+interface WalkDagContext<Strategy> {
+  cid: CID
+  queue: PQueue
+  strategy: Strategy
+  options?: ExportCarOptions
+}
+
+interface ExportWalkDagContext extends WalkDagContext<ExportStrategy> {
+  writer: Pick<CarWriter, 'put'>
+  recursive?: boolean
+}
+
+interface TraversalWalkDagContext extends WalkDagContext<TraversalStrategy> {
+  traversalContext: TraversalContext
+  parentPath: CID[]
+}
+
+export class Car implements CarInterface {
+  private readonly components: CarComponents
+  private readonly log: Logger
+
+  constructor (components: CarComponents, init: any) {
+    this.components = components
+    this.log = components.logger.forComponent('helia:car')
+  }
+
+  async import (reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void> {
+    await drain(this.components.blockstore.putMany(
+      map(reader.blocks(), ({ cid, bytes }) => ({ cid, block: bytes })),
+      options
+    ))
+  }
+
+  async export (root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void> {
+    const deferred = defer<Error | undefined>()
+    const roots = Array.isArray(root) ? root : [root]
+
+    // Create traversal-specific context
+    const traversalContext: TraversalContext = {
+      currentPath: [],
+      pathsToTarget: null
+    }
+
+    const traversalStrategy = options?.traversal
+    const exportStrategy = options?.exporter ?? new SubgraphExporter()
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very
+    // large DAGs
+    const queue = new PQueue({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+
+    let startedExport = false
+    queue.on('idle', () => {
+      if (startedExport) {
+        // idle event was called, and started exporting, so we are done.
+        deferred.resolve()
+      } else if (!startedExport && traversalContext.pathsToTarget?.length === roots.length) {
+        // queue is idle, we haven't started exporting yet, and we have path(s)
+        // to the target(s), so we can start the export process.
+        this.log.trace('starting export of blocks to the car file')
+        startedExport = true
+
+        for (const path of traversalContext.pathsToTarget) {
+          const targetIndex = path.length - 1
+          const targetCid = path[targetIndex]
+
+          // Process all verification blocks in the path except the target
+          path.slice(0, -1).forEach(cid => {
+            void queue.add(async () => {
+              await this.#exportDagNode({ cid, queue, writer, strategy: exportStrategy, options, recursive: false })
+            })
+              .catch((err) => {
+                this.log.error('error during queue operation - %e', err)
+              })
+          })
+
+          // Process the target block (which will recursively export its DAG)
+          void queue.add(async () => {
+            await this.#exportDagNode({ cid: targetCid, queue, writer, strategy: exportStrategy, options })
+          })
+            .catch((err) => {
+              this.log.error('error during queue operation - %e', err)
+            })
+        }
+      } else {
+        // queue is idle, we haven't started exporting yet, and we don't have
+        // path(s) to the target(s), so we can't start the export process.
+        // this should not happen without a separate error during traversal, but
+        // we'll handle it here anyway.
+        this.log.trace('no paths to target, skipping export')
+        deferred.reject(new Error('Could not traverse to target CID(s)'))
+      }
+    })
+    queue.on('error', (err) => {
+      queue.clear()
+      deferred.reject(err)
+    })
+
+    for (const root of roots) {
+      void queue.add(async () => {
+        this.log.trace('traversing dag from %c', root)
+        await this.#traverseDagNode({ cid: root, queue, strategy: traversalStrategy ?? new GraphSearch(root), traversalContext, parentPath: [], options })
+      })
+        .catch((err) => {
+          this.log.error('error during queue operation - %e', err)
+        })
+    }
+
+    // wait for the writer to end
+    try {
+      await deferred.promise
+    } finally {
+      await writer.close()
+    }
+  }
+
+  async * stream (root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined> {
+    const { writer, out } = CarWriter.create(root)
+
+    // has to be done async so we write to `writer` and read from `out` at the
+    // same time
+    this.export(root, writer, options)
+      .catch((err) => {
+        this.log.error('error during streaming export - %e', err)
+      })
+
+    for await (const buf of out) {
+      yield buf
+    }
+  }
+
+  /**
+   * Traverse a DAG and stop when we reach the target node
+   */
+  async #traverseDagNode ({ cid, queue, strategy, traversalContext, parentPath, options }: TraversalWalkDagContext): Promise<void> {
+    // if we are traversing, we need to gather path(s) to the target(s)
+    const currentPath = [...parentPath, cid]
+
+    if (strategy.isTarget(cid)) {
+      traversalContext.pathsToTarget = traversalContext.pathsToTarget ?? []
+      traversalContext.pathsToTarget.push([...currentPath])
+      this.log.trace('found path to target %c', cid)
+      return
+    }
+
+    const codec = await this.components.getCodec(cid.code)
+    const bytes = await this.components.blockstore.get(cid, options)
+
+    // we are recursively traversing the dag
+    const decodedBlock = createUnsafe({ bytes, cid, codec })
+
+    for await (const nextCid of strategy.traverse(cid, decodedBlock)) {
+      void queue.add(async () => {
+        await this.#traverseDagNode({ cid: nextCid, queue, strategy, traversalContext, parentPath: currentPath ?? [], options })
+      })
+        .catch((err) => {
+          this.log.error('error during traversal queue operation - %e', err)
+        })
+    }
+  }
+
+  /**
+   * Use an ExportStrategy to export part of all of a DAG
+   */
+  async #exportDagNode ({ cid, queue, writer, strategy, options, recursive = true }: ExportWalkDagContext): Promise<void> {
+    if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
+      return
+    }
+
+    const codec = await this.components.getCodec(cid.code)
+    const bytes = await this.components.blockstore.get(cid, options)
+
+    // Mark as processed
+    options?.blockFilter?.add(cid.multihash.bytes)
+
+    // Write to CAR
+    await writer.put({ cid, bytes })
+
+    if (recursive) {
+      // we are recursively traversing the dag
+      const decodedBlock = createUnsafe({ bytes, cid, codec })
+
+      for await (const nextCid of strategy.export(cid, decodedBlock)) {
+        void queue.add(async () => {
+          await this.#exportDagNode({ cid: nextCid, queue, writer, strategy, options })
+        })
+          .catch((err) => {
+            this.log.error('error during export queue operation - %e', err)
+          })
+      }
+    }
+  }
+}

package/src/constants.ts

@@ -0,0 +1,11 @@
+export const DAG_WALK_QUEUE_CONCURRENCY = 1
+
+/**
+ * The multicodec code for dag-pb binary data
+ */
+export const DAG_PB_CODEC_CODE = 0x70
+
+/**
+ * The multicodec code for raw binary data
+ */
+export const RAW_PB_CODEC_CODE = 0x55

package/src/errors.ts

@@ -0,0 +1,8 @@
+export class NotUnixFSError extends Error {
+  static code = 'ERR_NOT_UNIXFS'
+  static message = 'Not a UnixFS node'
+  static name = 'NotUnixFSError'
+  code = 'ERR_NOT_UNIXFS'
+  message = 'Not a UnixFS node'
+  name = 'NotUnixFSError'
+}

package/src/export-strategies/block-exporter.ts

@@ -0,0 +1,13 @@
+import { type CID } from 'multiformats/cid'
+import { type ExportStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+
+/**
+ * Yields the first block from the first CID and stops
+ */
+export class BlockExporter implements ExportStrategy {
+  async * export (cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+    // don't yield the block, index.ts will add it to the car file and then
+    // we're done
+  }
+}

package/src/export-strategies/index.ts

@@ -0,0 +1,3 @@
+export * from './subgraph-exporter.js'
+export * from './block-exporter.js'
+export * from './unixfs-exporter.js'

package/src/export-strategies/subgraph-exporter.ts

@@ -0,0 +1,18 @@
+import { type CID } from 'multiformats/cid'
+import { type ExportStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+
+/**
+ * Traverses the DAG breadth-first starting at the target CID and yields all
+ * encountered blocks.
+ *
+ * Blocks linked to from the target block are traversed using codecs defined in
+ * the helia config.
+ */
+export class SubgraphExporter implements ExportStrategy {
+  async * export (_cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+    for await (const [, linkedCid] of block.links()) {
+      yield linkedCid
+    }
+  }
+}

package/src/export-strategies/unixfs-exporter.ts

@@ -0,0 +1,22 @@
+import { DAG_PB_CODEC_CODE, RAW_PB_CODEC_CODE } from '../constants.js'
+import { NotUnixFSError } from '../errors.js'
+import type { ExportStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+import type { CID } from 'multiformats/cid'
+
+/**
+ * This exporter is used when you want to generate a car file that contains a
+ * single UnixFS file or directory
+ */
+export class UnixFSExporter implements ExportStrategy {
+  async * export (cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+    if (cid.code !== DAG_PB_CODEC_CODE && cid.code !== RAW_PB_CODEC_CODE) {
+      throw new NotUnixFSError('Target CID was not UnixFS - use the SubGraphExporter to export arbitrary graphs')
+    }
+
+    // yield all the blocks that make up the file or directory
+    for await (const [, linkedCid] of block.links()) {
+      yield linkedCid
+    }
+  }
+}