@helia/car 4.0.3 → 4.0.4-bb2ab74

package/dist/src/index.js

@@ -1,39 +1,56 @@
 /**
  * @packageDocumentation
  *
- * `@helia/car` provides `import` and `export` methods to read/write Car files to {@link https://github.com/ipfs/helia Helia}'s blockstore.
+ * `@helia/car` provides `import` and `export` methods to read/write Car files
+ * to {@link https://github.com/ipfs/helia Helia}'s blockstore.
  *
  * See the {@link Car} interface for all available operations.
  *
- * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more esoteric DAG walkers can be passed as an init option.
+ * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more
+ * esoteric DAG walkers can be passed as an init option.
  *
  * @example Exporting a DAG as a CAR file
  *
  * ```typescript
  * import { createHelia } from 'helia'
- * import { unixfs } from '@helia/unixfs'
  * import { car } from '@helia/car'
- * import { CarWriter } from '@ipld/car'
- * import { Readable } from 'node:stream'
+ * import { CID } from 'multiformats/cid'
  * import nodeFs from 'node:fs'
  *
- * const helia = await createHelia({
- *   // ... helia config
- * })
- * const fs = unixfs(helia)
+ * const helia = await createHelia()
+ * const cid = CID.parse('QmFoo...')
+ *
+ * const c = car(helia)
+ * const out = nodeFs.createWriteStream('example.car')
+ *
+ * for await (const buf of c.stream(cid)) {
+ *   out.write(buf)
+ * }
  *
- * // add some UnixFS data
- * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4]))
+ * out.end()
+ * ```
+ *
+ * @example Exporting a part of a UnixFS DAG as a CAR file
+ *
+ * ```typescript
+ * import { createHelia } from 'helia'
+ * import { car, UnixFSPath } from '@helia/car'
+ * import { CID } from 'multiformats/cid'
+ * import nodeFs from 'node:fs'
+ *
+ * const helia = await createHelia()
+ * const cid = CID.parse('QmFoo...')
  *
- * // export it as a Car
  * const c = car(helia)
- * const { writer, out } = await CarWriter.create(cid)
+ * const out = nodeFs.createWriteStream('example.car')
  *
- * // `out` needs to be directed somewhere, see the @ipld/car docs for more information
- * Readable.from(out).pipe(nodeFs.createWriteStream('example.car'))
+ * for await (const buf of c.stream(cid, {
+ *   traversal: new UnixFSPath('/foo/bar/baz.txt')
+ * })) {
+ *   out.write(buf)
+ * }
  *
- * // write the DAG behind `cid` into the writer
- * await c.export(cid, writer)
+ * out.end()
  * ```
  *
  * @example Importing all blocks from a CAR file
@@ -58,87 +75,13 @@
  * await c.import(reader)
  * ```
  */
-import { CarWriter } from '@ipld/car';
-import drain from 'it-drain';
-import map from 'it-map';
-import { createUnsafe } from 'multiformats/block';
-import defer from 'p-defer';
-import PQueue from 'p-queue';
-const DAG_WALK_QUEUE_CONCURRENCY = 1;
-class DefaultCar {
-    components;
-    constructor(components, init) {
-        this.components = components;
-    }
-    async import(reader, options) {
-        await drain(this.components.blockstore.putMany(map(reader.blocks(), ({ cid, bytes }) => ({ cid, block: bytes })), options));
-    }
-    async export(root, writer, options) {
-        const deferred = defer();
-        const roots = Array.isArray(root) ? root : [root];
-        // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
-        const queue = new PQueue({
-            concurrency: DAG_WALK_QUEUE_CONCURRENCY
-        });
-        queue.on('idle', () => {
-            deferred.resolve();
-        });
-        queue.on('error', (err) => {
-            queue.clear();
-            deferred.reject(err);
-        });
-        for (const root of roots) {
-            void queue.add(async () => {
-                await this.#walkDag(root, queue, async (cid, bytes) => {
-                    // if a filter has been passed, skip blocks that have already been written
-                    if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
-                        return;
-                    }
-                    options?.blockFilter?.add(cid.multihash.bytes);
-                    await writer.put({ cid, bytes });
-                }, options);
-            })
-                .catch(() => { });
-        }
-        // wait for the writer to end
-        try {
-            await deferred.promise;
-        }
-        finally {
-            await writer.close();
-        }
-    }
-    async *stream(root, options) {
-        const { writer, out } = CarWriter.create(root);
-        // has to be done async so we write to `writer` and read from `out` at the
-        // same time
-        this.export(root, writer, options)
-            .catch(() => { });
-        for await (const buf of out) {
-            yield buf;
-        }
-    }
-    /**
-     * Walk the DAG behind the passed CID, ensure all blocks are present in the blockstore
-     * and update the pin count for them
-     */
-    async #walkDag(cid, queue, withBlock, options) {
-        const codec = await this.components.getCodec(cid.code);
-        const bytes = await this.components.blockstore.get(cid, options);
-        await withBlock(cid, bytes);
-        const block = createUnsafe({ bytes, cid, codec });
-        // walk dag, ensure all blocks are present
-        for await (const [, cid] of block.links()) {
-            void queue.add(async () => {
-                await this.#walkDag(cid, queue, withBlock, options);
-            });
-        }
-    }
-}
+import { Car as CarClass } from './car.js';
+export * from './export-strategies/index.js';
+export * from './traversal-strategies/index.js';
 /**
  * Create a {@link Car} instance for use with {@link https://github.com/ipfs/helia Helia}
  */
 export function car(helia, init = {}) {
-    return new DefaultCar(helia, init);
+    return new CarClass(helia, init);
 }
 //# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,KAAK,MAAM,UAAU,CAAA;AAC5B,OAAO,GAAG,MAAM,QAAQ,CAAA;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,KAAK,MAAM,SAAS,CAAA;AAC3B,OAAO,MAAM,MAAM,SAAS,CAAA;AAwG5B,MAAM,0BAA0B,GAAG,CAAC,CAAA;AAEpC,MAAM,UAAU;IACG,UAAU,CAAe;IAE1C,YAAa,UAAyB,EAAE,IAAS;QAC/C,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,MAAM,CAAE,MAAiC,EAAE,OAAqE;QACpH,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO,CAC5C,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,EACjE,OAAO,CACR,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAE,IAAiB,EAAE,MAAwC,EAAE,OAA0B;QACnG,MAAM,QAAQ,GAAG,KAAK,EAAqB,CAAA;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEjD,sFAAsF;QACtF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC;YACvB,WAAW,EAAE,0BAA0B;SACxC,CAAC,CAAA;QACF,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,QAAQ,CAAC,OAAO,EAAE,CAAA;QACpB,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACtB,CAAC,CAAC,CAAA;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;oBACpD,0EAA0E;oBAC1E,IAAI,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;wBAC5D,OAAM;oBACR,CAAC;oBAED,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;oBAC9C,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;gBAClC,CAAC,EAAE,OAAO,CAAC,CAAA;YACb,CAAC,CAAC;iBACC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACpB,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,OAAO,CAAA;QACxB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACtB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAE,MAAM,CAAE,IAAiB,EAAE,OAA0B;QAC3D,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE9C,0EAA0E;QAC1E,YAAY;QACZ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC;aAC/B,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAElB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAA;QACX,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAE,GAAQ,EAAE,KAAa,EAAE,SAAyD,EAAE,OAAgE;QAClK,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QAEhE,MAAM,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAE3B,MAAM,KAAK,GAAG,YAAY,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;QAEjD,0CAA0C;QAC1C,IAAI,KAAK,EAAE,MAAM,CAAC,EAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YACzC,KAAK,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;YACrD,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CAAE,KAAoB,EAAE,OAAY,EAAE;IACvD,OAAO,IAAI,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;AACpC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AAEH,OAAO,EAAE,GAAG,IAAI,QAAQ,EAAE,MAAM,UAAU,CAAA;AAgD1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,iCAAiC,CAAA;AAgH/C;;GAEG;AACH,MAAM,UAAU,GAAG,CAAE,KAAoB,EAAE,OAAY,EAAE;IACvD,OAAO,IAAI,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC"}

package/dist/src/traversal-strategies/cid-path.d.ts

@@ -0,0 +1,16 @@
+import type { TraversalStrategy } from '../index.js';
+import type { BlockView } from 'multiformats/block/interface';
+import type { CID } from 'multiformats/cid';
+/**
+ * Simple strategy that traverses a known path to a target CID.
+ *
+ * All this strategy does is yield the next CID in the known path.
+ */
+export declare class CIDPath implements TraversalStrategy {
+    private readonly pathToTarget;
+    private readonly target;
+    constructor(pathToTarget: CID[]);
+    isTarget(cid: CID): boolean;
+    traverse<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, _block?: T): AsyncGenerator<CID, void, undefined>;
+}
+//# sourceMappingURL=cid-path.d.ts.map

package/dist/src/traversal-strategies/cid-path.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cid-path.d.ts","sourceRoot":"","sources":["../../../src/traversal-strategies/cid-path.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAE3C;;;;GAIG;AACH,qBAAa,OAAQ,YAAW,iBAAiB;IAC/C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAK;gBAEf,YAAY,EAAE,GAAG,EAAE;IAKhC,QAAQ,CAAE,GAAG,EAAE,GAAG,GAAG,OAAO;IAIpB,QAAQ,CAAE,CAAC,SAAS,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC;CAMzH"}

package/dist/src/traversal-strategies/cid-path.js

@@ -0,0 +1,22 @@
+/**
+ * Simple strategy that traverses a known path to a target CID.
+ *
+ * All this strategy does is yield the next CID in the known path.
+ */
+export class CIDPath {
+    pathToTarget;
+    target;
+    constructor(pathToTarget) {
+        this.pathToTarget = pathToTarget;
+        this.target = pathToTarget[pathToTarget.length - 1];
+    }
+    isTarget(cid) {
+        return this.target.equals(cid);
+    }
+    async *traverse(cid, _block) {
+        const givenCidIndex = this.pathToTarget.indexOf(cid);
+        const nextCid = this.pathToTarget[givenCidIndex + 1];
+        yield nextCid;
+    }
+}
+//# sourceMappingURL=cid-path.js.map

package/dist/src/traversal-strategies/cid-path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cid-path.js","sourceRoot":"","sources":["../../../src/traversal-strategies/cid-path.ts"],"names":[],"mappings":"AAIA;;;;GAIG;AACH,MAAM,OAAO,OAAO;IACD,YAAY,CAAO;IACnB,MAAM,CAAK;IAE5B,YAAa,YAAmB;QAC9B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,QAAQ,CAAE,GAAQ;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,CAAE,QAAQ,CAA6C,GAAQ,EAAE,MAAU;QAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,aAAa,GAAG,CAAC,CAAC,CAAA;QAEpD,MAAM,OAAO,CAAA;IACf,CAAC;CACF"}

package/dist/src/traversal-strategies/graph-search.d.ts

@@ -0,0 +1,14 @@
+import type { TraversalStrategy } from '../index.js';
+import type { BlockView } from 'multiformats/block/interface';
+import type { CID } from 'multiformats/cid';
+/**
+ * A traversal strategy that performs a breadth-first search (so as to not load blocks unnecessarily) looking for a
+ * target CID. Traversal stops when we reach the target CID or run out of nodes.
+ */
+export declare class GraphSearch implements TraversalStrategy {
+    private readonly target;
+    constructor(target: CID);
+    isTarget(cid: CID): boolean;
+    traverse<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>;
+}
+//# sourceMappingURL=graph-search.d.ts.map

package/dist/src/traversal-strategies/graph-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-search.d.ts","sourceRoot":"","sources":["../../../src/traversal-strategies/graph-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAE3C;;;GAGG;AACH,qBAAa,WAAY,YAAW,iBAAiB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAK;gBAEf,MAAM,EAAE,GAAG;IAIxB,QAAQ,CAAE,GAAG,EAAE,GAAG,GAAG,OAAO;IAIpB,QAAQ,CAAE,CAAC,SAAS,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC;CAKvH"}

package/dist/src/traversal-strategies/graph-search.js

@@ -0,0 +1,19 @@
+/**
+ * A traversal strategy that performs a breadth-first search (so as to not load blocks unnecessarily) looking for a
+ * target CID. Traversal stops when we reach the target CID or run out of nodes.
+ */
+export class GraphSearch {
+    target;
+    constructor(target) {
+        this.target = target;
+    }
+    isTarget(cid) {
+        return this.target.equals(cid);
+    }
+    async *traverse(cid, block) {
+        for await (const [, linkedCid] of block.links()) {
+            yield linkedCid;
+        }
+    }
+}
+//# sourceMappingURL=graph-search.js.map

package/dist/src/traversal-strategies/graph-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-search.js","sourceRoot":"","sources":["../../../src/traversal-strategies/graph-search.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,MAAM,OAAO,WAAW;IACL,MAAM,CAAK;IAE5B,YAAa,MAAW;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,QAAQ,CAAE,GAAQ;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,CAAE,QAAQ,CAA6C,GAAQ,EAAE,KAAQ;QAC7E,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChD,MAAM,SAAS,CAAA;QACjB,CAAC;IACH,CAAC;CACF"}

package/dist/src/traversal-strategies/index.d.ts

@@ -0,0 +1,4 @@
+export * from './graph-search.js';
+export * from './cid-path.js';
+export * from './unixfs-path.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/traversal-strategies/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/traversal-strategies/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA"}

package/dist/src/traversal-strategies/index.js

@@ -0,0 +1,4 @@
+export * from './graph-search.js';
+export * from './cid-path.js';
+export * from './unixfs-path.js';
+//# sourceMappingURL=index.js.map

package/dist/src/traversal-strategies/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/traversal-strategies/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA"}

package/dist/src/traversal-strategies/unixfs-path.d.ts

@@ -0,0 +1,13 @@
+import type { TraversalStrategy } from '../index.js';
+import type { BlockView } from 'multiformats/block/interface';
+import type { CID } from 'multiformats/cid';
+/**
+ * Traverses a DAG containing UnixFS directories
+ */
+export declare class UnixFSPath implements TraversalStrategy {
+    private readonly path;
+    constructor(path: string);
+    isTarget(): boolean;
+    traverse<T extends BlockView<any, any, any, 0 | 1>>(cid: CID, block: T): AsyncGenerator<CID, void, undefined>;
+}
+//# sourceMappingURL=unixfs-path.d.ts.map

package/dist/src/traversal-strategies/unixfs-path.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unixfs-path.d.ts","sourceRoot":"","sources":["../../../src/traversal-strategies/unixfs-path.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAE3C;;GAEG;AACH,qBAAa,UAAW,YAAW,iBAAiB;IAClD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAU;gBAElB,IAAI,EAAE,MAAM;IAKzB,QAAQ,IAAK,OAAO;IAIZ,QAAQ,CAAE,CAAC,SAAS,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC;CAkCvH"}

package/dist/src/traversal-strategies/unixfs-path.js

@@ -0,0 +1,42 @@
+import { decode } from '@ipld/dag-pb';
+import { UnixFS } from 'ipfs-unixfs';
+import { DAG_PB_CODEC_CODE } from '../constants.js';
+import { NotUnixFSError } from '../errors.js';
+/**
+ * Traverses a DAG containing UnixFS directories
+ */
+export class UnixFSPath {
+    path;
+    constructor(path) {
+        // "/foo/bar/baz.txt" -> ['foo', 'bar', 'baz.txt']
+        this.path = path.replace(/^\//, '').split('/');
+    }
+    isTarget() {
+        return this.path.length === 0;
+    }
+    async *traverse(cid, block) {
+        if (cid.code !== DAG_PB_CODEC_CODE) {
+            throw new NotUnixFSError('Target CID is not UnixFS');
+        }
+        const segment = this.path.shift();
+        if (segment == null) {
+            return;
+        }
+        const pb = decode(block.bytes);
+        if (pb.Data == null) {
+            throw new NotUnixFSError('Target CID has no UnixFS data in decoded bytes');
+        }
+        const unixfs = UnixFS.unmarshal(pb.Data);
+        if (unixfs.type === 'directory') {
+            const link = pb.Links.filter(link => link.Name === segment).pop();
+            if (link == null) {
+                throw new NotUnixFSError(`Target CID directory has no link with name ${segment}`);
+            }
+            yield link.Hash;
+            return;
+        }
+        // TODO: HAMT support
+        throw new NotUnixFSError('Target CID is not a UnixFS directory');
+    }
+}
+//# sourceMappingURL=unixfs-path.js.map

package/dist/src/traversal-strategies/unixfs-path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unixfs-path.js","sourceRoot":"","sources":["../../../src/traversal-strategies/unixfs-path.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAK7C;;GAEG;AACH,MAAM,OAAO,UAAU;IACJ,IAAI,CAAU;IAE/B,YAAa,IAAY;QACvB,kDAAkD;QAClD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,CAAE,QAAQ,CAA6C,GAAQ,EAAE,KAAQ;QAC7E,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,cAAc,CAAC,0BAA0B,CAAC,CAAA;QACtD,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;QAEjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAM;QACR,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,EAAE,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,IAAI,cAAc,CAAC,gDAAgD,CAAC,CAAA;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAExC,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,GAAG,EAAE,CAAA;YAEjE,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,cAAc,CAAC,8CAA8C,OAAO,EAAE,CAAC,CAAA;YACnF,CAAC;YAED,MAAM,IAAI,CAAC,IAAI,CAAA;YACf,OAAM;QACR,CAAC;QAED,qBAAqB;QAErB,MAAM,IAAI,cAAc,CAAC,sCAAsC,CAAC,CAAA;IAClE,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/car",
-  "version": "4.0.3",
+  "version": "4.0.4-bb2ab74",
   "description": "Import/export car files from Helia",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/car#readme",
@@ -54,11 +54,13 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/interface": "^5.2.1",
+    "@helia/interface": "5.2.1-bb2ab74",
     "@ipld/car": "^5.3.3",
+    "@ipld/dag-pb": "^4.1.3",
     "@libp2p/interface": "^2.2.1",
     "@libp2p/utils": "^6.2.1",
     "interface-blockstore": "^5.3.1",
+    "ipfs-unixfs": "^11.2.1",
     "it-drain": "^3.0.7",
     "it-map": "^3.1.1",
     "multiformats": "^13.3.1",
@@ -67,14 +69,18 @@
     "progress-events": "^1.0.1"
   },
   "devDependencies": {
-    "@helia/mfs": "^4.0.3",
-    "@helia/unixfs": "^4.0.3",
-    "@ipld/dag-pb": "^4.1.3",
+    "@helia/mfs": "5.0.0-bb2ab74",
+    "@helia/unixfs": "5.0.0-bb2ab74",
+    "@ipld/dag-cbor": "^9.2.2",
+    "@libp2p/logger": "^5.1.15",
     "aegir": "^45.1.1",
     "blockstore-core": "^5.0.2",
     "datastore-core": "^10.0.2",
     "ipfs-unixfs-importer": "^15.3.1",
-    "it-to-buffer": "^4.0.7"
+    "it-foreach": "^2.1.2",
+    "it-length": "^3.0.7",
+    "it-to-buffer": "^4.0.7",
+    "sinon": "^19.0.2"
   },
   "sideEffects": false
 }

package/src/car.ts

@@ -0,0 +1,218 @@
+import { CarWriter } from '@ipld/car'
+import drain from 'it-drain'
+import map from 'it-map'
+import { createUnsafe } from 'multiformats/block'
+import { type CID } from 'multiformats/cid'
+import defer from 'p-defer'
+import PQueue from 'p-queue'
+import { DAG_WALK_QUEUE_CONCURRENCY } from './constants.js'
+import { SubgraphExporter } from './export-strategies/subgraph-exporter.js'
+import { GraphSearch } from './traversal-strategies/graph-search.js'
+import type { CarComponents, Car as CarInterface, ExportCarOptions, ExportStrategy, TraversalStrategy } from './index.js'
+import type { PutManyBlocksProgressEvents } from '@helia/interface/blocks'
+import type { CarReader } from '@ipld/car'
+import type { AbortOptions, Logger } from '@libp2p/interface'
+import type { ProgressOptions } from 'progress-events'
+
+/**
+ * Context for the traversal process.
+ */
+interface TraversalContext {
+  currentPath: CID[]
+  pathsToTarget: CID[][] | null // collect all target paths
+}
+
+interface WalkDagContext<Strategy> {
+  cid: CID
+  queue: PQueue
+  strategy: Strategy
+  options?: ExportCarOptions
+}
+
+interface ExportWalkDagContext extends WalkDagContext<ExportStrategy> {
+  writer: Pick<CarWriter, 'put'>
+  recursive?: boolean
+}
+
+interface TraversalWalkDagContext extends WalkDagContext<TraversalStrategy> {
+  traversalContext: TraversalContext
+  parentPath: CID[]
+}
+
+export class Car implements CarInterface {
+  private readonly components: CarComponents
+  private readonly log: Logger
+
+  constructor (components: CarComponents, init: any) {
+    this.components = components
+    this.log = components.logger.forComponent('helia:car')
+  }
+
+  async import (reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void> {
+    await drain(this.components.blockstore.putMany(
+      map(reader.blocks(), ({ cid, bytes }) => ({ cid, block: bytes })),
+      options
+    ))
+  }
+
+  async export (root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void> {
+    const deferred = defer<Error | undefined>()
+    const roots = Array.isArray(root) ? root : [root]
+
+    // Create traversal-specific context
+    const traversalContext: TraversalContext = {
+      currentPath: [],
+      pathsToTarget: null
+    }
+
+    const traversalStrategy = options?.traversal
+    const exportStrategy = options?.exporter ?? new SubgraphExporter()
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very
+    // large DAGs
+    const queue = new PQueue({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+
+    let startedExport = false
+    queue.on('idle', () => {
+      if (startedExport) {
+        // idle event was called, and started exporting, so we are done.
+        deferred.resolve()
+      } else if (!startedExport && traversalContext.pathsToTarget?.length === roots.length) {
+        // queue is idle, we haven't started exporting yet, and we have path(s)
+        // to the target(s), so we can start the export process.
+        this.log.trace('starting export of blocks to the car file')
+        startedExport = true
+
+        for (const path of traversalContext.pathsToTarget) {
+          const targetIndex = path.length - 1
+          const targetCid = path[targetIndex]
+
+          // Process all verification blocks in the path except the target
+          path.slice(0, -1).forEach(cid => {
+            void queue.add(async () => {
+              await this.#exportDagNode({ cid, queue, writer, strategy: exportStrategy, options, recursive: false })
+            })
+              .catch((err) => {
+                this.log.error('error during queue operation - %e', err)
+              })
+          })
+
+          // Process the target block (which will recursively export its DAG)
+          void queue.add(async () => {
+            await this.#exportDagNode({ cid: targetCid, queue, writer, strategy: exportStrategy, options })
+          })
+            .catch((err) => {
+              this.log.error('error during queue operation - %e', err)
+            })
+        }
+      } else {
+        // queue is idle, we haven't started exporting yet, and we don't have
+        // path(s) to the target(s), so we can't start the export process.
+        // this should not happen without a separate error during traversal, but
+        // we'll handle it here anyway.
+        this.log.trace('no paths to target, skipping export')
+        deferred.reject(new Error('Could not traverse to target CID(s)'))
+      }
+    })
+    queue.on('error', (err) => {
+      queue.clear()
+      deferred.reject(err)
+    })
+
+    for (const root of roots) {
+      void queue.add(async () => {
+        this.log.trace('traversing dag from %c', root)
+        await this.#traverseDagNode({ cid: root, queue, strategy: traversalStrategy ?? new GraphSearch(root), traversalContext, parentPath: [], options })
+      })
+        .catch((err) => {
+          this.log.error('error during queue operation - %e', err)
+        })
+    }
+
+    // wait for the writer to end
+    try {
+      await deferred.promise
+    } finally {
+      await writer.close()
+    }
+  }
+
+  async * stream (root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined> {
+    const { writer, out } = CarWriter.create(root)
+
+    // has to be done async so we write to `writer` and read from `out` at the
+    // same time
+    this.export(root, writer, options)
+      .catch((err) => {
+        this.log.error('error during streaming export - %e', err)
+      })
+
+    for await (const buf of out) {
+      yield buf
+    }
+  }
+
+  /**
+   * Traverse a DAG and stop when we reach the target node
+   */
+  async #traverseDagNode ({ cid, queue, strategy, traversalContext, parentPath, options }: TraversalWalkDagContext): Promise<void> {
+    // if we are traversing, we need to gather path(s) to the target(s)
+    const currentPath = [...parentPath, cid]
+
+    if (strategy.isTarget(cid)) {
+      traversalContext.pathsToTarget = traversalContext.pathsToTarget ?? []
+      traversalContext.pathsToTarget.push([...currentPath])
+      this.log.trace('found path to target %c', cid)
+      return
+    }
+
+    const codec = await this.components.getCodec(cid.code)
+    const bytes = await this.components.blockstore.get(cid, options)
+
+    // we are recursively traversing the dag
+    const decodedBlock = createUnsafe({ bytes, cid, codec })
+
+    for await (const nextCid of strategy.traverse(cid, decodedBlock)) {
+      void queue.add(async () => {
+        await this.#traverseDagNode({ cid: nextCid, queue, strategy, traversalContext, parentPath: currentPath ?? [], options })
+      })
+        .catch((err) => {
+          this.log.error('error during traversal queue operation - %e', err)
+        })
+    }
+  }
+
+  /**
+   * Use an ExportStrategy to export part of all of a DAG
+   */
+  async #exportDagNode ({ cid, queue, writer, strategy, options, recursive = true }: ExportWalkDagContext): Promise<void> {
+    if (options?.blockFilter?.has(cid.multihash.bytes) === true) {
+      return
+    }
+
+    const codec = await this.components.getCodec(cid.code)
+    const bytes = await this.components.blockstore.get(cid, options)
+
+    // Mark as processed
+    options?.blockFilter?.add(cid.multihash.bytes)
+
+    // Write to CAR
+    await writer.put({ cid, bytes })
+
+    if (recursive) {
+      // we are recursively traversing the dag
+      const decodedBlock = createUnsafe({ bytes, cid, codec })
+
+      for await (const nextCid of strategy.export(cid, decodedBlock)) {
+        void queue.add(async () => {
+          await this.#exportDagNode({ cid: nextCid, queue, writer, strategy, options })
+        })
+          .catch((err) => {
+            this.log.error('error during export queue operation - %e', err)
+          })
+      }
+    }
+  }
+}

package/src/constants.ts

@@ -0,0 +1,11 @@
+export const DAG_WALK_QUEUE_CONCURRENCY = 1
+
+/**
+ * The multicodec code for dag-pb binary data
+ */
+export const DAG_PB_CODEC_CODE = 0x70
+
+/**
+ * The multicodec code for raw binary data
+ */
+export const RAW_PB_CODEC_CODE = 0x55

package/src/errors.ts

@@ -0,0 +1,8 @@
+export class NotUnixFSError extends Error {
+  static code = 'ERR_NOT_UNIXFS'
+  static message = 'Not a UnixFS node'
+  static name = 'NotUnixFSError'
+  code = 'ERR_NOT_UNIXFS'
+  message = 'Not a UnixFS node'
+  name = 'NotUnixFSError'
+}

package/src/export-strategies/block-exporter.ts

@@ -0,0 +1,13 @@
+import { type CID } from 'multiformats/cid'
+import { type ExportStrategy } from '../index.js'
+import type { BlockView } from 'multiformats/block/interface'
+
+/**
+ * Yields the first block from the first CID and stops
+ */
+export class BlockExporter implements ExportStrategy {
+  async * export (cid: CID, block: BlockView<any, any, any, 0 | 1>): AsyncGenerator<CID, void, undefined> {
+    // don't yield the block, index.ts will add it to the car file and then
+    // we're done
+  }
+}

package/src/export-strategies/index.ts

@@ -0,0 +1,3 @@
+export * from './subgraph-exporter.js'
+export * from './block-exporter.js'
+export * from './unixfs-exporter.js'