@helia/car 4.1.3 → 4.2.0-1f0e7f7d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.min.js +1 -1
- package/dist/index.min.js.map +4 -4
- package/dist/src/car.d.ts +2 -3
- package/dist/src/car.d.ts.map +1 -1
- package/dist/src/car.js +42 -25
- package/dist/src/car.js.map +1 -1
- package/dist/src/index.d.ts +8 -39
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -2
- package/dist/src/index.js.map +1 -1
- package/package.json +25 -33
- package/src/car.ts +49 -29
- package/src/index.ts +8 -40
- package/dist/typedoc-urls.json +0 -20
package/dist/src/car.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { CarWriter } from '@ipld/car';
|
|
2
1
|
import type { CarComponents, Car as CarInterface, ExportCarOptions } from './index.js';
|
|
3
2
|
import type { PutManyBlocksProgressEvents } from '@helia/interface/blocks';
|
|
4
3
|
import type { CarReader } from '@ipld/car';
|
|
@@ -11,7 +10,7 @@ export declare class Car implements CarInterface {
|
|
|
11
10
|
private readonly log;
|
|
12
11
|
constructor(components: CarComponents, init: any);
|
|
13
12
|
import(reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void>;
|
|
14
|
-
export(root: CID | CID[],
|
|
15
|
-
|
|
13
|
+
export(root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined>;
|
|
14
|
+
private _export;
|
|
16
15
|
}
|
|
17
16
|
//# sourceMappingURL=car.d.ts.map
|
package/dist/src/car.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../../src/car.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../../src/car.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,YAAY,EAAE,gBAAgB,EAAqC,MAAM,YAAY,CAAA;AACzH,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAA;AAC1E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAU,MAAM,mBAAmB,CAAA;AAC7D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AA2BtD,qBAAa,GAAI,YAAW,YAAY;;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAe;IAC1C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAQ;gBAEf,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG;IAK3C,MAAM,CAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,eAAe,CAAC,2BAA2B,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9H,MAAM,CAAE,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC;YAkC7F,OAAO;CAiJtB"}
|
package/dist/src/car.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { CarWriter } from '@ipld/car';
|
|
2
|
+
import { Queue } from '@libp2p/utils';
|
|
2
3
|
import drain from 'it-drain';
|
|
3
4
|
import map from 'it-map';
|
|
5
|
+
import toBuffer from 'it-to-buffer';
|
|
4
6
|
import { createUnsafe } from 'multiformats/block';
|
|
5
|
-
import
|
|
6
|
-
import PQueue from 'p-queue';
|
|
7
|
+
import { raceSignal } from 'race-signal';
|
|
7
8
|
import { DAG_WALK_QUEUE_CONCURRENCY } from './constants.js';
|
|
8
9
|
import { SubgraphExporter } from './export-strategies/subgraph-exporter.js';
|
|
9
10
|
import { GraphSearch } from './traversal-strategies/graph-search.js';
|
|
@@ -15,10 +16,38 @@ export class Car {
|
|
|
15
16
|
this.log = components.logger.forComponent('helia:car');
|
|
16
17
|
}
|
|
17
18
|
async import(reader, options) {
|
|
18
|
-
await drain(this.components.blockstore.putMany(map(reader.blocks(), ({ cid, bytes }) => ({ cid,
|
|
19
|
+
await drain(this.components.blockstore.putMany(map(reader.blocks(), ({ cid, bytes }) => ({ cid, bytes })), options));
|
|
19
20
|
}
|
|
20
|
-
async export(root,
|
|
21
|
-
const
|
|
21
|
+
async *export(root, options) {
|
|
22
|
+
const { writer, out } = CarWriter.create(root);
|
|
23
|
+
const iter = out[Symbol.asyncIterator]();
|
|
24
|
+
const controller = new AbortController();
|
|
25
|
+
// has to be done async so we write to `writer` and read from `out` at the
|
|
26
|
+
// same time
|
|
27
|
+
this._export(root, writer, options)
|
|
28
|
+
.catch((err) => {
|
|
29
|
+
this.log.error('error during streaming export - %e', err);
|
|
30
|
+
controller.abort(err);
|
|
31
|
+
});
|
|
32
|
+
while (true) {
|
|
33
|
+
const { done, value } = await raceSignal(iter.next(), controller.signal);
|
|
34
|
+
// the writer's `out` iterable can yield results synchronously, in which
|
|
35
|
+
// case the controller may have been aborted but the event may not have
|
|
36
|
+
// fired yet so check the signal status manually before processing the
|
|
37
|
+
// next iterable result
|
|
38
|
+
if (controller.signal.aborted) {
|
|
39
|
+
throw controller.signal.reason;
|
|
40
|
+
}
|
|
41
|
+
if (value != null) {
|
|
42
|
+
yield value;
|
|
43
|
+
}
|
|
44
|
+
if (done === true) {
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
async _export(root, writer, options) {
|
|
50
|
+
const deferred = Promise.withResolvers();
|
|
22
51
|
const roots = Array.isArray(root) ? root : [root];
|
|
23
52
|
// Create traversal-specific context
|
|
24
53
|
const traversalContext = {
|
|
@@ -29,11 +58,11 @@ export class Car {
|
|
|
29
58
|
const exportStrategy = options?.exporter ?? new SubgraphExporter();
|
|
30
59
|
// use a queue to walk the DAG instead of recursion so we can traverse very
|
|
31
60
|
// large DAGs
|
|
32
|
-
const queue = new
|
|
61
|
+
const queue = new Queue({
|
|
33
62
|
concurrency: DAG_WALK_QUEUE_CONCURRENCY
|
|
34
63
|
});
|
|
35
64
|
let startedExport = false;
|
|
36
|
-
queue.
|
|
65
|
+
queue.addEventListener('idle', () => {
|
|
37
66
|
if (startedExport) {
|
|
38
67
|
// idle event was called, and started exporting, so we are done.
|
|
39
68
|
deferred.resolve();
|
|
@@ -48,7 +77,7 @@ export class Car {
|
|
|
48
77
|
const targetCid = path[targetIndex];
|
|
49
78
|
// Process all verification blocks in the path except the target
|
|
50
79
|
path.slice(0, -1).forEach(cid => {
|
|
51
|
-
|
|
80
|
+
queue.add(async () => {
|
|
52
81
|
await this.#exportDagNode({ cid, queue, writer, strategy: exportStrategy, options, recursive: false });
|
|
53
82
|
})
|
|
54
83
|
.catch((err) => {
|
|
@@ -56,7 +85,7 @@ export class Car {
|
|
|
56
85
|
});
|
|
57
86
|
});
|
|
58
87
|
// Process the target block (which will recursively export its DAG)
|
|
59
|
-
|
|
88
|
+
queue.add(async () => {
|
|
60
89
|
await this.#exportDagNode({ cid: targetCid, queue, writer, strategy: exportStrategy, options });
|
|
61
90
|
})
|
|
62
91
|
.catch((err) => {
|
|
@@ -73,9 +102,9 @@ export class Car {
|
|
|
73
102
|
deferred.reject(new Error('Could not traverse to target CID(s)'));
|
|
74
103
|
}
|
|
75
104
|
});
|
|
76
|
-
queue.
|
|
105
|
+
queue.addEventListener('failure', (evt) => {
|
|
77
106
|
queue.clear();
|
|
78
|
-
deferred.reject(
|
|
107
|
+
deferred.reject(evt.detail.error);
|
|
79
108
|
});
|
|
80
109
|
for (const root of roots) {
|
|
81
110
|
void queue.add(async () => {
|
|
@@ -94,18 +123,6 @@ export class Car {
|
|
|
94
123
|
await writer.close();
|
|
95
124
|
}
|
|
96
125
|
}
|
|
97
|
-
async *stream(root, options) {
|
|
98
|
-
const { writer, out } = CarWriter.create(root);
|
|
99
|
-
// has to be done async so we write to `writer` and read from `out` at the
|
|
100
|
-
// same time
|
|
101
|
-
this.export(root, writer, options)
|
|
102
|
-
.catch((err) => {
|
|
103
|
-
this.log.error('error during streaming export - %e', err);
|
|
104
|
-
});
|
|
105
|
-
for await (const buf of out) {
|
|
106
|
-
yield buf;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
126
|
/**
|
|
110
127
|
* Traverse a DAG and stop when we reach the target node
|
|
111
128
|
*/
|
|
@@ -119,7 +136,7 @@ export class Car {
|
|
|
119
136
|
return;
|
|
120
137
|
}
|
|
121
138
|
const codec = await this.components.getCodec(cid.code);
|
|
122
|
-
const bytes = await this.components.blockstore.get(cid, options);
|
|
139
|
+
const bytes = await toBuffer(this.components.blockstore.get(cid, options));
|
|
123
140
|
// we are recursively traversing the dag
|
|
124
141
|
const decodedBlock = createUnsafe({ bytes, cid, codec });
|
|
125
142
|
for await (const nextCid of strategy.traverse(cid, decodedBlock)) {
|
|
@@ -139,7 +156,7 @@ export class Car {
|
|
|
139
156
|
return;
|
|
140
157
|
}
|
|
141
158
|
const codec = await this.components.getCodec(cid.code);
|
|
142
|
-
const bytes = await this.components.blockstore.get(cid, options);
|
|
159
|
+
const bytes = await toBuffer(this.components.blockstore.get(cid, options));
|
|
143
160
|
// Mark as processed
|
|
144
161
|
options?.blockFilter?.add(cid.multihash.bytes);
|
|
145
162
|
// Write to CAR
|
package/dist/src/car.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"car.js","sourceRoot":"","sources":["../../src/car.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,KAAK,MAAM,UAAU,CAAA;AAC5B,OAAO,GAAG,MAAM,QAAQ,CAAA;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,
|
|
1
|
+
{"version":3,"file":"car.js","sourceRoot":"","sources":["../../src/car.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AACrC,OAAO,KAAK,MAAM,UAAU,CAAA;AAC5B,OAAO,GAAG,MAAM,QAAQ,CAAA;AACxB,OAAO,QAAQ,MAAM,cAAc,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,0CAA0C,CAAA;AAC3E,OAAO,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAA;AAiCpE,MAAM,OAAO,GAAG;IACG,UAAU,CAAe;IACzB,GAAG,CAAQ;IAE5B,YAAa,UAAyB,EAAE,IAAS;QAC/C,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,CAAA;IACxD,CAAC;IAED,KAAK,CAAC,MAAM,CAAE,MAAiC,EAAE,OAAqE;QACpH,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO,CAC5C,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,EAC1D,OAAO,CACR,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,CAAE,MAAM,CAAE,IAAiB,EAAE,OAA0B;QAC3D,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QAExC,0EAA0E;QAC1E,YAAY;QACZ,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC;aAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAA;YACzD,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACvB,CAAC,CAAC,CAAA;QAEJ,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;YAExE,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,uBAAuB;YACvB,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC9B,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,CAAA;YAChC,CAAC;YAED,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,KAAK,CAAA;YACb,CAAC;YAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,MAAK;YACP,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAE,IAAiB,EAAE,MAAwC,EAAE,OAA0B;QAC5G,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,EAAgB,CAAA;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEjD,oCAAoC;QACpC,MAAM,gBAAgB,GAAqB;YACzC,WAAW,EAAE,EAAE;YACf,aAAa,EAAE,IAAI;SACpB,CAAA;QAED,MAAM,iBAAiB,GAAG,OAAO,EAAE,SAAS,CAAA;QAC5C,MAAM,cAAc,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,gBAAgB,EAAE,CAAA;QAElE,2EAA2E;QAC3E,aAAa;QACb,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACtB,WAAW,EAAE,0BAA0B;SACxC,CAAC,CAAA;QAEF,IAAI,aAAa,GAAG,KAAK,CAAA;QACzB,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE;YAClC,IAAI,aAAa,EAAE,CAAC;gBAClB,gEAAgE;gBAChE,QAAQ,CAAC,OAAO,EAAE,CAAA;YACpB,CAAC;iBAAM,IAAI,CAAC,aAAa,IAAI,gBAAgB,CAAC,aAAa,EAAE,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrF,uEAAuE;gBACvE,wDAAwD;gBACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;gBAC3D,aAAa,GAAG,IAAI,CAAA;gBAEpB,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,aAAa,EAAE,CAAC;oBAClD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;oBACnC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAA;oBAEnC,gEAAgE;oBAChE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;wBAC9B,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;4BACnB,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAA;wBACxG,CAAC,CAAC;6BACC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;4BACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAA;wBAC1D,CAAC,CAAC,CAAA;oBACN,CAAC,CAAC,CAAA;oBAEF,mEAAmE;oBACnE,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;wBACnB,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAA;oBACjG,CAAC,CAAC;yBACC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAA;oBAC1D,CAAC,CAAC,CAAA;gBACN,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,qEAAqE;gBACrE,kEAAkE;gBAClE,wEAAwE;gBACxE,+BAA+B;gBAC/B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;gBACrD,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC,CAAA;YACnE,CAAC;QACH,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YACxC,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAA;gBAC9C,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,iBAAiB,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,gBAAgB,EAAE,UAAU,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;YACpJ,CAAC,CAAC;iBACC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAA;YAC1D,CAAC,CAAC,CAAA;QACN,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,OAAO,CAAA;QACxB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACtB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAE,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,UAAU,EAAE,OAAO,EAA2B;QAC9G,mEAAmE;QACnE,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,CAAC,CAAA;QAExC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,aAAa,GAAG,gBAAgB,CAAC,aAAa,IAAI,EAAE,CAAA;YACrE,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAA;YACrD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,GAAG,CAAC,CAAA;YAC9C,OAAM;QACR,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;QAE1E,wCAAwC;QACxC,MAAM,YAAY,GAAG,YAAY,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;QAExD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,CAAC;YACjE,KAAK,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,UAAU,EAAE,WAAW,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;YAC1H,CAAC,CAAC;iBACC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAA;YACpE,CAAC,CAAC,CAAA;QACN,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,GAAG,IAAI,EAAwB;QACrG,IAAI,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5D,OAAM;QACR,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;QAE1E,oBAAoB;QACpB,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAE9C,eAAe;QACf,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;QAEhC,IAAI,SAAS,EAAE,CAAC;YACd,wCAAwC;YACxC,MAAM,YAAY,GAAG,YAAY,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;YAExD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC/D,KAAK,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;oBACxB,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;gBAC/E,CAAC,CAAC;qBACC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,EAAE,GAAG,CAAC,CAAA;gBACjE,CAAC,CAAC,CAAA;YACN,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* const c = car(helia)
|
|
24
24
|
* const out = nodeFs.createWriteStream('example.car')
|
|
25
25
|
*
|
|
26
|
-
* for await (const buf of c.
|
|
26
|
+
* for await (const buf of c.export(cid)) {
|
|
27
27
|
* out.write(buf)
|
|
28
28
|
* }
|
|
29
29
|
*
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* const c = car(helia)
|
|
45
45
|
* const out = nodeFs.createWriteStream('example.car')
|
|
46
46
|
*
|
|
47
|
-
* for await (const buf of c.
|
|
47
|
+
* for await (const buf of c.export(cid, {
|
|
48
48
|
* traversal: new UnixFSPath('/foo/bar/baz.txt')
|
|
49
49
|
* })) {
|
|
50
50
|
* out.write(buf)
|
|
@@ -76,10 +76,10 @@
|
|
|
76
76
|
* ```
|
|
77
77
|
*/
|
|
78
78
|
import type { CodecLoader } from '@helia/interface';
|
|
79
|
-
import type { PutManyBlocksProgressEvents, GetBlockProgressEvents } from '@helia/interface/blocks';
|
|
80
|
-
import type {
|
|
79
|
+
import type { PutManyBlocksProgressEvents, GetBlockProgressEvents, ProviderOptions } from '@helia/interface/blocks';
|
|
80
|
+
import type { CarReader } from '@ipld/car';
|
|
81
81
|
import type { AbortOptions, ComponentLogger } from '@libp2p/interface';
|
|
82
|
-
import type { Filter } from '@libp2p/utils
|
|
82
|
+
import type { Filter } from '@libp2p/utils';
|
|
83
83
|
import type { Blockstore } from 'interface-blockstore';
|
|
84
84
|
import type { BlockView } from 'multiformats/block/interface';
|
|
85
85
|
import type { CID } from 'multiformats/cid';
|
|
@@ -119,7 +119,7 @@ export interface ExportStrategy {
|
|
|
119
119
|
}
|
|
120
120
|
export * from './export-strategies/index.js';
|
|
121
121
|
export * from './traversal-strategies/index.js';
|
|
122
|
-
export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents
|
|
122
|
+
export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents>, ProviderOptions {
|
|
123
123
|
/**
|
|
124
124
|
* If true, the blockstore will not do any network requests.
|
|
125
125
|
*
|
|
@@ -168,37 +168,6 @@ export interface Car {
|
|
|
168
168
|
* ```
|
|
169
169
|
*/
|
|
170
170
|
import(reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void>;
|
|
171
|
-
/**
|
|
172
|
-
* Store all blocks that make up one or more DAGs in a car file.
|
|
173
|
-
*
|
|
174
|
-
* @example
|
|
175
|
-
*
|
|
176
|
-
* ```typescript
|
|
177
|
-
* import fs from 'node:fs'
|
|
178
|
-
* import { Readable } from 'node:stream'
|
|
179
|
-
* import { car } from '@helia/car'
|
|
180
|
-
* import { CarWriter } from '@ipld/car'
|
|
181
|
-
* import { createHelia } from 'helia'
|
|
182
|
-
* import { CID } from 'multiformats/cid'
|
|
183
|
-
* import { pEvent } from 'p-event'
|
|
184
|
-
*
|
|
185
|
-
* const helia = await createHelia()
|
|
186
|
-
* const cid = CID.parse('QmFoo...')
|
|
187
|
-
*
|
|
188
|
-
* const c = car(helia)
|
|
189
|
-
* const { writer, out } = CarWriter.create(cid)
|
|
190
|
-
* const output = fs.createWriteStream('example.car')
|
|
191
|
-
* const stream = Readable.from(out).pipe(output)
|
|
192
|
-
*
|
|
193
|
-
* await Promise.all([
|
|
194
|
-
* c.export(cid, writer),
|
|
195
|
-
* pEvent(stream, 'close')
|
|
196
|
-
* ])
|
|
197
|
-
* ```
|
|
198
|
-
*
|
|
199
|
-
* @deprecated Use `stream` instead. In a future release `stream` will be renamed `export`.
|
|
200
|
-
*/
|
|
201
|
-
export(root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void>;
|
|
202
171
|
/**
|
|
203
172
|
* Returns an AsyncGenerator that yields CAR file bytes.
|
|
204
173
|
*
|
|
@@ -214,12 +183,12 @@ export interface Car {
|
|
|
214
183
|
*
|
|
215
184
|
* const c = car(helia)
|
|
216
185
|
*
|
|
217
|
-
* for (const buf of c.
|
|
186
|
+
* for (const buf of c.export(cid)) {
|
|
218
187
|
* // store or send `buf` somewhere
|
|
219
188
|
* }
|
|
220
189
|
* ```
|
|
221
190
|
*/
|
|
222
|
-
|
|
191
|
+
export(root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined>;
|
|
223
192
|
}
|
|
224
193
|
/**
|
|
225
194
|
* Create a {@link Car} instance for use with {@link https://github.com/ipfs/helia Helia}
|
package/dist/src/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,KAAK,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,KAAK,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACnH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAEtD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,eAAe,CAAA;IACvB,UAAU,EAAE,UAAU,CAAA;IACtB,QAAQ,EAAE,WAAW,CAAA;CACtB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,CAAC,SAAS,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IAE7G;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAA;CAC5B;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,MAAM,CAAC,CAAC,SAAS,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;CAC5G;AAED,cAAc,8BAA8B,CAAA;AAC5C,cAAc,iCAAiC,CAAA;AAE/C,MAAM,WAAW,gBAAiB,SAAQ,YAAY,EAAE,eAAe,CAAC,sBAAsB,CAAC,EAAE,eAAe;IAE9G;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;IAEjB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;OAGG;IACH,SAAS,CAAC,EAAE,iBAAiB,CAAA;IAE7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,GAAG;IAClB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,eAAe,CAAC,2BAA2B,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE/H;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;CACnG;AAED;;GAEG;AACH,wBAAgB,GAAG,CAAE,KAAK,EAAE,aAAa,EAAE,IAAI,GAAE,GAAQ,GAAG,GAAG,CAE9D"}
|
package/dist/src/index.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* const c = car(helia)
|
|
24
24
|
* const out = nodeFs.createWriteStream('example.car')
|
|
25
25
|
*
|
|
26
|
-
* for await (const buf of c.
|
|
26
|
+
* for await (const buf of c.export(cid)) {
|
|
27
27
|
* out.write(buf)
|
|
28
28
|
* }
|
|
29
29
|
*
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* const c = car(helia)
|
|
45
45
|
* const out = nodeFs.createWriteStream('example.car')
|
|
46
46
|
*
|
|
47
|
-
* for await (const buf of c.
|
|
47
|
+
* for await (const buf of c.export(cid, {
|
|
48
48
|
* traversal: new UnixFSPath('/foo/bar/baz.txt')
|
|
49
49
|
* })) {
|
|
50
50
|
* out.write(buf)
|
package/dist/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AAEH,OAAO,EAAE,GAAG,IAAI,QAAQ,EAAE,MAAM,UAAU,CAAA;AAgD1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,iCAAiC,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AAEH,OAAO,EAAE,GAAG,IAAI,QAAQ,EAAE,MAAM,UAAU,CAAA;AAgD1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,iCAAiC,CAAA;AAgF/C;;GAEG;AACH,MAAM,UAAU,GAAG,CAAE,KAAoB,EAAE,OAAY,EAAE;IACvD,OAAO,IAAI,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@helia/car",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0-1f0e7f7d",
|
|
4
4
|
"description": "Import/export car files from Helia",
|
|
5
5
|
"license": "Apache-2.0 OR MIT",
|
|
6
6
|
"homepage": "https://github.com/ipfs/helia/tree/main/packages/car#readme",
|
|
@@ -32,13 +32,6 @@
|
|
|
32
32
|
"import": "./dist/src/index.js"
|
|
33
33
|
}
|
|
34
34
|
},
|
|
35
|
-
"eslintConfig": {
|
|
36
|
-
"extends": "ipfs",
|
|
37
|
-
"parserOptions": {
|
|
38
|
-
"project": true,
|
|
39
|
-
"sourceType": "module"
|
|
40
|
-
}
|
|
41
|
-
},
|
|
42
35
|
"scripts": {
|
|
43
36
|
"clean": "aegir clean",
|
|
44
37
|
"lint": "aegir lint",
|
|
@@ -54,33 +47,32 @@
|
|
|
54
47
|
"test:electron-main": "aegir test -t electron-main"
|
|
55
48
|
},
|
|
56
49
|
"dependencies": {
|
|
57
|
-
"@helia/interface": "
|
|
58
|
-
"@ipld/car": "^5.
|
|
59
|
-
"@ipld/dag-pb": "^4.1.
|
|
60
|
-
"@libp2p/interface": "^
|
|
61
|
-
"@libp2p/utils": "^
|
|
62
|
-
"interface-blockstore": "^
|
|
63
|
-
"ipfs-unixfs": "^
|
|
64
|
-
"it-drain": "^3.0.
|
|
65
|
-
"it-map": "^3.1.
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"
|
|
50
|
+
"@helia/interface": "5.4.0-1f0e7f7d",
|
|
51
|
+
"@ipld/car": "^5.4.2",
|
|
52
|
+
"@ipld/dag-pb": "^4.1.5",
|
|
53
|
+
"@libp2p/interface": "^3.0.2",
|
|
54
|
+
"@libp2p/utils": "^7.0.5",
|
|
55
|
+
"interface-blockstore": "^6.0.1",
|
|
56
|
+
"ipfs-unixfs": "^12.0.0",
|
|
57
|
+
"it-drain": "^3.0.10",
|
|
58
|
+
"it-map": "^3.1.4",
|
|
59
|
+
"it-to-buffer": "^4.0.10",
|
|
60
|
+
"multiformats": "^13.4.1",
|
|
61
|
+
"progress-events": "^1.0.1",
|
|
62
|
+
"race-signal": "^2.0.0"
|
|
70
63
|
},
|
|
71
64
|
"devDependencies": {
|
|
72
|
-
"@helia/mfs": "
|
|
73
|
-
"@helia/unixfs": "
|
|
74
|
-
"@ipld/dag-cbor": "^9.2.
|
|
75
|
-
"@libp2p/logger": "^
|
|
76
|
-
"aegir": "^47.0.
|
|
77
|
-
"blockstore-core": "^
|
|
78
|
-
"datastore-core": "^
|
|
79
|
-
"ipfs-unixfs-importer": "^
|
|
80
|
-
"it-foreach": "^2.1.
|
|
81
|
-
"it-length": "^3.0.
|
|
82
|
-
"
|
|
83
|
-
"sinon": "^20.0.0"
|
|
65
|
+
"@helia/mfs": "5.1.0-1f0e7f7d",
|
|
66
|
+
"@helia/unixfs": "5.1.0-1f0e7f7d",
|
|
67
|
+
"@ipld/dag-cbor": "^9.2.5",
|
|
68
|
+
"@libp2p/logger": "^6.0.5",
|
|
69
|
+
"aegir": "^47.0.22",
|
|
70
|
+
"blockstore-core": "^6.0.2",
|
|
71
|
+
"datastore-core": "^11.0.2",
|
|
72
|
+
"ipfs-unixfs-importer": "^16.0.1",
|
|
73
|
+
"it-foreach": "^2.1.4",
|
|
74
|
+
"it-length": "^3.0.9",
|
|
75
|
+
"sinon": "^21.0.0"
|
|
84
76
|
},
|
|
85
77
|
"sideEffects": false
|
|
86
78
|
}
|
package/src/car.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { CarWriter } from '@ipld/car'
|
|
2
|
+
import { Queue } from '@libp2p/utils'
|
|
2
3
|
import drain from 'it-drain'
|
|
3
4
|
import map from 'it-map'
|
|
5
|
+
import toBuffer from 'it-to-buffer'
|
|
4
6
|
import { createUnsafe } from 'multiformats/block'
|
|
5
|
-
import
|
|
6
|
-
import PQueue from 'p-queue'
|
|
7
|
+
import { raceSignal } from 'race-signal'
|
|
7
8
|
import { DAG_WALK_QUEUE_CONCURRENCY } from './constants.js'
|
|
8
9
|
import { SubgraphExporter } from './export-strategies/subgraph-exporter.js'
|
|
9
10
|
import { GraphSearch } from './traversal-strategies/graph-search.js'
|
|
@@ -24,7 +25,7 @@ interface TraversalContext {
|
|
|
24
25
|
|
|
25
26
|
interface WalkDagContext<Strategy> {
|
|
26
27
|
cid: CID
|
|
27
|
-
queue:
|
|
28
|
+
queue: Queue
|
|
28
29
|
strategy: Strategy
|
|
29
30
|
options?: ExportCarOptions
|
|
30
31
|
}
|
|
@@ -50,13 +51,47 @@ export class Car implements CarInterface {
|
|
|
50
51
|
|
|
51
52
|
async import (reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void> {
|
|
52
53
|
await drain(this.components.blockstore.putMany(
|
|
53
|
-
map(reader.blocks(), ({ cid, bytes }) => ({ cid,
|
|
54
|
+
map(reader.blocks(), ({ cid, bytes }) => ({ cid, bytes })),
|
|
54
55
|
options
|
|
55
56
|
))
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
async export (root: CID | CID[],
|
|
59
|
-
const
|
|
59
|
+
async * export (root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined> {
|
|
60
|
+
const { writer, out } = CarWriter.create(root)
|
|
61
|
+
const iter = out[Symbol.asyncIterator]()
|
|
62
|
+
const controller = new AbortController()
|
|
63
|
+
|
|
64
|
+
// has to be done async so we write to `writer` and read from `out` at the
|
|
65
|
+
// same time
|
|
66
|
+
this._export(root, writer, options)
|
|
67
|
+
.catch((err) => {
|
|
68
|
+
this.log.error('error during streaming export - %e', err)
|
|
69
|
+
controller.abort(err)
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
while (true) {
|
|
73
|
+
const { done, value } = await raceSignal(iter.next(), controller.signal)
|
|
74
|
+
|
|
75
|
+
// the writer's `out` iterable can yield results synchronously, in which
|
|
76
|
+
// case the controller may have been aborted but the event may not have
|
|
77
|
+
// fired yet so check the signal status manually before processing the
|
|
78
|
+
// next iterable result
|
|
79
|
+
if (controller.signal.aborted) {
|
|
80
|
+
throw controller.signal.reason
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (value != null) {
|
|
84
|
+
yield value
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (done === true) {
|
|
88
|
+
break
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private async _export (root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void> {
|
|
94
|
+
const deferred = Promise.withResolvers<Error | void>()
|
|
60
95
|
const roots = Array.isArray(root) ? root : [root]
|
|
61
96
|
|
|
62
97
|
// Create traversal-specific context
|
|
@@ -70,12 +105,12 @@ export class Car implements CarInterface {
|
|
|
70
105
|
|
|
71
106
|
// use a queue to walk the DAG instead of recursion so we can traverse very
|
|
72
107
|
// large DAGs
|
|
73
|
-
const queue = new
|
|
108
|
+
const queue = new Queue({
|
|
74
109
|
concurrency: DAG_WALK_QUEUE_CONCURRENCY
|
|
75
110
|
})
|
|
76
111
|
|
|
77
112
|
let startedExport = false
|
|
78
|
-
queue.
|
|
113
|
+
queue.addEventListener('idle', () => {
|
|
79
114
|
if (startedExport) {
|
|
80
115
|
// idle event was called, and started exporting, so we are done.
|
|
81
116
|
deferred.resolve()
|
|
@@ -91,7 +126,7 @@ export class Car implements CarInterface {
|
|
|
91
126
|
|
|
92
127
|
// Process all verification blocks in the path except the target
|
|
93
128
|
path.slice(0, -1).forEach(cid => {
|
|
94
|
-
|
|
129
|
+
queue.add(async () => {
|
|
95
130
|
await this.#exportDagNode({ cid, queue, writer, strategy: exportStrategy, options, recursive: false })
|
|
96
131
|
})
|
|
97
132
|
.catch((err) => {
|
|
@@ -100,7 +135,7 @@ export class Car implements CarInterface {
|
|
|
100
135
|
})
|
|
101
136
|
|
|
102
137
|
// Process the target block (which will recursively export its DAG)
|
|
103
|
-
|
|
138
|
+
queue.add(async () => {
|
|
104
139
|
await this.#exportDagNode({ cid: targetCid, queue, writer, strategy: exportStrategy, options })
|
|
105
140
|
})
|
|
106
141
|
.catch((err) => {
|
|
@@ -116,9 +151,9 @@ export class Car implements CarInterface {
|
|
|
116
151
|
deferred.reject(new Error('Could not traverse to target CID(s)'))
|
|
117
152
|
}
|
|
118
153
|
})
|
|
119
|
-
queue.
|
|
154
|
+
queue.addEventListener('failure', (evt) => {
|
|
120
155
|
queue.clear()
|
|
121
|
-
deferred.reject(
|
|
156
|
+
deferred.reject(evt.detail.error)
|
|
122
157
|
})
|
|
123
158
|
|
|
124
159
|
for (const root of roots) {
|
|
@@ -139,21 +174,6 @@ export class Car implements CarInterface {
|
|
|
139
174
|
}
|
|
140
175
|
}
|
|
141
176
|
|
|
142
|
-
async * stream (root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined> {
|
|
143
|
-
const { writer, out } = CarWriter.create(root)
|
|
144
|
-
|
|
145
|
-
// has to be done async so we write to `writer` and read from `out` at the
|
|
146
|
-
// same time
|
|
147
|
-
this.export(root, writer, options)
|
|
148
|
-
.catch((err) => {
|
|
149
|
-
this.log.error('error during streaming export - %e', err)
|
|
150
|
-
})
|
|
151
|
-
|
|
152
|
-
for await (const buf of out) {
|
|
153
|
-
yield buf
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
|
|
157
177
|
/**
|
|
158
178
|
* Traverse a DAG and stop when we reach the target node
|
|
159
179
|
*/
|
|
@@ -169,7 +189,7 @@ export class Car implements CarInterface {
|
|
|
169
189
|
}
|
|
170
190
|
|
|
171
191
|
const codec = await this.components.getCodec(cid.code)
|
|
172
|
-
const bytes = await this.components.blockstore.get(cid, options)
|
|
192
|
+
const bytes = await toBuffer(this.components.blockstore.get(cid, options))
|
|
173
193
|
|
|
174
194
|
// we are recursively traversing the dag
|
|
175
195
|
const decodedBlock = createUnsafe({ bytes, cid, codec })
|
|
@@ -193,7 +213,7 @@ export class Car implements CarInterface {
|
|
|
193
213
|
}
|
|
194
214
|
|
|
195
215
|
const codec = await this.components.getCodec(cid.code)
|
|
196
|
-
const bytes = await this.components.blockstore.get(cid, options)
|
|
216
|
+
const bytes = await toBuffer(this.components.blockstore.get(cid, options))
|
|
197
217
|
|
|
198
218
|
// Mark as processed
|
|
199
219
|
options?.blockFilter?.add(cid.multihash.bytes)
|
package/src/index.ts
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* const c = car(helia)
|
|
24
24
|
* const out = nodeFs.createWriteStream('example.car')
|
|
25
25
|
*
|
|
26
|
-
* for await (const buf of c.
|
|
26
|
+
* for await (const buf of c.export(cid)) {
|
|
27
27
|
* out.write(buf)
|
|
28
28
|
* }
|
|
29
29
|
*
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* const c = car(helia)
|
|
45
45
|
* const out = nodeFs.createWriteStream('example.car')
|
|
46
46
|
*
|
|
47
|
-
* for await (const buf of c.
|
|
47
|
+
* for await (const buf of c.export(cid, {
|
|
48
48
|
* traversal: new UnixFSPath('/foo/bar/baz.txt')
|
|
49
49
|
* })) {
|
|
50
50
|
* out.write(buf)
|
|
@@ -78,10 +78,10 @@
|
|
|
78
78
|
|
|
79
79
|
import { Car as CarClass } from './car.js'
|
|
80
80
|
import type { CodecLoader } from '@helia/interface'
|
|
81
|
-
import type { PutManyBlocksProgressEvents, GetBlockProgressEvents } from '@helia/interface/blocks'
|
|
82
|
-
import type {
|
|
81
|
+
import type { PutManyBlocksProgressEvents, GetBlockProgressEvents, ProviderOptions } from '@helia/interface/blocks'
|
|
82
|
+
import type { CarReader } from '@ipld/car'
|
|
83
83
|
import type { AbortOptions, ComponentLogger } from '@libp2p/interface'
|
|
84
|
-
import type { Filter } from '@libp2p/utils
|
|
84
|
+
import type { Filter } from '@libp2p/utils'
|
|
85
85
|
import type { Blockstore } from 'interface-blockstore'
|
|
86
86
|
import type { BlockView } from 'multiformats/block/interface'
|
|
87
87
|
import type { CID } from 'multiformats/cid'
|
|
@@ -127,7 +127,7 @@ export interface ExportStrategy {
|
|
|
127
127
|
export * from './export-strategies/index.js'
|
|
128
128
|
export * from './traversal-strategies/index.js'
|
|
129
129
|
|
|
130
|
-
export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents
|
|
130
|
+
export interface ExportCarOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents>, ProviderOptions {
|
|
131
131
|
|
|
132
132
|
/**
|
|
133
133
|
* If true, the blockstore will not do any network requests.
|
|
@@ -182,38 +182,6 @@ export interface Car {
|
|
|
182
182
|
*/
|
|
183
183
|
import(reader: Pick<CarReader, 'blocks'>, options?: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents>): Promise<void>
|
|
184
184
|
|
|
185
|
-
/**
|
|
186
|
-
* Store all blocks that make up one or more DAGs in a car file.
|
|
187
|
-
*
|
|
188
|
-
* @example
|
|
189
|
-
*
|
|
190
|
-
* ```typescript
|
|
191
|
-
* import fs from 'node:fs'
|
|
192
|
-
* import { Readable } from 'node:stream'
|
|
193
|
-
* import { car } from '@helia/car'
|
|
194
|
-
* import { CarWriter } from '@ipld/car'
|
|
195
|
-
* import { createHelia } from 'helia'
|
|
196
|
-
* import { CID } from 'multiformats/cid'
|
|
197
|
-
* import { pEvent } from 'p-event'
|
|
198
|
-
*
|
|
199
|
-
* const helia = await createHelia()
|
|
200
|
-
* const cid = CID.parse('QmFoo...')
|
|
201
|
-
*
|
|
202
|
-
* const c = car(helia)
|
|
203
|
-
* const { writer, out } = CarWriter.create(cid)
|
|
204
|
-
* const output = fs.createWriteStream('example.car')
|
|
205
|
-
* const stream = Readable.from(out).pipe(output)
|
|
206
|
-
*
|
|
207
|
-
* await Promise.all([
|
|
208
|
-
* c.export(cid, writer),
|
|
209
|
-
* pEvent(stream, 'close')
|
|
210
|
-
* ])
|
|
211
|
-
* ```
|
|
212
|
-
*
|
|
213
|
-
* @deprecated Use `stream` instead. In a future release `stream` will be renamed `export`.
|
|
214
|
-
*/
|
|
215
|
-
export(root: CID | CID[], writer: Pick<CarWriter, 'put' | 'close'>, options?: ExportCarOptions): Promise<void>
|
|
216
|
-
|
|
217
185
|
/**
|
|
218
186
|
* Returns an AsyncGenerator that yields CAR file bytes.
|
|
219
187
|
*
|
|
@@ -229,12 +197,12 @@ export interface Car {
|
|
|
229
197
|
*
|
|
230
198
|
* const c = car(helia)
|
|
231
199
|
*
|
|
232
|
-
* for (const buf of c.
|
|
200
|
+
* for (const buf of c.export(cid)) {
|
|
233
201
|
* // store or send `buf` somewhere
|
|
234
202
|
* }
|
|
235
203
|
* ```
|
|
236
204
|
*/
|
|
237
|
-
|
|
205
|
+
export(root: CID | CID[], options?: ExportCarOptions): AsyncGenerator<Uint8Array, void, undefined>
|
|
238
206
|
}
|
|
239
207
|
|
|
240
208
|
/**
|