ipfs-unixfs-importer 9.0.4 → 9.0.7-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -14
- package/cjs/src/chunker/fixed-size.js +3 -3
- package/cjs/src/chunker/rabin.js +3 -3
- package/cjs/src/dag-builder/file/balanced.js +1 -1
- package/cjs/src/dag-builder/file/buffer-importer.js +2 -4
- package/cjs/src/dag-builder/file/flat.js +1 -1
- package/cjs/src/dag-builder/file/index.js +8 -13
- package/cjs/src/dag-builder/file/trickle.js +1 -1
- package/cjs/src/dag-builder/index.js +8 -9
- package/cjs/src/dag-builder/validate-chunks.js +2 -2
- package/cjs/src/index.js +7 -9
- package/cjs/src/options.js +2 -2
- package/cjs/src/utils/persist.js +2 -4
- package/esm/src/dag-builder/file/index.js +2 -1
- package/esm/src/dag-builder/index.js +6 -3
- package/esm/src/index.js +4 -2
- package/esm/src/options.js +2 -2
- package/package.json +5 -4
- package/types/src/dag-builder/file/index.d.ts.map +1 -1
- package/types/src/dag-builder/index.d.ts.map +1 -1
- package/types/src/index.d.ts.map +1 -1
- package/types/src/options.d.ts.map +1 -1
- package/types/src/types.d.ts +142 -132
- package/types/src/types.d.ts.map +1 -0
- package/types/tsconfig-types.aegir.tsbuildinfo +1 -1
- package/types/tsconfig.tsbuildinfo +1 -1
- package/types/src/chunker/index.d.ts +0 -3
- package/types/src/chunker/index.d.ts.map +0 -1
- package/types/tsconfig-check.aegir.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
- [Usage](#usage)
|
|
19
19
|
- [Example](#example)
|
|
20
20
|
- [API](#api)
|
|
21
|
-
- [const stream = importer(source,
|
|
21
|
+
- [const stream = importer(source, blockstore [, options])](#const-stream--importersource-blockstore--options)
|
|
22
22
|
- [Overriding internals](#overriding-internals)
|
|
23
23
|
- [Contribute](#contribute)
|
|
24
24
|
- [License](#license)
|
|
@@ -45,7 +45,11 @@ Let's create a little directory to import:
|
|
|
45
45
|
And write the importing logic:
|
|
46
46
|
|
|
47
47
|
```js
|
|
48
|
-
|
|
48
|
+
import { importer } from 'ipfs-unixfs-importer'
|
|
49
|
+
import { MemoryBlockstore } from 'blockstore-core/memory'
|
|
50
|
+
|
|
51
|
+
// Where the blocks will be stored
|
|
52
|
+
const blockstore = new MemoryBlockstore()
|
|
49
53
|
|
|
50
54
|
// Import path /tmp/foo/bar
|
|
51
55
|
const source = [{
|
|
@@ -56,9 +60,7 @@ const source = [{
|
|
|
56
60
|
content: fs.createReadStream(file2)
|
|
57
61
|
}]
|
|
58
62
|
|
|
59
|
-
|
|
60
|
-
// https://github.com/ipld/js-ipld-resolver
|
|
61
|
-
for await (const entry of importer(source, ipld, options)) {
|
|
63
|
+
for await (const entry of importer(source, blockstore, options)) {
|
|
62
64
|
console.info(entry)
|
|
63
65
|
}
|
|
64
66
|
```
|
|
@@ -91,10 +93,10 @@ When run, metadata about DAGNodes in the created tree is printed until the root:
|
|
|
91
93
|
#### API
|
|
92
94
|
|
|
93
95
|
```js
|
|
94
|
-
|
|
96
|
+
import { importer } from 'ipfs-unixfs-importer'
|
|
95
97
|
```
|
|
96
98
|
|
|
97
|
-
#### const stream = importer(source,
|
|
99
|
+
#### const stream = importer(source, blockstore [, options])
|
|
98
100
|
|
|
99
101
|
The `importer` function returns an async iterator takes a source async iterator that yields objects of the form:
|
|
100
102
|
|
|
@@ -109,9 +111,9 @@ The `importer` function returns an async iterator takes a source async iterator
|
|
|
109
111
|
|
|
110
112
|
`stream` will output file info objects as files get stored in IPFS. When stats on a node are emitted they are guaranteed to have been written.
|
|
111
113
|
|
|
112
|
-
`
|
|
114
|
+
`blockstore` is an instance of a [blockstore][]
|
|
113
115
|
|
|
114
|
-
The input's file paths and directory structure will be preserved in the [`dag-pb`](https://github.com/ipld/js-
|
|
116
|
+
The input's file paths and directory structure will be preserved in the [`dag-pb`](https://github.com/ipld/js-dag-pb) created nodes.
|
|
115
117
|
|
|
116
118
|
`options` is an JavaScript option that might include the following keys:
|
|
117
119
|
|
|
@@ -150,20 +152,20 @@ Several aspects of the importer are overridable by specifying functions as part
|
|
|
150
152
|
- It should yield `Buffer` objects constructed from the `source` or throw an `Error`
|
|
151
153
|
- `chunker` (function): Optional function that supports the signature `async function * (source, options)` where `source` is an async generator and `options` is an options object
|
|
152
154
|
- It should yield `Buffer` objects.
|
|
153
|
-
- `bufferImporter` (function): Optional function that supports the signature `async function * (entry,
|
|
154
|
-
- This function should read `Buffer`s from `source` and persist them using `
|
|
155
|
+
- `bufferImporter` (function): Optional function that supports the signature `async function * (entry, blockstore, options)`
|
|
156
|
+
- This function should read `Buffer`s from `source` and persist them using `blockstore.put` or similar
|
|
155
157
|
- `entry` is the `{ path, content }` entry, where `entry.content` is an async generator that yields Buffers
|
|
156
158
|
- It should yield functions that return a Promise that resolves to an object with the properties `{ cid, unixfs, size }` where `cid` is a [CID], `unixfs` is a [UnixFS] entry and `size` is a `Number` that represents the serialized size of the [IPLD] node that holds the buffer data.
|
|
157
159
|
- Values will be pulled from this generator in parallel - the amount of parallelisation is controlled by the `blockWriteConcurrency` option (default: 10)
|
|
158
|
-
- `dagBuilder` (function): Optional function that supports the signature `async function * (source,
|
|
160
|
+
- `dagBuilder` (function): Optional function that supports the signature `async function * (source, blockstore, options)`
|
|
159
161
|
- This function should read `{ path, content }` entries from `source` and turn them into DAGs
|
|
160
162
|
- It should yield a `function` that returns a `Promise` that resolves to `{ cid, path, unixfs, node }` where `cid` is a `CID`, `path` is a string, `unixfs` is a UnixFS entry and `node` is a `DAGNode`.
|
|
161
163
|
- Values will be pulled from this generator in parallel - the amount of parallelisation is controlled by the `fileImportConcurrency` option (default: 50)
|
|
162
|
-
- `treeBuilder` (function): Optional function that supports the signature `async function * (source,
|
|
164
|
+
- `treeBuilder` (function): Optional function that supports the signature `async function * (source, blockstore, options)`
|
|
163
165
|
- This function should read `{ cid, path, unixfs, node }` entries from `source` and place them in a directory structure
|
|
164
166
|
- It should yield an object with the properties `{ cid, path, unixfs, size }` where `cid` is a `CID`, `path` is a string, `unixfs` is a UnixFS entry and `size` is a `Number`.
|
|
165
167
|
|
|
166
|
-
[
|
|
168
|
+
[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme
|
|
167
169
|
[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs
|
|
168
170
|
[IPLD]: https://github.com/ipld/js-ipld
|
|
169
171
|
[CID]: https://github.com/multiformats/js-cid
|
|
@@ -7,7 +7,7 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau
|
|
|
7
7
|
var BufferList__default = /*#__PURE__*/_interopDefaultLegacy(BufferList);
|
|
8
8
|
|
|
9
9
|
async function* fixedSizeChunker(source, options) {
|
|
10
|
-
let bl = new BufferList__default[
|
|
10
|
+
let bl = new BufferList__default["default"]();
|
|
11
11
|
let currentLength = 0;
|
|
12
12
|
let emitted = false;
|
|
13
13
|
const maxChunkSize = options.maxChunkSize;
|
|
@@ -18,10 +18,10 @@ async function* fixedSizeChunker(source, options) {
|
|
|
18
18
|
yield bl.slice(0, maxChunkSize);
|
|
19
19
|
emitted = true;
|
|
20
20
|
if (maxChunkSize === bl.length) {
|
|
21
|
-
bl = new BufferList__default[
|
|
21
|
+
bl = new BufferList__default["default"]();
|
|
22
22
|
currentLength = 0;
|
|
23
23
|
} else {
|
|
24
|
-
const newBl = new BufferList__default[
|
|
24
|
+
const newBl = new BufferList__default["default"]();
|
|
25
25
|
newBl.append(bl.shallowSlice(maxChunkSize));
|
|
26
26
|
bl = newBl;
|
|
27
27
|
currentLength -= maxChunkSize;
|
package/cjs/src/chunker/rabin.js
CHANGED
|
@@ -16,14 +16,14 @@ async function* rabinChunker(source, options) {
|
|
|
16
16
|
min = options.minChunkSize;
|
|
17
17
|
max = options.maxChunkSize;
|
|
18
18
|
} else if (!options.avgChunkSize) {
|
|
19
|
-
throw errCode__default[
|
|
19
|
+
throw errCode__default["default"](new Error('please specify an average chunk size'), 'ERR_INVALID_AVG_CHUNK_SIZE');
|
|
20
20
|
} else {
|
|
21
21
|
avg = options.avgChunkSize;
|
|
22
22
|
min = avg / 3;
|
|
23
23
|
max = avg + avg / 2;
|
|
24
24
|
}
|
|
25
25
|
if (min < 16) {
|
|
26
|
-
throw errCode__default[
|
|
26
|
+
throw errCode__default["default"](new Error('rabin min must be greater than 16'), 'ERR_INVALID_MIN_CHUNK_SIZE');
|
|
27
27
|
}
|
|
28
28
|
if (max < min) {
|
|
29
29
|
max = min;
|
|
@@ -44,7 +44,7 @@ async function* rabinChunker(source, options) {
|
|
|
44
44
|
}
|
|
45
45
|
async function* rabin(source, options) {
|
|
46
46
|
const r = await rabinWasm.create(options.bits, options.min, options.max, options.window);
|
|
47
|
-
const buffers = new BufferList__default[
|
|
47
|
+
const buffers = new BufferList__default["default"]();
|
|
48
48
|
for await (const chunk of source) {
|
|
49
49
|
buffers.append(chunk);
|
|
50
50
|
const sizes = r.fingerprint(chunk);
|
|
@@ -11,7 +11,7 @@ function balanced(source, reduce, options) {
|
|
|
11
11
|
}
|
|
12
12
|
async function reduceToParents(source, reduce, options) {
|
|
13
13
|
const roots = [];
|
|
14
|
-
for await (const chunked of batch__default[
|
|
14
|
+
for await (const chunked of batch__default["default"](source, options.maxChildrenPerNode)) {
|
|
15
15
|
roots.push(await reduce(chunked));
|
|
16
16
|
}
|
|
17
17
|
if (roots.length > 1) {
|
|
@@ -14,14 +14,12 @@ function _interopNamespace(e) {
|
|
|
14
14
|
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
15
15
|
Object.defineProperty(n, k, d.get ? d : {
|
|
16
16
|
enumerable: true,
|
|
17
|
-
get: function () {
|
|
18
|
-
return e[k];
|
|
19
|
-
}
|
|
17
|
+
get: function () { return e[k]; }
|
|
20
18
|
});
|
|
21
19
|
}
|
|
22
20
|
});
|
|
23
21
|
}
|
|
24
|
-
n[
|
|
22
|
+
n["default"] = e;
|
|
25
23
|
return Object.freeze(n);
|
|
26
24
|
}
|
|
27
25
|
|
|
@@ -7,7 +7,7 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau
|
|
|
7
7
|
var all__default = /*#__PURE__*/_interopDefaultLegacy(all);
|
|
8
8
|
|
|
9
9
|
async function flat(source, reduce) {
|
|
10
|
-
return reduce(await all__default[
|
|
10
|
+
return reduce(await all__default["default"](source));
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
module.exports = flat;
|
|
@@ -9,13 +9,10 @@ var rawCodec = require('multiformats/codecs/raw');
|
|
|
9
9
|
var flat = require('./flat.js');
|
|
10
10
|
var balanced = require('./balanced.js');
|
|
11
11
|
var trickle = require('./trickle.js');
|
|
12
|
+
var bufferImporter = require('./buffer-importer.js');
|
|
12
13
|
|
|
13
14
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
14
15
|
|
|
15
|
-
function _interopNamespaceDefaultOnly(e) {
|
|
16
|
-
return Object.freeze({__proto__: null, 'default': e});
|
|
17
|
-
}
|
|
18
|
-
|
|
19
16
|
function _interopNamespace(e) {
|
|
20
17
|
if (e && e.__esModule) return e;
|
|
21
18
|
var n = Object.create(null);
|
|
@@ -25,14 +22,12 @@ function _interopNamespace(e) {
|
|
|
25
22
|
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
26
23
|
Object.defineProperty(n, k, d.get ? d : {
|
|
27
24
|
enumerable: true,
|
|
28
|
-
get: function () {
|
|
29
|
-
return e[k];
|
|
30
|
-
}
|
|
25
|
+
get: function () { return e[k]; }
|
|
31
26
|
});
|
|
32
27
|
}
|
|
33
28
|
});
|
|
34
29
|
}
|
|
35
|
-
n[
|
|
30
|
+
n["default"] = e;
|
|
36
31
|
return Object.freeze(n);
|
|
37
32
|
}
|
|
38
33
|
|
|
@@ -49,13 +44,13 @@ const dagBuilders = {
|
|
|
49
44
|
async function* buildFileBatch(file, blockstore, options) {
|
|
50
45
|
let count = -1;
|
|
51
46
|
let previous;
|
|
52
|
-
let bufferImporter;
|
|
47
|
+
let bufferImporter$1;
|
|
53
48
|
if (typeof options.bufferImporter === 'function') {
|
|
54
|
-
bufferImporter = options.bufferImporter;
|
|
49
|
+
bufferImporter$1 = options.bufferImporter;
|
|
55
50
|
} else {
|
|
56
|
-
bufferImporter =
|
|
51
|
+
bufferImporter$1 = bufferImporter;
|
|
57
52
|
}
|
|
58
|
-
for await (const entry of parallelBatch__default[
|
|
53
|
+
for await (const entry of parallelBatch__default["default"](bufferImporter$1(file, blockstore, options), options.blockWriteConcurrency)) {
|
|
59
54
|
count++;
|
|
60
55
|
if (count === 0) {
|
|
61
56
|
previous = entry;
|
|
@@ -150,7 +145,7 @@ const reduce = (file, blockstore, options) => {
|
|
|
150
145
|
function fileBuilder(file, block, options) {
|
|
151
146
|
const dagBuilder = dagBuilders[options.strategy];
|
|
152
147
|
if (!dagBuilder) {
|
|
153
|
-
throw errCode__default[
|
|
148
|
+
throw errCode__default["default"](new Error(`Unknown importer build strategy name: ${ options.strategy }`), 'ERR_BAD_STRATEGY');
|
|
154
149
|
}
|
|
155
150
|
return dagBuilder(buildFileBatch(file, block, options), reduce(file, block, options), options);
|
|
156
151
|
}
|
|
@@ -11,7 +11,7 @@ async function trickleStream(source, reduce, options) {
|
|
|
11
11
|
let iteration = 0;
|
|
12
12
|
let maxDepth = 1;
|
|
13
13
|
let subTree = root;
|
|
14
|
-
for await (const layer of batch__default[
|
|
14
|
+
for await (const layer of batch__default["default"](source, options.maxChildrenPerNode)) {
|
|
15
15
|
if (subTree.isFull()) {
|
|
16
16
|
if (subTree !== root) {
|
|
17
17
|
root.addChild(await subTree.reduce(reduce));
|
|
@@ -3,13 +3,12 @@
|
|
|
3
3
|
var dir = require('./dir.js');
|
|
4
4
|
var index = require('./file/index.js');
|
|
5
5
|
var errCode = require('err-code');
|
|
6
|
+
var rabin = require('../chunker/rabin.js');
|
|
7
|
+
var fixedSize = require('../chunker/fixed-size.js');
|
|
8
|
+
var validateChunks = require('./validate-chunks.js');
|
|
6
9
|
|
|
7
10
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
8
11
|
|
|
9
|
-
function _interopNamespaceDefaultOnly(e) {
|
|
10
|
-
return Object.freeze({__proto__: null, 'default': e});
|
|
11
|
-
}
|
|
12
|
-
|
|
13
12
|
var errCode__default = /*#__PURE__*/_interopDefaultLegacy(errCode);
|
|
14
13
|
|
|
15
14
|
function isIterable(thing) {
|
|
@@ -32,9 +31,9 @@ function contentAsAsyncIterable(content) {
|
|
|
32
31
|
return content;
|
|
33
32
|
}
|
|
34
33
|
} catch {
|
|
35
|
-
throw errCode__default[
|
|
34
|
+
throw errCode__default["default"](new Error('Content was invalid'), 'ERR_INVALID_CONTENT');
|
|
36
35
|
}
|
|
37
|
-
throw errCode__default[
|
|
36
|
+
throw errCode__default["default"](new Error('Content was invalid'), 'ERR_INVALID_CONTENT');
|
|
38
37
|
}
|
|
39
38
|
async function* dagBuilder(source, blockstore, options) {
|
|
40
39
|
for await (const entry of source) {
|
|
@@ -49,15 +48,15 @@ async function* dagBuilder(source, blockstore, options) {
|
|
|
49
48
|
if (typeof options.chunker === 'function') {
|
|
50
49
|
chunker = options.chunker;
|
|
51
50
|
} else if (options.chunker === 'rabin') {
|
|
52
|
-
chunker =
|
|
51
|
+
chunker = rabin;
|
|
53
52
|
} else {
|
|
54
|
-
chunker =
|
|
53
|
+
chunker = fixedSize;
|
|
55
54
|
}
|
|
56
55
|
let chunkValidator;
|
|
57
56
|
if (typeof options.chunkValidator === 'function') {
|
|
58
57
|
chunkValidator = options.chunkValidator;
|
|
59
58
|
} else {
|
|
60
|
-
chunkValidator =
|
|
59
|
+
chunkValidator = validateChunks;
|
|
61
60
|
}
|
|
62
61
|
const file = {
|
|
63
62
|
path: entry.path,
|
|
@@ -10,7 +10,7 @@ var errCode__default = /*#__PURE__*/_interopDefaultLegacy(errCode);
|
|
|
10
10
|
async function* validateChunks(source) {
|
|
11
11
|
for await (const content of source) {
|
|
12
12
|
if (content.length === undefined) {
|
|
13
|
-
throw errCode__default[
|
|
13
|
+
throw errCode__default["default"](new Error('Content was invalid'), 'ERR_INVALID_CONTENT');
|
|
14
14
|
}
|
|
15
15
|
if (typeof content === 'string' || content instanceof String) {
|
|
16
16
|
yield fromString.fromString(content.toString());
|
|
@@ -19,7 +19,7 @@ async function* validateChunks(source) {
|
|
|
19
19
|
} else if (content instanceof Uint8Array) {
|
|
20
20
|
yield content;
|
|
21
21
|
} else {
|
|
22
|
-
throw errCode__default[
|
|
22
|
+
throw errCode__default["default"](new Error('Content was invalid'), 'ERR_INVALID_CONTENT');
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
}
|
package/cjs/src/index.js
CHANGED
|
@@ -4,13 +4,11 @@ Object.defineProperty(exports, '__esModule', { value: true });
|
|
|
4
4
|
|
|
5
5
|
var parallelBatch = require('it-parallel-batch');
|
|
6
6
|
var options = require('./options.js');
|
|
7
|
+
var index = require('./dag-builder/index.js');
|
|
8
|
+
var treeBuilder = require('./tree-builder.js');
|
|
7
9
|
|
|
8
10
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
9
11
|
|
|
10
|
-
function _interopNamespaceDefaultOnly(e) {
|
|
11
|
-
return Object.freeze({__proto__: null, 'default': e});
|
|
12
|
-
}
|
|
13
|
-
|
|
14
12
|
var parallelBatch__default = /*#__PURE__*/_interopDefaultLegacy(parallelBatch);
|
|
15
13
|
|
|
16
14
|
async function* importer(source, blockstore, options$1 = {}) {
|
|
@@ -19,13 +17,13 @@ async function* importer(source, blockstore, options$1 = {}) {
|
|
|
19
17
|
if (typeof options$1.dagBuilder === 'function') {
|
|
20
18
|
dagBuilder = options$1.dagBuilder;
|
|
21
19
|
} else {
|
|
22
|
-
dagBuilder =
|
|
20
|
+
dagBuilder = index;
|
|
23
21
|
}
|
|
24
|
-
let treeBuilder;
|
|
22
|
+
let treeBuilder$1;
|
|
25
23
|
if (typeof options$1.treeBuilder === 'function') {
|
|
26
|
-
treeBuilder = options$1.treeBuilder;
|
|
24
|
+
treeBuilder$1 = options$1.treeBuilder;
|
|
27
25
|
} else {
|
|
28
|
-
treeBuilder =
|
|
26
|
+
treeBuilder$1 = treeBuilder;
|
|
29
27
|
}
|
|
30
28
|
let candidates;
|
|
31
29
|
if (Symbol.asyncIterator in source || Symbol.iterator in source) {
|
|
@@ -33,7 +31,7 @@ async function* importer(source, blockstore, options$1 = {}) {
|
|
|
33
31
|
} else {
|
|
34
32
|
candidates = [source];
|
|
35
33
|
}
|
|
36
|
-
for await (const entry of treeBuilder(parallelBatch__default[
|
|
34
|
+
for await (const entry of treeBuilder$1(parallelBatch__default["default"](dagBuilder(candidates, blockstore, opts), opts.fileImportConcurrency), blockstore, opts)) {
|
|
37
35
|
yield {
|
|
38
36
|
cid: entry.cid,
|
|
39
37
|
path: entry.path,
|
package/cjs/src/options.js
CHANGED
|
@@ -8,7 +8,6 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau
|
|
|
8
8
|
|
|
9
9
|
var mergeOptions__default = /*#__PURE__*/_interopDefaultLegacy(mergeOptions);
|
|
10
10
|
|
|
11
|
-
mergeOptions__default['default'].bind({ ignoreUndefined: true });
|
|
12
11
|
async function hamtHashFn(buf) {
|
|
13
12
|
return (await murmur3.murmur3128.encode(buf)).slice(0, 8).reverse();
|
|
14
13
|
}
|
|
@@ -42,7 +41,8 @@ const defaultOptions = {
|
|
|
42
41
|
hamtBucketBits: 8
|
|
43
42
|
};
|
|
44
43
|
var defaultOptions$1 = (options = {}) => {
|
|
45
|
-
|
|
44
|
+
const defaults = mergeOptions__default["default"].bind({ ignoreUndefined: true });
|
|
45
|
+
return defaults(defaultOptions, options);
|
|
46
46
|
};
|
|
47
47
|
|
|
48
48
|
module.exports = defaultOptions$1;
|
package/cjs/src/utils/persist.js
CHANGED
|
@@ -13,14 +13,12 @@ function _interopNamespace(e) {
|
|
|
13
13
|
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
14
14
|
Object.defineProperty(n, k, d.get ? d : {
|
|
15
15
|
enumerable: true,
|
|
16
|
-
get: function () {
|
|
17
|
-
return e[k];
|
|
18
|
-
}
|
|
16
|
+
get: function () { return e[k]; }
|
|
19
17
|
});
|
|
20
18
|
}
|
|
21
19
|
});
|
|
22
20
|
}
|
|
23
|
-
n[
|
|
21
|
+
n["default"] = e;
|
|
24
22
|
return Object.freeze(n);
|
|
25
23
|
}
|
|
26
24
|
|
|
@@ -11,6 +11,7 @@ import * as dagPb from '@ipld/dag-pb';
|
|
|
11
11
|
import dagFlat from './flat.js';
|
|
12
12
|
import dagBalanced from './balanced.js';
|
|
13
13
|
import dagTrickle from './trickle.js';
|
|
14
|
+
import bufferImporterFn from './buffer-importer.js';
|
|
14
15
|
const dagBuilders = {
|
|
15
16
|
flat: dagFlat,
|
|
16
17
|
balanced: dagBalanced,
|
|
@@ -23,7 +24,7 @@ async function* buildFileBatch(file, blockstore, options) {
|
|
|
23
24
|
if (typeof options.bufferImporter === 'function') {
|
|
24
25
|
bufferImporter = options.bufferImporter;
|
|
25
26
|
} else {
|
|
26
|
-
bufferImporter =
|
|
27
|
+
bufferImporter = bufferImporterFn;
|
|
27
28
|
}
|
|
28
29
|
for await (const entry of parallelBatch(bufferImporter(file, blockstore, options), options.blockWriteConcurrency)) {
|
|
29
30
|
count++;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import dirBuilder from './dir.js';
|
|
2
2
|
import fileBuilder from './file/index.js';
|
|
3
3
|
import errCode from 'err-code';
|
|
4
|
+
import rabin from '../chunker/rabin.js';
|
|
5
|
+
import fixedSize from '../chunker/fixed-size.js';
|
|
6
|
+
import validateChunks from './validate-chunks.js';
|
|
4
7
|
function isIterable(thing) {
|
|
5
8
|
return Symbol.iterator in thing;
|
|
6
9
|
}
|
|
@@ -38,15 +41,15 @@ async function* dagBuilder(source, blockstore, options) {
|
|
|
38
41
|
if (typeof options.chunker === 'function') {
|
|
39
42
|
chunker = options.chunker;
|
|
40
43
|
} else if (options.chunker === 'rabin') {
|
|
41
|
-
chunker =
|
|
44
|
+
chunker = rabin;
|
|
42
45
|
} else {
|
|
43
|
-
chunker =
|
|
46
|
+
chunker = fixedSize;
|
|
44
47
|
}
|
|
45
48
|
let chunkValidator;
|
|
46
49
|
if (typeof options.chunkValidator === 'function') {
|
|
47
50
|
chunkValidator = options.chunkValidator;
|
|
48
51
|
} else {
|
|
49
|
-
chunkValidator =
|
|
52
|
+
chunkValidator = validateChunks;
|
|
50
53
|
}
|
|
51
54
|
const file = {
|
|
52
55
|
path: entry.path,
|
package/esm/src/index.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import parallelBatch from 'it-parallel-batch';
|
|
2
2
|
import defaultOptions from './options.js';
|
|
3
|
+
import dagBuilderFn from './dag-builder/index.js';
|
|
4
|
+
import treeBuilderFn from './tree-builder.js';
|
|
3
5
|
export async function* importer(source, blockstore, options = {}) {
|
|
4
6
|
const opts = defaultOptions(options);
|
|
5
7
|
let dagBuilder;
|
|
6
8
|
if (typeof options.dagBuilder === 'function') {
|
|
7
9
|
dagBuilder = options.dagBuilder;
|
|
8
10
|
} else {
|
|
9
|
-
dagBuilder =
|
|
11
|
+
dagBuilder = dagBuilderFn;
|
|
10
12
|
}
|
|
11
13
|
let treeBuilder;
|
|
12
14
|
if (typeof options.treeBuilder === 'function') {
|
|
13
15
|
treeBuilder = options.treeBuilder;
|
|
14
16
|
} else {
|
|
15
|
-
treeBuilder =
|
|
17
|
+
treeBuilder = treeBuilderFn;
|
|
16
18
|
}
|
|
17
19
|
let candidates;
|
|
18
20
|
if (Symbol.asyncIterator in source || Symbol.iterator in source) {
|
package/esm/src/options.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import mergeOptions from 'merge-options';
|
|
2
2
|
import { sha256 } from 'multiformats/hashes/sha2';
|
|
3
3
|
import { murmur3128 } from '@multiformats/murmur3';
|
|
4
|
-
mergeOptions.bind({ ignoreUndefined: true });
|
|
5
4
|
async function hamtHashFn(buf) {
|
|
6
5
|
return (await murmur3128.encode(buf)).slice(0, 8).reverse();
|
|
7
6
|
}
|
|
@@ -35,5 +34,6 @@ const defaultOptions = {
|
|
|
35
34
|
hamtBucketBits: 8
|
|
36
35
|
};
|
|
37
36
|
export default (options = {}) => {
|
|
38
|
-
|
|
37
|
+
const defaults = mergeOptions.bind({ ignoreUndefined: true });
|
|
38
|
+
return defaults(defaultOptions, options);
|
|
39
39
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ipfs-unixfs-importer",
|
|
3
|
-
"version": "9.0.
|
|
3
|
+
"version": "9.0.7-rc.5+273a141",
|
|
4
4
|
"description": "JavaScript implementation of the UnixFs importer used by IPFS",
|
|
5
5
|
"leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
|
|
6
6
|
"main": "./cjs/src/index.js",
|
|
@@ -48,7 +48,8 @@
|
|
|
48
48
|
"nyc": "^15.0.0",
|
|
49
49
|
"readable-stream": "^3.6.0",
|
|
50
50
|
"rimraf": "^3.0.2",
|
|
51
|
-
"util": "^0.12.3"
|
|
51
|
+
"util": "^0.12.3",
|
|
52
|
+
"wherearewe": "^1.0.0"
|
|
52
53
|
},
|
|
53
54
|
"dependencies": {
|
|
54
55
|
"@ipld/dag-pb": "^2.0.2",
|
|
@@ -57,7 +58,7 @@
|
|
|
57
58
|
"err-code": "^3.0.1",
|
|
58
59
|
"hamt-sharding": "^2.0.0",
|
|
59
60
|
"interface-blockstore": "^1.0.0",
|
|
60
|
-
"ipfs-unixfs": "^6.0.
|
|
61
|
+
"ipfs-unixfs": "^6.0.7-rc.5+273a141",
|
|
61
62
|
"it-all": "^1.0.5",
|
|
62
63
|
"it-batch": "^1.0.8",
|
|
63
64
|
"it-first": "^1.0.6",
|
|
@@ -74,7 +75,7 @@
|
|
|
74
75
|
"sourceType": "module"
|
|
75
76
|
}
|
|
76
77
|
},
|
|
77
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "273a141b5ee3805bd0ef2dc8ed7870f8c6c8a820",
|
|
78
79
|
"exports": {
|
|
79
80
|
".": {
|
|
80
81
|
"browser": "./esm/src/index.js",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/dag-builder/file/index.js"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/dag-builder/file/index.js"],"names":[],"mappings":";yBAca,OAAO,sBAAsB,EAAE,UAAU;mBACzC,OAAO,aAAa,EAAE,IAAI;8BAC1B,OAAO,aAAa,EAAE,eAAe;sBACrC,OAAO,aAAa,EAAE,OAAO;yBAC7B,OAAO,aAAa,EAAE,UAAU;6BAChC,OAAO,aAAa,EAAE,cAAc;AA+KjD;;GAEG;AACH,uOAQC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/dag-builder/index.js"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/dag-builder/index.js"],"names":[],"mappings":";mBAQa,OAAO,UAAU,EAAE,IAAI;wBACvB,OAAO,UAAU,EAAE,SAAS;yBAC5B,OAAO,UAAU,EAAE,UAAU;sBAC7B,OAAO,UAAU,EAAE,OAAO;6BAC1B,OAAO,UAAU,EAAE,cAAc;AA2C9C;;GAEG;AACH,kUA4DC"}
|
package/types/src/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.js"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;GAgBG;AAEH;;;;GAIG;AACH,iCAJW,cAAc,eAAe,CAAC,GAAG,SAAS,eAAe,CAAC,GAAG,eAAe,cAC5E,UAAU,YACV,mBAAmB;;;;;kBAwC7B;yBA5DY,OAAO,sBAAsB,EAAE,UAAU;8BACzC,OAAO,SAAS,EAAE,eAAe;kCACjC,OAAO,SAAS,EAAE,mBAAmB;8BACrC,OAAO,SAAS,EAAE,eAAe;wBACjC,OAAO,SAAS,EAAE,SAAS;mBAC3B,OAAO,SAAS,EAAE,IAAI;2BACtB,OAAO,SAAS,EAAE,YAAY;sBAE9B,OAAO,SAAS,EAAE,OAAO;yBACzB,OAAO,SAAS,EAAE,UAAU;0BAC5B,OAAO,SAAS,EAAE,WAAW;6BAC7B,OAAO,SAAS,EAAE,cAAc;6BAChC,OAAO,SAAS,EAAE,cAAc;sBAChC,OAAO,SAAS,EAAE,OAAO;8BACzB,OAAO,SAAS,EAAE,eAAe"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/options.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/options.js"],"names":[],"mappings":"AA4De,oCAHJ,mBAAmB,GACjB,eAAe,CAK3B;;kCA7CY,OAAO,SAAS,EAAE,mBAAmB;8BACrC,OAAO,SAAS,EAAE,eAAe"}
|