@ipld/car 3.2.4 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -2
- package/api.ts +22 -4
- package/buffer-writer +1 -0
- package/cjs/browser-test/common.js +78 -3
- package/cjs/browser-test/node-test-large.js +8 -8
- package/cjs/browser-test/test-buffer-writer.js +330 -0
- package/cjs/browser-test/test-errors.js +57 -34
- package/cjs/browser-test/test-indexer.js +12 -0
- package/cjs/browser-test/test-reader.js +83 -0
- package/cjs/browser-test/test-writer.js +3 -3
- package/cjs/lib/buffer-writer.js +161 -0
- package/cjs/lib/decoder.js +72 -15
- package/cjs/lib/encoder.js +2 -2
- package/cjs/lib/header-validator.js +29 -0
- package/cjs/lib/reader-browser.js +7 -7
- package/cjs/lib/writer-browser.js +1 -1
- package/cjs/node-test/common.js +78 -3
- package/cjs/node-test/node-test-large.js +8 -8
- package/cjs/node-test/test-buffer-writer.js +330 -0
- package/cjs/node-test/test-errors.js +57 -34
- package/cjs/node-test/test-indexer.js +12 -0
- package/cjs/node-test/test-reader.js +83 -0
- package/cjs/node-test/test-writer.js +3 -3
- package/esm/browser-test/common.js +76 -1
- package/esm/browser-test/test-buffer-writer.js +311 -0
- package/esm/browser-test/test-errors.js +57 -33
- package/esm/browser-test/test-indexer.js +15 -0
- package/esm/browser-test/test-reader.js +90 -1
- package/esm/browser-test/test-writer.js +3 -3
- package/esm/lib/buffer-writer.js +126 -0
- package/esm/lib/decoder.js +69 -13
- package/esm/lib/header-validator.js +23 -0
- package/esm/lib/reader-browser.js +7 -8
- package/esm/lib/writer-browser.js +1 -1
- package/esm/node-test/common.js +76 -1
- package/esm/node-test/test-buffer-writer.js +311 -0
- package/esm/node-test/test-errors.js +57 -33
- package/esm/node-test/test-indexer.js +15 -0
- package/esm/node-test/test-reader.js +90 -1
- package/esm/node-test/test-writer.js +3 -3
- package/examples/car-to-fixture.js +1 -4
- package/examples/dump-index.js +24 -0
- package/examples/test-examples.js +33 -0
- package/lib/buffer-writer.js +286 -0
- package/lib/coding.ts +17 -2
- package/lib/decoder.js +130 -14
- package/lib/header-validator.js +33 -0
- package/lib/header.ipldsch +6 -0
- package/lib/reader-browser.js +11 -11
- package/lib/writer-browser.js +1 -1
- package/package.json +16 -6
- package/test/_fixtures_to_js.mjs +24 -0
- package/test/common.js +49 -3
- package/test/go.carv2 +0 -0
- package/test/test-buffer-writer.js +256 -0
- package/test/test-errors.js +52 -30
- package/test/test-indexer.js +24 -1
- package/test/test-reader.js +94 -1
- package/test/test-writer.js +3 -3
- package/tsconfig.json +3 -1
- package/types/api.d.ts +16 -0
- package/types/api.d.ts.map +1 -1
- package/types/lib/buffer-writer.d.ts +86 -0
- package/types/lib/buffer-writer.d.ts.map +1 -0
- package/types/lib/coding.d.ts +14 -4
- package/types/lib/coding.d.ts.map +1 -1
- package/types/lib/decoder.d.ts +38 -2
- package/types/lib/decoder.d.ts.map +1 -1
- package/types/lib/header-validator.d.ts +2 -0
- package/types/lib/header-validator.d.ts.map +1 -0
- package/types/lib/reader-browser.d.ts +15 -7
- package/types/lib/reader-browser.d.ts.map +1 -1
- package/types/test/_fixtures_to_js.d.mts +3 -0
- package/types/test/_fixtures_to_js.d.mts.map +1 -0
- package/types/test/common.d.ts +13 -0
- package/types/test/common.d.ts.map +1 -1
- package/types/test/fixtures-expectations.d.ts +63 -0
- package/types/test/fixtures-expectations.d.ts.map +1 -0
- package/types/test/fixtures.d.ts +3 -0
- package/types/test/fixtures.d.ts.map +1 -0
- package/types/test/test-buffer-writer.d.ts +2 -0
- package/types/test/test-buffer-writer.d.ts.map +1 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
|
+
|
|
5
|
+
var varint = require('varint');
|
|
6
|
+
var cborg = require('cborg');
|
|
7
|
+
var length = require('cborg/length');
|
|
8
|
+
var CBOR = require('@ipld/dag-cbor');
|
|
9
|
+
|
|
10
|
+
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
11
|
+
|
|
12
|
+
function _interopNamespace(e) {
|
|
13
|
+
if (e && e.__esModule) return e;
|
|
14
|
+
var n = Object.create(null);
|
|
15
|
+
if (e) {
|
|
16
|
+
Object.keys(e).forEach(function (k) {
|
|
17
|
+
if (k !== 'default') {
|
|
18
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
19
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
20
|
+
enumerable: true,
|
|
21
|
+
get: function () { return e[k]; }
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
n["default"] = e;
|
|
27
|
+
return Object.freeze(n);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
var varint__default = /*#__PURE__*/_interopDefaultLegacy(varint);
|
|
31
|
+
var CBOR__namespace = /*#__PURE__*/_interopNamespace(CBOR);
|
|
32
|
+
|
|
33
|
+
class CarBufferWriter {
|
|
34
|
+
constructor(bytes, headerSize) {
|
|
35
|
+
this.bytes = bytes;
|
|
36
|
+
this.byteOffset = headerSize;
|
|
37
|
+
this.roots = [];
|
|
38
|
+
this.headerSize = headerSize;
|
|
39
|
+
}
|
|
40
|
+
addRoot(root, options) {
|
|
41
|
+
addRoot(this, root, options);
|
|
42
|
+
return this;
|
|
43
|
+
}
|
|
44
|
+
write(block) {
|
|
45
|
+
addBlock(this, block);
|
|
46
|
+
return this;
|
|
47
|
+
}
|
|
48
|
+
close(options) {
|
|
49
|
+
return close(this, options);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const addRoot = (writer, root, {
|
|
53
|
+
resize = false
|
|
54
|
+
} = {}) => {
|
|
55
|
+
const {bytes, headerSize, byteOffset, roots} = writer;
|
|
56
|
+
writer.roots.push(root);
|
|
57
|
+
const size = headerLength(writer);
|
|
58
|
+
if (size > headerSize) {
|
|
59
|
+
if (size - headerSize + byteOffset < bytes.byteLength) {
|
|
60
|
+
if (resize) {
|
|
61
|
+
resizeHeader(writer, size);
|
|
62
|
+
} else {
|
|
63
|
+
roots.pop();
|
|
64
|
+
throw new RangeError(`Header of size ${ headerSize } has no capacity for new root ${ root }.
|
|
65
|
+
However there is a space in the buffer and you could call addRoot(root, { resize: root }) to resize header to make a space for this root.`);
|
|
66
|
+
}
|
|
67
|
+
} else {
|
|
68
|
+
roots.pop();
|
|
69
|
+
throw new RangeError(`Buffer has no capacity for a new root ${ root }`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
const blockLength = ({cid, bytes}) => {
|
|
74
|
+
const size = cid.bytes.byteLength + bytes.byteLength;
|
|
75
|
+
return varint__default["default"].encodingLength(size) + size;
|
|
76
|
+
};
|
|
77
|
+
const addBlock = (writer, {cid, bytes}) => {
|
|
78
|
+
const byteLength = cid.bytes.byteLength + bytes.byteLength;
|
|
79
|
+
const size = varint__default["default"].encode(byteLength);
|
|
80
|
+
if (writer.byteOffset + size.length + byteLength > writer.bytes.byteLength) {
|
|
81
|
+
throw new RangeError('Buffer has no capacity for this block');
|
|
82
|
+
} else {
|
|
83
|
+
writeBytes(writer, size);
|
|
84
|
+
writeBytes(writer, cid.bytes);
|
|
85
|
+
writeBytes(writer, bytes);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
const close = (writer, {
|
|
89
|
+
resize = false
|
|
90
|
+
} = {}) => {
|
|
91
|
+
const {roots, bytes, byteOffset, headerSize} = writer;
|
|
92
|
+
const headerBytes = CBOR__namespace.encode({
|
|
93
|
+
version: 1,
|
|
94
|
+
roots
|
|
95
|
+
});
|
|
96
|
+
const varintBytes = varint__default["default"].encode(headerBytes.length);
|
|
97
|
+
const size = varintBytes.length + headerBytes.byteLength;
|
|
98
|
+
const offset = headerSize - size;
|
|
99
|
+
if (offset === 0) {
|
|
100
|
+
writeHeader(writer, varintBytes, headerBytes);
|
|
101
|
+
return bytes.subarray(0, byteOffset);
|
|
102
|
+
} else if (resize) {
|
|
103
|
+
resizeHeader(writer, size);
|
|
104
|
+
writeHeader(writer, varintBytes, headerBytes);
|
|
105
|
+
return bytes.subarray(0, writer.byteOffset);
|
|
106
|
+
} else {
|
|
107
|
+
throw new RangeError(`Header size was overestimated.
|
|
108
|
+
You can use close({ resize: true }) to resize header`);
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
const resizeHeader = (writer, byteLength) => {
|
|
112
|
+
const {bytes, headerSize} = writer;
|
|
113
|
+
bytes.set(bytes.subarray(headerSize, writer.byteOffset), byteLength);
|
|
114
|
+
writer.byteOffset += byteLength - headerSize;
|
|
115
|
+
writer.headerSize = byteLength;
|
|
116
|
+
};
|
|
117
|
+
const writeBytes = (writer, bytes) => {
|
|
118
|
+
writer.bytes.set(bytes, writer.byteOffset);
|
|
119
|
+
writer.byteOffset += bytes.length;
|
|
120
|
+
};
|
|
121
|
+
const writeHeader = ({bytes}, varint, header) => {
|
|
122
|
+
bytes.set(varint);
|
|
123
|
+
bytes.set(header, varint.length);
|
|
124
|
+
};
|
|
125
|
+
const headerPreludeTokens = [
|
|
126
|
+
new cborg.Token(cborg.Type.map, 2),
|
|
127
|
+
new cborg.Token(cborg.Type.string, 'version'),
|
|
128
|
+
new cborg.Token(cborg.Type.uint, 1),
|
|
129
|
+
new cborg.Token(cborg.Type.string, 'roots')
|
|
130
|
+
];
|
|
131
|
+
const CID_TAG = new cborg.Token(cborg.Type.tag, 42);
|
|
132
|
+
const calculateHeaderLength = rootLengths => {
|
|
133
|
+
const tokens = [...headerPreludeTokens];
|
|
134
|
+
tokens.push(new cborg.Token(cborg.Type.array, rootLengths.length));
|
|
135
|
+
for (const rootLength of rootLengths) {
|
|
136
|
+
tokens.push(CID_TAG);
|
|
137
|
+
tokens.push(new cborg.Token(cborg.Type.bytes, { length: rootLength + 1 }));
|
|
138
|
+
}
|
|
139
|
+
const length$1 = length.tokensToLength(tokens);
|
|
140
|
+
return varint__default["default"].encodingLength(length$1) + length$1;
|
|
141
|
+
};
|
|
142
|
+
const headerLength = ({roots}) => calculateHeaderLength(roots.map(cid => cid.bytes.byteLength));
|
|
143
|
+
const estimateHeaderLength = (rootCount, rootByteLength = 36) => calculateHeaderLength(new Array(rootCount).fill(rootByteLength));
|
|
144
|
+
const createWriter = (buffer, {roots = [], byteOffset = 0, byteLength = buffer.byteLength, headerSize = headerLength({ roots })} = {}) => {
|
|
145
|
+
const bytes = new Uint8Array(buffer, byteOffset, byteLength);
|
|
146
|
+
const writer = new CarBufferWriter(bytes, headerSize);
|
|
147
|
+
for (const root of roots) {
|
|
148
|
+
writer.addRoot(root);
|
|
149
|
+
}
|
|
150
|
+
return writer;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
exports.addBlock = addBlock;
|
|
154
|
+
exports.addRoot = addRoot;
|
|
155
|
+
exports.blockLength = blockLength;
|
|
156
|
+
exports.calculateHeaderLength = calculateHeaderLength;
|
|
157
|
+
exports.close = close;
|
|
158
|
+
exports.createWriter = createWriter;
|
|
159
|
+
exports.estimateHeaderLength = estimateHeaderLength;
|
|
160
|
+
exports.headerLength = headerLength;
|
|
161
|
+
exports.resizeHeader = resizeHeader;
|
package/cjs/lib/decoder.js
CHANGED
|
@@ -5,7 +5,8 @@ Object.defineProperty(exports, '__esModule', { value: true });
|
|
|
5
5
|
var varint = require('varint');
|
|
6
6
|
var cid = require('multiformats/cid');
|
|
7
7
|
var Digest = require('multiformats/hashes/digest');
|
|
8
|
-
var
|
|
8
|
+
var CBOR = require('@ipld/dag-cbor');
|
|
9
|
+
var headerValidator = require('./header-validator.js');
|
|
9
10
|
|
|
10
11
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
11
12
|
|
|
@@ -35,36 +36,58 @@ const CIDV0_BYTES = {
|
|
|
35
36
|
LENGTH: 32,
|
|
36
37
|
DAG_PB: 112
|
|
37
38
|
};
|
|
39
|
+
const V2_HEADER_LENGTH = 16 + 8 + 8 + 8;
|
|
38
40
|
async function readVarint(reader) {
|
|
39
41
|
const bytes = await reader.upTo(8);
|
|
42
|
+
if (!bytes.length) {
|
|
43
|
+
throw new Error('Unexpected end of data');
|
|
44
|
+
}
|
|
40
45
|
const i = varint__default["default"].decode(bytes);
|
|
41
46
|
reader.seek(varint__default["default"].decode.bytes);
|
|
42
47
|
return i;
|
|
43
48
|
}
|
|
44
|
-
async function
|
|
49
|
+
async function readV2Header(reader) {
|
|
50
|
+
const bytes = await reader.exactly(V2_HEADER_LENGTH);
|
|
51
|
+
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
52
|
+
let offset = 0;
|
|
53
|
+
const header = {
|
|
54
|
+
version: 2,
|
|
55
|
+
characteristics: [
|
|
56
|
+
dv.getBigUint64(offset, true),
|
|
57
|
+
dv.getBigUint64(offset += 8, true)
|
|
58
|
+
],
|
|
59
|
+
dataOffset: Number(dv.getBigUint64(offset += 8, true)),
|
|
60
|
+
dataSize: Number(dv.getBigUint64(offset += 8, true)),
|
|
61
|
+
indexOffset: Number(dv.getBigUint64(offset += 8, true))
|
|
62
|
+
};
|
|
63
|
+
reader.seek(V2_HEADER_LENGTH);
|
|
64
|
+
return header;
|
|
65
|
+
}
|
|
66
|
+
async function readHeader(reader, strictVersion) {
|
|
45
67
|
const length = await readVarint(reader);
|
|
46
68
|
if (length === 0) {
|
|
47
69
|
throw new Error('Invalid CAR header (zero length)');
|
|
48
70
|
}
|
|
49
71
|
const header = await reader.exactly(length);
|
|
50
72
|
reader.seek(length);
|
|
51
|
-
const block =
|
|
52
|
-
if (
|
|
73
|
+
const block = CBOR.decode(header);
|
|
74
|
+
if (!headerValidator.CarHeader(block)) {
|
|
53
75
|
throw new Error('Invalid CAR header format');
|
|
54
76
|
}
|
|
55
|
-
if (block.version !== 1) {
|
|
56
|
-
|
|
57
|
-
throw new Error(`Invalid CAR version: "${ block.version }"`);
|
|
58
|
-
}
|
|
59
|
-
throw new Error(`Invalid CAR version: ${ block.version }`);
|
|
77
|
+
if (block.version !== 1 && block.version !== 2 || strictVersion !== undefined && block.version !== strictVersion) {
|
|
78
|
+
throw new Error(`Invalid CAR version: ${ block.version }${ strictVersion !== undefined ? ` (expected ${ strictVersion })` : '' }`);
|
|
60
79
|
}
|
|
61
|
-
|
|
80
|
+
const hasRoots = Array.isArray(block.roots);
|
|
81
|
+
if (block.version === 1 && !hasRoots || block.version === 2 && hasRoots) {
|
|
62
82
|
throw new Error('Invalid CAR header format');
|
|
63
83
|
}
|
|
64
|
-
if (
|
|
65
|
-
|
|
84
|
+
if (block.version === 1) {
|
|
85
|
+
return block;
|
|
66
86
|
}
|
|
67
|
-
|
|
87
|
+
const v2Header = await readV2Header(reader);
|
|
88
|
+
reader.seek(v2Header.dataOffset - reader.pos);
|
|
89
|
+
const v1Header = await readHeader(reader, 1);
|
|
90
|
+
return Object.assign(v1Header, v2Header);
|
|
68
91
|
}
|
|
69
92
|
async function readMultihash(reader) {
|
|
70
93
|
const bytes = await reader.upTo(8);
|
|
@@ -102,7 +125,7 @@ async function readBlockHead(reader) {
|
|
|
102
125
|
}
|
|
103
126
|
length += reader.pos - start;
|
|
104
127
|
const cid = await readCid(reader);
|
|
105
|
-
const blockLength = length - (reader.pos - start);
|
|
128
|
+
const blockLength = length - Number(reader.pos - start);
|
|
106
129
|
return {
|
|
107
130
|
cid,
|
|
108
131
|
length,
|
|
@@ -132,7 +155,14 @@ async function readBlockIndex(reader) {
|
|
|
132
155
|
return index;
|
|
133
156
|
}
|
|
134
157
|
function createDecoder(reader) {
|
|
135
|
-
const headerPromise =
|
|
158
|
+
const headerPromise = (async () => {
|
|
159
|
+
const header = await readHeader(reader);
|
|
160
|
+
if (header.version === 2) {
|
|
161
|
+
const v1length = reader.pos - header.dataOffset;
|
|
162
|
+
reader = limitReader(reader, header.dataSize - v1length);
|
|
163
|
+
}
|
|
164
|
+
return header;
|
|
165
|
+
})();
|
|
136
166
|
return {
|
|
137
167
|
header: () => headerPromise,
|
|
138
168
|
async *blocks() {
|
|
@@ -235,10 +265,37 @@ function asyncIterableReader(asyncIterable) {
|
|
|
235
265
|
}
|
|
236
266
|
return chunkReader(readChunk);
|
|
237
267
|
}
|
|
268
|
+
function limitReader(reader, byteLimit) {
|
|
269
|
+
let bytesRead = 0;
|
|
270
|
+
return {
|
|
271
|
+
async upTo(length) {
|
|
272
|
+
let bytes = await reader.upTo(length);
|
|
273
|
+
if (bytes.length + bytesRead > byteLimit) {
|
|
274
|
+
bytes = bytes.subarray(0, byteLimit - bytesRead);
|
|
275
|
+
}
|
|
276
|
+
return bytes;
|
|
277
|
+
},
|
|
278
|
+
async exactly(length) {
|
|
279
|
+
const bytes = await reader.exactly(length);
|
|
280
|
+
if (bytes.length + bytesRead > byteLimit) {
|
|
281
|
+
throw new Error('Unexpected end of data');
|
|
282
|
+
}
|
|
283
|
+
return bytes;
|
|
284
|
+
},
|
|
285
|
+
seek(length) {
|
|
286
|
+
bytesRead += length;
|
|
287
|
+
reader.seek(length);
|
|
288
|
+
},
|
|
289
|
+
get pos() {
|
|
290
|
+
return reader.pos;
|
|
291
|
+
}
|
|
292
|
+
};
|
|
293
|
+
}
|
|
238
294
|
|
|
239
295
|
exports.asyncIterableReader = asyncIterableReader;
|
|
240
296
|
exports.bytesReader = bytesReader;
|
|
241
297
|
exports.chunkReader = chunkReader;
|
|
242
298
|
exports.createDecoder = createDecoder;
|
|
299
|
+
exports.limitReader = limitReader;
|
|
243
300
|
exports.readBlockHead = readBlockHead;
|
|
244
301
|
exports.readHeader = readHeader;
|
package/cjs/lib/encoder.js
CHANGED
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
5
|
var varint = require('varint');
|
|
6
|
-
var
|
|
6
|
+
var CBOR = require('@ipld/dag-cbor');
|
|
7
7
|
|
|
8
8
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
9
9
|
|
|
10
10
|
var varint__default = /*#__PURE__*/_interopDefaultLegacy(varint);
|
|
11
11
|
|
|
12
12
|
function createHeader(roots) {
|
|
13
|
-
const headerBytes =
|
|
13
|
+
const headerBytes = CBOR.encode({
|
|
14
14
|
version: 1,
|
|
15
15
|
roots
|
|
16
16
|
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
|
+
|
|
5
|
+
const Kinds = {
|
|
6
|
+
Null: obj => obj === null,
|
|
7
|
+
Int: obj => Number.isInteger(obj),
|
|
8
|
+
Float: obj => typeof obj === 'number' && Number.isFinite(obj),
|
|
9
|
+
String: obj => typeof obj === 'string',
|
|
10
|
+
Bool: obj => typeof obj === 'boolean',
|
|
11
|
+
Bytes: obj => obj instanceof Uint8Array,
|
|
12
|
+
Link: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
|
|
13
|
+
List: obj => Array.isArray(obj),
|
|
14
|
+
Map: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
|
|
15
|
+
};
|
|
16
|
+
const Types = {
|
|
17
|
+
Int: Kinds.Int,
|
|
18
|
+
'CarHeader > version': obj => Types.Int(obj),
|
|
19
|
+
'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
|
|
20
|
+
'CarHeader > roots (anon)': obj => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
|
|
21
|
+
'CarHeader > roots': obj => Types['CarHeader > roots (anon)'](obj),
|
|
22
|
+
CarHeader: obj => {
|
|
23
|
+
const keys = obj && Object.keys(obj);
|
|
24
|
+
return Kinds.Map(obj) && ['version'].every(k => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value));
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
const CarHeader = Types.CarHeader;
|
|
28
|
+
|
|
29
|
+
exports.CarHeader = CarHeader;
|
|
@@ -5,17 +5,16 @@ Object.defineProperty(exports, '__esModule', { value: true });
|
|
|
5
5
|
var decoder = require('./decoder.js');
|
|
6
6
|
|
|
7
7
|
class CarReader {
|
|
8
|
-
constructor(
|
|
9
|
-
this.
|
|
10
|
-
this._roots = roots;
|
|
8
|
+
constructor(header, blocks) {
|
|
9
|
+
this._header = header;
|
|
11
10
|
this._blocks = blocks;
|
|
12
11
|
this._keys = blocks.map(b => b.cid.toString());
|
|
13
12
|
}
|
|
14
13
|
get version() {
|
|
15
|
-
return this.
|
|
14
|
+
return this._header.version;
|
|
16
15
|
}
|
|
17
16
|
async getRoots() {
|
|
18
|
-
return this.
|
|
17
|
+
return this._header.roots;
|
|
19
18
|
}
|
|
20
19
|
async has(key) {
|
|
21
20
|
return this._keys.indexOf(key.toString()) > -1;
|
|
@@ -49,14 +48,15 @@ class CarReader {
|
|
|
49
48
|
}
|
|
50
49
|
async function decodeReaderComplete(reader) {
|
|
51
50
|
const decoder$1 = decoder.createDecoder(reader);
|
|
52
|
-
const
|
|
51
|
+
const header = await decoder$1.header();
|
|
53
52
|
const blocks = [];
|
|
54
53
|
for await (const block of decoder$1.blocks()) {
|
|
55
54
|
blocks.push(block);
|
|
56
55
|
}
|
|
57
|
-
return new CarReader(
|
|
56
|
+
return new CarReader(header, blocks);
|
|
58
57
|
}
|
|
59
58
|
const __browser = true;
|
|
60
59
|
|
|
61
60
|
exports.CarReader = CarReader;
|
|
62
61
|
exports.__browser = __browser;
|
|
62
|
+
exports.decodeReaderComplete = decodeReaderComplete;
|
|
@@ -62,7 +62,7 @@ class CarWriter {
|
|
|
62
62
|
const reader = decoder.bytesReader(bytes);
|
|
63
63
|
await decoder.readHeader(reader);
|
|
64
64
|
const newHeader = encoder.createHeader(roots);
|
|
65
|
-
if (reader.pos !== newHeader.length) {
|
|
65
|
+
if (Number(reader.pos) !== newHeader.length) {
|
|
66
66
|
throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${ reader.pos } bytes, new header is ${ newHeader.length } bytes)`);
|
|
67
67
|
}
|
|
68
68
|
bytes.set(newHeader, 0);
|
package/cjs/node-test/common.js
CHANGED
|
@@ -5,7 +5,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
|
|
|
5
5
|
var multiformats = require('multiformats');
|
|
6
6
|
var sha2 = require('multiformats/hashes/sha2');
|
|
7
7
|
var raw = require('multiformats/codecs/raw');
|
|
8
|
-
var
|
|
8
|
+
var CBOR = require('@ipld/dag-cbor');
|
|
9
9
|
var dagPb = require('@ipld/dag-pb');
|
|
10
10
|
var chai = require('chai');
|
|
11
11
|
var chaiAsPromised = require('chai-as-promised');
|
|
@@ -31,7 +31,7 @@ function _interopNamespace(e) {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
var raw__namespace = /*#__PURE__*/_interopNamespace(raw);
|
|
34
|
-
var
|
|
34
|
+
var CBOR__namespace = /*#__PURE__*/_interopNamespace(CBOR);
|
|
35
35
|
var dagPb__namespace = /*#__PURE__*/_interopNamespace(dagPb);
|
|
36
36
|
var chai__default = /*#__PURE__*/_interopDefaultLegacy(chai);
|
|
37
37
|
var chaiAsPromised__default = /*#__PURE__*/_interopDefaultLegacy(chaiAsPromised);
|
|
@@ -101,7 +101,7 @@ async function makeData() {
|
|
|
101
101
|
cborBlocks.push(await toBlock({
|
|
102
102
|
name: b[0],
|
|
103
103
|
link: b[1]
|
|
104
|
-
},
|
|
104
|
+
}, CBOR__namespace));
|
|
105
105
|
}
|
|
106
106
|
allBlocks = [
|
|
107
107
|
[
|
|
@@ -214,12 +214,87 @@ const goCarIndex = [
|
|
|
214
214
|
blockLength: 18
|
|
215
215
|
}
|
|
216
216
|
];
|
|
217
|
+
const goCarV2Bytes = multiformats.bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000');
|
|
218
|
+
const goCarV2Roots = [multiformats.CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')];
|
|
219
|
+
const goCarV2Index = [
|
|
220
|
+
{
|
|
221
|
+
blockLength: 47,
|
|
222
|
+
blockOffset: 143,
|
|
223
|
+
cid: multiformats.CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'),
|
|
224
|
+
length: 82,
|
|
225
|
+
offset: 108
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
blockLength: 99,
|
|
229
|
+
blockOffset: 226,
|
|
230
|
+
cid: multiformats.CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
|
|
231
|
+
length: 135,
|
|
232
|
+
offset: 190
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
blockLength: 54,
|
|
236
|
+
blockOffset: 360,
|
|
237
|
+
cid: multiformats.CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
|
|
238
|
+
length: 89,
|
|
239
|
+
offset: 325
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
blockLength: 4,
|
|
243
|
+
blockOffset: 451,
|
|
244
|
+
cid: multiformats.CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
|
|
245
|
+
length: 41,
|
|
246
|
+
offset: 414
|
|
247
|
+
},
|
|
248
|
+
{
|
|
249
|
+
blockLength: 7,
|
|
250
|
+
blockOffset: 492,
|
|
251
|
+
cid: multiformats.CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
|
|
252
|
+
length: 44,
|
|
253
|
+
offset: 455
|
|
254
|
+
}
|
|
255
|
+
];
|
|
256
|
+
const goCarV2Contents = {
|
|
257
|
+
QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
|
|
258
|
+
Links: [{
|
|
259
|
+
Hash: multiformats.CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
|
|
260
|
+
Name: '\uD83C\uDF64',
|
|
261
|
+
Tsize: 164
|
|
262
|
+
}]
|
|
263
|
+
},
|
|
264
|
+
QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
|
|
265
|
+
Links: [
|
|
266
|
+
{
|
|
267
|
+
Hash: multiformats.CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
|
|
268
|
+
Name: 'barreleye',
|
|
269
|
+
Tsize: 58
|
|
270
|
+
},
|
|
271
|
+
{
|
|
272
|
+
Hash: multiformats.CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
|
|
273
|
+
Name: '\uD83D\uDC21',
|
|
274
|
+
Tsize: 7
|
|
275
|
+
}
|
|
276
|
+
]
|
|
277
|
+
},
|
|
278
|
+
Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
|
|
279
|
+
Links: [{
|
|
280
|
+
Hash: multiformats.CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
|
|
281
|
+
Name: 'fishmonger',
|
|
282
|
+
Tsize: 4
|
|
283
|
+
}]
|
|
284
|
+
},
|
|
285
|
+
bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
|
|
286
|
+
bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
|
|
287
|
+
};
|
|
217
288
|
|
|
218
289
|
exports.assert = assert;
|
|
219
290
|
exports.carBytes = carBytes;
|
|
220
291
|
exports.goCarBytes = goCarBytes;
|
|
221
292
|
exports.goCarIndex = goCarIndex;
|
|
222
293
|
exports.goCarRoots = goCarRoots;
|
|
294
|
+
exports.goCarV2Bytes = goCarV2Bytes;
|
|
295
|
+
exports.goCarV2Contents = goCarV2Contents;
|
|
296
|
+
exports.goCarV2Index = goCarV2Index;
|
|
297
|
+
exports.goCarV2Roots = goCarV2Roots;
|
|
223
298
|
exports.makeData = makeData;
|
|
224
299
|
exports.makeIterable = makeIterable;
|
|
225
300
|
exports.rndCid = rndCid;
|
|
@@ -4,7 +4,7 @@ var fs = require('fs');
|
|
|
4
4
|
var stream = require('stream');
|
|
5
5
|
var ipldGarbage = require('ipld-garbage');
|
|
6
6
|
var varint = require('varint');
|
|
7
|
-
var
|
|
7
|
+
var CBOR = require('@ipld/dag-cbor');
|
|
8
8
|
var sha2 = require('multiformats/hashes/sha2');
|
|
9
9
|
var cid = require('multiformats/cid');
|
|
10
10
|
require('../car.js');
|
|
@@ -36,7 +36,7 @@ function _interopNamespace(e) {
|
|
|
36
36
|
|
|
37
37
|
var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs);
|
|
38
38
|
var varint__default = /*#__PURE__*/_interopDefaultLegacy(varint);
|
|
39
|
-
var
|
|
39
|
+
var CBOR__namespace = /*#__PURE__*/_interopNamespace(CBOR);
|
|
40
40
|
|
|
41
41
|
describe('Large CAR', () => {
|
|
42
42
|
const objects = [];
|
|
@@ -45,7 +45,7 @@ describe('Large CAR', () => {
|
|
|
45
45
|
it('create, no roots', async () => {
|
|
46
46
|
const {writer: writer$1, out} = writer.CarWriter.create([]);
|
|
47
47
|
stream.Readable.from(out).pipe(fs__default["default"].createWriteStream('./test.car'));
|
|
48
|
-
let offset =
|
|
48
|
+
let offset = CBOR__namespace.encode({
|
|
49
49
|
version: 1,
|
|
50
50
|
roots: []
|
|
51
51
|
}).length;
|
|
@@ -53,9 +53,9 @@ describe('Large CAR', () => {
|
|
|
53
53
|
for (let i = 0; i < 500; i++) {
|
|
54
54
|
const obj = ipldGarbage.garbage(1000);
|
|
55
55
|
objects.push(obj);
|
|
56
|
-
const bytes =
|
|
56
|
+
const bytes = CBOR__namespace.encode(obj);
|
|
57
57
|
const hash = await sha2.sha256.digest(bytes);
|
|
58
|
-
const cid$1 = cid.CID.create(1,
|
|
58
|
+
const cid$1 = cid.CID.create(1, CBOR__namespace.code, hash);
|
|
59
59
|
cids.push(cid$1.toString());
|
|
60
60
|
const blockLength = bytes.length;
|
|
61
61
|
let length = cid$1.bytes.length + blockLength;
|
|
@@ -101,7 +101,7 @@ describe('Large CAR', () => {
|
|
|
101
101
|
let i = 0;
|
|
102
102
|
for await (const {cid, bytes} of reader$1.blocks()) {
|
|
103
103
|
common.assert.strictEqual(cid.toString(), cids[i], `cid #${ i } ${ cid } <> ${ cids[i] }`);
|
|
104
|
-
const obj =
|
|
104
|
+
const obj = CBOR__namespace.decode(bytes);
|
|
105
105
|
common.assert.deepStrictEqual(obj, objects[i], `object #${ i }`);
|
|
106
106
|
i++;
|
|
107
107
|
}
|
|
@@ -112,7 +112,7 @@ describe('Large CAR', () => {
|
|
|
112
112
|
let i = 0;
|
|
113
113
|
for await (const {cid, bytes} of reader$1.blocks()) {
|
|
114
114
|
common.assert.strictEqual(cid.toString(), cids[i], `cid #${ i } ${ cid } <> ${ cids[i] }`);
|
|
115
|
-
const obj =
|
|
115
|
+
const obj = CBOR__namespace.decode(bytes);
|
|
116
116
|
common.assert.deepStrictEqual(obj, objects[i], `object #${ i }`);
|
|
117
117
|
i++;
|
|
118
118
|
}
|
|
@@ -123,7 +123,7 @@ describe('Large CAR', () => {
|
|
|
123
123
|
let i = 0;
|
|
124
124
|
for await (const {cid, bytes} of reader.blocks()) {
|
|
125
125
|
common.assert.strictEqual(cid.toString(), cids[i], `cid #${ i } ${ cid } <> ${ cids[i] }`);
|
|
126
|
-
const obj =
|
|
126
|
+
const obj = CBOR__namespace.decode(bytes);
|
|
127
127
|
common.assert.deepStrictEqual(obj, objects[i], `object #${ i }`);
|
|
128
128
|
i++;
|
|
129
129
|
}
|