@ipld/car 3.2.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -2
- package/cjs/browser-test/common.js +75 -0
- package/cjs/browser-test/test-errors.js +55 -32
- package/cjs/browser-test/test-indexer.js +12 -0
- package/cjs/browser-test/test-reader.js +83 -0
- package/cjs/lib/decoder.js +70 -13
- package/cjs/lib/header-validator.js +29 -0
- package/cjs/lib/reader-browser.js +7 -7
- package/cjs/lib/writer-browser.js +1 -1
- package/cjs/node-test/common.js +75 -0
- package/cjs/node-test/test-errors.js +55 -32
- package/cjs/node-test/test-indexer.js +12 -0
- package/cjs/node-test/test-reader.js +83 -0
- package/esm/browser-test/common.js +76 -1
- package/esm/browser-test/test-errors.js +57 -33
- package/esm/browser-test/test-indexer.js +15 -0
- package/esm/browser-test/test-reader.js +90 -1
- package/esm/lib/decoder.js +69 -13
- package/esm/lib/header-validator.js +23 -0
- package/esm/lib/reader-browser.js +7 -8
- package/esm/lib/writer-browser.js +1 -1
- package/esm/node-test/common.js +76 -1
- package/esm/node-test/test-errors.js +57 -33
- package/esm/node-test/test-indexer.js +15 -0
- package/esm/node-test/test-reader.js +90 -1
- package/examples/car-to-fixture.js +1 -4
- package/examples/dump-index.js +24 -0
- package/examples/package.json +1 -1
- package/examples/test-examples.js +33 -0
- package/lib/coding.ts +17 -2
- package/lib/decoder.js +130 -14
- package/lib/header-validator.js +33 -0
- package/lib/header.ipldsch +6 -0
- package/lib/reader-browser.js +11 -11
- package/lib/writer-browser.js +1 -1
- package/package.json +7 -7
- package/test/_fixtures_to_js.mjs +24 -0
- package/test/common.js +49 -3
- package/test/go.carv2 +0 -0
- package/test/test-errors.js +52 -30
- package/test/test-indexer.js +24 -1
- package/test/test-reader.js +94 -1
- package/tsconfig.json +2 -1
- package/types/lib/coding.d.ts +14 -4
- package/types/lib/coding.d.ts.map +1 -1
- package/types/lib/decoder.d.ts +38 -2
- package/types/lib/decoder.d.ts.map +1 -1
- package/types/lib/header-validator.d.ts +2 -0
- package/types/lib/header-validator.d.ts.map +1 -0
- package/types/lib/reader-browser.d.ts +15 -7
- package/types/lib/reader-browser.d.ts.map +1 -1
- package/types/test/_fixtures_to_js.d.mts +3 -0
- package/types/test/_fixtures_to_js.d.mts.map +1 -0
- package/types/test/common.d.ts +13 -0
- package/types/test/common.d.ts.map +1 -1
- package/types/test/fixtures-expectations.d.ts +63 -0
- package/types/test/fixtures-expectations.d.ts.map +1 -0
- package/types/test/fixtures.d.ts +3 -0
- package/types/test/fixtures.d.ts.map +1 -0
package/esm/lib/decoder.js
CHANGED
|
@@ -2,18 +2,40 @@ import varint from 'varint';
|
|
|
2
2
|
import { CID } from 'multiformats/cid';
|
|
3
3
|
import * as Digest from 'multiformats/hashes/digest';
|
|
4
4
|
import { decode as decodeDagCbor } from '@ipld/dag-cbor';
|
|
5
|
+
import { CarHeader as headerValidator } from './header-validator.js';
|
|
5
6
|
const CIDV0_BYTES = {
|
|
6
7
|
SHA2_256: 18,
|
|
7
8
|
LENGTH: 32,
|
|
8
9
|
DAG_PB: 112
|
|
9
10
|
};
|
|
11
|
+
const V2_HEADER_LENGTH = 16 + 8 + 8 + 8;
|
|
10
12
|
async function readVarint(reader) {
|
|
11
13
|
const bytes = await reader.upTo(8);
|
|
14
|
+
if (!bytes.length) {
|
|
15
|
+
throw new Error('Unexpected end of data');
|
|
16
|
+
}
|
|
12
17
|
const i = varint.decode(bytes);
|
|
13
18
|
reader.seek(varint.decode.bytes);
|
|
14
19
|
return i;
|
|
15
20
|
}
|
|
16
|
-
|
|
21
|
+
async function readV2Header(reader) {
|
|
22
|
+
const bytes = await reader.exactly(V2_HEADER_LENGTH);
|
|
23
|
+
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
24
|
+
let offset = 0;
|
|
25
|
+
const header = {
|
|
26
|
+
version: 2,
|
|
27
|
+
characteristics: [
|
|
28
|
+
dv.getBigUint64(offset, true),
|
|
29
|
+
dv.getBigUint64(offset += 8, true)
|
|
30
|
+
],
|
|
31
|
+
dataOffset: Number(dv.getBigUint64(offset += 8, true)),
|
|
32
|
+
dataSize: Number(dv.getBigUint64(offset += 8, true)),
|
|
33
|
+
indexOffset: Number(dv.getBigUint64(offset += 8, true))
|
|
34
|
+
};
|
|
35
|
+
reader.seek(V2_HEADER_LENGTH);
|
|
36
|
+
return header;
|
|
37
|
+
}
|
|
38
|
+
export async function readHeader(reader, strictVersion) {
|
|
17
39
|
const length = await readVarint(reader);
|
|
18
40
|
if (length === 0) {
|
|
19
41
|
throw new Error('Invalid CAR header (zero length)');
|
|
@@ -21,22 +43,23 @@ export async function readHeader(reader) {
|
|
|
21
43
|
const header = await reader.exactly(length);
|
|
22
44
|
reader.seek(length);
|
|
23
45
|
const block = decodeDagCbor(header);
|
|
24
|
-
if (
|
|
46
|
+
if (!headerValidator(block)) {
|
|
25
47
|
throw new Error('Invalid CAR header format');
|
|
26
48
|
}
|
|
27
|
-
if (block.version !== 1) {
|
|
28
|
-
|
|
29
|
-
throw new Error(`Invalid CAR version: "${ block.version }"`);
|
|
30
|
-
}
|
|
31
|
-
throw new Error(`Invalid CAR version: ${ block.version }`);
|
|
49
|
+
if (block.version !== 1 && block.version !== 2 || strictVersion !== undefined && block.version !== strictVersion) {
|
|
50
|
+
throw new Error(`Invalid CAR version: ${ block.version }${ strictVersion !== undefined ? ` (expected ${ strictVersion })` : '' }`);
|
|
32
51
|
}
|
|
33
|
-
|
|
52
|
+
const hasRoots = Array.isArray(block.roots);
|
|
53
|
+
if (block.version === 1 && !hasRoots || block.version === 2 && hasRoots) {
|
|
34
54
|
throw new Error('Invalid CAR header format');
|
|
35
55
|
}
|
|
36
|
-
if (
|
|
37
|
-
|
|
56
|
+
if (block.version === 1) {
|
|
57
|
+
return block;
|
|
38
58
|
}
|
|
39
|
-
|
|
59
|
+
const v2Header = await readV2Header(reader);
|
|
60
|
+
reader.seek(v2Header.dataOffset - reader.pos);
|
|
61
|
+
const v1Header = await readHeader(reader, 1);
|
|
62
|
+
return Object.assign(v1Header, v2Header);
|
|
40
63
|
}
|
|
41
64
|
async function readMultihash(reader) {
|
|
42
65
|
const bytes = await reader.upTo(8);
|
|
@@ -74,7 +97,7 @@ export async function readBlockHead(reader) {
|
|
|
74
97
|
}
|
|
75
98
|
length += reader.pos - start;
|
|
76
99
|
const cid = await readCid(reader);
|
|
77
|
-
const blockLength = length - (reader.pos - start);
|
|
100
|
+
const blockLength = length - Number(reader.pos - start);
|
|
78
101
|
return {
|
|
79
102
|
cid,
|
|
80
103
|
length,
|
|
@@ -104,7 +127,14 @@ async function readBlockIndex(reader) {
|
|
|
104
127
|
return index;
|
|
105
128
|
}
|
|
106
129
|
export function createDecoder(reader) {
|
|
107
|
-
const headerPromise =
|
|
130
|
+
const headerPromise = (async () => {
|
|
131
|
+
const header = await readHeader(reader);
|
|
132
|
+
if (header.version === 2) {
|
|
133
|
+
const v1length = reader.pos - header.dataOffset;
|
|
134
|
+
reader = limitReader(reader, header.dataSize - v1length);
|
|
135
|
+
}
|
|
136
|
+
return header;
|
|
137
|
+
})();
|
|
108
138
|
return {
|
|
109
139
|
header: () => headerPromise,
|
|
110
140
|
async *blocks() {
|
|
@@ -206,4 +236,30 @@ export function asyncIterableReader(asyncIterable) {
|
|
|
206
236
|
return next.value;
|
|
207
237
|
}
|
|
208
238
|
return chunkReader(readChunk);
|
|
239
|
+
}
|
|
240
|
+
export function limitReader(reader, byteLimit) {
|
|
241
|
+
let bytesRead = 0;
|
|
242
|
+
return {
|
|
243
|
+
async upTo(length) {
|
|
244
|
+
let bytes = await reader.upTo(length);
|
|
245
|
+
if (bytes.length + bytesRead > byteLimit) {
|
|
246
|
+
bytes = bytes.subarray(0, byteLimit - bytesRead);
|
|
247
|
+
}
|
|
248
|
+
return bytes;
|
|
249
|
+
},
|
|
250
|
+
async exactly(length) {
|
|
251
|
+
const bytes = await reader.exactly(length);
|
|
252
|
+
if (bytes.length + bytesRead > byteLimit) {
|
|
253
|
+
throw new Error('Unexpected end of data');
|
|
254
|
+
}
|
|
255
|
+
return bytes;
|
|
256
|
+
},
|
|
257
|
+
seek(length) {
|
|
258
|
+
bytesRead += length;
|
|
259
|
+
reader.seek(length);
|
|
260
|
+
},
|
|
261
|
+
get pos() {
|
|
262
|
+
return reader.pos;
|
|
263
|
+
}
|
|
264
|
+
};
|
|
209
265
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const Kinds = {
|
|
2
|
+
Null: obj => obj === null,
|
|
3
|
+
Int: obj => Number.isInteger(obj),
|
|
4
|
+
Float: obj => typeof obj === 'number' && Number.isFinite(obj),
|
|
5
|
+
String: obj => typeof obj === 'string',
|
|
6
|
+
Bool: obj => typeof obj === 'boolean',
|
|
7
|
+
Bytes: obj => obj instanceof Uint8Array,
|
|
8
|
+
Link: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
|
|
9
|
+
List: obj => Array.isArray(obj),
|
|
10
|
+
Map: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
|
|
11
|
+
};
|
|
12
|
+
const Types = {
|
|
13
|
+
Int: Kinds.Int,
|
|
14
|
+
'CarHeader > version': obj => Types.Int(obj),
|
|
15
|
+
'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
|
|
16
|
+
'CarHeader > roots (anon)': obj => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
|
|
17
|
+
'CarHeader > roots': obj => Types['CarHeader > roots (anon)'](obj),
|
|
18
|
+
CarHeader: obj => {
|
|
19
|
+
const keys = obj && Object.keys(obj);
|
|
20
|
+
return Kinds.Map(obj) && ['version'].every(k => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value));
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
export const CarHeader = Types.CarHeader;
|
|
@@ -4,17 +4,16 @@ import {
|
|
|
4
4
|
createDecoder
|
|
5
5
|
} from './decoder.js';
|
|
6
6
|
export class CarReader {
|
|
7
|
-
constructor(
|
|
8
|
-
this.
|
|
9
|
-
this._roots = roots;
|
|
7
|
+
constructor(header, blocks) {
|
|
8
|
+
this._header = header;
|
|
10
9
|
this._blocks = blocks;
|
|
11
10
|
this._keys = blocks.map(b => b.cid.toString());
|
|
12
11
|
}
|
|
13
12
|
get version() {
|
|
14
|
-
return this.
|
|
13
|
+
return this._header.version;
|
|
15
14
|
}
|
|
16
15
|
async getRoots() {
|
|
17
|
-
return this.
|
|
16
|
+
return this._header.roots;
|
|
18
17
|
}
|
|
19
18
|
async has(key) {
|
|
20
19
|
return this._keys.indexOf(key.toString()) > -1;
|
|
@@ -46,13 +45,13 @@ export class CarReader {
|
|
|
46
45
|
return decodeReaderComplete(asyncIterableReader(asyncIterable));
|
|
47
46
|
}
|
|
48
47
|
}
|
|
49
|
-
async function decodeReaderComplete(reader) {
|
|
48
|
+
export async function decodeReaderComplete(reader) {
|
|
50
49
|
const decoder = createDecoder(reader);
|
|
51
|
-
const
|
|
50
|
+
const header = await decoder.header();
|
|
52
51
|
const blocks = [];
|
|
53
52
|
for await (const block of decoder.blocks()) {
|
|
54
53
|
blocks.push(block);
|
|
55
54
|
}
|
|
56
|
-
return new CarReader(
|
|
55
|
+
return new CarReader(header, blocks);
|
|
57
56
|
}
|
|
58
57
|
export const __browser = true;
|
|
@@ -63,7 +63,7 @@ export class CarWriter {
|
|
|
63
63
|
const reader = bytesReader(bytes);
|
|
64
64
|
await readHeader(reader);
|
|
65
65
|
const newHeader = createHeader(roots);
|
|
66
|
-
if (reader.pos !== newHeader.length) {
|
|
66
|
+
if (Number(reader.pos) !== newHeader.length) {
|
|
67
67
|
throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${ reader.pos } bytes, new header is ${ newHeader.length } bytes)`);
|
|
68
68
|
}
|
|
69
69
|
bytes.set(newHeader, 0);
|
package/esm/node-test/common.js
CHANGED
|
@@ -186,6 +186,77 @@ const goCarIndex = [
|
|
|
186
186
|
blockLength: 18
|
|
187
187
|
}
|
|
188
188
|
];
|
|
189
|
+
const goCarV2Bytes = bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000');
|
|
190
|
+
const goCarV2Roots = [CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')];
|
|
191
|
+
const goCarV2Index = [
|
|
192
|
+
{
|
|
193
|
+
blockLength: 47,
|
|
194
|
+
blockOffset: 143,
|
|
195
|
+
cid: CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'),
|
|
196
|
+
length: 82,
|
|
197
|
+
offset: 108
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
blockLength: 99,
|
|
201
|
+
blockOffset: 226,
|
|
202
|
+
cid: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
|
|
203
|
+
length: 135,
|
|
204
|
+
offset: 190
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
blockLength: 54,
|
|
208
|
+
blockOffset: 360,
|
|
209
|
+
cid: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
|
|
210
|
+
length: 89,
|
|
211
|
+
offset: 325
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
blockLength: 4,
|
|
215
|
+
blockOffset: 451,
|
|
216
|
+
cid: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
|
|
217
|
+
length: 41,
|
|
218
|
+
offset: 414
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
blockLength: 7,
|
|
222
|
+
blockOffset: 492,
|
|
223
|
+
cid: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
|
|
224
|
+
length: 44,
|
|
225
|
+
offset: 455
|
|
226
|
+
}
|
|
227
|
+
];
|
|
228
|
+
const goCarV2Contents = {
|
|
229
|
+
QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
|
|
230
|
+
Links: [{
|
|
231
|
+
Hash: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
|
|
232
|
+
Name: '\uD83C\uDF64',
|
|
233
|
+
Tsize: 164
|
|
234
|
+
}]
|
|
235
|
+
},
|
|
236
|
+
QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
|
|
237
|
+
Links: [
|
|
238
|
+
{
|
|
239
|
+
Hash: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
|
|
240
|
+
Name: 'barreleye',
|
|
241
|
+
Tsize: 58
|
|
242
|
+
},
|
|
243
|
+
{
|
|
244
|
+
Hash: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
|
|
245
|
+
Name: '\uD83D\uDC21',
|
|
246
|
+
Tsize: 7
|
|
247
|
+
}
|
|
248
|
+
]
|
|
249
|
+
},
|
|
250
|
+
Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
|
|
251
|
+
Links: [{
|
|
252
|
+
Hash: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
|
|
253
|
+
Name: 'fishmonger',
|
|
254
|
+
Tsize: 4
|
|
255
|
+
}]
|
|
256
|
+
},
|
|
257
|
+
bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
|
|
258
|
+
bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
|
|
259
|
+
};
|
|
189
260
|
export {
|
|
190
261
|
toBlock,
|
|
191
262
|
assert,
|
|
@@ -195,5 +266,9 @@ export {
|
|
|
195
266
|
carBytes,
|
|
196
267
|
goCarBytes,
|
|
197
268
|
goCarRoots,
|
|
198
|
-
goCarIndex
|
|
269
|
+
goCarIndex,
|
|
270
|
+
goCarV2Bytes,
|
|
271
|
+
goCarV2Roots,
|
|
272
|
+
goCarV2Index,
|
|
273
|
+
goCarV2Contents
|
|
199
274
|
};
|
|
@@ -4,7 +4,8 @@ import { encode as vEncode } from 'varint';
|
|
|
4
4
|
import { CarReader } from '../lib/reader.js';
|
|
5
5
|
import {
|
|
6
6
|
carBytes,
|
|
7
|
-
assert
|
|
7
|
+
assert,
|
|
8
|
+
goCarV2Bytes
|
|
8
9
|
} from './common.js';
|
|
9
10
|
function makeHeader(block) {
|
|
10
11
|
const u = cbEncode(block);
|
|
@@ -26,42 +27,65 @@ describe('Misc errors', () => {
|
|
|
26
27
|
});
|
|
27
28
|
});
|
|
28
29
|
it('bad version', async () => {
|
|
29
|
-
const buf2 = bytes.fromHex('
|
|
30
|
-
assert.strictEqual(bytes.toHex(makeHeader({ version:
|
|
31
|
-
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version:
|
|
30
|
+
const buf2 = bytes.fromHex('0aa16776657273696f6e03');
|
|
31
|
+
assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03');
|
|
32
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3');
|
|
32
33
|
});
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
describe('bad header', async () => {
|
|
35
|
+
it('sanity check', async () => {
|
|
36
|
+
const buf2 = makeHeader({
|
|
37
|
+
version: 1,
|
|
38
|
+
roots: []
|
|
39
|
+
});
|
|
40
|
+
await assert.isFulfilled(CarReader.fromBytes(buf2));
|
|
37
41
|
});
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
buf2 = makeHeader({
|
|
42
|
-
version: '1',
|
|
43
|
-
roots: []
|
|
42
|
+
it('no \'version\' array', async () => {
|
|
43
|
+
const buf2 = makeHeader({ roots: [] });
|
|
44
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
44
45
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
46
|
+
it('bad \'version\' type', async () => {
|
|
47
|
+
const buf2 = makeHeader({
|
|
48
|
+
version: '1',
|
|
49
|
+
roots: []
|
|
50
|
+
});
|
|
51
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
51
52
|
});
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
it('no \'roots\' array', async () => {
|
|
54
|
+
const buf2 = makeHeader({ version: 1 });
|
|
55
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
56
|
+
});
|
|
57
|
+
it('bad \'roots\' type', async () => {
|
|
58
|
+
const buf2 = makeHeader({
|
|
59
|
+
version: 1,
|
|
60
|
+
roots: {}
|
|
61
|
+
});
|
|
62
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
63
|
+
});
|
|
64
|
+
it('extraneous properties', async () => {
|
|
65
|
+
const buf2 = makeHeader({
|
|
66
|
+
version: 1,
|
|
67
|
+
roots: [],
|
|
68
|
+
blip: true
|
|
69
|
+
});
|
|
70
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
71
|
+
});
|
|
72
|
+
it('not an object', async () => {
|
|
73
|
+
const buf2 = makeHeader([
|
|
74
|
+
1,
|
|
75
|
+
[]
|
|
76
|
+
]);
|
|
77
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
78
|
+
});
|
|
79
|
+
it('not an object', async () => {
|
|
80
|
+
const buf2 = makeHeader(null);
|
|
81
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
82
|
+
});
|
|
83
|
+
it('recursive v2 header', async () => {
|
|
84
|
+
const v2Header = goCarV2Bytes.slice(0, 51);
|
|
85
|
+
const buf2 = new Uint8Array(51 * 2);
|
|
86
|
+
buf2.set(v2Header, 0);
|
|
87
|
+
buf2.set(v2Header, 51);
|
|
88
|
+
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)');
|
|
57
89
|
});
|
|
58
|
-
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
59
|
-
buf2 = makeHeader([
|
|
60
|
-
1,
|
|
61
|
-
[]
|
|
62
|
-
]);
|
|
63
|
-
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
64
|
-
buf2 = makeHeader(null);
|
|
65
|
-
await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
|
|
66
90
|
});
|
|
67
91
|
});
|
|
@@ -2,6 +2,9 @@ import { CarIndexer } from '../lib/indexer.js';
|
|
|
2
2
|
import {
|
|
3
3
|
goCarBytes,
|
|
4
4
|
goCarIndex,
|
|
5
|
+
goCarV2Bytes,
|
|
6
|
+
goCarV2Roots,
|
|
7
|
+
goCarV2Index,
|
|
5
8
|
makeIterable,
|
|
6
9
|
assert
|
|
7
10
|
} from './common.js';
|
|
@@ -17,6 +20,18 @@ describe('CarIndexer fromBytes()', () => {
|
|
|
17
20
|
}
|
|
18
21
|
assert.deepStrictEqual(indexData, goCarIndex);
|
|
19
22
|
});
|
|
23
|
+
it('v2 complete', async () => {
|
|
24
|
+
const indexer = await CarIndexer.fromBytes(goCarV2Bytes);
|
|
25
|
+
const roots = await indexer.getRoots();
|
|
26
|
+
assert.strictEqual(roots.length, 1);
|
|
27
|
+
assert(goCarV2Roots[0].equals(roots[0]));
|
|
28
|
+
assert.strictEqual(indexer.version, 2);
|
|
29
|
+
const indexData = [];
|
|
30
|
+
for await (const index of indexer) {
|
|
31
|
+
indexData.push(index);
|
|
32
|
+
}
|
|
33
|
+
assert.deepStrictEqual(indexData, goCarV2Index);
|
|
34
|
+
});
|
|
20
35
|
it('bad argument', async () => {
|
|
21
36
|
for (const arg of [
|
|
22
37
|
true,
|
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
import { CarReader } from '../lib/reader.js';
|
|
2
2
|
import { CarWriter } from '../lib/writer.js';
|
|
3
|
+
import {
|
|
4
|
+
bytesReader,
|
|
5
|
+
readHeader
|
|
6
|
+
} from '../lib/decoder.js';
|
|
3
7
|
import * as Block from 'multiformats/block';
|
|
4
8
|
import { sha256 } from 'multiformats/hashes/sha2';
|
|
5
9
|
import * as raw from 'multiformats/codecs/raw';
|
|
10
|
+
import { base64 } from 'multiformats/bases/base64';
|
|
11
|
+
import * as dagPb from '@ipld/dag-pb';
|
|
6
12
|
import {
|
|
7
13
|
carBytes,
|
|
8
14
|
makeIterable,
|
|
9
|
-
assert
|
|
15
|
+
assert,
|
|
16
|
+
goCarV2Bytes,
|
|
17
|
+
goCarV2Roots,
|
|
18
|
+
goCarV2Index,
|
|
19
|
+
goCarV2Contents
|
|
10
20
|
} from './common.js';
|
|
11
21
|
import {
|
|
12
22
|
verifyRoots,
|
|
@@ -15,6 +25,8 @@ import {
|
|
|
15
25
|
verifyBlocks,
|
|
16
26
|
verifyCids
|
|
17
27
|
} from './verify-store-reader.js';
|
|
28
|
+
import { data as fixtures } from './fixtures.js';
|
|
29
|
+
import { expectations as fixtureExpectations } from './fixtures-expectations.js';
|
|
18
30
|
describe('CarReader fromBytes()', () => {
|
|
19
31
|
it('complete', async () => {
|
|
20
32
|
const reader = await CarReader.fromBytes(carBytes);
|
|
@@ -52,6 +64,29 @@ describe('CarReader fromBytes()', () => {
|
|
|
52
64
|
message: 'Unexpected end of data'
|
|
53
65
|
});
|
|
54
66
|
});
|
|
67
|
+
it('v2 complete', async () => {
|
|
68
|
+
const reader = await CarReader.fromBytes(goCarV2Bytes);
|
|
69
|
+
const roots = await reader.getRoots();
|
|
70
|
+
assert.strictEqual(roots.length, 1);
|
|
71
|
+
assert(goCarV2Roots[0].equals(roots[0]));
|
|
72
|
+
assert.strictEqual(reader.version, 2);
|
|
73
|
+
for (const {cid} of goCarV2Index) {
|
|
74
|
+
const block = await reader.get(cid);
|
|
75
|
+
assert.isDefined(block);
|
|
76
|
+
if (block) {
|
|
77
|
+
assert(cid.equals(block.cid));
|
|
78
|
+
let content;
|
|
79
|
+
if (cid.code === dagPb.code) {
|
|
80
|
+
content = dagPb.decode(block.bytes);
|
|
81
|
+
} else if (cid.code === 85) {
|
|
82
|
+
content = new TextDecoder().decode(block.bytes);
|
|
83
|
+
} else {
|
|
84
|
+
assert.fail('Unexpected codec');
|
|
85
|
+
}
|
|
86
|
+
assert.deepStrictEqual(content, goCarV2Contents[cid.toString()]);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
});
|
|
55
90
|
it('decode error - trailing null bytes', async () => {
|
|
56
91
|
const bytes = new Uint8Array(carBytes.length + 5);
|
|
57
92
|
bytes.set(carBytes);
|
|
@@ -178,4 +213,58 @@ describe('CarReader fromIterable()', () => {
|
|
|
178
213
|
message: 'Unexpected end of data'
|
|
179
214
|
});
|
|
180
215
|
});
|
|
216
|
+
it('v2 decode error - truncated', async () => {
|
|
217
|
+
const bytes = goCarV2Bytes.slice();
|
|
218
|
+
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
219
|
+
dv.setBigUint64(35, BigInt(448 - 10), true);
|
|
220
|
+
await assert.isRejected(CarReader.fromIterable(makeIterable(bytes, 64)), {
|
|
221
|
+
name: 'Error',
|
|
222
|
+
message: 'Unexpected end of data'
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
});
|
|
226
|
+
describe('Shared fixtures', () => {
|
|
227
|
+
describe('Header', () => {
|
|
228
|
+
for (const [name, {
|
|
229
|
+
version: expectedVersion,
|
|
230
|
+
err: expectedError
|
|
231
|
+
}] of Object.entries(fixtureExpectations)) {
|
|
232
|
+
it(name, async () => {
|
|
233
|
+
const data = base64.baseDecode(fixtures[name]);
|
|
234
|
+
let header;
|
|
235
|
+
try {
|
|
236
|
+
header = await readHeader(bytesReader(data));
|
|
237
|
+
} catch (err) {
|
|
238
|
+
if (expectedError != null) {
|
|
239
|
+
assert.equal(err.message, expectedError);
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
assert.ifError(err);
|
|
243
|
+
}
|
|
244
|
+
if (expectedError != null) {
|
|
245
|
+
assert.fail(`Expected error: ${ expectedError }`);
|
|
246
|
+
}
|
|
247
|
+
assert.isDefined(header, 'did not decode header');
|
|
248
|
+
if (expectedVersion != null && header != null) {
|
|
249
|
+
assert.strictEqual(header.version, expectedVersion);
|
|
250
|
+
}
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
describe('Contents', () => {
|
|
255
|
+
for (const [name, {cids: expectedCids}] of Object.entries(fixtureExpectations)) {
|
|
256
|
+
if (expectedCids == null) {
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
it(name, async () => {
|
|
260
|
+
const data = base64.baseDecode(fixtures[name]);
|
|
261
|
+
const reader = await CarReader.fromBytes(data);
|
|
262
|
+
let i = 0;
|
|
263
|
+
for await (const cid of reader.cids()) {
|
|
264
|
+
assert.strictEqual(cid.toString(), expectedCids[i++]);
|
|
265
|
+
}
|
|
266
|
+
assert.strictEqual(i, expectedCids.length);
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
});
|
|
181
270
|
});
|
|
@@ -42,10 +42,7 @@ async function run () {
|
|
|
42
42
|
const indexer = await CarIndexer.fromBytes(bytes)
|
|
43
43
|
const reader = await CarReader.fromBytes(bytes)
|
|
44
44
|
const fixture = {
|
|
45
|
-
header:
|
|
46
|
-
roots: await reader.getRoots(),
|
|
47
|
-
version: reader.version
|
|
48
|
-
},
|
|
45
|
+
header: reader._header, // a little naughty but we need gory details
|
|
49
46
|
blocks: []
|
|
50
47
|
}
|
|
51
48
|
let i = 0
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Take a .car file and dump its index in DAG-JSON format, one line per block
|
|
4
|
+
|
|
5
|
+
import fs from 'fs'
|
|
6
|
+
import { CarIndexer } from '@ipld/car/indexer'
|
|
7
|
+
import * as dagJson from '@ipld/dag-json'
|
|
8
|
+
|
|
9
|
+
if (!process.argv[2]) {
|
|
10
|
+
console.log('Usage: dump-index.js <path/to/car>')
|
|
11
|
+
process.exit(1)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async function run () {
|
|
15
|
+
const indexer = await CarIndexer.fromIterable(fs.createReadStream(process.argv[2]))
|
|
16
|
+
for await (const blockIndex of indexer) {
|
|
17
|
+
console.log(new TextDecoder().decode(dagJson.encode(blockIndex)))
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
run().catch((err) => {
|
|
22
|
+
console.error(err)
|
|
23
|
+
process.exit(1)
|
|
24
|
+
})
|
package/examples/package.json
CHANGED
|
@@ -76,6 +76,39 @@ Blocks:
|
|
|
76
76
|
assert.strictEqual(stdout, '{"blocks":[{"blockLength":55,"blockOffset":137,"cid":{"/":"bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"},"content":{"link":{"/":"QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"},"name":"blip"},"length":92,"offset":100},{"blockLength":97,"blockOffset":228,"cid":{"/":"QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"},"content":{"Links":[{"Hash":{"/":"bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"},"Name":"bear","Tsize":4},{"Hash":{"/":"QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"},"Name":"second","Tsize":149}]},"length":133,"offset":192},{"blockLength":4,"blockOffset":362,"cid":{"/":"bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"},"content":{"/":{"bytes":"Y2NjYw"}},"length":41,"offset":325},{"blockLength":94,"blockOffset":402,"cid":{"/":"QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"},"content":{"Links":[{"Hash":{"/":"bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"},"Name":"dog","Tsize":4},{"Hash":{"/":"QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"},"Name":"first","Tsize":51}]},"length":130,"offset":366},{"blockLength":4,"blockOffset":533,"cid":{"/":"bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"},"content":{"/":{"bytes":"YmJiYg"}},"length":41,"offset":496},{"blockLength":47,"blockOffset":572,"cid":{"/":"QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"},"content":{"Links":[{"Hash":{"/":"bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"},"Name":"cat","Tsize":4}]},"length":82,"offset":537},{"blockLength":4,"blockOffset":656,"cid":{"/":"bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"},"content":{"/":{"bytes":"YWFhYQ"}},"length":41,"offset":619},{"blockLength":18,"blockOffset":697,"cid":{"/":"bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"},"content":{"link":null,"name":"limbo"},"length":55,"offset":660}],"header":{"roots":[{"/":"bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"},{"/":"bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"}],"version":1}}\n')
|
|
77
77
|
console.log('\u001b[32m✔\u001b[39m [example] car-to-fixture ../test/go.car')
|
|
78
78
|
})
|
|
79
|
+
}).then(async () => {
|
|
80
|
+
await runExample('dump-index', ['example.car']).then(({ stdout, stderr }) => {
|
|
81
|
+
assert.strictEqual(stderr, '')
|
|
82
|
+
assert.strictEqual(stdout, '{"blockLength":24,"blockOffset":96,"cid":{"/":"bafkreihwkf6mtnjobdqrkiksr7qhp6tiiqywux64aylunbvmfhzeql2coa"},"length":61,"offset":59}\n')
|
|
83
|
+
console.log('\u001b[32m✔\u001b[39m [example] dump-index example.car')
|
|
84
|
+
})
|
|
85
|
+
}).then(async () => {
|
|
86
|
+
await runExample('dump-index', ['../test/go.car']).then(({ stdout, stderr }) => {
|
|
87
|
+
assert.strictEqual(stderr, '')
|
|
88
|
+
assert.strictEqual(stdout,
|
|
89
|
+
`{"blockLength":55,"blockOffset":137,"cid":{"/":"bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"},"length":92,"offset":100}
|
|
90
|
+
{"blockLength":97,"blockOffset":228,"cid":{"/":"QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"},"length":133,"offset":192}
|
|
91
|
+
{"blockLength":4,"blockOffset":362,"cid":{"/":"bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"},"length":41,"offset":325}
|
|
92
|
+
{"blockLength":94,"blockOffset":402,"cid":{"/":"QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"},"length":130,"offset":366}
|
|
93
|
+
{"blockLength":4,"blockOffset":533,"cid":{"/":"bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"},"length":41,"offset":496}
|
|
94
|
+
{"blockLength":47,"blockOffset":572,"cid":{"/":"QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"},"length":82,"offset":537}
|
|
95
|
+
{"blockLength":4,"blockOffset":656,"cid":{"/":"bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"},"length":41,"offset":619}
|
|
96
|
+
{"blockLength":18,"blockOffset":697,"cid":{"/":"bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"},"length":55,"offset":660}
|
|
97
|
+
`)
|
|
98
|
+
console.log('\u001b[32m✔\u001b[39m [example] dump-index ../test/go.carv2')
|
|
99
|
+
})
|
|
100
|
+
}).then(async () => {
|
|
101
|
+
await runExample('dump-index', ['../test/go.carv2']).then(({ stdout, stderr }) => {
|
|
102
|
+
assert.strictEqual(stderr, '')
|
|
103
|
+
assert.strictEqual(stdout,
|
|
104
|
+
`{"blockLength":47,"blockOffset":143,"cid":{"/":"QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z"},"length":82,"offset":108}
|
|
105
|
+
{"blockLength":99,"blockOffset":226,"cid":{"/":"QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM"},"length":135,"offset":190}
|
|
106
|
+
{"blockLength":54,"blockOffset":360,"cid":{"/":"Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE"},"length":89,"offset":325}
|
|
107
|
+
{"blockLength":4,"blockOffset":451,"cid":{"/":"bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu"},"length":41,"offset":414}
|
|
108
|
+
{"blockLength":7,"blockOffset":492,"cid":{"/":"bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju"},"length":44,"offset":455}
|
|
109
|
+
`)
|
|
110
|
+
console.log('\u001b[32m✔\u001b[39m [example] dump-index ../test/go.carv2')
|
|
111
|
+
})
|
|
79
112
|
}).catch((err) => {
|
|
80
113
|
console.error(err.stack)
|
|
81
114
|
process.exit(1)
|