@ipld/car 3.2.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +74 -2
  2. package/cjs/browser-test/common.js +75 -0
  3. package/cjs/browser-test/test-errors.js +55 -32
  4. package/cjs/browser-test/test-indexer.js +12 -0
  5. package/cjs/browser-test/test-reader.js +83 -0
  6. package/cjs/lib/decoder.js +70 -13
  7. package/cjs/lib/header-validator.js +29 -0
  8. package/cjs/lib/reader-browser.js +7 -7
  9. package/cjs/lib/writer-browser.js +1 -1
  10. package/cjs/node-test/common.js +75 -0
  11. package/cjs/node-test/test-errors.js +55 -32
  12. package/cjs/node-test/test-indexer.js +12 -0
  13. package/cjs/node-test/test-reader.js +83 -0
  14. package/esm/browser-test/common.js +76 -1
  15. package/esm/browser-test/test-errors.js +57 -33
  16. package/esm/browser-test/test-indexer.js +15 -0
  17. package/esm/browser-test/test-reader.js +90 -1
  18. package/esm/lib/decoder.js +69 -13
  19. package/esm/lib/header-validator.js +23 -0
  20. package/esm/lib/reader-browser.js +7 -8
  21. package/esm/lib/writer-browser.js +1 -1
  22. package/esm/node-test/common.js +76 -1
  23. package/esm/node-test/test-errors.js +57 -33
  24. package/esm/node-test/test-indexer.js +15 -0
  25. package/esm/node-test/test-reader.js +90 -1
  26. package/examples/car-to-fixture.js +1 -4
  27. package/examples/dump-index.js +24 -0
  28. package/examples/package.json +1 -1
  29. package/examples/test-examples.js +33 -0
  30. package/lib/coding.ts +17 -2
  31. package/lib/decoder.js +130 -14
  32. package/lib/header-validator.js +33 -0
  33. package/lib/header.ipldsch +6 -0
  34. package/lib/reader-browser.js +11 -11
  35. package/lib/writer-browser.js +1 -1
  36. package/package.json +7 -7
  37. package/test/_fixtures_to_js.mjs +24 -0
  38. package/test/common.js +49 -3
  39. package/test/go.carv2 +0 -0
  40. package/test/test-errors.js +52 -30
  41. package/test/test-indexer.js +24 -1
  42. package/test/test-reader.js +94 -1
  43. package/tsconfig.json +2 -1
  44. package/types/lib/coding.d.ts +14 -4
  45. package/types/lib/coding.d.ts.map +1 -1
  46. package/types/lib/decoder.d.ts +38 -2
  47. package/types/lib/decoder.d.ts.map +1 -1
  48. package/types/lib/header-validator.d.ts +2 -0
  49. package/types/lib/header-validator.d.ts.map +1 -0
  50. package/types/lib/reader-browser.d.ts +15 -7
  51. package/types/lib/reader-browser.d.ts.map +1 -1
  52. package/types/test/_fixtures_to_js.d.mts +3 -0
  53. package/types/test/_fixtures_to_js.d.mts.map +1 -0
  54. package/types/test/common.d.ts +13 -0
  55. package/types/test/common.d.ts.map +1 -1
  56. package/types/test/fixtures-expectations.d.ts +63 -0
  57. package/types/test/fixtures-expectations.d.ts.map +1 -0
  58. package/types/test/fixtures.d.ts +3 -0
  59. package/types/test/fixtures.d.ts.map +1 -0
@@ -2,18 +2,40 @@ import varint from 'varint';
2
2
  import { CID } from 'multiformats/cid';
3
3
  import * as Digest from 'multiformats/hashes/digest';
4
4
  import { decode as decodeDagCbor } from '@ipld/dag-cbor';
5
+ import { CarHeader as headerValidator } from './header-validator.js';
5
6
  const CIDV0_BYTES = {
6
7
  SHA2_256: 18,
7
8
  LENGTH: 32,
8
9
  DAG_PB: 112
9
10
  };
11
+ const V2_HEADER_LENGTH = 16 + 8 + 8 + 8;
10
12
  async function readVarint(reader) {
11
13
  const bytes = await reader.upTo(8);
14
+ if (!bytes.length) {
15
+ throw new Error('Unexpected end of data');
16
+ }
12
17
  const i = varint.decode(bytes);
13
18
  reader.seek(varint.decode.bytes);
14
19
  return i;
15
20
  }
16
- export async function readHeader(reader) {
21
+ async function readV2Header(reader) {
22
+ const bytes = await reader.exactly(V2_HEADER_LENGTH);
23
+ const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
24
+ let offset = 0;
25
+ const header = {
26
+ version: 2,
27
+ characteristics: [
28
+ dv.getBigUint64(offset, true),
29
+ dv.getBigUint64(offset += 8, true)
30
+ ],
31
+ dataOffset: Number(dv.getBigUint64(offset += 8, true)),
32
+ dataSize: Number(dv.getBigUint64(offset += 8, true)),
33
+ indexOffset: Number(dv.getBigUint64(offset += 8, true))
34
+ };
35
+ reader.seek(V2_HEADER_LENGTH);
36
+ return header;
37
+ }
38
+ export async function readHeader(reader, strictVersion) {
17
39
  const length = await readVarint(reader);
18
40
  if (length === 0) {
19
41
  throw new Error('Invalid CAR header (zero length)');
@@ -21,22 +43,23 @@ export async function readHeader(reader) {
21
43
  const header = await reader.exactly(length);
22
44
  reader.seek(length);
23
45
  const block = decodeDagCbor(header);
24
- if (block == null || Array.isArray(block) || typeof block !== 'object') {
46
+ if (!headerValidator(block)) {
25
47
  throw new Error('Invalid CAR header format');
26
48
  }
27
- if (block.version !== 1) {
28
- if (typeof block.version === 'string') {
29
- throw new Error(`Invalid CAR version: "${ block.version }"`);
30
- }
31
- throw new Error(`Invalid CAR version: ${ block.version }`);
49
+ if (block.version !== 1 && block.version !== 2 || strictVersion !== undefined && block.version !== strictVersion) {
50
+ throw new Error(`Invalid CAR version: ${ block.version }${ strictVersion !== undefined ? ` (expected ${ strictVersion })` : '' }`);
32
51
  }
33
- if (!Array.isArray(block.roots)) {
52
+ const hasRoots = Array.isArray(block.roots);
53
+ if (block.version === 1 && !hasRoots || block.version === 2 && hasRoots) {
34
54
  throw new Error('Invalid CAR header format');
35
55
  }
36
- if (Object.keys(block).filter(p => p !== 'roots' && p !== 'version').length) {
37
- throw new Error('Invalid CAR header format');
56
+ if (block.version === 1) {
57
+ return block;
38
58
  }
39
- return block;
59
+ const v2Header = await readV2Header(reader);
60
+ reader.seek(v2Header.dataOffset - reader.pos);
61
+ const v1Header = await readHeader(reader, 1);
62
+ return Object.assign(v1Header, v2Header);
40
63
  }
41
64
  async function readMultihash(reader) {
42
65
  const bytes = await reader.upTo(8);
@@ -74,7 +97,7 @@ export async function readBlockHead(reader) {
74
97
  }
75
98
  length += reader.pos - start;
76
99
  const cid = await readCid(reader);
77
- const blockLength = length - (reader.pos - start);
100
+ const blockLength = length - Number(reader.pos - start);
78
101
  return {
79
102
  cid,
80
103
  length,
@@ -104,7 +127,14 @@ async function readBlockIndex(reader) {
104
127
  return index;
105
128
  }
106
129
  export function createDecoder(reader) {
107
- const headerPromise = readHeader(reader);
130
+ const headerPromise = (async () => {
131
+ const header = await readHeader(reader);
132
+ if (header.version === 2) {
133
+ const v1length = reader.pos - header.dataOffset;
134
+ reader = limitReader(reader, header.dataSize - v1length);
135
+ }
136
+ return header;
137
+ })();
108
138
  return {
109
139
  header: () => headerPromise,
110
140
  async *blocks() {
@@ -206,4 +236,30 @@ export function asyncIterableReader(asyncIterable) {
206
236
  return next.value;
207
237
  }
208
238
  return chunkReader(readChunk);
239
+ }
240
+ export function limitReader(reader, byteLimit) {
241
+ let bytesRead = 0;
242
+ return {
243
+ async upTo(length) {
244
+ let bytes = await reader.upTo(length);
245
+ if (bytes.length + bytesRead > byteLimit) {
246
+ bytes = bytes.subarray(0, byteLimit - bytesRead);
247
+ }
248
+ return bytes;
249
+ },
250
+ async exactly(length) {
251
+ const bytes = await reader.exactly(length);
252
+ if (bytes.length + bytesRead > byteLimit) {
253
+ throw new Error('Unexpected end of data');
254
+ }
255
+ return bytes;
256
+ },
257
+ seek(length) {
258
+ bytesRead += length;
259
+ reader.seek(length);
260
+ },
261
+ get pos() {
262
+ return reader.pos;
263
+ }
264
+ };
209
265
  }
@@ -0,0 +1,23 @@
1
+ const Kinds = {
2
+ Null: obj => obj === null,
3
+ Int: obj => Number.isInteger(obj),
4
+ Float: obj => typeof obj === 'number' && Number.isFinite(obj),
5
+ String: obj => typeof obj === 'string',
6
+ Bool: obj => typeof obj === 'boolean',
7
+ Bytes: obj => obj instanceof Uint8Array,
8
+ Link: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
9
+ List: obj => Array.isArray(obj),
10
+ Map: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
11
+ };
12
+ const Types = {
13
+ Int: Kinds.Int,
14
+ 'CarHeader > version': obj => Types.Int(obj),
15
+ 'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
16
+ 'CarHeader > roots (anon)': obj => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
17
+ 'CarHeader > roots': obj => Types['CarHeader > roots (anon)'](obj),
18
+ CarHeader: obj => {
19
+ const keys = obj && Object.keys(obj);
20
+ return Kinds.Map(obj) && ['version'].every(k => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value));
21
+ }
22
+ };
23
+ export const CarHeader = Types.CarHeader;
@@ -4,17 +4,16 @@ import {
4
4
  createDecoder
5
5
  } from './decoder.js';
6
6
  export class CarReader {
7
- constructor(version, roots, blocks) {
8
- this._version = version;
9
- this._roots = roots;
7
+ constructor(header, blocks) {
8
+ this._header = header;
10
9
  this._blocks = blocks;
11
10
  this._keys = blocks.map(b => b.cid.toString());
12
11
  }
13
12
  get version() {
14
- return this._version;
13
+ return this._header.version;
15
14
  }
16
15
  async getRoots() {
17
- return this._roots;
16
+ return this._header.roots;
18
17
  }
19
18
  async has(key) {
20
19
  return this._keys.indexOf(key.toString()) > -1;
@@ -46,13 +45,13 @@ export class CarReader {
46
45
  return decodeReaderComplete(asyncIterableReader(asyncIterable));
47
46
  }
48
47
  }
49
- async function decodeReaderComplete(reader) {
48
+ export async function decodeReaderComplete(reader) {
50
49
  const decoder = createDecoder(reader);
51
- const {version, roots} = await decoder.header();
50
+ const header = await decoder.header();
52
51
  const blocks = [];
53
52
  for await (const block of decoder.blocks()) {
54
53
  blocks.push(block);
55
54
  }
56
- return new CarReader(version, roots, blocks);
55
+ return new CarReader(header, blocks);
57
56
  }
58
57
  export const __browser = true;
@@ -63,7 +63,7 @@ export class CarWriter {
63
63
  const reader = bytesReader(bytes);
64
64
  await readHeader(reader);
65
65
  const newHeader = createHeader(roots);
66
- if (reader.pos !== newHeader.length) {
66
+ if (Number(reader.pos) !== newHeader.length) {
67
67
  throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${ reader.pos } bytes, new header is ${ newHeader.length } bytes)`);
68
68
  }
69
69
  bytes.set(newHeader, 0);
@@ -186,6 +186,77 @@ const goCarIndex = [
186
186
  blockLength: 18
187
187
  }
188
188
  ];
189
+ const goCarV2Bytes = bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000');
190
+ const goCarV2Roots = [CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')];
191
+ const goCarV2Index = [
192
+ {
193
+ blockLength: 47,
194
+ blockOffset: 143,
195
+ cid: CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'),
196
+ length: 82,
197
+ offset: 108
198
+ },
199
+ {
200
+ blockLength: 99,
201
+ blockOffset: 226,
202
+ cid: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
203
+ length: 135,
204
+ offset: 190
205
+ },
206
+ {
207
+ blockLength: 54,
208
+ blockOffset: 360,
209
+ cid: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
210
+ length: 89,
211
+ offset: 325
212
+ },
213
+ {
214
+ blockLength: 4,
215
+ blockOffset: 451,
216
+ cid: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
217
+ length: 41,
218
+ offset: 414
219
+ },
220
+ {
221
+ blockLength: 7,
222
+ blockOffset: 492,
223
+ cid: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
224
+ length: 44,
225
+ offset: 455
226
+ }
227
+ ];
228
+ const goCarV2Contents = {
229
+ QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
230
+ Links: [{
231
+ Hash: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
232
+ Name: '\uD83C\uDF64',
233
+ Tsize: 164
234
+ }]
235
+ },
236
+ QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
237
+ Links: [
238
+ {
239
+ Hash: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
240
+ Name: 'barreleye',
241
+ Tsize: 58
242
+ },
243
+ {
244
+ Hash: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
245
+ Name: '\uD83D\uDC21',
246
+ Tsize: 7
247
+ }
248
+ ]
249
+ },
250
+ Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
251
+ Links: [{
252
+ Hash: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
253
+ Name: 'fishmonger',
254
+ Tsize: 4
255
+ }]
256
+ },
257
+ bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
258
+ bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
259
+ };
189
260
  export {
190
261
  toBlock,
191
262
  assert,
@@ -195,5 +266,9 @@ export {
195
266
  carBytes,
196
267
  goCarBytes,
197
268
  goCarRoots,
198
- goCarIndex
269
+ goCarIndex,
270
+ goCarV2Bytes,
271
+ goCarV2Roots,
272
+ goCarV2Index,
273
+ goCarV2Contents
199
274
  };
@@ -4,7 +4,8 @@ import { encode as vEncode } from 'varint';
4
4
  import { CarReader } from '../lib/reader.js';
5
5
  import {
6
6
  carBytes,
7
- assert
7
+ assert,
8
+ goCarV2Bytes
8
9
  } from './common.js';
9
10
  function makeHeader(block) {
10
11
  const u = cbEncode(block);
@@ -26,42 +27,65 @@ describe('Misc errors', () => {
26
27
  });
27
28
  });
28
29
  it('bad version', async () => {
29
- const buf2 = bytes.fromHex('0aa16776657273696f6e02');
30
- assert.strictEqual(bytes.toHex(makeHeader({ version: 2 })), '0aa16776657273696f6e02');
31
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2');
30
+ const buf2 = bytes.fromHex('0aa16776657273696f6e03');
31
+ assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03');
32
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3');
32
33
  });
33
- it('bad header', async () => {
34
- let buf2 = makeHeader({
35
- version: 1,
36
- roots: []
34
+ describe('bad header', async () => {
35
+ it('sanity check', async () => {
36
+ const buf2 = makeHeader({
37
+ version: 1,
38
+ roots: []
39
+ });
40
+ await assert.isFulfilled(CarReader.fromBytes(buf2));
37
41
  });
38
- await assert.isFulfilled(CarReader.fromBytes(buf2));
39
- buf2 = makeHeader({ roots: [] });
40
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: undefined');
41
- buf2 = makeHeader({
42
- version: '1',
43
- roots: []
42
+ it('no \'version\' array', async () => {
43
+ const buf2 = makeHeader({ roots: [] });
44
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
44
45
  });
45
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: "1"');
46
- buf2 = makeHeader({ version: 1 });
47
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
48
- buf2 = makeHeader({
49
- version: 1,
50
- roots: {}
46
+ it('bad \'version\' type', async () => {
47
+ const buf2 = makeHeader({
48
+ version: '1',
49
+ roots: []
50
+ });
51
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
51
52
  });
52
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
53
- buf2 = makeHeader({
54
- version: 1,
55
- roots: [],
56
- blip: true
53
+ it('no \'roots\' array', async () => {
54
+ const buf2 = makeHeader({ version: 1 });
55
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
56
+ });
57
+ it('bad \'roots\' type', async () => {
58
+ const buf2 = makeHeader({
59
+ version: 1,
60
+ roots: {}
61
+ });
62
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
63
+ });
64
+ it('extraneous properties', async () => {
65
+ const buf2 = makeHeader({
66
+ version: 1,
67
+ roots: [],
68
+ blip: true
69
+ });
70
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
71
+ });
72
+ it('not an object', async () => {
73
+ const buf2 = makeHeader([
74
+ 1,
75
+ []
76
+ ]);
77
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
78
+ });
79
+ it('not an object', async () => {
80
+ const buf2 = makeHeader(null);
81
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
82
+ });
83
+ it('recursive v2 header', async () => {
84
+ const v2Header = goCarV2Bytes.slice(0, 51);
85
+ const buf2 = new Uint8Array(51 * 2);
86
+ buf2.set(v2Header, 0);
87
+ buf2.set(v2Header, 51);
88
+ await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)');
57
89
  });
58
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
59
- buf2 = makeHeader([
60
- 1,
61
- []
62
- ]);
63
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
64
- buf2 = makeHeader(null);
65
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
66
90
  });
67
91
  });
@@ -2,6 +2,9 @@ import { CarIndexer } from '../lib/indexer.js';
2
2
  import {
3
3
  goCarBytes,
4
4
  goCarIndex,
5
+ goCarV2Bytes,
6
+ goCarV2Roots,
7
+ goCarV2Index,
5
8
  makeIterable,
6
9
  assert
7
10
  } from './common.js';
@@ -17,6 +20,18 @@ describe('CarIndexer fromBytes()', () => {
17
20
  }
18
21
  assert.deepStrictEqual(indexData, goCarIndex);
19
22
  });
23
+ it('v2 complete', async () => {
24
+ const indexer = await CarIndexer.fromBytes(goCarV2Bytes);
25
+ const roots = await indexer.getRoots();
26
+ assert.strictEqual(roots.length, 1);
27
+ assert(goCarV2Roots[0].equals(roots[0]));
28
+ assert.strictEqual(indexer.version, 2);
29
+ const indexData = [];
30
+ for await (const index of indexer) {
31
+ indexData.push(index);
32
+ }
33
+ assert.deepStrictEqual(indexData, goCarV2Index);
34
+ });
20
35
  it('bad argument', async () => {
21
36
  for (const arg of [
22
37
  true,
@@ -1,12 +1,22 @@
1
1
  import { CarReader } from '../lib/reader.js';
2
2
  import { CarWriter } from '../lib/writer.js';
3
+ import {
4
+ bytesReader,
5
+ readHeader
6
+ } from '../lib/decoder.js';
3
7
  import * as Block from 'multiformats/block';
4
8
  import { sha256 } from 'multiformats/hashes/sha2';
5
9
  import * as raw from 'multiformats/codecs/raw';
10
+ import { base64 } from 'multiformats/bases/base64';
11
+ import * as dagPb from '@ipld/dag-pb';
6
12
  import {
7
13
  carBytes,
8
14
  makeIterable,
9
- assert
15
+ assert,
16
+ goCarV2Bytes,
17
+ goCarV2Roots,
18
+ goCarV2Index,
19
+ goCarV2Contents
10
20
  } from './common.js';
11
21
  import {
12
22
  verifyRoots,
@@ -15,6 +25,8 @@ import {
15
25
  verifyBlocks,
16
26
  verifyCids
17
27
  } from './verify-store-reader.js';
28
+ import { data as fixtures } from './fixtures.js';
29
+ import { expectations as fixtureExpectations } from './fixtures-expectations.js';
18
30
  describe('CarReader fromBytes()', () => {
19
31
  it('complete', async () => {
20
32
  const reader = await CarReader.fromBytes(carBytes);
@@ -52,6 +64,29 @@ describe('CarReader fromBytes()', () => {
52
64
  message: 'Unexpected end of data'
53
65
  });
54
66
  });
67
+ it('v2 complete', async () => {
68
+ const reader = await CarReader.fromBytes(goCarV2Bytes);
69
+ const roots = await reader.getRoots();
70
+ assert.strictEqual(roots.length, 1);
71
+ assert(goCarV2Roots[0].equals(roots[0]));
72
+ assert.strictEqual(reader.version, 2);
73
+ for (const {cid} of goCarV2Index) {
74
+ const block = await reader.get(cid);
75
+ assert.isDefined(block);
76
+ if (block) {
77
+ assert(cid.equals(block.cid));
78
+ let content;
79
+ if (cid.code === dagPb.code) {
80
+ content = dagPb.decode(block.bytes);
81
+ } else if (cid.code === 85) {
82
+ content = new TextDecoder().decode(block.bytes);
83
+ } else {
84
+ assert.fail('Unexpected codec');
85
+ }
86
+ assert.deepStrictEqual(content, goCarV2Contents[cid.toString()]);
87
+ }
88
+ }
89
+ });
55
90
  it('decode error - trailing null bytes', async () => {
56
91
  const bytes = new Uint8Array(carBytes.length + 5);
57
92
  bytes.set(carBytes);
@@ -178,4 +213,58 @@ describe('CarReader fromIterable()', () => {
178
213
  message: 'Unexpected end of data'
179
214
  });
180
215
  });
216
+ it('v2 decode error - truncated', async () => {
217
+ const bytes = goCarV2Bytes.slice();
218
+ const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
219
+ dv.setBigUint64(35, BigInt(448 - 10), true);
220
+ await assert.isRejected(CarReader.fromIterable(makeIterable(bytes, 64)), {
221
+ name: 'Error',
222
+ message: 'Unexpected end of data'
223
+ });
224
+ });
225
+ });
226
+ describe('Shared fixtures', () => {
227
+ describe('Header', () => {
228
+ for (const [name, {
229
+ version: expectedVersion,
230
+ err: expectedError
231
+ }] of Object.entries(fixtureExpectations)) {
232
+ it(name, async () => {
233
+ const data = base64.baseDecode(fixtures[name]);
234
+ let header;
235
+ try {
236
+ header = await readHeader(bytesReader(data));
237
+ } catch (err) {
238
+ if (expectedError != null) {
239
+ assert.equal(err.message, expectedError);
240
+ return;
241
+ }
242
+ assert.ifError(err);
243
+ }
244
+ if (expectedError != null) {
245
+ assert.fail(`Expected error: ${ expectedError }`);
246
+ }
247
+ assert.isDefined(header, 'did not decode header');
248
+ if (expectedVersion != null && header != null) {
249
+ assert.strictEqual(header.version, expectedVersion);
250
+ }
251
+ });
252
+ }
253
+ });
254
+ describe('Contents', () => {
255
+ for (const [name, {cids: expectedCids}] of Object.entries(fixtureExpectations)) {
256
+ if (expectedCids == null) {
257
+ continue;
258
+ }
259
+ it(name, async () => {
260
+ const data = base64.baseDecode(fixtures[name]);
261
+ const reader = await CarReader.fromBytes(data);
262
+ let i = 0;
263
+ for await (const cid of reader.cids()) {
264
+ assert.strictEqual(cid.toString(), expectedCids[i++]);
265
+ }
266
+ assert.strictEqual(i, expectedCids.length);
267
+ });
268
+ }
269
+ });
181
270
  });
@@ -42,10 +42,7 @@ async function run () {
42
42
  const indexer = await CarIndexer.fromBytes(bytes)
43
43
  const reader = await CarReader.fromBytes(bytes)
44
44
  const fixture = {
45
- header: {
46
- roots: await reader.getRoots(),
47
- version: reader.version
48
- },
45
+ header: reader._header, // a little naughty but we need gory details
49
46
  blocks: []
50
47
  }
51
48
  let i = 0
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env node
2
+
3
+ // Take a .car file and dump its index in DAG-JSON format, one line per block
4
+
5
+ import fs from 'fs'
6
+ import { CarIndexer } from '@ipld/car/indexer'
7
+ import * as dagJson from '@ipld/dag-json'
8
+
9
+ if (!process.argv[2]) {
10
+ console.log('Usage: dump-index.js <path/to/car>')
11
+ process.exit(1)
12
+ }
13
+
14
+ async function run () {
15
+ const indexer = await CarIndexer.fromIterable(fs.createReadStream(process.argv[2]))
16
+ for await (const blockIndex of indexer) {
17
+ console.log(new TextDecoder().decode(dagJson.encode(blockIndex)))
18
+ }
19
+ }
20
+
21
+ run().catch((err) => {
22
+ console.error(err)
23
+ process.exit(1)
24
+ })
@@ -16,7 +16,7 @@
16
16
  "license": "(Apache-2.0 AND MIT)",
17
17
  "dependencies": {
18
18
  "@ipld/car": "file:..",
19
- "@ipld/dag-cbor": "^6.0.4",
19
+ "@ipld/dag-cbor": "^7.0.0",
20
20
  "@ipld/dag-json": "^8.0.0",
21
21
  "@ipld/dag-pb": "^2.0.2",
22
22
  "@multiformats/blake2": "^1.0.1",
@@ -76,6 +76,39 @@ Blocks:
76
76
  assert.strictEqual(stdout, '{"blocks":[{"blockLength":55,"blockOffset":137,"cid":{"/":"bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"},"content":{"link":{"/":"QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"},"name":"blip"},"length":92,"offset":100},{"blockLength":97,"blockOffset":228,"cid":{"/":"QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"},"content":{"Links":[{"Hash":{"/":"bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"},"Name":"bear","Tsize":4},{"Hash":{"/":"QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"},"Name":"second","Tsize":149}]},"length":133,"offset":192},{"blockLength":4,"blockOffset":362,"cid":{"/":"bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"},"content":{"/":{"bytes":"Y2NjYw"}},"length":41,"offset":325},{"blockLength":94,"blockOffset":402,"cid":{"/":"QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"},"content":{"Links":[{"Hash":{"/":"bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"},"Name":"dog","Tsize":4},{"Hash":{"/":"QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"},"Name":"first","Tsize":51}]},"length":130,"offset":366},{"blockLength":4,"blockOffset":533,"cid":{"/":"bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"},"content":{"/":{"bytes":"YmJiYg"}},"length":41,"offset":496},{"blockLength":47,"blockOffset":572,"cid":{"/":"QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"},"content":{"Links":[{"Hash":{"/":"bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"},"Name":"cat","Tsize":4}]},"length":82,"offset":537},{"blockLength":4,"blockOffset":656,"cid":{"/":"bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"},"content":{"/":{"bytes":"YWFhYQ"}},"length":41,"offset":619},{"blockLength":18,"blockOffset":697,"cid":{"/":"bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"},"content":{"link":null,"name":"limbo"},"length":55,"offset":660}],"header":{"roots":[{"/":"bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"},{"/":"bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"}],"version":1}}\n')
77
77
  console.log('\u001b[32m✔\u001b[39m [example] car-to-fixture ../test/go.car')
78
78
  })
79
+ }).then(async () => {
80
+ await runExample('dump-index', ['example.car']).then(({ stdout, stderr }) => {
81
+ assert.strictEqual(stderr, '')
82
+ assert.strictEqual(stdout, '{"blockLength":24,"blockOffset":96,"cid":{"/":"bafkreihwkf6mtnjobdqrkiksr7qhp6tiiqywux64aylunbvmfhzeql2coa"},"length":61,"offset":59}\n')
83
+ console.log('\u001b[32m✔\u001b[39m [example] dump-index example.car')
84
+ })
85
+ }).then(async () => {
86
+ await runExample('dump-index', ['../test/go.car']).then(({ stdout, stderr }) => {
87
+ assert.strictEqual(stderr, '')
88
+ assert.strictEqual(stdout,
89
+ `{"blockLength":55,"blockOffset":137,"cid":{"/":"bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"},"length":92,"offset":100}
90
+ {"blockLength":97,"blockOffset":228,"cid":{"/":"QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"},"length":133,"offset":192}
91
+ {"blockLength":4,"blockOffset":362,"cid":{"/":"bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"},"length":41,"offset":325}
92
+ {"blockLength":94,"blockOffset":402,"cid":{"/":"QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"},"length":130,"offset":366}
93
+ {"blockLength":4,"blockOffset":533,"cid":{"/":"bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"},"length":41,"offset":496}
94
+ {"blockLength":47,"blockOffset":572,"cid":{"/":"QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"},"length":82,"offset":537}
95
+ {"blockLength":4,"blockOffset":656,"cid":{"/":"bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"},"length":41,"offset":619}
96
+ {"blockLength":18,"blockOffset":697,"cid":{"/":"bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"},"length":55,"offset":660}
97
+ `)
98
+ console.log('\u001b[32m✔\u001b[39m [example] dump-index ../test/go.carv2')
99
+ })
100
+ }).then(async () => {
101
+ await runExample('dump-index', ['../test/go.carv2']).then(({ stdout, stderr }) => {
102
+ assert.strictEqual(stderr, '')
103
+ assert.strictEqual(stdout,
104
+ `{"blockLength":47,"blockOffset":143,"cid":{"/":"QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z"},"length":82,"offset":108}
105
+ {"blockLength":99,"blockOffset":226,"cid":{"/":"QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM"},"length":135,"offset":190}
106
+ {"blockLength":54,"blockOffset":360,"cid":{"/":"Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE"},"length":89,"offset":325}
107
+ {"blockLength":4,"blockOffset":451,"cid":{"/":"bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu"},"length":41,"offset":414}
108
+ {"blockLength":7,"blockOffset":492,"cid":{"/":"bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju"},"length":44,"offset":455}
109
+ `)
110
+ console.log('\u001b[32m✔\u001b[39m [example] dump-index ../test/go.carv2')
111
+ })
79
112
  }).catch((err) => {
80
113
  console.error(err.stack)
81
114
  process.exit(1)