@ipld/car 3.2.4 → 4.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ipld/car",
-  "version": "3.2.4",
+  "version": "4.1.1",
   "description": "Content Addressable aRchive format reader and writer",
   "main": "./cjs/car.js",
   "types": "./types/car.d.ts",
@@ -8,17 +8,17 @@
     "lint": "standard",
     "build": "npm run build:js && npm run build:types",
     "build:js": "ipjs build --tests --main && npm run build:copy",
-    "build:copy": "mkdir -p dist/examples/ && cp -a tsconfig.json *.js *.ts lib test dist/ && cp examples/*.* dist/examples/",
+    "build:copy": "mkdir -p dist/examples/ && cp -a tsconfig.json .npmignore *.js *.ts lib test dist/ && cp examples/*.* dist/examples/ && rm -rf dist/test/fixtures/",
     "build:types": "tsc --build && mv types dist",
     "test:cjs": "rm -rf dist && npm run build && cp test/go.car dist/cjs/node-test/ && mocha dist/cjs/node-test/test-*.js && mocha dist/cjs/node-test/node-test-*.js && npm run test:cjs:browser",
     "test:esm": "rm -rf dist && npm run build && cp test/go.car dist/esm/node-test/ && mocha dist/esm/node-test/test-*.js && mocha dist/esm/node-test/node-test-*.js && npm run test:esm:browser",
-    "test:node": "c8 --check-coverage --branches 100 --functions 100 --lines 100 mocha test/test-*.js test/node-test-*.js",
+    "test:node": "c8 --check-coverage --branches 100 --functions 100 --lines 100 --exclude lib/header-validator.js --exclude test/ mocha test/test-*.js test/node-test-*.js",
     "test:cjs:browser": "polendina --page --worker --serviceworker --cleanup dist/cjs/browser-test/test-*.js",
     "test:esm:browser": "polendina --page --worker --serviceworker --cleanup dist/esm/browser-test/test-*.js",
     "test": "npm run lint && npm run test:node && npm run test:cjs && npm run test --prefix examples/",
     "test:ci": "npm run lint && npm run test:node && npm run test:esm && npm run test:cjs && npm run test --prefix examples/",
     "coverage": "c8 --reporter=html --reporter=text mocha test/test-*.js && npx st -d coverage -p 8888",
-    "docs": "jsdoc4readme --readme --description-only lib/reader*.js lib/indexed-reader.js lib/iterator.js lib/indexer.js lib/writer*.js"
+    "docs": "jsdoc4readme --readme --description-only lib/reader*.js lib/indexed-reader.js lib/iterator.js lib/indexer.js lib/writer*.js lib/buffer-writer.js lib/decoder.js"
   },
   "keywords": [
     "car",
@@ -63,10 +63,16 @@
       "browser": "./esm/lib/writer-browser.js",
       "require": "./cjs/lib/writer.js",
       "import": "./esm/lib/writer.js"
+    },
+    "./buffer-writer": {
+      "browser": "./esm/lib/buffer-writer.js",
+      "require": "./cjs/lib/buffer-writer.js",
+      "import": "./esm/lib/buffer-writer.js"
     }
   },
   "dependencies": {
     "@ipld/dag-cbor": "^7.0.0",
+    "cborg": "^1.9.0",
     "multiformats": "^9.5.4",
     "varint": "^6.0.0"
   },
@@ -87,7 +93,7 @@
     "jsdoc4readme": "^1.4.0",
     "mocha": "^9.1.3",
     "polendina": "~2.0.1",
-    "standard": "^16.0.4",
+    "standard": "^17.0.0",
     "typescript": "~4.6.2"
   },
   "standard": {
@@ -126,6 +132,9 @@
       "writer": [
         "types/lib/writer.d.ts"
       ],
+      "buffer-writer": [
+        "types/lib/buffer-writer.d.ts"
+      ],
       "*": [
         "types/*"
       ],
@@ -235,6 +244,7 @@
     "./iterator": "./cjs/lib/iterator.js",
     "./writer": "./cjs/lib/writer-browser.js",
     "./esm/lib/writer.js": "./esm/lib/writer-browser.js",
-    "./cjs/lib/writer.js": "./cjs/lib/writer-browser.js"
+    "./cjs/lib/writer.js": "./cjs/lib/writer-browser.js",
+    "./buffer-writer": "./cjs/lib/buffer-writer.js"
   }
 }

package/test/_fixtures_to_js.mjs

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+
+import { readdir, readFile, writeFile } from 'fs/promises'
+import { dirname, join } from 'path'
+
+async function main () {
+  const thisdir = dirname(new URL(import.meta.url).pathname)
+  const outfile = join(thisdir, 'fixtures.js')
+  const fixturesdir = join(thisdir, 'fixtures')
+  const files = await readdir(fixturesdir)
+  let content = '/** @type {Record<string, string>} */\nexport const data = {\n'
+  for (const f of files) {
+    content += `  '${f}': '`
+    content += (await readFile(join(fixturesdir, f))).toString('base64')
+    content += '\',\n'
+  }
+  content += '  _: \'\'\n}\n'
+  await writeFile(join(outfile), content, 'utf8')
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

package/test/common.js

@@ -132,14 +132,13 @@ function makeIterable (data, chunkSize) {
 }
 
 const carBytes = bytes.fromHex('63a265726f6f747382d82a58250001711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8bd82a5825000171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365b6776657273696f6e01280155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b461616161280155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e646877626262622801551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a2745163636363511220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d0122d0a240155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b4120363617418048001122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a122d0a240155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e6468771203646f671804122d0a221220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d01205666972737418338301122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de122e0a2401551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a274511204626561721804122f0a22122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a12067365636f6e641895015b01711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8ba2646c696e6bd82a582300122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de646e616d6564626c6970360171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365ba2646c696e6bf6646e616d65656c696d626f')
+
 // go.car is written as a graph, not by the allBlocks ordering here, so ordering is slightly out
 const goCarBytes = bytes.fromHex('63a265726f6f747382d82a58250001711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8bd82a5825000171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365b6776657273696f6e015b01711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8ba2646c696e6bd82a582300122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de646e616d6564626c69708301122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de122e0a2401551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a274511204626561721804122f0a22122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a12067365636f6e641895012801551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a27451636363638001122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a122d0a240155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e6468771203646f671804122d0a221220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d0120566697273741833280155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e64687762626262511220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d0122d0a240155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b412036361741804280155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b461616161360171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365ba2646c696e6bf6646e616d65656c696d626f')
-
 const goCarRoots = [
   CID.parse('bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm'),
   CID.parse('bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm')
 ]
-
 const goCarIndex = [
   { cid: CID.parse('bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm'), offset: 100, length: 92, blockOffset: 137, blockLength: 55 },
   { cid: CID.parse('QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d'), offset: 192, length: 133, blockOffset: 228, blockLength: 97 },
@@ -151,6 +150,49 @@ const goCarIndex = [
   { cid: CID.parse('bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm'), offset: 660, length: 55, blockOffset: 697, blockLength: 18 }
 ]
 
+const goCarV2Bytes = bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000')
+const goCarV2Roots = [CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')]
+const goCarV2Index = [
+  { blockLength: 47, blockOffset: 143, cid: CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'), length: 82, offset: 108 },
+  { blockLength: 99, blockOffset: 226, cid: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'), length: 135, offset: 190 },
+  { blockLength: 54, blockOffset: 360, cid: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'), length: 89, offset: 325 },
+  { blockLength: 4, blockOffset: 451, cid: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'), length: 41, offset: 414 },
+  { blockLength: 7, blockOffset: 492, cid: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'), length: 44, offset: 455 }
+]
+/** @type {{[k in string]: any}} */
+const goCarV2Contents = {
+  QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
+    Links: [{
+      Hash: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
+      Name: '🍤',
+      Tsize: 164
+    }]
+  },
+  QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
+    Links: [
+      {
+        Hash: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
+        Name: 'barreleye',
+        Tsize: 58
+      },
+      {
+        Hash: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
+        Name: '🐡',
+        Tsize: 7
+      }
+    ]
+  },
+  Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
+    Links: [{
+      Hash: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
+      Name: 'fishmonger',
+      Tsize: 4
+    }]
+  },
+  bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
+  bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
+}
+
 export {
   toBlock,
   assert,
@@ -160,5 +202,9 @@ export {
   carBytes,
   goCarBytes,
   goCarRoots,
-  goCarIndex
+  goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  goCarV2Contents
 }

package/test/test-buffer-writer.js

@@ -0,0 +1,256 @@
+/* eslint-env mocha */
+
+import * as CarBufferWriter from '@ipld/car/buffer-writer'
+import { CarReader } from '@ipld/car/reader'
+import { createHeader } from '../lib/encoder.js'
+import { assert } from './common.js'
+import { CID, varint } from 'multiformats'
+import * as CBOR from '@ipld/dag-cbor'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
+import { identity } from 'multiformats/hashes/identity'
+import * as Raw from 'multiformats/codecs/raw'
+import * as Block from 'multiformats/block'
+
+describe('CarBufferWriter', () => {
+  const cid = CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu')
+  describe('calculateHeaderLength', async () => {
+    for (const count of [0, 1, 10, 18, 24, 48, 124, 255, 258, 65536 - 1, 65536]) {
+      it(`calculateHeaderLength(new Array(${count}).fill(36))`, () => {
+        const roots = new Array(count).fill(cid)
+        const sizes = new Array(count).fill(cid.bytes.byteLength)
+        assert.deepEqual(
+          CarBufferWriter.calculateHeaderLength(sizes),
+          createHeader(roots).byteLength
+        )
+      })
+      it(`calculateHeaderLength(new Array(${count}).fill(36))`, () => {
+        const roots = new Array(count).fill(cid)
+        const rootLengths = roots.map((c) => c.bytes.byteLength)
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(rootLengths), createHeader(roots).byteLength)
+      })
+    }
+    it('estimate on large CIDs', () => {
+      const largeCID = CID.parse(`bafkqbbac${'a'.repeat(416)}`)
+      assert.equal(
+        CarBufferWriter.calculateHeaderLength([
+          cid.bytes.byteLength,
+          largeCID.bytes.byteLength
+        ]),
+        createHeader([
+          cid,
+          largeCID
+        ]).byteLength
+      )
+    })
+
+    it('estimate on large CIDs 2', () => {
+      const largeCID = CID.createV1(Raw.code, identity.digest(new Uint8Array(512).fill(1)))
+      assert.equal(
+        CarBufferWriter.calculateHeaderLength([
+          cid.bytes.byteLength,
+          largeCID.bytes.byteLength
+        ]),
+        createHeader([cid, largeCID]).byteLength
+      )
+    })
+  })
+
+  describe('writer', () => {
+    it('estimate header and write blocks', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(1)
+      const dataSize = 256
+      const buffer = new ArrayBuffer(headerSize + dataSize)
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize })
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      })
+
+      writer.write(b1)
+
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      })
+      writer.write(b2)
+
+      writer.addRoot(b1.cid)
+      const bytes = writer.close()
+
+      const reader = await CarReader.fromBytes(bytes)
+      assert.deepEqual(await reader.getRoots(), [b1.cid])
+      assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
+    })
+
+    it('overestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2)
+      const dataSize = 256
+      const buffer = new ArrayBuffer(headerSize + dataSize)
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize })
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      })
+
+      writer.write(b1)
+
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      })
+      writer.write(b2)
+
+      writer.addRoot(b1.cid)
+      assert.throws(() => writer.close(), /Header size was overestimate/)
+      const bytes = writer.close({ resize: true })
+
+      const reader = await CarReader.fromBytes(bytes)
+      assert.deepEqual(await reader.getRoots(), [b1.cid])
+      assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
+    })
+
+    it('underestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2)
+      const dataSize = 300
+      const buffer = new ArrayBuffer(headerSize + dataSize)
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize })
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      })
+
+      writer.write(b1)
+      writer.addRoot(b1.cid)
+
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha512
+      })
+      writer.write(b2)
+      assert.throws(() => writer.addRoot(b2.cid), /has no capacity/)
+      writer.addRoot(b2.cid, { resize: true })
+
+      const bytes = writer.close()
+
+      const reader = await CarReader.fromBytes(bytes)
+      assert.deepEqual(await reader.getRoots(), [b1.cid, b2.cid])
+      assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
+    })
+  })
+
+  it('has no space for the root', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1)
+    const dataSize = 100
+    const buffer = new ArrayBuffer(headerSize + dataSize)
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize })
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    })
+
+    writer.write(b1)
+    writer.addRoot(b1.cid)
+
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    })
+    writer.write(b2)
+    assert.throws(() => writer.addRoot(b2.cid), /Buffer has no capacity for a new root/)
+    assert.throws(() => writer.addRoot(b2.cid, { resize: true }), /Buffer has no capacity for a new root/)
+
+    const bytes = writer.close()
+
+    const reader = await CarReader.fromBytes(bytes)
+    assert.deepEqual(await reader.getRoots(), [b1.cid])
+    assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
+  })
+
+  it('has no space for the block', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1)
+    const dataSize = 58
+    const buffer = new ArrayBuffer(headerSize + dataSize)
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize })
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    })
+
+    writer.write(b1)
+    writer.addRoot(b1.cid)
+
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    })
+    assert.throws(() => writer.write(b2), /Buffer has no capacity for this block/)
+
+    const bytes = writer.close()
+
+    const reader = await CarReader.fromBytes(bytes)
+    assert.deepEqual(await reader.getRoots(), [b1.cid])
+    assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }])
+  })
+
+  it('provide roots', async () => {
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    })
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha512
+    })
+
+    const buffer = new ArrayBuffer(300)
+    const writer = CarBufferWriter.createWriter(buffer, { roots: [b1.cid, b2.cid] })
+
+    writer.write(b1)
+    writer.write(b2)
+
+    const bytes = writer.close()
+
+    const reader = await CarReader.fromBytes(bytes)
+    assert.deepEqual(await reader.getRoots(), [b1.cid, b2.cid])
+    assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
+  })
+
+  it('provide large CID root', async () => {
+    const bytes = new Uint8Array(512).fill(1)
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    })
+
+    const b2 = {
+      cid: CID.createV1(Raw.code, identity.digest(bytes)),
+      bytes
+    }
+
+    const headerSize = CBOR.encode({ version: 1, roots: [b1.cid, b2.cid] }).byteLength
+    const bodySize = CarBufferWriter.blockLength(b1) + CarBufferWriter.blockLength(b2)
+    const varintSize = varint.encodingLength(headerSize)
+
+    const writer = CarBufferWriter.createWriter(new ArrayBuffer(varintSize + headerSize + bodySize), { roots: [b1.cid, b2.cid] })
+
+    writer.write(b1)
+    writer.write(b2)
+    const car = writer.close()
+    const reader = await CarReader.fromBytes(car)
+    assert.deepEqual(await reader.getRoots(), [b1.cid, b2.cid])
+    assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
+  })
+})

package/test/test-errors.js

@@ -3,7 +3,7 @@ import { bytes } from 'multiformats'
 import { encode as cbEncode } from '@ipld/dag-cbor'
 import { encode as vEncode } from 'varint'
 import { CarReader } from '@ipld/car/reader'
-import { carBytes, assert } from './common.js'
+import { carBytes, assert, goCarV2Bytes } from './common.js'
 
 /**
  * @param {any} block
@@ -34,43 +34,65 @@ describe('Misc errors', () => {
 
   it('bad version', async () => {
     // quick sanity check that makeHeader() works properly!
-    const buf2 = bytes.fromHex('0aa16776657273696f6e02')
-    // {version:2} - fixed string, likely to be used by CARv2 to escape header parsing rules
-    assert.strictEqual(bytes.toHex(makeHeader({ version: 2 })), '0aa16776657273696f6e02')
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2')
+    const buf2 = bytes.fromHex('0aa16776657273696f6e03')
+    assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03')
+    // {version:3}
+    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3')
   })
 
-  it('bad header', async () => {
-    // sanity check, this should be fine
-    let buf2 = makeHeader({ version: 1, roots: [] })
-    await assert.isFulfilled(CarReader.fromBytes(buf2))
+  describe('bad header', async () => {
+    it('sanity check', async () => {
+      // sanity check, this should be fine
+      const buf2 = makeHeader({ version: 1, roots: [] })
+      await assert.isFulfilled(CarReader.fromBytes(buf2))
+    })
 
-    // no 'version' array
-    buf2 = makeHeader({ roots: [] })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: undefined')
+    it('no \'version\' array', async () => {
+      const buf2 = makeHeader({ roots: [] })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // bad 'version' type
-    buf2 = makeHeader({ version: '1', roots: [] })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: "1"')
+    it('bad \'version\' type', async () => {
+      const buf2 = makeHeader({ version: '1', roots: [] })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
+
+    it('no \'roots\' array', async () => {
+      const buf2 = makeHeader({ version: 1 })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // no 'roots' array
-    buf2 = makeHeader({ version: 1 })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('bad \'roots\' type', async () => {
+      const buf2 = makeHeader({ version: 1, roots: {} })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // bad 'roots' type
-    buf2 = makeHeader({ version: 1, roots: {} })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('extraneous properties', async () => {
+      const buf2 = makeHeader({ version: 1, roots: [], blip: true })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // extraneous properties
-    buf2 = makeHeader({ version: 1, roots: [], blip: true })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('not an object', async () => {
+      const buf2 = makeHeader([1, []])
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // not an object
-    buf2 = makeHeader([1, []])
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('not an object', async () => {
+      const buf2 = makeHeader(null)
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // not an object
-    buf2 = makeHeader(null)
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('recursive v2 header', async () => {
+      // first 51 bytes are the carv2 header:
+      //   11b prefix, 16b characteristics, 8b data offset, 8b data size, 8b index offset
+      const v2Header = goCarV2Bytes.slice(0, 51)
+      // parser should expect to get a carv1 header at the data offset, but it uses the same
+      // code to check the carv2 header so let's make sure it doesn't allow recursive carv2
+      // headers
+      const buf2 = new Uint8Array(51 * 2)
+      buf2.set(v2Header, 0)
+      buf2.set(v2Header, 51)
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)')
+    })
   })
 })

package/test/test-indexer.js

@@ -1,7 +1,15 @@
 /* eslint-env mocha */
 
 import { CarIndexer } from '@ipld/car/indexer'
-import { goCarBytes, goCarIndex, makeIterable, assert } from './common.js'
+import {
+  goCarBytes,
+  goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  makeIterable,
+  assert
+} from './common.js'
 import { verifyRoots } from './verify-store-reader.js'
 
 describe('CarIndexer fromBytes()', () => {
@@ -18,6 +26,21 @@ describe('CarIndexer fromBytes()', () => {
     assert.deepStrictEqual(indexData, goCarIndex)
   })
 
+  it('v2 complete', async () => {
+    const indexer = await CarIndexer.fromBytes(goCarV2Bytes)
+    const roots = await indexer.getRoots()
+    assert.strictEqual(roots.length, 1)
+    assert.ok(goCarV2Roots[0].equals(roots[0]))
+    assert.strictEqual(indexer.version, 2)
+
+    const indexData = []
+    for await (const index of indexer) {
+      indexData.push(index)
+    }
+
+    assert.deepStrictEqual(indexData, goCarV2Index)
+  })
+
   it('bad argument', async () => {
     for (const arg of [true, false, null, undefined, 'string', 100, { obj: 'nope' }]) {
       // @ts-ignore