@ipld/car 3.2.4 → 4.0.0

package/lib/decoder.js

@@ -2,6 +2,7 @@ import varint from 'varint'
 import { CID } from 'multiformats/cid'
 import * as Digest from 'multiformats/hashes/digest'
 import { decode as decodeDagCbor } from '@ipld/dag-cbor'
+import { CarHeader as headerValidator } from './header-validator.js'
 
 /**
  * @typedef {import('../api').Block} Block
@@ -9,6 +10,8 @@ import { decode as decodeDagCbor } from '@ipld/dag-cbor'
  * @typedef {import('../api').BlockIndex} BlockIndex
  * @typedef {import('./coding').BytesReader} BytesReader
  * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
+ * @typedef {import('./coding').CarV2FixedHeader} CarV2FixedHeader
  * @typedef {import('./coding').CarDecoder} CarDecoder
  */
 
@@ -18,12 +21,17 @@ const CIDV0_BYTES = {
   DAG_PB: 0x70
 }
 
+const V2_HEADER_LENGTH = /* characteristics */ 16 /* v1 offset */ + 8 /* v1 size */ + 8 /* index offset */ + 8
+
 /**
  * @param {BytesReader} reader
  * @returns {Promise<number>}
  */
 async function readVarint (reader) {
   const bytes = await reader.upTo(8)
+  if (!bytes.length) {
+    throw new Error('Unexpected end of data')
+  }
   const i = varint.decode(bytes)
   reader.seek(varint.decode.bytes)
   return i
@@ -33,9 +41,40 @@ async function readVarint (reader) {
 
 /**
  * @param {BytesReader} reader
- * @returns {Promise<CarHeader>}
+ * @returns {Promise<CarV2FixedHeader>}
+ */
+async function readV2Header (reader) {
+  /** @type {Uint8Array} */
+  const bytes = await reader.exactly(V2_HEADER_LENGTH)
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  let offset = 0
+  const header = {
+    version: 2,
+    /** @type {[bigint, bigint]} */
+    characteristics: [
+      dv.getBigUint64(offset, true),
+      dv.getBigUint64(offset += 8, true)
+    ],
+    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
+    dataSize: Number(dv.getBigUint64(offset += 8, true)),
+    indexOffset: Number(dv.getBigUint64(offset += 8, true))
+  }
+  reader.seek(V2_HEADER_LENGTH)
+  return header
+  /* c8 ignore next 2 */
+  // Node.js 12 c8 bug
+}
+
+/**
+ * Reads header data from a `BytesReader`. The header may either be in the form
+ * of a `CarHeader` or `CarV2Header` depending on the CAR being read.
+ *
+ * @name async decoder.readHeader(reader)
+ * @param {BytesReader} reader
+ * @param {number} [strictVersion]
+ * @returns {Promise<CarHeader|CarV2Header>}
  */
-export async function readHeader (reader) {
+export async function readHeader (reader, strictVersion) {
   const length = await readVarint(reader)
   if (length === 0) {
     throw new Error('Invalid CAR header (zero length)')
@@ -43,22 +82,26 @@ export async function readHeader (reader) {
   const header = await reader.exactly(length)
   reader.seek(length)
   const block = decodeDagCbor(header)
-  if (block == null || Array.isArray(block) || typeof block !== 'object') {
+  if (!headerValidator(block)) {
     throw new Error('Invalid CAR header format')
   }
-  if (block.version !== 1) {
-    if (typeof block.version === 'string') {
-      throw new Error(`Invalid CAR version: "${block.version}"`)
-    }
-    throw new Error(`Invalid CAR version: ${block.version}`)
+  if ((block.version !== 1 && block.version !== 2) || (strictVersion !== undefined && block.version !== strictVersion)) {
+    throw new Error(`Invalid CAR version: ${block.version}${strictVersion !== undefined ? ` (expected ${strictVersion})` : ''}`)
   }
-  if (!Array.isArray(block.roots)) {
+  // we've made 'roots' optional in the schema so we can do the version check
+  // before rejecting the block as invalid if there is no version
+  const hasRoots = Array.isArray(block.roots)
+  if ((block.version === 1 && !hasRoots) || (block.version === 2 && hasRoots)) {
     throw new Error('Invalid CAR header format')
   }
-  if (Object.keys(block).filter((p) => p !== 'roots' && p !== 'version').length) {
-    throw new Error('Invalid CAR header format')
+  if (block.version === 1) {
+    return block
   }
-  return block
+  // version 2
+  const v2Header = await readV2Header(reader)
+  reader.seek(v2Header.dataOffset - reader.pos)
+  const v1Header = await readHeader(reader, 1)
+  return Object.assign(v1Header, v2Header)
   /* c8 ignore next 2 */
   // Node.js 12 c8 bug
 }
@@ -112,6 +155,12 @@ async function readCid (reader) {
 }
 
 /**
+ * Reads the leading data of an individual block from CAR data from a
+ * `BytesReader`. Returns a `BlockHeader` object which contains
+ * `{ cid, length, blockLength }` which can be used to either index the block
+ * or read the block binary data.
+ *
+ * @name async decoder.readBlockHead(reader)
  * @param {BytesReader} reader
  * @returns {Promise<BlockHeader>}
  */
@@ -125,7 +174,7 @@ export async function readBlockHead (reader) {
   }
   length += (reader.pos - start)
   const cid = await readCid(reader)
-  const blockLength = length - (reader.pos - start) // subtract CID length
+  const blockLength = length - Number(reader.pos - start) // subtract CID length
 
   return { cid, length, blockLength }
   /* c8 ignore next 2 */
@@ -160,11 +209,25 @@ async function readBlockIndex (reader) {
 }
 
 /**
+ * Creates a `CarDecoder` from a `BytesReader`. The `CarDecoder` is as async
+ * interface that will consume the bytes from the `BytesReader` to yield a
+ * `header()` and either `blocks()` or `blocksIndex()` data.
+ *
+ * @name decoder.createDecoder(reader)
  * @param {BytesReader} reader
  * @returns {CarDecoder}
  */
 export function createDecoder (reader) {
-  const headerPromise = readHeader(reader)
+  const headerPromise = (async () => {
+    const header = await readHeader(reader)
+    if (header.version === 2) {
+      const v1length = reader.pos - header.dataOffset
+      reader = limitReader(reader, header.dataSize - v1length)
+    }
+    return header
+    /* c8 ignore next 2 */
+    // Node.js 12 c8 bug
+  })()
 
   return {
     header: () => headerPromise,
@@ -186,6 +249,9 @@ export function createDecoder (reader) {
 }
 
 /**
+ * Creates a `BytesReader` from a `Uint8Array`.
+ *
+ * @name decoder.bytesReader(bytes)
  * @param {Uint8Array} bytes
  * @returns {BytesReader}
  */
@@ -298,6 +364,10 @@ export function chunkReader (readChunk /*, closer */) {
 }
 
 /**
+ * Creates a `BytesReader` from an `AsyncIterable<Uint8Array>`, which allows for
+ * consumption of CAR data from a streaming source.
+ *
+ * @name decoder.asyncIterableReader(asyncIterable)
  * @param {AsyncIterable<Uint8Array>} asyncIterable
  * @returns {BytesReader}
  */
@@ -316,3 +386,49 @@ export function asyncIterableReader (asyncIterable) {
 
   return chunkReader(readChunk)
 }
+
+/**
+ * Wraps a `BytesReader` in a limiting `BytesReader` which limits maximum read
+ * to `byteLimit` bytes. It _does not_ update `pos` of the original
+ * `BytesReader`.
+ *
+ * @name decoder.limitReader(reader, byteLimit)
+ * @param {BytesReader} reader
+ * @param {number} byteLimit
+ * @returns {BytesReader}
+ */
+export function limitReader (reader, byteLimit) {
+  let bytesRead = 0
+
+  /** @type {BytesReader} */
+  return {
+    async upTo (length) {
+      let bytes = await reader.upTo(length)
+      if (bytes.length + bytesRead > byteLimit) {
+        bytes = bytes.subarray(0, byteLimit - bytesRead)
+      }
+      return bytes
+      /* c8 ignore next 2 */
+      // Node.js 12 c8 bug
+    },
+
+    async exactly (length) {
+      const bytes = await reader.exactly(length)
+      if (bytes.length + bytesRead > byteLimit) {
+        throw new Error('Unexpected end of data')
+      }
+      return bytes
+      /* c8 ignore next 2 */
+      // Node.js 12 c8 bug
+    },
+
+    seek (length) {
+      bytesRead += length
+      reader.seek(length)
+    },
+
+    get pos () {
+      return reader.pos
+    }
+  }
+}

package/lib/header-validator.js

@@ -0,0 +1,33 @@
+/** Auto-generated with ipld-schema-validator@0.0.0-dev at Thu Jun 17 2021 from IPLD Schema:
+ *
+ * type CarHeader struct {
+ *   version Int
+ *   roots optional [&Any]
+ *   # roots is _not_ optional for CarV1 but we defer that check within code to
+ *   # gracefully handle the >V1 case where it's just {version:X}
+ * }
+ *
+ */
+
+const Kinds = {
+  Null: /** @returns {boolean} */ (/** @type {any} */ obj) => obj === null,
+  Int: /** @returns {boolean} */ (/** @type {any} */ obj) => Number.isInteger(obj),
+  Float: /** @returns {boolean} */ (/** @type {any} */ obj) => typeof obj === 'number' && Number.isFinite(obj),
+  String: /** @returns {boolean} */ (/** @type {any} */ obj) => typeof obj === 'string',
+  Bool: /** @returns {boolean} */ (/** @type {any} */ obj) => typeof obj === 'boolean',
+  Bytes: /** @returns {boolean} */ (/** @type {any} */ obj) => obj instanceof Uint8Array,
+  Link: /** @returns {boolean} */ (/** @type {any} */ obj) => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
+  List: /** @returns {boolean} */ (/** @type {any} */ obj) => Array.isArray(obj),
+  Map: /** @returns {boolean} */ (/** @type {any} */ obj) => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
+}
+/** @type {{ [k in string]: (obj:any)=>boolean}} */
+const Types = {
+  Int: Kinds.Int,
+  'CarHeader > version': /** @returns {boolean} */ (/** @type {any} */ obj) => Types.Int(obj),
+  'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
+  'CarHeader > roots (anon)': /** @returns {boolean} */ (/** @type {any} */ obj) => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
+  'CarHeader > roots': /** @returns {boolean} */ (/** @type {any} */ obj) => Types['CarHeader > roots (anon)'](obj),
+  CarHeader: /** @returns {boolean} */ (/** @type {any} */ obj) => { const keys = obj && Object.keys(obj); return Kinds.Map(obj) && ['version'].every((k) => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value)) }
+}
+
+export const CarHeader = Types.CarHeader

package/lib/header.ipldsch

@@ -0,0 +1,6 @@
+type CarHeader struct {
+  version Int
+  roots optional [&Any]
+  # roots is _not_ optional for CarV1 but we defer that check within code to
+  # gracefully handle the >V1 case where it's just {version:X}
+}

package/lib/reader-browser.js

@@ -5,6 +5,8 @@ import { asyncIterableReader, bytesReader, createDecoder } from './decoder.js'
  * @typedef {import('../api').Block} Block
  * @typedef {import('../api').CarReader} CarReaderIface
  * @typedef {import('./coding').BytesReader} BytesReader
+ * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
  */
 
 /**
@@ -26,18 +28,16 @@ import { asyncIterableReader, bytesReader, createDecoder } from './decoder.js'
  * @class
  * @implements {CarReaderIface}
  * @property {number} version The version number of the CAR referenced by this
- * reader (should be `1`).
+ * reader (should be `1` or `2`).
  */
 export class CarReader {
   /**
    * @constructs CarReader
-   * @param {number} version
-   * @param {CID[]} roots
+   * @param {CarHeader|CarV2Header} header
    * @param {Block[]} blocks
    */
-  constructor (version, roots, blocks) {
-    this._version = version
-    this._roots = roots
+  constructor (header, blocks) {
+    this._header = header
     this._blocks = blocks
     this._keys = blocks.map((b) => b.cid.toString())
   }
@@ -48,7 +48,7 @@ export class CarReader {
    * @instance
    */
   get version () {
-    return this._version
+    return this._header.version
   }
 
   /**
@@ -62,7 +62,7 @@ export class CarReader {
    * @returns {Promise<CID[]>}
    */
   async getRoots () {
-    return this._roots
+    return this._header.roots
     /* c8 ignore next 2 */
     // Node.js 12 c8 bug
   }
@@ -190,15 +190,15 @@ export class CarReader {
  * @param {BytesReader} reader
  * @returns {Promise<CarReader>}
  */
-async function decodeReaderComplete (reader) {
+export async function decodeReaderComplete (reader) {
   const decoder = createDecoder(reader)
-  const { version, roots } = await decoder.header()
+  const header = await decoder.header()
   const blocks = []
   for await (const block of decoder.blocks()) {
     blocks.push(block)
   }
 
-  return new CarReader(version, roots, blocks)
+  return new CarReader(header, blocks)
   /* c8 ignore next 2 */
   // Node.js 12 c8 bug
 }

package/lib/writer-browser.js

@@ -176,7 +176,7 @@ export class CarWriter {
     const reader = bytesReader(bytes)
     await readHeader(reader)
     const newHeader = createHeader(roots)
-    if (reader.pos !== newHeader.length) {
+    if (Number(reader.pos) !== newHeader.length) {
       throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${reader.pos} bytes, new header is ${newHeader.length} bytes)`)
     }
     bytes.set(newHeader, 0)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ipld/car",
-  "version": "3.2.4",
+  "version": "4.0.0",
   "description": "Content Addressable aRchive format reader and writer",
   "main": "./cjs/car.js",
   "types": "./types/car.d.ts",
@@ -8,17 +8,17 @@
     "lint": "standard",
     "build": "npm run build:js && npm run build:types",
     "build:js": "ipjs build --tests --main && npm run build:copy",
-    "build:copy": "mkdir -p dist/examples/ && cp -a tsconfig.json *.js *.ts lib test dist/ && cp examples/*.* dist/examples/",
+    "build:copy": "mkdir -p dist/examples/ && cp -a tsconfig.json .npmignore *.js *.ts lib test dist/ && cp examples/*.* dist/examples/ && rm -rf dist/test/fixtures/",
     "build:types": "tsc --build && mv types dist",
     "test:cjs": "rm -rf dist && npm run build && cp test/go.car dist/cjs/node-test/ && mocha dist/cjs/node-test/test-*.js && mocha dist/cjs/node-test/node-test-*.js && npm run test:cjs:browser",
     "test:esm": "rm -rf dist && npm run build && cp test/go.car dist/esm/node-test/ && mocha dist/esm/node-test/test-*.js && mocha dist/esm/node-test/node-test-*.js && npm run test:esm:browser",
-    "test:node": "c8 --check-coverage --branches 100 --functions 100 --lines 100 mocha test/test-*.js test/node-test-*.js",
+    "test:node": "c8 --check-coverage --branches 100 --functions 100 --lines 100 --exclude lib/header-validator.js --exclude test/ mocha test/test-*.js test/node-test-*.js",
     "test:cjs:browser": "polendina --page --worker --serviceworker --cleanup dist/cjs/browser-test/test-*.js",
     "test:esm:browser": "polendina --page --worker --serviceworker --cleanup dist/esm/browser-test/test-*.js",
     "test": "npm run lint && npm run test:node && npm run test:cjs && npm run test --prefix examples/",
     "test:ci": "npm run lint && npm run test:node && npm run test:esm && npm run test:cjs && npm run test --prefix examples/",
     "coverage": "c8 --reporter=html --reporter=text mocha test/test-*.js && npx st -d coverage -p 8888",
-    "docs": "jsdoc4readme --readme --description-only lib/reader*.js lib/indexed-reader.js lib/iterator.js lib/indexer.js lib/writer*.js"
+    "docs": "jsdoc4readme --readme --description-only lib/reader*.js lib/indexed-reader.js lib/iterator.js lib/indexer.js lib/writer*.js lib/decoder.js"
   },
   "keywords": [
     "car",

package/test/_fixtures_to_js.mjs

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+
+import { readdir, readFile, writeFile } from 'fs/promises'
+import { dirname, join } from 'path'
+
+async function main () {
+  const thisdir = dirname(new URL(import.meta.url).pathname)
+  const outfile = join(thisdir, 'fixtures.js')
+  const fixturesdir = join(thisdir, 'fixtures')
+  const files = await readdir(fixturesdir)
+  let content = '/** @type {Record<string, string>} */\nexport const data = {\n'
+  for (const f of files) {
+    content += `  '${f}': '`
+    content += (await readFile(join(fixturesdir, f))).toString('base64')
+    content += '\',\n'
+  }
+  content += '  _: \'\'\n}\n'
+  await writeFile(join(outfile), content, 'utf8')
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

package/test/common.js

@@ -132,14 +132,13 @@ function makeIterable (data, chunkSize) {
 }
 
 const carBytes = bytes.fromHex('63a265726f6f747382d82a58250001711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8bd82a5825000171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365b6776657273696f6e01280155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b461616161280155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e646877626262622801551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a2745163636363511220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d0122d0a240155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b4120363617418048001122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a122d0a240155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e6468771203646f671804122d0a221220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d01205666972737418338301122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de122e0a2401551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a274511204626561721804122f0a22122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a12067365636f6e641895015b01711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8ba2646c696e6bd82a582300122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de646e616d6564626c6970360171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365ba2646c696e6bf6646e616d65656c696d626f')
+
 // go.car is written as a graph, not by the allBlocks ordering here, so ordering is slightly out
 const goCarBytes = bytes.fromHex('63a265726f6f747382d82a58250001711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8bd82a5825000171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365b6776657273696f6e015b01711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8ba2646c696e6bd82a582300122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de646e616d6564626c69708301122002acecc5de2438ea4126a3010ecb1f8a599c8eff22fff1a1dcffe999b27fd3de122e0a2401551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a274511204626561721804122f0a22122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a12067365636f6e641895012801551220b6fbd675f98e2abd22d4ed29fdc83150fedc48597e92dd1a7a24381d44a27451636363638001122079a982de3c9907953d4d323cee1d0fb1ed8f45f8ef02870c0cb9e09246bd530a122d0a240155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e6468771203646f671804122d0a221220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d0120566697273741833280155122081cc5b17018674b401b42f35ba07bb79e211239c23bffe658da1577e3e64687762626262511220e7dc486e97e6ebe5cdabab3e392bdad128b6e09acc94bb4e2aa2af7b986d24d0122d0a240155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b412036361741804280155122061be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b461616161360171122069ea0740f9807a28f4d932c62e7c1c83be055e55072c90266ab3e79df63a365ba2646c696e6bf6646e616d65656c696d626f')
-
 const goCarRoots = [
   CID.parse('bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm'),
   CID.parse('bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm')
 ]
-
 const goCarIndex = [
   { cid: CID.parse('bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm'), offset: 100, length: 92, blockOffset: 137, blockLength: 55 },
   { cid: CID.parse('QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d'), offset: 192, length: 133, blockOffset: 228, blockLength: 97 },
@@ -151,6 +150,49 @@ const goCarIndex = [
   { cid: CID.parse('bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm'), offset: 660, length: 55, blockOffset: 697, blockLength: 18 }
 ]
 
+const goCarV2Bytes = bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000')
+const goCarV2Roots = [CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')]
+const goCarV2Index = [
+  { blockLength: 47, blockOffset: 143, cid: CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'), length: 82, offset: 108 },
+  { blockLength: 99, blockOffset: 226, cid: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'), length: 135, offset: 190 },
+  { blockLength: 54, blockOffset: 360, cid: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'), length: 89, offset: 325 },
+  { blockLength: 4, blockOffset: 451, cid: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'), length: 41, offset: 414 },
+  { blockLength: 7, blockOffset: 492, cid: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'), length: 44, offset: 455 }
+]
+/** @type {{[k in string]: any}} */
+const goCarV2Contents = {
+  QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
+    Links: [{
+      Hash: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
+      Name: '🍤',
+      Tsize: 164
+    }]
+  },
+  QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
+    Links: [
+      {
+        Hash: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
+        Name: 'barreleye',
+        Tsize: 58
+      },
+      {
+        Hash: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
+        Name: '🐡',
+        Tsize: 7
+      }
+    ]
+  },
+  Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
+    Links: [{
+      Hash: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
+      Name: 'fishmonger',
+      Tsize: 4
+    }]
+  },
+  bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
+  bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
+}
+
 export {
   toBlock,
   assert,
@@ -160,5 +202,9 @@ export {
   carBytes,
   goCarBytes,
   goCarRoots,
-  goCarIndex
+  goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  goCarV2Contents
 }

package/test/test-errors.js

@@ -3,7 +3,7 @@ import { bytes } from 'multiformats'
 import { encode as cbEncode } from '@ipld/dag-cbor'
 import { encode as vEncode } from 'varint'
 import { CarReader } from '@ipld/car/reader'
-import { carBytes, assert } from './common.js'
+import { carBytes, assert, goCarV2Bytes } from './common.js'
 
 /**
  * @param {any} block
@@ -34,43 +34,65 @@ describe('Misc errors', () => {
 
   it('bad version', async () => {
     // quick sanity check that makeHeader() works properly!
-    const buf2 = bytes.fromHex('0aa16776657273696f6e02')
-    // {version:2} - fixed string, likely to be used by CARv2 to escape header parsing rules
-    assert.strictEqual(bytes.toHex(makeHeader({ version: 2 })), '0aa16776657273696f6e02')
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2')
+    const buf2 = bytes.fromHex('0aa16776657273696f6e03')
+    assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03')
+    // {version:3}
+    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3')
   })
 
-  it('bad header', async () => {
-    // sanity check, this should be fine
-    let buf2 = makeHeader({ version: 1, roots: [] })
-    await assert.isFulfilled(CarReader.fromBytes(buf2))
+  describe('bad header', async () => {
+    it('sanity check', async () => {
+      // sanity check, this should be fine
+      const buf2 = makeHeader({ version: 1, roots: [] })
+      await assert.isFulfilled(CarReader.fromBytes(buf2))
+    })
 
-    // no 'version' array
-    buf2 = makeHeader({ roots: [] })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: undefined')
+    it('no \'version\' array', async () => {
+      const buf2 = makeHeader({ roots: [] })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // bad 'version' type
-    buf2 = makeHeader({ version: '1', roots: [] })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: "1"')
+    it('bad \'version\' type', async () => {
+      const buf2 = makeHeader({ version: '1', roots: [] })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
+
+    it('no \'roots\' array', async () => {
+      const buf2 = makeHeader({ version: 1 })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // no 'roots' array
-    buf2 = makeHeader({ version: 1 })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('bad \'roots\' type', async () => {
+      const buf2 = makeHeader({ version: 1, roots: {} })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // bad 'roots' type
-    buf2 = makeHeader({ version: 1, roots: {} })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('extraneous properties', async () => {
+      const buf2 = makeHeader({ version: 1, roots: [], blip: true })
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // extraneous properties
-    buf2 = makeHeader({ version: 1, roots: [], blip: true })
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('not an object', async () => {
+      const buf2 = makeHeader([1, []])
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // not an object
-    buf2 = makeHeader([1, []])
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('not an object', async () => {
+      const buf2 = makeHeader(null)
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    })
 
-    // not an object
-    buf2 = makeHeader(null)
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
+    it('recursive v2 header', async () => {
+      // first 51 bytes are the carv2 header:
+      //   11b prefix, 16b characteristics, 8b data offset, 8b data size, 8b index offset
+      const v2Header = goCarV2Bytes.slice(0, 51)
+      // parser should expect to get a carv1 header at the data offset, but it uses the same
+      // code to check the carv2 header so let's make sure it doesn't allow recursive carv2
+      // headers
+      const buf2 = new Uint8Array(51 * 2)
+      buf2.set(v2Header, 0)
+      buf2.set(v2Header, 51)
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)')
+    })
   })
 })

package/test/test-indexer.js

@@ -1,7 +1,15 @@
 /* eslint-env mocha */
 
 import { CarIndexer } from '@ipld/car/indexer'
-import { goCarBytes, goCarIndex, makeIterable, assert } from './common.js'
+import {
+  goCarBytes,
+  goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  makeIterable,
+  assert
+} from './common.js'
 import { verifyRoots } from './verify-store-reader.js'
 
 describe('CarIndexer fromBytes()', () => {
@@ -18,6 +26,21 @@ describe('CarIndexer fromBytes()', () => {
     assert.deepStrictEqual(indexData, goCarIndex)
   })
 
+  it('v2 complete', async () => {
+    const indexer = await CarIndexer.fromBytes(goCarV2Bytes)
+    const roots = await indexer.getRoots()
+    assert.strictEqual(roots.length, 1)
+    assert(goCarV2Roots[0].equals(roots[0]))
+    assert.strictEqual(indexer.version, 2)
+
+    const indexData = []
+    for await (const index of indexer) {
+      indexData.push(index)
+    }
+
+    assert.deepStrictEqual(indexData, goCarV2Index)
+  })
+
   it('bad argument', async () => {
     for (const arg of [true, false, null, undefined, 'string', 100, { obj: 'nope' }]) {
       // @ts-ignore