@ipld/car 3.2.3 → 4.1.0

package/esm/lib/decoder.js

@@ -2,18 +2,40 @@ import varint from 'varint';
 import { CID } from 'multiformats/cid';
 import * as Digest from 'multiformats/hashes/digest';
 import { decode as decodeDagCbor } from '@ipld/dag-cbor';
+import { CarHeader as headerValidator } from './header-validator.js';
 const CIDV0_BYTES = {
   SHA2_256: 18,
   LENGTH: 32,
   DAG_PB: 112
 };
+const V2_HEADER_LENGTH = 16 + 8 + 8 + 8;
 async function readVarint(reader) {
   const bytes = await reader.upTo(8);
+  if (!bytes.length) {
+    throw new Error('Unexpected end of data');
+  }
   const i = varint.decode(bytes);
   reader.seek(varint.decode.bytes);
   return i;
 }
-export async function readHeader(reader) {
+async function readV2Header(reader) {
+  const bytes = await reader.exactly(V2_HEADER_LENGTH);
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  let offset = 0;
+  const header = {
+    version: 2,
+    characteristics: [
+      dv.getBigUint64(offset, true),
+      dv.getBigUint64(offset += 8, true)
+    ],
+    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
+    dataSize: Number(dv.getBigUint64(offset += 8, true)),
+    indexOffset: Number(dv.getBigUint64(offset += 8, true))
+  };
+  reader.seek(V2_HEADER_LENGTH);
+  return header;
+}
+export async function readHeader(reader, strictVersion) {
   const length = await readVarint(reader);
   if (length === 0) {
     throw new Error('Invalid CAR header (zero length)');
@@ -21,22 +43,23 @@ export async function readHeader(reader) {
   const header = await reader.exactly(length);
   reader.seek(length);
   const block = decodeDagCbor(header);
-  if (block == null || Array.isArray(block) || typeof block !== 'object') {
+  if (!headerValidator(block)) {
     throw new Error('Invalid CAR header format');
   }
-  if (block.version !== 1) {
-    if (typeof block.version === 'string') {
-      throw new Error(`Invalid CAR version: "${ block.version }"`);
-    }
-    throw new Error(`Invalid CAR version: ${ block.version }`);
+  if (block.version !== 1 && block.version !== 2 || strictVersion !== undefined && block.version !== strictVersion) {
+    throw new Error(`Invalid CAR version: ${ block.version }${ strictVersion !== undefined ? ` (expected ${ strictVersion })` : '' }`);
   }
-  if (!Array.isArray(block.roots)) {
+  const hasRoots = Array.isArray(block.roots);
+  if (block.version === 1 && !hasRoots || block.version === 2 && hasRoots) {
     throw new Error('Invalid CAR header format');
   }
-  if (Object.keys(block).filter(p => p !== 'roots' && p !== 'version').length) {
-    throw new Error('Invalid CAR header format');
+  if (block.version === 1) {
+    return block;
   }
-  return block;
+  const v2Header = await readV2Header(reader);
+  reader.seek(v2Header.dataOffset - reader.pos);
+  const v1Header = await readHeader(reader, 1);
+  return Object.assign(v1Header, v2Header);
 }
 async function readMultihash(reader) {
   const bytes = await reader.upTo(8);
@@ -74,7 +97,7 @@ export async function readBlockHead(reader) {
   }
   length += reader.pos - start;
   const cid = await readCid(reader);
-  const blockLength = length - (reader.pos - start);
+  const blockLength = length - Number(reader.pos - start);
   return {
     cid,
     length,
@@ -104,7 +127,14 @@ async function readBlockIndex(reader) {
   return index;
 }
 export function createDecoder(reader) {
-  const headerPromise = readHeader(reader);
+  const headerPromise = (async () => {
+    const header = await readHeader(reader);
+    if (header.version === 2) {
+      const v1length = reader.pos - header.dataOffset;
+      reader = limitReader(reader, header.dataSize - v1length);
+    }
+    return header;
+  })();
   return {
     header: () => headerPromise,
     async *blocks() {
@@ -206,4 +236,30 @@ export function asyncIterableReader(asyncIterable) {
     return next.value;
   }
   return chunkReader(readChunk);
+}
+export function limitReader(reader, byteLimit) {
+  let bytesRead = 0;
+  return {
+    async upTo(length) {
+      let bytes = await reader.upTo(length);
+      if (bytes.length + bytesRead > byteLimit) {
+        bytes = bytes.subarray(0, byteLimit - bytesRead);
+      }
+      return bytes;
+    },
+    async exactly(length) {
+      const bytes = await reader.exactly(length);
+      if (bytes.length + bytesRead > byteLimit) {
+        throw new Error('Unexpected end of data');
+      }
+      return bytes;
+    },
+    seek(length) {
+      bytesRead += length;
+      reader.seek(length);
+    },
+    get pos() {
+      return reader.pos;
+    }
+  };
 }

package/esm/lib/header-validator.js

@@ -0,0 +1,23 @@
+const Kinds = {
+  Null: obj => obj === null,
+  Int: obj => Number.isInteger(obj),
+  Float: obj => typeof obj === 'number' && Number.isFinite(obj),
+  String: obj => typeof obj === 'string',
+  Bool: obj => typeof obj === 'boolean',
+  Bytes: obj => obj instanceof Uint8Array,
+  Link: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
+  List: obj => Array.isArray(obj),
+  Map: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
+};
+const Types = {
+  Int: Kinds.Int,
+  'CarHeader > version': obj => Types.Int(obj),
+  'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
+  'CarHeader > roots (anon)': obj => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
+  'CarHeader > roots': obj => Types['CarHeader > roots (anon)'](obj),
+  CarHeader: obj => {
+    const keys = obj && Object.keys(obj);
+    return Kinds.Map(obj) && ['version'].every(k => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value));
+  }
+};
+export const CarHeader = Types.CarHeader;

package/esm/lib/reader-browser.js

@@ -4,17 +4,16 @@ import {
   createDecoder
 } from './decoder.js';
 export class CarReader {
-  constructor(version, roots, blocks) {
-    this._version = version;
-    this._roots = roots;
+  constructor(header, blocks) {
+    this._header = header;
     this._blocks = blocks;
     this._keys = blocks.map(b => b.cid.toString());
   }
   get version() {
-    return this._version;
+    return this._header.version;
   }
   async getRoots() {
-    return this._roots;
+    return this._header.roots;
   }
   async has(key) {
     return this._keys.indexOf(key.toString()) > -1;
@@ -46,13 +45,13 @@ export class CarReader {
     return decodeReaderComplete(asyncIterableReader(asyncIterable));
   }
 }
-async function decodeReaderComplete(reader) {
+export async function decodeReaderComplete(reader) {
   const decoder = createDecoder(reader);
-  const {version, roots} = await decoder.header();
+  const header = await decoder.header();
   const blocks = [];
   for await (const block of decoder.blocks()) {
     blocks.push(block);
   }
-  return new CarReader(version, roots, blocks);
+  return new CarReader(header, blocks);
 }
 export const __browser = true;

package/esm/lib/writer-browser.js

@@ -63,7 +63,7 @@ export class CarWriter {
     const reader = bytesReader(bytes);
     await readHeader(reader);
     const newHeader = createHeader(roots);
-    if (reader.pos !== newHeader.length) {
+    if (Number(reader.pos) !== newHeader.length) {
       throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${ reader.pos } bytes, new header is ${ newHeader.length } bytes)`);
     }
     bytes.set(newHeader, 0);

package/esm/node-test/common.js

@@ -186,6 +186,77 @@ const goCarIndex = [
     blockLength: 18
   }
 ];
+const goCarV2Bytes = bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000');
+const goCarV2Roots = [CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')];
+const goCarV2Index = [
+  {
+    blockLength: 47,
+    blockOffset: 143,
+    cid: CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'),
+    length: 82,
+    offset: 108
+  },
+  {
+    blockLength: 99,
+    blockOffset: 226,
+    cid: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
+    length: 135,
+    offset: 190
+  },
+  {
+    blockLength: 54,
+    blockOffset: 360,
+    cid: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
+    length: 89,
+    offset: 325
+  },
+  {
+    blockLength: 4,
+    blockOffset: 451,
+    cid: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
+    length: 41,
+    offset: 414
+  },
+  {
+    blockLength: 7,
+    blockOffset: 492,
+    cid: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
+    length: 44,
+    offset: 455
+  }
+];
+const goCarV2Contents = {
+  QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
+    Links: [{
+        Hash: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
+        Name: '\uD83C\uDF64',
+        Tsize: 164
+      }]
+  },
+  QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
+    Links: [
+      {
+        Hash: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
+        Name: 'barreleye',
+        Tsize: 58
+      },
+      {
+        Hash: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
+        Name: '\uD83D\uDC21',
+        Tsize: 7
+      }
+    ]
+  },
+  Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
+    Links: [{
+        Hash: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
+        Name: 'fishmonger',
+        Tsize: 4
+      }]
+  },
+  bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
+  bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
+};
 export {
   toBlock,
   assert,
@@ -195,5 +266,9 @@ export {
   carBytes,
   goCarBytes,
   goCarRoots,
-  goCarIndex
+  goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  goCarV2Contents
 };

package/esm/node-test/test-buffer-writer.js

@@ -0,0 +1,311 @@
+import * as CarBufferWriter from '../lib/buffer-writer.js';
+import { CarReader } from '../lib/reader.js';
+import { createHeader } from '../lib/encoder.js';
+import { assert } from './common.js';
+import {
+  CID,
+  varint
+} from 'multiformats';
+import * as CBOR from '@ipld/dag-cbor';
+import {
+  sha256,
+  sha512
+} from 'multiformats/hashes/sha2';
+import { identity } from 'multiformats/hashes/identity';
+import * as Raw from 'multiformats/codecs/raw';
+import * as Block from 'multiformats/block';
+describe('CarBufferWriter', () => {
+  const cid = CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu');
+  describe('calculateHeaderLength', async () => {
+    for (const count of [
+        0,
+        1,
+        10,
+        18,
+        24,
+        48,
+        124,
+        255,
+        258,
+        65536 - 1,
+        65536
+      ]) {
+      it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
+        const roots = new Array(count).fill(cid);
+        const sizes = new Array(count).fill(cid.bytes.byteLength);
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(sizes), createHeader(roots).byteLength);
+      });
+      it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
+        const roots = new Array(count).fill(cid);
+        const rootLengths = roots.map(c => c.bytes.byteLength);
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(rootLengths), createHeader(roots).byteLength);
+      });
+    }
+    it('estimate on large CIDs', () => {
+      const largeCID = CID.parse(`bafkqbbac${ 'a'.repeat(416) }`);
+      assert.equal(CarBufferWriter.calculateHeaderLength([
+        cid.bytes.byteLength,
+        largeCID.bytes.byteLength
+      ]), createHeader([
+        cid,
+        largeCID
+      ]).byteLength);
+    });
+    it('estimate on large CIDs 2', () => {
+      const largeCID = CID.createV1(Raw.code, identity.digest(new Uint8Array(512).fill(1)));
+      assert.equal(CarBufferWriter.calculateHeaderLength([
+        cid.bytes.byteLength,
+        largeCID.bytes.byteLength
+      ]), createHeader([
+        cid,
+        largeCID
+      ]).byteLength);
+    });
+  });
+  describe('writer', () => {
+    it('estimate header and write blocks', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(1);
+      const dataSize = 256;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b2);
+      writer.addRoot(b1.cid);
+      const bytes = writer.close();
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [b1.cid]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+    it('overestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2);
+      const dataSize = 256;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b2);
+      writer.addRoot(b1.cid);
+      assert.throws(() => writer.close(), /Header size was overestimate/);
+      const bytes = writer.close({ resize: true });
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [b1.cid]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+    it('underestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2);
+      const dataSize = 300;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      writer.addRoot(b1.cid);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha512
+      });
+      writer.write(b2);
+      assert.throws(() => writer.addRoot(b2.cid), /has no capacity/);
+      writer.addRoot(b2.cid, { resize: true });
+      const bytes = writer.close();
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [
+        b1.cid,
+        b2.cid
+      ]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+  });
+  it('has no space for the root', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1);
+    const dataSize = 100;
+    const buffer = new ArrayBuffer(headerSize + dataSize);
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b1);
+    writer.addRoot(b1.cid);
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b2);
+    assert.throws(() => writer.addRoot(b2.cid), /Buffer has no capacity for a new root/);
+    assert.throws(() => writer.addRoot(b2.cid, { resize: true }), /Buffer has no capacity for a new root/);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [b1.cid]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+  it('has no space for the block', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1);
+    const dataSize = 58;
+    const buffer = new ArrayBuffer(headerSize + dataSize);
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b1);
+    writer.addRoot(b1.cid);
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    assert.throws(() => writer.write(b2), /Buffer has no capacity for this block/);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [b1.cid]);
+    assert.deepEqual(reader._blocks, [{
+        cid: b1.cid,
+        bytes: b1.bytes
+      }]);
+  });
+  it('provide roots', async () => {
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha512
+    });
+    const buffer = new ArrayBuffer(300);
+    const writer = CarBufferWriter.createWriter(buffer, {
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    });
+    writer.write(b1);
+    writer.write(b2);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [
+      b1.cid,
+      b2.cid
+    ]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+  it('provide large CID root', async () => {
+    const bytes = new Uint8Array(512).fill(1);
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    const b2 = {
+      cid: CID.createV1(Raw.code, identity.digest(bytes)),
+      bytes
+    };
+    const headerSize = CBOR.encode({
+      version: 1,
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    }).byteLength;
+    const bodySize = CarBufferWriter.blockLength(b1) + CarBufferWriter.blockLength(b2);
+    const varintSize = varint.encodingLength(headerSize);
+    const writer = CarBufferWriter.createWriter(new ArrayBuffer(varintSize + headerSize + bodySize), {
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    });
+    writer.write(b1);
+    writer.write(b2);
+    const car = writer.close();
+    const reader = await CarReader.fromBytes(car);
+    assert.deepEqual(await reader.getRoots(), [
+      b1.cid,
+      b2.cid
+    ]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+});