@ipld/car 3.2.4 → 4.1.1

package/esm/browser-test/test-buffer-writer.js

@@ -0,0 +1,311 @@
+import * as CarBufferWriter from '../lib/buffer-writer.js';
+import { CarReader } from '../lib/reader-browser.js';
+import { createHeader } from '../lib/encoder.js';
+import { assert } from './common.js';
+import {
+  CID,
+  varint
+} from 'multiformats';
+import * as CBOR from '@ipld/dag-cbor';
+import {
+  sha256,
+  sha512
+} from 'multiformats/hashes/sha2';
+import { identity } from 'multiformats/hashes/identity';
+import * as Raw from 'multiformats/codecs/raw';
+import * as Block from 'multiformats/block';
+describe('CarBufferWriter', () => {
+  const cid = CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu');
+  describe('calculateHeaderLength', async () => {
+    for (const count of [
+        0,
+        1,
+        10,
+        18,
+        24,
+        48,
+        124,
+        255,
+        258,
+        65536 - 1,
+        65536
+      ]) {
+      it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
+        const roots = new Array(count).fill(cid);
+        const sizes = new Array(count).fill(cid.bytes.byteLength);
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(sizes), createHeader(roots).byteLength);
+      });
+      it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
+        const roots = new Array(count).fill(cid);
+        const rootLengths = roots.map(c => c.bytes.byteLength);
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(rootLengths), createHeader(roots).byteLength);
+      });
+    }
+    it('estimate on large CIDs', () => {
+      const largeCID = CID.parse(`bafkqbbac${ 'a'.repeat(416) }`);
+      assert.equal(CarBufferWriter.calculateHeaderLength([
+        cid.bytes.byteLength,
+        largeCID.bytes.byteLength
+      ]), createHeader([
+        cid,
+        largeCID
+      ]).byteLength);
+    });
+    it('estimate on large CIDs 2', () => {
+      const largeCID = CID.createV1(Raw.code, identity.digest(new Uint8Array(512).fill(1)));
+      assert.equal(CarBufferWriter.calculateHeaderLength([
+        cid.bytes.byteLength,
+        largeCID.bytes.byteLength
+      ]), createHeader([
+        cid,
+        largeCID
+      ]).byteLength);
+    });
+  });
+  describe('writer', () => {
+    it('estimate header and write blocks', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(1);
+      const dataSize = 256;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b2);
+      writer.addRoot(b1.cid);
+      const bytes = writer.close();
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [b1.cid]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+    it('overestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2);
+      const dataSize = 256;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b2);
+      writer.addRoot(b1.cid);
+      assert.throws(() => writer.close(), /Header size was overestimate/);
+      const bytes = writer.close({ resize: true });
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [b1.cid]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+    it('underestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2);
+      const dataSize = 300;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      writer.addRoot(b1.cid);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha512
+      });
+      writer.write(b2);
+      assert.throws(() => writer.addRoot(b2.cid), /has no capacity/);
+      writer.addRoot(b2.cid, { resize: true });
+      const bytes = writer.close();
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [
+        b1.cid,
+        b2.cid
+      ]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+  });
+  it('has no space for the root', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1);
+    const dataSize = 100;
+    const buffer = new ArrayBuffer(headerSize + dataSize);
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b1);
+    writer.addRoot(b1.cid);
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b2);
+    assert.throws(() => writer.addRoot(b2.cid), /Buffer has no capacity for a new root/);
+    assert.throws(() => writer.addRoot(b2.cid, { resize: true }), /Buffer has no capacity for a new root/);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [b1.cid]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+  it('has no space for the block', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1);
+    const dataSize = 58;
+    const buffer = new ArrayBuffer(headerSize + dataSize);
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b1);
+    writer.addRoot(b1.cid);
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    assert.throws(() => writer.write(b2), /Buffer has no capacity for this block/);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [b1.cid]);
+    assert.deepEqual(reader._blocks, [{
+        cid: b1.cid,
+        bytes: b1.bytes
+      }]);
+  });
+  it('provide roots', async () => {
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha512
+    });
+    const buffer = new ArrayBuffer(300);
+    const writer = CarBufferWriter.createWriter(buffer, {
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    });
+    writer.write(b1);
+    writer.write(b2);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [
+      b1.cid,
+      b2.cid
+    ]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+  it('provide large CID root', async () => {
+    const bytes = new Uint8Array(512).fill(1);
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    const b2 = {
+      cid: CID.createV1(Raw.code, identity.digest(bytes)),
+      bytes
+    };
+    const headerSize = CBOR.encode({
+      version: 1,
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    }).byteLength;
+    const bodySize = CarBufferWriter.blockLength(b1) + CarBufferWriter.blockLength(b2);
+    const varintSize = varint.encodingLength(headerSize);
+    const writer = CarBufferWriter.createWriter(new ArrayBuffer(varintSize + headerSize + bodySize), {
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    });
+    writer.write(b1);
+    writer.write(b2);
+    const car = writer.close();
+    const reader = await CarReader.fromBytes(car);
+    assert.deepEqual(await reader.getRoots(), [
+      b1.cid,
+      b2.cid
+    ]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+});

package/esm/browser-test/test-errors.js

@@ -4,7 +4,8 @@ import { encode as vEncode } from 'varint';
 import { CarReader } from '../lib/reader-browser.js';
 import {
   carBytes,
-  assert
+  assert,
+  goCarV2Bytes
 } from './common.js';
 function makeHeader(block) {
   const u = cbEncode(block);
@@ -26,42 +27,65 @@ describe('Misc errors', () => {
     });
   });
   it('bad version', async () => {
-    const buf2 = bytes.fromHex('0aa16776657273696f6e02');
-    assert.strictEqual(bytes.toHex(makeHeader({ version: 2 })), '0aa16776657273696f6e02');
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2');
+    const buf2 = bytes.fromHex('0aa16776657273696f6e03');
+    assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03');
+    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3');
   });
-  it('bad header', async () => {
-    let buf2 = makeHeader({
-      version: 1,
-      roots: []
+  describe('bad header', async () => {
+    it('sanity check', async () => {
+      const buf2 = makeHeader({
+        version: 1,
+        roots: []
+      });
+      await assert.isFulfilled(CarReader.fromBytes(buf2));
     });
-    await assert.isFulfilled(CarReader.fromBytes(buf2));
-    buf2 = makeHeader({ roots: [] });
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: undefined');
-    buf2 = makeHeader({
-      version: '1',
-      roots: []
+    it('no \'version\' array', async () => {
+      const buf2 = makeHeader({ roots: [] });
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
     });
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: "1"');
-    buf2 = makeHeader({ version: 1 });
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
-    buf2 = makeHeader({
-      version: 1,
-      roots: {}
+    it('bad \'version\' type', async () => {
+      const buf2 = makeHeader({
+        version: '1',
+        roots: []
+      });
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
     });
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
-    buf2 = makeHeader({
-      version: 1,
-      roots: [],
-      blip: true
+    it('no \'roots\' array', async () => {
+      const buf2 = makeHeader({ version: 1 });
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
+    });
+    it('bad \'roots\' type', async () => {
+      const buf2 = makeHeader({
+        version: 1,
+        roots: {}
+      });
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
+    });
+    it('extraneous properties', async () => {
+      const buf2 = makeHeader({
+        version: 1,
+        roots: [],
+        blip: true
+      });
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
+    });
+    it('not an object', async () => {
+      const buf2 = makeHeader([
+        1,
+        []
+      ]);
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
+    });
+    it('not an object', async () => {
+      const buf2 = makeHeader(null);
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
+    });
+    it('recursive v2 header', async () => {
+      const v2Header = goCarV2Bytes.slice(0, 51);
+      const buf2 = new Uint8Array(51 * 2);
+      buf2.set(v2Header, 0);
+      buf2.set(v2Header, 51);
+      await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)');
     });
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
-    buf2 = makeHeader([
-      1,
-      []
-    ]);
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
-    buf2 = makeHeader(null);
-    await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
   });
 });

package/esm/browser-test/test-indexer.js

@@ -2,6 +2,9 @@ import { CarIndexer } from '../lib/indexer.js';
 import {
   goCarBytes,
   goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
   makeIterable,
   assert
 } from './common.js';
@@ -17,6 +20,18 @@ describe('CarIndexer fromBytes()', () => {
     }
     assert.deepStrictEqual(indexData, goCarIndex);
   });
+  it('v2 complete', async () => {
+    const indexer = await CarIndexer.fromBytes(goCarV2Bytes);
+    const roots = await indexer.getRoots();
+    assert.strictEqual(roots.length, 1);
+    assert.ok(goCarV2Roots[0].equals(roots[0]));
+    assert.strictEqual(indexer.version, 2);
+    const indexData = [];
+    for await (const index of indexer) {
+      indexData.push(index);
+    }
+    assert.deepStrictEqual(indexData, goCarV2Index);
+  });
   it('bad argument', async () => {
     for (const arg of [
         true,

package/esm/browser-test/test-reader.js

@@ -1,12 +1,22 @@
 import { CarReader } from '../lib/reader-browser.js';
 import { CarWriter } from '../lib/writer-browser.js';
+import {
+  bytesReader,
+  readHeader
+} from '../lib/decoder.js';
 import * as Block from 'multiformats/block';
 import { sha256 } from 'multiformats/hashes/sha2';
 import * as raw from 'multiformats/codecs/raw';
+import { base64 } from 'multiformats/bases/base64';
+import * as dagPb from '@ipld/dag-pb';
 import {
   carBytes,
   makeIterable,
-  assert
+  assert,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  goCarV2Contents
 } from './common.js';
 import {
   verifyRoots,
@@ -15,6 +25,8 @@ import {
   verifyBlocks,
   verifyCids
 } from './verify-store-reader.js';
+import { data as fixtures } from './fixtures.js';
+import { expectations as fixtureExpectations } from './fixtures-expectations.js';
 describe('CarReader fromBytes()', () => {
   it('complete', async () => {
     const reader = await CarReader.fromBytes(carBytes);
@@ -52,6 +64,29 @@ describe('CarReader fromBytes()', () => {
       message: 'Unexpected end of data'
     });
   });
+  it('v2 complete', async () => {
+    const reader = await CarReader.fromBytes(goCarV2Bytes);
+    const roots = await reader.getRoots();
+    assert.strictEqual(roots.length, 1);
+    assert.ok(goCarV2Roots[0].equals(roots[0]));
+    assert.strictEqual(reader.version, 2);
+    for (const {cid} of goCarV2Index) {
+      const block = await reader.get(cid);
+      assert.isDefined(block);
+      if (block) {
+        assert.ok(cid.equals(block.cid));
+        let content;
+        if (cid.code === dagPb.code) {
+          content = dagPb.decode(block.bytes);
+        } else if (cid.code === 85) {
+          content = new TextDecoder().decode(block.bytes);
+        } else {
+          assert.fail('Unexpected codec');
+        }
+        assert.deepStrictEqual(content, goCarV2Contents[cid.toString()]);
+      }
+    }
+  });
   it('decode error - trailing null bytes', async () => {
     const bytes = new Uint8Array(carBytes.length + 5);
     bytes.set(carBytes);
@@ -178,4 +213,58 @@ describe('CarReader fromIterable()', () => {
       message: 'Unexpected end of data'
     });
   });
+  it('v2 decode error - truncated', async () => {
+    const bytes = goCarV2Bytes.slice();
+    const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    dv.setBigUint64(35, BigInt(448 - 10), true);
+    await assert.isRejected(CarReader.fromIterable(makeIterable(bytes, 64)), {
+      name: 'Error',
+      message: 'Unexpected end of data'
+    });
+  });
+});
+describe('Shared fixtures', () => {
+  describe('Header', () => {
+    for (const [name, {
+            version: expectedVersion,
+            err: expectedError
+          }] of Object.entries(fixtureExpectations)) {
+      it(name, async () => {
+        const data = base64.baseDecode(fixtures[name]);
+        let header;
+        try {
+          header = await readHeader(bytesReader(data));
+        } catch (err) {
+          if (expectedError != null) {
+            assert.equal(err.message, expectedError);
+            return;
+          }
+          assert.ifError(err);
+        }
+        if (expectedError != null) {
+          assert.fail(`Expected error: ${ expectedError }`);
+        }
+        assert.isDefined(header, 'did not decode header');
+        if (expectedVersion != null && header != null) {
+          assert.strictEqual(header.version, expectedVersion);
+        }
+      });
+    }
+  });
+  describe('Contents', () => {
+    for (const [name, {cids: expectedCids}] of Object.entries(fixtureExpectations)) {
+      if (expectedCids == null) {
+        continue;
+      }
+      it(name, async () => {
+        const data = base64.baseDecode(fixtures[name]);
+        const reader = await CarReader.fromBytes(data);
+        let i = 0;
+        for await (const cid of reader.cids()) {
+          assert.strictEqual(cid.toString(), expectedCids[i++]);
+        }
+        assert.strictEqual(i, expectedCids.length);
+      });
+    }
+  });
 });

package/esm/browser-test/test-writer.js

@@ -193,9 +193,9 @@ describe('CarWriter', () => {
     const rawBytes = await append(0);
     const pbBytes = await append(1);
     const cborBytes = await append(2);
-    assert(rawBytes.length > 0);
-    assert(pbBytes.length > 0);
-    assert(cborBytes.length > 0);
+    assert.ok(rawBytes.length > 0);
+    assert.ok(pbBytes.length > 0);
+    assert.ok(cborBytes.length > 0);
     const reassembled = concatBytes([
       headerBytes,
       rawBytes,