@ipld/car 4.1.5 → 5.0.0

package/esm/browser-test/test-reader.js

@@ -1,270 +0,0 @@
-import { CarReader } from '../lib/reader-browser.js';
-import { CarWriter } from '../lib/writer-browser.js';
-import {
-  bytesReader,
-  readHeader
-} from '../lib/decoder.js';
-import * as Block from 'multiformats/block';
-import { sha256 } from 'multiformats/hashes/sha2';
-import * as raw from 'multiformats/codecs/raw';
-import { base64 } from 'multiformats/bases/base64';
-import * as dagPb from '@ipld/dag-pb';
-import {
-  carBytes,
-  makeIterable,
-  assert,
-  goCarV2Bytes,
-  goCarV2Roots,
-  goCarV2Index,
-  goCarV2Contents
-} from './common.js';
-import {
-  verifyRoots,
-  verifyHas,
-  verifyGet,
-  verifyBlocks,
-  verifyCids
-} from './verify-store-reader.js';
-import { data as fixtures } from './fixtures.js';
-import { expectations as fixtureExpectations } from './fixtures-expectations.js';
-describe('CarReader fromBytes()', () => {
-  it('complete', async () => {
-    const reader = await CarReader.fromBytes(carBytes);
-    await verifyRoots(reader);
-    await verifyHas(reader);
-    await verifyGet(reader);
-    await verifyBlocks(reader.blocks());
-    await verifyCids(reader.cids());
-    assert.strictEqual(reader.version, 1);
-  });
-  it('complete (get before has) switch', async () => {
-    const reader = await CarReader.fromBytes(carBytes);
-    await verifyRoots(reader);
-    await verifyGet(reader);
-    await verifyHas(reader);
-    await verifyBlocks(reader.blocks());
-    await verifyCids(reader.cids());
-  });
-  it('bad argument', async () => {
-    for (const arg of [
-        true,
-        false,
-        null,
-        undefined,
-        'string',
-        100,
-        { obj: 'nope' }
-      ]) {
-      await assert.isRejected(CarReader.fromBytes(arg));
-    }
-  });
-  it('decode error - truncated', async () => {
-    await assert.isRejected(CarReader.fromBytes(carBytes.slice(0, carBytes.length - 10)), {
-      name: 'Error',
-      message: 'Unexpected end of data'
-    });
-  });
-  it('v2 complete', async () => {
-    const reader = await CarReader.fromBytes(goCarV2Bytes);
-    const roots = await reader.getRoots();
-    assert.strictEqual(roots.length, 1);
-    assert.ok(goCarV2Roots[0].equals(roots[0]));
-    assert.strictEqual(reader.version, 2);
-    for (const {cid} of goCarV2Index) {
-      const block = await reader.get(cid);
-      assert.isDefined(block);
-      if (block) {
-        assert.ok(cid.equals(block.cid));
-        let content;
-        if (cid.code === dagPb.code) {
-          content = dagPb.decode(block.bytes);
-        } else if (cid.code === 85) {
-          content = new TextDecoder().decode(block.bytes);
-        } else {
-          assert.fail('Unexpected codec');
-        }
-        assert.deepStrictEqual(content, goCarV2Contents[cid.toString()]);
-      }
-    }
-  });
-  it('decode error - trailing null bytes', async () => {
-    const bytes = new Uint8Array(carBytes.length + 5);
-    bytes.set(carBytes);
-    try {
-      await CarReader.fromBytes(bytes);
-    } catch (err) {
-      assert.strictEqual(err.message, 'Invalid CAR section (zero length)');
-      return;
-    }
-    assert.fail('Did not throw');
-  });
-  it('decode error - bad first byte', async () => {
-    const bytes = new Uint8Array(carBytes.length + 5);
-    bytes.set(carBytes);
-    bytes[0] = 0;
-    try {
-      await CarReader.fromBytes(bytes);
-    } catch (err) {
-      assert.strictEqual(err.message, 'Invalid CAR header (zero length)');
-      return;
-    }
-    assert.fail('Did not throw');
-  });
-});
-describe('CarReader fromIterable()', () => {
-  it('complete (single chunk)', async () => {
-    const reader = await CarReader.fromIterable(makeIterable(carBytes, carBytes.length));
-    await verifyRoots(reader);
-    await verifyHas(reader);
-    await verifyGet(reader);
-    await verifyBlocks(reader.blocks());
-    await verifyCids(reader.cids());
-  });
-  it('complete (101-byte chunks)', async () => {
-    const reader = await CarReader.fromIterable(makeIterable(carBytes, 101));
-    await verifyRoots(reader);
-    await verifyHas(reader);
-    await verifyGet(reader);
-    await verifyBlocks(reader.blocks());
-    await verifyCids(reader.cids());
-  });
-  it('complete (64-byte chunks)', async () => {
-    const reader = await CarReader.fromIterable(makeIterable(carBytes, 64));
-    await verifyRoots(reader);
-    await verifyHas(reader);
-    await verifyGet(reader);
-    await verifyBlocks(reader.blocks());
-    await verifyCids(reader.cids());
-  });
-  it('complete (32-byte chunks)', async () => {
-    const reader = await CarReader.fromIterable(makeIterable(carBytes, 32));
-    await verifyRoots(reader);
-    await verifyHas(reader);
-    await verifyGet(reader);
-    await verifyBlocks(reader.blocks());
-    await verifyCids(reader.cids());
-  });
-  it('handle zero-byte chunks', async () => {
-    const {writer, out} = await CarWriter.create([]);
-    const b1 = await Block.encode({
-      value: Uint8Array.from([
-        0,
-        1,
-        2
-      ]),
-      hasher: sha256,
-      codec: raw
-    });
-    writer.put(b1);
-    const b2 = await Block.encode({
-      value: Uint8Array.from([]),
-      hasher: sha256,
-      codec: raw
-    });
-    writer.put(b2);
-    const b3 = await Block.encode({
-      value: Uint8Array.from([
-        3,
-        4,
-        5
-      ]),
-      hasher: sha256,
-      codec: raw
-    });
-    writer.put(b3);
-    const closePromise = writer.close();
-    const reader = await CarReader.fromIterable(out);
-    const b1a = await reader.get(b1.cid);
-    assert.isDefined(b1a);
-    assert.deepStrictEqual(b1a && Array.from(b1a.bytes), [
-      0,
-      1,
-      2
-    ]);
-    const b2a = await reader.get(b2.cid);
-    assert.isDefined(b2a);
-    assert.deepStrictEqual(b2a && Array.from(b2a.bytes), []);
-    const b3a = await reader.get(b3.cid);
-    assert.isDefined(b3a);
-    assert.deepStrictEqual(b3a && Array.from(b3a.bytes), [
-      3,
-      4,
-      5
-    ]);
-    await closePromise;
-  });
-  it('bad argument', async () => {
-    for (const arg of [
-        new Uint8Array(0),
-        true,
-        false,
-        null,
-        undefined,
-        'string',
-        100,
-        { obj: 'nope' }
-      ]) {
-      await assert.isRejected(CarReader.fromIterable(arg));
-    }
-  });
-  it('decode error - truncated', async () => {
-    await assert.isRejected(CarReader.fromIterable(makeIterable(carBytes.slice(0, carBytes.length - 10), 64)), {
-      name: 'Error',
-      message: 'Unexpected end of data'
-    });
-  });
-  it('v2 decode error - truncated', async () => {
-    const bytes = goCarV2Bytes.slice();
-    const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    dv.setBigUint64(35, BigInt(448 - 10), true);
-    await assert.isRejected(CarReader.fromIterable(makeIterable(bytes, 64)), {
-      name: 'Error',
-      message: 'Unexpected end of data'
-    });
-  });
-});
-describe('Shared fixtures', () => {
-  describe('Header', () => {
-    for (const [name, {
-            version: expectedVersion,
-            err: expectedError
-          }] of Object.entries(fixtureExpectations)) {
-      it(name, async () => {
-        const data = base64.baseDecode(fixtures[name]);
-        let header;
-        try {
-          header = await readHeader(bytesReader(data));
-        } catch (err) {
-          if (expectedError != null) {
-            assert.equal(err.message, expectedError);
-            return;
-          }
-          assert.ifError(err);
-        }
-        if (expectedError != null) {
-          assert.fail(`Expected error: ${ expectedError }`);
-        }
-        assert.isDefined(header, 'did not decode header');
-        if (expectedVersion != null && header != null) {
-          assert.strictEqual(header.version, expectedVersion);
-        }
-      });
-    }
-  });
-  describe('Contents', () => {
-    for (const [name, {cids: expectedCids}] of Object.entries(fixtureExpectations)) {
-      if (expectedCids == null) {
-        continue;
-      }
-      it(name, async () => {
-        const data = base64.baseDecode(fixtures[name]);
-        const reader = await CarReader.fromBytes(data);
-        let i = 0;
-        for await (const cid of reader.cids()) {
-          assert.strictEqual(cid.toString(), expectedCids[i++]);
-        }
-        assert.strictEqual(i, expectedCids.length);
-      });
-    }
-  });
-});

package/esm/browser-test/test-writer.js

@@ -1,297 +0,0 @@
-import { CarWriter } from '../lib/writer-browser.js';
-import { CarReader } from '../lib/reader-browser.js';
-import {
-  bytes,
-  CID
-} from 'multiformats';
-import {
-  carBytes,
-  makeData,
-  assert,
-  rndCid
-} from './common.js';
-import {
-  verifyRoots,
-  verifyHas,
-  verifyGet,
-  verifyBlocks,
-  verifyCids
-} from './verify-store-reader.js';
-const {toHex} = bytes;
-function concatBytes(chunks) {
-  const length = chunks.reduce((p, c) => p + c.length, 0);
-  const bytes = new Uint8Array(length);
-  let off = 0;
-  for (const chunk of chunks) {
-    bytes.set(chunk, off);
-    off += chunk.length;
-  }
-  return bytes;
-}
-function collector(iterable) {
-  const chunks = [];
-  const cfn = (async () => {
-    for await (const chunk of iterable) {
-      chunks.push(chunk);
-    }
-    return concatBytes(chunks);
-  })();
-  return cfn;
-}
-const newRoots = [
-  CID.parse('bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq'),
-  CID.parse('bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4')
-];
-async function verifyUpdateRoots(bytes) {
-  const reader = await CarReader.fromBytes(bytes);
-  await assert.isRejected(verifyRoots(reader));
-  assert.deepEqual(await reader.getRoots(), newRoots);
-  await verifyHas(reader);
-  await verifyGet(reader);
-  await verifyBlocks(reader.blocks(), true);
-  await verifyCids(reader.cids(), true);
-}
-describe('CarWriter', () => {
-  let cborBlocks;
-  let allBlocks;
-  let allBlocksFlattened;
-  let roots;
-  function assertCarData(actual) {
-    assert.strictEqual(toHex(actual), toHex(carBytes), 'got expected bytes');
-  }
-  before(async () => {
-    const data = await makeData();
-    cborBlocks = data.cborBlocks;
-    allBlocks = data.allBlocks;
-    allBlocksFlattened = data.allBlocksFlattened;
-    roots = [
-      cborBlocks[0].cid,
-      cborBlocks[1].cid
-    ];
-  });
-  it('complete', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    assert.strictEqual(typeof out[Symbol.asyncIterator], 'function');
-    const collection = collector(out);
-    const writeQueue = [];
-    for (const block of allBlocksFlattened) {
-      writeQueue.push(writer.put(block));
-    }
-    writeQueue.push(writer.close());
-    let collected = false;
-    collection.then(bytes => {
-      collected = true;
-      assertCarData(bytes);
-    });
-    await Promise.all(writeQueue);
-    assert.strictEqual(collected, true);
-  });
-  it('complete, deferred collection', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    const writeQueue = [];
-    for (const block of allBlocksFlattened) {
-      writeQueue.push(writer.put(block));
-    }
-    writeQueue.push(writer.close());
-    let collected = false;
-    collector(out).then(bytes => {
-      collected = true;
-      assertCarData(bytes);
-    });
-    await Promise.all(writeQueue);
-    assert.strictEqual(collected, true);
-  });
-  it('complete, close after write', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    assert.strictEqual(typeof out[Symbol.asyncIterator], 'function');
-    const collection = collector(out);
-    const writeQueue = [];
-    for (const block of allBlocksFlattened) {
-      writeQueue.push(writer.put(block));
-    }
-    writeQueue.push(writer.close());
-    let written = false;
-    Promise.all(writeQueue).then(() => {
-      written = true;
-    });
-    const bytes = await collection;
-    assert.strictEqual(written, false);
-    await Promise.resolve();
-    assertCarData(bytes);
-  });
-  it('complete, no queue', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      await writer.put(block);
-    }
-    await writer.close();
-    const bytes = await collection;
-    assertCarData(bytes);
-  });
-  it('complete, slow drip', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    assert.strictEqual(typeof out[Symbol.asyncIterator], 'function');
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      writer.put(block);
-      await new Promise(resolve => setTimeout(resolve, 100));
-    }
-    await writer.close();
-    await new Promise(resolve => setTimeout(resolve, 100));
-    const bytes = await collection;
-    assertCarData(bytes);
-  });
-  it('complete, no queue, deferred collection', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    for (const block of allBlocksFlattened) {
-      writer.put(block);
-    }
-    writer.close();
-    const collection = collector(out);
-    const bytes = await collection;
-    assertCarData(bytes);
-  });
-  it('single root', async () => {
-    const {writer, out} = CarWriter.create(roots[0]);
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      await writer.put(block);
-    }
-    await writer.close();
-    const bytes = await collection;
-    const expectedRootDef = 'a265726f6f747381d82a58250001711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8b6776657273696f6e01';
-    const expectedStart = (expectedRootDef.length / 2).toString(16) + expectedRootDef + '28';
-    assert.strictEqual(toHex(bytes).substring(0, expectedStart.length), expectedStart);
-  });
-  it('no roots', async () => {
-    const {writer, out} = CarWriter.create();
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      await writer.put(block);
-    }
-    await writer.close();
-    const bytes = await collection;
-    const expectedRootDef = 'a265726f6f7473806776657273696f6e01';
-    const expectedStart = (expectedRootDef.length / 2).toString(16) + expectedRootDef + '28';
-    assert.strictEqual(toHex(bytes).substring(0, expectedStart.length), expectedStart);
-  });
-  it('appender', async () => {
-    let writerOut = CarWriter.create(roots);
-    let collection = collector(writerOut.out);
-    await writerOut.writer.close();
-    const headerBytes = await collection;
-    const append = async index => {
-      writerOut = CarWriter.createAppender();
-      collection = collector(writerOut.out);
-      for (const block of allBlocks[index][1]) {
-        await writerOut.writer.put(block);
-      }
-      await writerOut.writer.close();
-      return collection;
-    };
-    const rawBytes = await append(0);
-    const pbBytes = await append(1);
-    const cborBytes = await append(2);
-    assert.ok(rawBytes.length > 0);
-    assert.ok(pbBytes.length > 0);
-    assert.ok(cborBytes.length > 0);
-    const reassembled = concatBytes([
-      headerBytes,
-      rawBytes,
-      pbBytes,
-      cborBytes
-    ]);
-    assert.strictEqual(toHex(reassembled), toHex(carBytes));
-  });
-  it('bad argument for create()', () => {
-    for (const arg of [
-        new Uint8Array(0),
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      assert.throws(() => CarWriter.create(arg));
-    }
-  });
-  it('bad argument for put()', async () => {
-    const {writer} = CarWriter.create();
-    for (const arg of [
-        new Uint8Array(0),
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      await assert.isRejected(writer.put(arg));
-    }
-    for (const arg of [
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      await assert.isRejected(writer.put({
-        bytes: new Uint8Array(0),
-        cid: arg
-      }));
-    }
-    for (const arg of [
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      await assert.isRejected(writer.put({
-        cid: rndCid,
-        bytes: arg
-      }));
-    }
-  });
-  it('bad write after closed', async () => {
-    const {writer, out} = CarWriter.create();
-    const collection = collector(out);
-    await writer.put(allBlocksFlattened[0]);
-    await writer.close();
-    await assert.isRejected(writer.put(allBlocksFlattened[1]), /closed/);
-    await collection;
-  });
-  it('bad attempt to multiple iterate', async () => {
-    const {out} = CarWriter.create();
-    collector(out);
-    await assert.isRejected(collector(out), /multiple iterator/i);
-  });
-  it('bad attempt to multiple close', async () => {
-    const {writer, out} = CarWriter.create();
-    collector(out);
-    await writer.close();
-    await assert.isRejected(writer.close(), /closed/i);
-  });
-  it('update roots (fd)', async () => {
-    const bytes = carBytes.slice();
-    await CarWriter.updateRootsInBytes(bytes, newRoots);
-    await verifyUpdateRoots(bytes);
-  });
-  it('update roots error: wrong header size', async () => {
-    const bytes = carBytes.slice();
-    await assert.isRejected(CarWriter.updateRootsInBytes(bytes, [
-      ...newRoots,
-      newRoots[0]
-    ]), /can only overwrite a header of the same length/);
-    await assert.isRejected(CarWriter.updateRootsInBytes(bytes, [newRoots[0]]), /can only overwrite a header of the same length/);
-    await assert.isRejected(CarWriter.updateRootsInBytes(bytes, []), /can only overwrite a header of the same length/);
-  });
-});

package/esm/browser-test/verify-store-reader.js

@@ -1,122 +0,0 @@
-import { bytes } from 'multiformats';
-import * as raw from 'multiformats/codecs/raw';
-import {
-  toBlock,
-  assert,
-  makeData
-} from './common.js';
-function compareBlockData(actual, expected, id) {
-  assert.strictEqual(bytes.toHex(actual.bytes), bytes.toHex(expected.bytes), `comparing block as hex ${ id || '' }`);
-}
-function compareCids(actual, expected) {
-  assert.strictEqual(actual.toString(), expected.toString());
-}
-async function verifyRoots(reader) {
-  const {cborBlocks} = await makeData();
-  const expected = [
-    cborBlocks[0].cid.toString(),
-    cborBlocks[1].cid.toString()
-  ];
-  assert.deepStrictEqual((await reader.getRoots()).map(c => c.toString()), expected);
-}
-async function verifyHas(reader) {
-  const {allBlocks} = await makeData();
-  const verifyHas = async (cid, name) => {
-    assert.ok(await reader.has(cid), `reader doesn't have expected key for ${ name }`);
-  };
-  const verifyHasnt = async (cid, name) => {
-    assert.ok(!await reader.has(cid), `reader has unexpected key for ${ name }`);
-    assert.strictEqual(await reader.get(cid), undefined);
-  };
-  for (const [type, blocks] of allBlocks) {
-    for (let i = 0; i < blocks.length; i++) {
-      await verifyHas(blocks[i].cid, `block #${ i } (${ type } / ${ blocks[i].cid })`);
-    }
-  }
-  await verifyHasnt((await toBlock(new TextEncoder().encode('dddd'), raw)).cid, 'dddd');
-}
-async function verifyGet(reader) {
-  const {allBlocks} = await makeData();
-  const verifyBlock = async (expected, index, type) => {
-    let actual;
-    try {
-      actual = await reader.get(expected.cid);
-      assert.isDefined(actual);
-      if (actual) {
-        compareBlockData(actual, expected, `#${ index } (${ type })`);
-      }
-    } catch (err) {
-      assert.ifError(err, `get block length #${ index } (${ type })`);
-    }
-  };
-  for (const [type, blocks] of allBlocks) {
-    for (let i = 0; i < blocks.length; i++) {
-      await verifyBlock(blocks[i], i, type);
-    }
-  }
-}
-async function verifyBlocks(iterator, unordered) {
-  const {allBlocksFlattened} = await makeData();
-  if (!unordered) {
-    const expected = allBlocksFlattened.slice();
-    for await (const actual of iterator) {
-      const next = expected.shift();
-      assert.isDefined(next);
-      if (next) {
-        compareBlockData(actual, next);
-      }
-    }
-  } else {
-    const expected = {};
-    for (const block of allBlocksFlattened) {
-      expected[block.cid.toString()] = block;
-    }
-    for await (const actual of iterator) {
-      const {cid} = actual;
-      const exp = expected[cid.toString()];
-      if (!exp) {
-        throw new Error(`Unexpected block: ${ cid.toString() }`);
-      }
-      compareBlockData(actual, exp);
-      delete expected[cid.toString()];
-    }
-    if (Object.keys(expected).length) {
-      throw new Error('Did not find all expected blocks');
-    }
-  }
-}
-async function verifyCids(iterator, unordered) {
-  const {allBlocksFlattened} = await makeData();
-  if (!unordered) {
-    const expected = allBlocksFlattened.slice();
-    for await (const actual of iterator) {
-      const next = expected.shift();
-      assert.isDefined(next);
-      if (next) {
-        compareCids(actual, next.cid);
-      }
-    }
-  } else {
-    const expected = {};
-    for (const block of allBlocksFlattened) {
-      expected[block.cid.toString()] = block;
-    }
-    for await (const cid of iterator) {
-      const exp = expected[cid.toString()];
-      if (!exp) {
-        throw new Error(`Unexpected cid: ${ cid.toString() }`);
-      }
-      delete expected[cid.toString()];
-    }
-    if (Object.keys(expected).length) {
-      throw new Error('Did not find all expected cids');
-    }
-  }
-}
-export {
-  verifyRoots,
-  verifyHas,
-  verifyGet,
-  verifyBlocks,
-  verifyCids
-};

package/esm/car-browser.js

@@ -1,16 +0,0 @@
-import { CarReader } from './lib/reader-browser.js';
-import { CarIndexer } from './lib/indexer.js';
-import {
-  CarBlockIterator,
-  CarCIDIterator
-} from './lib/iterator.js';
-import { CarWriter } from './lib/writer-browser.js';
-import { CarIndexedReader } from './lib/indexed-reader-browser.js';
-export {
-  CarReader,
-  CarIndexer,
-  CarBlockIterator,
-  CarCIDIterator,
-  CarWriter,
-  CarIndexedReader
-};