@ipld/car 4.1.6 → 5.0.1

package/esm/node-test/test-writer.js

@@ -1,297 +0,0 @@
-import { CarWriter } from '../lib/writer.js';
-import { CarReader } from '../lib/reader.js';
-import {
-  bytes,
-  CID
-} from 'multiformats';
-import {
-  carBytes,
-  makeData,
-  assert,
-  rndCid
-} from './common.js';
-import {
-  verifyRoots,
-  verifyHas,
-  verifyGet,
-  verifyBlocks,
-  verifyCids
-} from './verify-store-reader.js';
-const {toHex} = bytes;
-function concatBytes(chunks) {
-  const length = chunks.reduce((p, c) => p + c.length, 0);
-  const bytes = new Uint8Array(length);
-  let off = 0;
-  for (const chunk of chunks) {
-    bytes.set(chunk, off);
-    off += chunk.length;
-  }
-  return bytes;
-}
-function collector(iterable) {
-  const chunks = [];
-  const cfn = (async () => {
-    for await (const chunk of iterable) {
-      chunks.push(chunk);
-    }
-    return concatBytes(chunks);
-  })();
-  return cfn;
-}
-const newRoots = [
-  CID.parse('bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq'),
-  CID.parse('bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4')
-];
-async function verifyUpdateRoots(bytes) {
-  const reader = await CarReader.fromBytes(bytes);
-  await assert.isRejected(verifyRoots(reader));
-  assert.deepEqual(await reader.getRoots(), newRoots);
-  await verifyHas(reader);
-  await verifyGet(reader);
-  await verifyBlocks(reader.blocks(), true);
-  await verifyCids(reader.cids(), true);
-}
-describe('CarWriter', () => {
-  let cborBlocks;
-  let allBlocks;
-  let allBlocksFlattened;
-  let roots;
-  function assertCarData(actual) {
-    assert.strictEqual(toHex(actual), toHex(carBytes), 'got expected bytes');
-  }
-  before(async () => {
-    const data = await makeData();
-    cborBlocks = data.cborBlocks;
-    allBlocks = data.allBlocks;
-    allBlocksFlattened = data.allBlocksFlattened;
-    roots = [
-      cborBlocks[0].cid,
-      cborBlocks[1].cid
-    ];
-  });
-  it('complete', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    assert.strictEqual(typeof out[Symbol.asyncIterator], 'function');
-    const collection = collector(out);
-    const writeQueue = [];
-    for (const block of allBlocksFlattened) {
-      writeQueue.push(writer.put(block));
-    }
-    writeQueue.push(writer.close());
-    let collected = false;
-    collection.then(bytes => {
-      collected = true;
-      assertCarData(bytes);
-    });
-    await Promise.all(writeQueue);
-    assert.strictEqual(collected, true);
-  });
-  it('complete, deferred collection', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    const writeQueue = [];
-    for (const block of allBlocksFlattened) {
-      writeQueue.push(writer.put(block));
-    }
-    writeQueue.push(writer.close());
-    let collected = false;
-    collector(out).then(bytes => {
-      collected = true;
-      assertCarData(bytes);
-    });
-    await Promise.all(writeQueue);
-    assert.strictEqual(collected, true);
-  });
-  it('complete, close after write', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    assert.strictEqual(typeof out[Symbol.asyncIterator], 'function');
-    const collection = collector(out);
-    const writeQueue = [];
-    for (const block of allBlocksFlattened) {
-      writeQueue.push(writer.put(block));
-    }
-    writeQueue.push(writer.close());
-    let written = false;
-    Promise.all(writeQueue).then(() => {
-      written = true;
-    });
-    const bytes = await collection;
-    assert.strictEqual(written, false);
-    await Promise.resolve();
-    assertCarData(bytes);
-  });
-  it('complete, no queue', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      await writer.put(block);
-    }
-    await writer.close();
-    const bytes = await collection;
-    assertCarData(bytes);
-  });
-  it('complete, slow drip', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    assert.strictEqual(typeof out[Symbol.asyncIterator], 'function');
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      writer.put(block);
-      await new Promise(resolve => setTimeout(resolve, 100));
-    }
-    await writer.close();
-    await new Promise(resolve => setTimeout(resolve, 100));
-    const bytes = await collection;
-    assertCarData(bytes);
-  });
-  it('complete, no queue, deferred collection', async () => {
-    const {writer, out} = CarWriter.create(roots);
-    for (const block of allBlocksFlattened) {
-      writer.put(block);
-    }
-    writer.close();
-    const collection = collector(out);
-    const bytes = await collection;
-    assertCarData(bytes);
-  });
-  it('single root', async () => {
-    const {writer, out} = CarWriter.create(roots[0]);
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      await writer.put(block);
-    }
-    await writer.close();
-    const bytes = await collection;
-    const expectedRootDef = 'a265726f6f747381d82a58250001711220f88bc853804cf294fe417e4fa83028689fcdb1b1592c5102e1474dbc200fab8b6776657273696f6e01';
-    const expectedStart = (expectedRootDef.length / 2).toString(16) + expectedRootDef + '28';
-    assert.strictEqual(toHex(bytes).substring(0, expectedStart.length), expectedStart);
-  });
-  it('no roots', async () => {
-    const {writer, out} = CarWriter.create();
-    const collection = collector(out);
-    for (const block of allBlocksFlattened) {
-      await writer.put(block);
-    }
-    await writer.close();
-    const bytes = await collection;
-    const expectedRootDef = 'a265726f6f7473806776657273696f6e01';
-    const expectedStart = (expectedRootDef.length / 2).toString(16) + expectedRootDef + '28';
-    assert.strictEqual(toHex(bytes).substring(0, expectedStart.length), expectedStart);
-  });
-  it('appender', async () => {
-    let writerOut = CarWriter.create(roots);
-    let collection = collector(writerOut.out);
-    await writerOut.writer.close();
-    const headerBytes = await collection;
-    const append = async index => {
-      writerOut = CarWriter.createAppender();
-      collection = collector(writerOut.out);
-      for (const block of allBlocks[index][1]) {
-        await writerOut.writer.put(block);
-      }
-      await writerOut.writer.close();
-      return collection;
-    };
-    const rawBytes = await append(0);
-    const pbBytes = await append(1);
-    const cborBytes = await append(2);
-    assert.ok(rawBytes.length > 0);
-    assert.ok(pbBytes.length > 0);
-    assert.ok(cborBytes.length > 0);
-    const reassembled = concatBytes([
-      headerBytes,
-      rawBytes,
-      pbBytes,
-      cborBytes
-    ]);
-    assert.strictEqual(toHex(reassembled), toHex(carBytes));
-  });
-  it('bad argument for create()', () => {
-    for (const arg of [
-        new Uint8Array(0),
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      assert.throws(() => CarWriter.create(arg));
-    }
-  });
-  it('bad argument for put()', async () => {
-    const {writer} = CarWriter.create();
-    for (const arg of [
-        new Uint8Array(0),
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      await assert.isRejected(writer.put(arg));
-    }
-    for (const arg of [
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      await assert.isRejected(writer.put({
-        bytes: new Uint8Array(0),
-        cid: arg
-      }));
-    }
-    for (const arg of [
-        true,
-        false,
-        null,
-        'string',
-        100,
-        { obj: 'nope' },
-        [false]
-      ]) {
-      await assert.isRejected(writer.put({
-        cid: rndCid,
-        bytes: arg
-      }));
-    }
-  });
-  it('bad write after closed', async () => {
-    const {writer, out} = CarWriter.create();
-    const collection = collector(out);
-    await writer.put(allBlocksFlattened[0]);
-    await writer.close();
-    await assert.isRejected(writer.put(allBlocksFlattened[1]), /closed/);
-    await collection;
-  });
-  it('bad attempt to multiple iterate', async () => {
-    const {out} = CarWriter.create();
-    collector(out);
-    await assert.isRejected(collector(out), /multiple iterator/i);
-  });
-  it('bad attempt to multiple close', async () => {
-    const {writer, out} = CarWriter.create();
-    collector(out);
-    await writer.close();
-    await assert.isRejected(writer.close(), /closed/i);
-  });
-  it('update roots (fd)', async () => {
-    const bytes = carBytes.slice();
-    await CarWriter.updateRootsInBytes(bytes, newRoots);
-    await verifyUpdateRoots(bytes);
-  });
-  it('update roots error: wrong header size', async () => {
-    const bytes = carBytes.slice();
-    await assert.isRejected(CarWriter.updateRootsInBytes(bytes, [
-      ...newRoots,
-      newRoots[0]
-    ]), /can only overwrite a header of the same length/);
-    await assert.isRejected(CarWriter.updateRootsInBytes(bytes, [newRoots[0]]), /can only overwrite a header of the same length/);
-    await assert.isRejected(CarWriter.updateRootsInBytes(bytes, []), /can only overwrite a header of the same length/);
-  });
-});

package/esm/node-test/verify-store-reader.js

@@ -1,122 +0,0 @@
-import { bytes } from 'multiformats';
-import * as raw from 'multiformats/codecs/raw';
-import {
-  toBlock,
-  assert,
-  makeData
-} from './common.js';
-function compareBlockData(actual, expected, id) {
-  assert.strictEqual(bytes.toHex(actual.bytes), bytes.toHex(expected.bytes), `comparing block as hex ${ id || '' }`);
-}
-function compareCids(actual, expected) {
-  assert.strictEqual(actual.toString(), expected.toString());
-}
-async function verifyRoots(reader) {
-  const {cborBlocks} = await makeData();
-  const expected = [
-    cborBlocks[0].cid.toString(),
-    cborBlocks[1].cid.toString()
-  ];
-  assert.deepStrictEqual((await reader.getRoots()).map(c => c.toString()), expected);
-}
-async function verifyHas(reader) {
-  const {allBlocks} = await makeData();
-  const verifyHas = async (cid, name) => {
-    assert.ok(await reader.has(cid), `reader doesn't have expected key for ${ name }`);
-  };
-  const verifyHasnt = async (cid, name) => {
-    assert.ok(!await reader.has(cid), `reader has unexpected key for ${ name }`);
-    assert.strictEqual(await reader.get(cid), undefined);
-  };
-  for (const [type, blocks] of allBlocks) {
-    for (let i = 0; i < blocks.length; i++) {
-      await verifyHas(blocks[i].cid, `block #${ i } (${ type } / ${ blocks[i].cid })`);
-    }
-  }
-  await verifyHasnt((await toBlock(new TextEncoder().encode('dddd'), raw)).cid, 'dddd');
-}
-async function verifyGet(reader) {
-  const {allBlocks} = await makeData();
-  const verifyBlock = async (expected, index, type) => {
-    let actual;
-    try {
-      actual = await reader.get(expected.cid);
-      assert.isDefined(actual);
-      if (actual) {
-        compareBlockData(actual, expected, `#${ index } (${ type })`);
-      }
-    } catch (err) {
-      assert.ifError(err, `get block length #${ index } (${ type })`);
-    }
-  };
-  for (const [type, blocks] of allBlocks) {
-    for (let i = 0; i < blocks.length; i++) {
-      await verifyBlock(blocks[i], i, type);
-    }
-  }
-}
-async function verifyBlocks(iterator, unordered) {
-  const {allBlocksFlattened} = await makeData();
-  if (!unordered) {
-    const expected = allBlocksFlattened.slice();
-    for await (const actual of iterator) {
-      const next = expected.shift();
-      assert.isDefined(next);
-      if (next) {
-        compareBlockData(actual, next);
-      }
-    }
-  } else {
-    const expected = {};
-    for (const block of allBlocksFlattened) {
-      expected[block.cid.toString()] = block;
-    }
-    for await (const actual of iterator) {
-      const {cid} = actual;
-      const exp = expected[cid.toString()];
-      if (!exp) {
-        throw new Error(`Unexpected block: ${ cid.toString() }`);
-      }
-      compareBlockData(actual, exp);
-      delete expected[cid.toString()];
-    }
-    if (Object.keys(expected).length) {
-      throw new Error('Did not find all expected blocks');
-    }
-  }
-}
-async function verifyCids(iterator, unordered) {
-  const {allBlocksFlattened} = await makeData();
-  if (!unordered) {
-    const expected = allBlocksFlattened.slice();
-    for await (const actual of iterator) {
-      const next = expected.shift();
-      assert.isDefined(next);
-      if (next) {
-        compareCids(actual, next.cid);
-      }
-    }
-  } else {
-    const expected = {};
-    for (const block of allBlocksFlattened) {
-      expected[block.cid.toString()] = block;
-    }
-    for await (const cid of iterator) {
-      const exp = expected[cid.toString()];
-      if (!exp) {
-        throw new Error(`Unexpected cid: ${ cid.toString() }`);
-      }
-      delete expected[cid.toString()];
-    }
-    if (Object.keys(expected).length) {
-      throw new Error('Did not find all expected cids');
-    }
-  }
-}
-export {
-  verifyRoots,
-  verifyHas,
-  verifyGet,
-  verifyBlocks,
-  verifyCids
-};

package/esm/package.json

@@ -1,9 +0,0 @@
-{
-  "type": "module",
-  "browser": {
-    "./car.js": "./car-browser.js",
-    "./lib/reader.js": "./lib/reader-browser.js",
-    "./lib/indexed-reader.js": "./lib/indexed-reader-browser.js",
-    "./lib/writer.js": "./lib/writer-browser.js"
-  }
-}

package/examples/car-to-fixture.js

@@ -1,66 +0,0 @@
-#!/usr/bin/env node
-
-// Take a .car file and dump its contents to stdout as a single DAG-JSON format
-// block. The format is standardised for describing CAR fixtures at
-// https://ipld.io/specs/transport/car/fixture/
-
-import fs from 'fs'
-import { CarReader } from '@ipld/car/reader'
-import { CarIndexer } from '@ipld/car/indexer'
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagPb from '@ipld/dag-pb'
-import * as dagJson from '@ipld/dag-json'
-import * as raw from 'multiformats/codecs/raw'
-import * as json from 'multiformats/codecs/json'
-
-if (!process.argv[2]) {
-  console.log('Usage: dump-car.js <path/to/car>')
-  process.exit(1)
-}
-
-const codecs = {
-  [dagCbor.code]: dagCbor,
-  [dagPb.code]: dagPb,
-  [dagJson.code]: dagJson,
-  [raw.code]: raw,
-  [json.code]: json
-}
-
-function decode (cid, bytes) {
-  if (!codecs[cid.code]) {
-    throw new Error(`Unknown codec code: 0x${cid.code.toString(16)}`)
-  }
-  return codecs[cid.code].decode(bytes)
-}
-
-async function run () {
-  const bytes = await fs.promises.readFile(process.argv[2])
-  // this is not the most optimal way to get both an index and a reader,
-  // nor is reading in the bytes into memory necessarily the best thing
-  // to be doing, but this is fine for small files and where efficiency
-  // isn't critical
-  const indexer = await CarIndexer.fromBytes(bytes)
-  const reader = await CarReader.fromBytes(bytes)
-  const fixture = {
-    header: reader._header, // a little naughty but we need gory details
-    blocks: []
-  }
-  let i = 0
-  for await (const blockIndex of indexer) {
-    fixture.blocks[i] = blockIndex
-    const block = await reader.get(blockIndex.cid)
-    fixture.blocks[i].content = decode(blockIndex.cid, block.bytes)
-    i++
-  }
-  const json = new TextDecoder().decode(dagJson.encode(fixture))
-  if (process.argv.includes('--pretty')) {
-    console.log(JSON.stringify(JSON.parse(json), null, 2))
-  } else {
-    console.log(json)
-  }
-}
-
-run().catch((err) => {
-  console.error(err)
-  process.exit(1)
-})

package/examples/dump-car.js

@@ -1,58 +0,0 @@
-#!/usr/bin/env node
-
-// Take a .car file and dump its contents into one file per block, with the
-// filename being the CID of that block.
-// Also prints a DAG-JSON form of the block and its CID to stdout.
-// If `--inspect` is supplied, don't write the blocks, just print them to stdout.
-
-import fs from 'fs'
-import { CarBlockIterator } from '@ipld/car/iterator'
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagPb from '@ipld/dag-pb'
-import * as dagJson from '@ipld/dag-json'
-import * as raw from 'multiformats/codecs/raw'
-import * as json from 'multiformats/codecs/json'
-
-if (!process.argv[2]) {
-  console.log('Usage: dump-car.js [--inspect] <path/to/car>')
-  process.exit(1)
-}
-
-const codecs = {
-  [dagCbor.code]: dagCbor,
-  [dagPb.code]: dagPb,
-  [dagJson.code]: dagJson,
-  [raw.code]: raw,
-  [json.code]: json
-}
-
-function decode (cid, bytes) {
-  if (!codecs[cid.code]) {
-    throw new Error(`Unknown codec code: 0x${cid.code.toString(16)}`)
-  }
-  return codecs[cid.code].decode(bytes)
-}
-
-async function run () {
-  const inspect = process.argv.includes('--inspect')
-  const inStream = fs.createReadStream(process.argv.filter((a) => a !== '--inspect')[2])
-  const reader = await CarBlockIterator.fromIterable(inStream)
-  console.log(`Version: ${reader.version}`)
-  console.log(`Roots: [${(await reader.getRoots()).map((r) => r.toString()).join(', ')}]`)
-  console.log('Blocks:')
-  let i = 1
-  for await (const { cid, bytes } of reader) {
-    if (!inspect) {
-      await fs.promises.writeFile(cid.toString(), bytes)
-    }
-
-    const decoded = decode(cid, bytes)
-    console.log(`#${i++} ${cid} [${codecs[cid.code].name}]`)
-    console.dir(new TextDecoder().decode(dagJson.encode(decoded)))
-  }
-}
-
-run().catch((err) => {
-  console.error(err)
-  process.exit(1)
-})

package/examples/dump-index.js

@@ -1,24 +0,0 @@
-#!/usr/bin/env node
-
-// Take a .car file and dump its index in DAG-JSON format, one line per block
-
-import fs from 'fs'
-import { CarIndexer } from '@ipld/car/indexer'
-import * as dagJson from '@ipld/dag-json'
-
-if (!process.argv[2]) {
-  console.log('Usage: dump-index.js <path/to/car>')
-  process.exit(1)
-}
-
-async function run () {
-  const indexer = await CarIndexer.fromIterable(fs.createReadStream(process.argv[2]))
-  for await (const blockIndex of indexer) {
-    console.log(new TextDecoder().decode(dagJson.encode(blockIndex)))
-  }
-}
-
-run().catch((err) => {
-  console.error(err)
-  process.exit(1)
-})

package/examples/package.json

@@ -1,27 +0,0 @@
-{
-  "name": "@ipld/car-examples",
-  "version": "0.0.0-dev",
-  "description": "How to use the Content Addressable aRchive format reader and writer",
-  "main": "example.js",
-  "type": "module",
-  "private": true,
-  "scripts": {
-    "start": "node round-trip.js",
-    "round-trip": "node round-trip.js",
-    "verify": "node verify-car.js",
-    "dump": "node dump-car.js",
-    "test": "npm install && node test-examples.js"
-  },
-  "author": "Rod <rod@vagg.org> (http://r.va.gg/)",
-  "license": "(Apache-2.0 AND MIT)",
-  "dependencies": {
-    "@ipld/car": "file:..",
-    "@ipld/dag-cbor": "^7.0.0",
-    "@ipld/dag-json": "^8.0.0",
-    "@ipld/dag-pb": "^2.0.2",
-    "@multiformats/blake2": "^1.0.1",
-    "@types/varint": "^6.0.0",
-    "multiformats": "^10.0.0",
-    "varint": "^6.0.0"
-  }
-}

package/examples/round-trip.js

@@ -1,45 +0,0 @@
-#!/usr/bin/env node
-
-// Create a simple .car file with a single block and that block's CID as the
-// single root. Then read the .car and fetch the block again.
-
-import fs from 'fs'
-import { Readable } from 'stream'
-import { CarReader, CarWriter } from '@ipld/car'
-import * as raw from 'multiformats/codecs/raw'
-import { CID } from 'multiformats/cid'
-import { sha256 } from 'multiformats/hashes/sha2'
-
-async function example () {
-  const bytes = new TextEncoder().encode('random meaningless bytes')
-  const hash = await sha256.digest(raw.encode(bytes))
-  const cid = CID.create(1, raw.code, hash)
-
-  // create the writer and set the header with a single root
-  const { writer, out } = await CarWriter.create([cid])
-  Readable.from(out).pipe(fs.createWriteStream('example.car'))
-
-  // store a new block, creates a new file entry in the CAR archive
-  await writer.put({ cid, bytes })
-  await writer.close()
-
-  const inStream = fs.createReadStream('example.car')
-  // read and parse the entire stream in one go, this will cache the contents of
-  // the car in memory so is not suitable for large files.
-  const reader = await CarReader.fromIterable(inStream)
-
-  // read the list of roots from the header
-  const roots = await reader.getRoots()
-  // retrieve a block, as a { cid:CID, bytes:UInt8Array } pair from the archive
-  const got = await reader.get(roots[0])
-  // also possible: for await (const { cid, bytes } of CarIterator.fromIterable(inStream)) { ... }
-
-  console.log('Retrieved [%s] from example.car with CID [%s]',
-    new TextDecoder().decode(got.bytes),
-    roots[0].toString())
-}
-
-example().catch((err) => {
-  console.error(err)
-  process.exit(1)
-})