@ipld/car 4.0.0 → 4.1.2

package/esm/browser-test/test-buffer-writer.js

@@ -0,0 +1,311 @@
+import * as CarBufferWriter from '../lib/buffer-writer.js';
+import { CarReader } from '../lib/reader-browser.js';
+import { createHeader } from '../lib/encoder.js';
+import { assert } from './common.js';
+import {
+  CID,
+  varint
+} from 'multiformats';
+import * as CBOR from '@ipld/dag-cbor';
+import {
+  sha256,
+  sha512
+} from 'multiformats/hashes/sha2';
+import { identity } from 'multiformats/hashes/identity';
+import * as Raw from 'multiformats/codecs/raw';
+import * as Block from 'multiformats/block';
+describe('CarBufferWriter', () => {
+  const cid = CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu');
+  describe('calculateHeaderLength', async () => {
+    for (const count of [
+        0,
+        1,
+        10,
+        18,
+        24,
+        48,
+        124,
+        255,
+        258,
+        65536 - 1,
+        65536
+      ]) {
+      it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
+        const roots = new Array(count).fill(cid);
+        const sizes = new Array(count).fill(cid.bytes.byteLength);
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(sizes), createHeader(roots).byteLength);
+      });
+      it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
+        const roots = new Array(count).fill(cid);
+        const rootLengths = roots.map(c => c.bytes.byteLength);
+        assert.deepEqual(CarBufferWriter.calculateHeaderLength(rootLengths), createHeader(roots).byteLength);
+      });
+    }
+    it('estimate on large CIDs', () => {
+      const largeCID = CID.parse(`bafkqbbac${ 'a'.repeat(416) }`);
+      assert.equal(CarBufferWriter.calculateHeaderLength([
+        cid.bytes.byteLength,
+        largeCID.bytes.byteLength
+      ]), createHeader([
+        cid,
+        largeCID
+      ]).byteLength);
+    });
+    it('estimate on large CIDs 2', () => {
+      const largeCID = CID.createV1(Raw.code, identity.digest(new Uint8Array(512).fill(1)));
+      assert.equal(CarBufferWriter.calculateHeaderLength([
+        cid.bytes.byteLength,
+        largeCID.bytes.byteLength
+      ]), createHeader([
+        cid,
+        largeCID
+      ]).byteLength);
+    });
+  });
+  describe('writer', () => {
+    it('estimate header and write blocks', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(1);
+      const dataSize = 256;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b2);
+      writer.addRoot(b1.cid);
+      const bytes = writer.close();
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [b1.cid]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+    it('overestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2);
+      const dataSize = 256;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b2);
+      writer.addRoot(b1.cid);
+      assert.throws(() => writer.close(), /Header size was overestimate/);
+      const bytes = writer.close({ resize: true });
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [b1.cid]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+    it('underestimate header', async () => {
+      const headerSize = CarBufferWriter.estimateHeaderLength(2);
+      const dataSize = 300;
+      const buffer = new ArrayBuffer(headerSize + dataSize);
+      const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+      const b1 = await Block.encode({
+        value: { hello: 'world' },
+        codec: CBOR,
+        hasher: sha256
+      });
+      writer.write(b1);
+      writer.addRoot(b1.cid);
+      const b2 = await Block.encode({
+        value: { bye: 'world' },
+        codec: CBOR,
+        hasher: sha512
+      });
+      writer.write(b2);
+      assert.throws(() => writer.addRoot(b2.cid), /has no capacity/);
+      writer.addRoot(b2.cid, { resize: true });
+      const bytes = writer.close();
+      const reader = await CarReader.fromBytes(bytes);
+      assert.deepEqual(await reader.getRoots(), [
+        b1.cid,
+        b2.cid
+      ]);
+      assert.deepEqual(reader._blocks, [
+        {
+          cid: b1.cid,
+          bytes: b1.bytes
+        },
+        {
+          cid: b2.cid,
+          bytes: b2.bytes
+        }
+      ]);
+    });
+  });
+  it('has no space for the root', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1);
+    const dataSize = 100;
+    const buffer = new ArrayBuffer(headerSize + dataSize);
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b1);
+    writer.addRoot(b1.cid);
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b2);
+    assert.throws(() => writer.addRoot(b2.cid), /Buffer has no capacity for a new root/);
+    assert.throws(() => writer.addRoot(b2.cid, { resize: true }), /Buffer has no capacity for a new root/);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [b1.cid]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+  it('has no space for the block', async () => {
+    const headerSize = CarBufferWriter.estimateHeaderLength(1);
+    const dataSize = 58;
+    const buffer = new ArrayBuffer(headerSize + dataSize);
+    const writer = CarBufferWriter.createWriter(buffer, { headerSize });
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    writer.write(b1);
+    writer.addRoot(b1.cid);
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    assert.throws(() => writer.write(b2), /Buffer has no capacity for this block/);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [b1.cid]);
+    assert.deepEqual(reader._blocks, [{
+        cid: b1.cid,
+        bytes: b1.bytes
+      }]);
+  });
+  it('provide roots', async () => {
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    const b2 = await Block.encode({
+      value: { bye: 'world' },
+      codec: CBOR,
+      hasher: sha512
+    });
+    const buffer = new ArrayBuffer(300);
+    const writer = CarBufferWriter.createWriter(buffer, {
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    });
+    writer.write(b1);
+    writer.write(b2);
+    const bytes = writer.close();
+    const reader = await CarReader.fromBytes(bytes);
+    assert.deepEqual(await reader.getRoots(), [
+      b1.cid,
+      b2.cid
+    ]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+  it('provide large CID root', async () => {
+    const bytes = new Uint8Array(512).fill(1);
+    const b1 = await Block.encode({
+      value: { hello: 'world' },
+      codec: CBOR,
+      hasher: sha256
+    });
+    const b2 = {
+      cid: CID.createV1(Raw.code, identity.digest(bytes)),
+      bytes
+    };
+    const headerSize = CBOR.encode({
+      version: 1,
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    }).byteLength;
+    const bodySize = CarBufferWriter.blockLength(b1) + CarBufferWriter.blockLength(b2);
+    const varintSize = varint.encodingLength(headerSize);
+    const writer = CarBufferWriter.createWriter(new ArrayBuffer(varintSize + headerSize + bodySize), {
+      roots: [
+        b1.cid,
+        b2.cid
+      ]
+    });
+    writer.write(b1);
+    writer.write(b2);
+    const car = writer.close();
+    const reader = await CarReader.fromBytes(car);
+    assert.deepEqual(await reader.getRoots(), [
+      b1.cid,
+      b2.cid
+    ]);
+    assert.deepEqual(reader._blocks, [
+      {
+        cid: b1.cid,
+        bytes: b1.bytes
+      },
+      {
+        cid: b2.cid,
+        bytes: b2.bytes
+      }
+    ]);
+  });
+});

package/esm/browser-test/test-indexer.js

@@ -24,7 +24,7 @@ describe('CarIndexer fromBytes()', () => {
     const indexer = await CarIndexer.fromBytes(goCarV2Bytes);
     const roots = await indexer.getRoots();
     assert.strictEqual(roots.length, 1);
-    assert(goCarV2Roots[0].equals(roots[0]));
+    assert.ok(goCarV2Roots[0].equals(roots[0]));
     assert.strictEqual(indexer.version, 2);
     const indexData = [];
     for await (const index of indexer) {

package/esm/browser-test/test-reader.js

@@ -68,13 +68,13 @@ describe('CarReader fromBytes()', () => {
     const reader = await CarReader.fromBytes(goCarV2Bytes);
     const roots = await reader.getRoots();
     assert.strictEqual(roots.length, 1);
-    assert(goCarV2Roots[0].equals(roots[0]));
+    assert.ok(goCarV2Roots[0].equals(roots[0]));
     assert.strictEqual(reader.version, 2);
     for (const {cid} of goCarV2Index) {
       const block = await reader.get(cid);
       assert.isDefined(block);
       if (block) {
-        assert(cid.equals(block.cid));
+        assert.ok(cid.equals(block.cid));
         let content;
         if (cid.code === dagPb.code) {
           content = dagPb.decode(block.bytes);

package/esm/browser-test/test-writer.js

@@ -193,9 +193,9 @@ describe('CarWriter', () => {
     const rawBytes = await append(0);
     const pbBytes = await append(1);
     const cborBytes = await append(2);
-    assert(rawBytes.length > 0);
-    assert(pbBytes.length > 0);
-    assert(cborBytes.length > 0);
+    assert.ok(rawBytes.length > 0);
+    assert.ok(pbBytes.length > 0);
+    assert.ok(cborBytes.length > 0);
     const reassembled = concatBytes([
       headerBytes,
       rawBytes,

package/esm/lib/buffer-writer.js

@@ -0,0 +1,126 @@
+import varint from 'varint';
+import {
+  Token,
+  Type
+} from 'cborg';
+import { tokensToLength } from 'cborg/length';
+import * as CBOR from '@ipld/dag-cbor';
+class CarBufferWriter {
+  constructor(bytes, headerSize) {
+    this.bytes = bytes;
+    this.byteOffset = headerSize;
+    this.roots = [];
+    this.headerSize = headerSize;
+  }
+  addRoot(root, options) {
+    addRoot(this, root, options);
+    return this;
+  }
+  write(block) {
+    addBlock(this, block);
+    return this;
+  }
+  close(options) {
+    return close(this, options);
+  }
+}
+export const addRoot = (writer, root, {
+  resize = false
+} = {}) => {
+  const {bytes, headerSize, byteOffset, roots} = writer;
+  writer.roots.push(root);
+  const size = headerLength(writer);
+  if (size > headerSize) {
+    if (size - headerSize + byteOffset < bytes.byteLength) {
+      if (resize) {
+        resizeHeader(writer, size);
+      } else {
+        roots.pop();
+        throw new RangeError(`Header of size ${ headerSize } has no capacity for new root ${ root }.
+  However there is a space in the buffer and you could call addRoot(root, { resize: root }) to resize header to make a space for this root.`);
+      }
+    } else {
+      roots.pop();
+      throw new RangeError(`Buffer has no capacity for a new root ${ root }`);
+    }
+  }
+};
+export const blockLength = ({cid, bytes}) => {
+  const size = cid.bytes.byteLength + bytes.byteLength;
+  return varint.encodingLength(size) + size;
+};
+export const addBlock = (writer, {cid, bytes}) => {
+  const byteLength = cid.bytes.byteLength + bytes.byteLength;
+  const size = varint.encode(byteLength);
+  if (writer.byteOffset + size.length + byteLength > writer.bytes.byteLength) {
+    throw new RangeError('Buffer has no capacity for this block');
+  } else {
+    writeBytes(writer, size);
+    writeBytes(writer, cid.bytes);
+    writeBytes(writer, bytes);
+  }
+};
+export const close = (writer, {
+  resize = false
+} = {}) => {
+  const {roots, bytes, byteOffset, headerSize} = writer;
+  const headerBytes = CBOR.encode({
+    version: 1,
+    roots
+  });
+  const varintBytes = varint.encode(headerBytes.length);
+  const size = varintBytes.length + headerBytes.byteLength;
+  const offset = headerSize - size;
+  if (offset === 0) {
+    writeHeader(writer, varintBytes, headerBytes);
+    return bytes.subarray(0, byteOffset);
+  } else if (resize) {
+    resizeHeader(writer, size);
+    writeHeader(writer, varintBytes, headerBytes);
+    return bytes.subarray(0, writer.byteOffset);
+  } else {
+    throw new RangeError(`Header size was overestimated.
+You can use close({ resize: true }) to resize header`);
+  }
+};
+export const resizeHeader = (writer, byteLength) => {
+  const {bytes, headerSize} = writer;
+  bytes.set(bytes.subarray(headerSize, writer.byteOffset), byteLength);
+  writer.byteOffset += byteLength - headerSize;
+  writer.headerSize = byteLength;
+};
+const writeBytes = (writer, bytes) => {
+  writer.bytes.set(bytes, writer.byteOffset);
+  writer.byteOffset += bytes.length;
+};
+const writeHeader = ({bytes}, varint, header) => {
+  bytes.set(varint);
+  bytes.set(header, varint.length);
+};
+const headerPreludeTokens = [
+  new Token(Type.map, 2),
+  new Token(Type.string, 'version'),
+  new Token(Type.uint, 1),
+  new Token(Type.string, 'roots')
+];
+const CID_TAG = new Token(Type.tag, 42);
+export const calculateHeaderLength = rootLengths => {
+  const tokens = [...headerPreludeTokens];
+  tokens.push(new Token(Type.array, rootLengths.length));
+  for (const rootLength of rootLengths) {
+    tokens.push(CID_TAG);
+    tokens.push(new Token(Type.bytes, { length: rootLength + 1 }));
+  }
+  const length = tokensToLength(tokens);
+  return varint.encodingLength(length) + length;
+};
+export const headerLength = ({roots}) => calculateHeaderLength(roots.map(cid => cid.bytes.byteLength));
+export const estimateHeaderLength = (rootCount, rootByteLength = 36) => calculateHeaderLength(new Array(rootCount).fill(rootByteLength));
+export const createWriter = (buffer, {roots = [], byteOffset = 0, byteLength = buffer.byteLength, headerSize = headerLength({ roots })} = {}) => {
+  const bytes = new Uint8Array(buffer, byteOffset, byteLength);
+  const writer = new CarBufferWriter(bytes, headerSize);
+  for (const root of roots) {
+    writer.addRoot(root);
+  }
+  return writer;
+};

package/esm/lib/encoder.js

@@ -26,7 +26,7 @@ function createEncoder(writer) {
       }
     },
     async close() {
-      return writer.end();
+      await writer.end();
     }
   };
 }

package/esm/lib/iterator-channel.js

@@ -30,7 +30,7 @@ export function create() {
       ended = true;
       const drainer = makeDrainer();
       outWaitResolver();
-      return drainer;
+      await drainer;
     }
   };
   const iterator = {