@ipld/car 3.2.4 → 4.1.1

package/esm/lib/buffer-writer.js

@@ -0,0 +1,126 @@
+import varint from 'varint';
+import {
+  Token,
+  Type
+} from 'cborg';
+import { tokensToLength } from 'cborg/length';
+import * as CBOR from '@ipld/dag-cbor';
+class CarBufferWriter {
+  constructor(bytes, headerSize) {
+    this.bytes = bytes;
+    this.byteOffset = headerSize;
+    this.roots = [];
+    this.headerSize = headerSize;
+  }
+  addRoot(root, options) {
+    addRoot(this, root, options);
+    return this;
+  }
+  write(block) {
+    addBlock(this, block);
+    return this;
+  }
+  close(options) {
+    return close(this, options);
+  }
+}
+export const addRoot = (writer, root, {
+  resize = false
+} = {}) => {
+  const {bytes, headerSize, byteOffset, roots} = writer;
+  writer.roots.push(root);
+  const size = headerLength(writer);
+  if (size > headerSize) {
+    if (size - headerSize + byteOffset < bytes.byteLength) {
+      if (resize) {
+        resizeHeader(writer, size);
+      } else {
+        roots.pop();
+        throw new RangeError(`Header of size ${ headerSize } has no capacity for new root ${ root }.
+  However there is a space in the buffer and you could call addRoot(root, { resize: root }) to resize header to make a space for this root.`);
+      }
+    } else {
+      roots.pop();
+      throw new RangeError(`Buffer has no capacity for a new root ${ root }`);
+    }
+  }
+};
+export const blockLength = ({cid, bytes}) => {
+  const size = cid.bytes.byteLength + bytes.byteLength;
+  return varint.encodingLength(size) + size;
+};
+export const addBlock = (writer, {cid, bytes}) => {
+  const byteLength = cid.bytes.byteLength + bytes.byteLength;
+  const size = varint.encode(byteLength);
+  if (writer.byteOffset + size.length + byteLength > writer.bytes.byteLength) {
+    throw new RangeError('Buffer has no capacity for this block');
+  } else {
+    writeBytes(writer, size);
+    writeBytes(writer, cid.bytes);
+    writeBytes(writer, bytes);
+  }
+};
+export const close = (writer, {
+  resize = false
+} = {}) => {
+  const {roots, bytes, byteOffset, headerSize} = writer;
+  const headerBytes = CBOR.encode({
+    version: 1,
+    roots
+  });
+  const varintBytes = varint.encode(headerBytes.length);
+  const size = varintBytes.length + headerBytes.byteLength;
+  const offset = headerSize - size;
+  if (offset === 0) {
+    writeHeader(writer, varintBytes, headerBytes);
+    return bytes.subarray(0, byteOffset);
+  } else if (resize) {
+    resizeHeader(writer, size);
+    writeHeader(writer, varintBytes, headerBytes);
+    return bytes.subarray(0, writer.byteOffset);
+  } else {
+    throw new RangeError(`Header size was overestimated.
+You can use close({ resize: true }) to resize header`);
+  }
+};
+export const resizeHeader = (writer, byteLength) => {
+  const {bytes, headerSize} = writer;
+  bytes.set(bytes.subarray(headerSize, writer.byteOffset), byteLength);
+  writer.byteOffset += byteLength - headerSize;
+  writer.headerSize = byteLength;
+};
+const writeBytes = (writer, bytes) => {
+  writer.bytes.set(bytes, writer.byteOffset);
+  writer.byteOffset += bytes.length;
+};
+const writeHeader = ({bytes}, varint, header) => {
+  bytes.set(varint);
+  bytes.set(header, varint.length);
+};
+const headerPreludeTokens = [
+  new Token(Type.map, 2),
+  new Token(Type.string, 'version'),
+  new Token(Type.uint, 1),
+  new Token(Type.string, 'roots')
+];
+const CID_TAG = new Token(Type.tag, 42);
+export const calculateHeaderLength = rootLengths => {
+  const tokens = [...headerPreludeTokens];
+  tokens.push(new Token(Type.array, rootLengths.length));
+  for (const rootLength of rootLengths) {
+    tokens.push(CID_TAG);
+    tokens.push(new Token(Type.bytes, { length: rootLength + 1 }));
+  }
+  const length = tokensToLength(tokens);
+  return varint.encodingLength(length) + length;
+};
+export const headerLength = ({roots}) => calculateHeaderLength(roots.map(cid => cid.bytes.byteLength));
+export const estimateHeaderLength = (rootCount, rootByteLength = 36) => calculateHeaderLength(new Array(rootCount).fill(rootByteLength));
+export const createWriter = (buffer, {roots = [], byteOffset = 0, byteLength = buffer.byteLength, headerSize = headerLength({ roots })} = {}) => {
+  const bytes = new Uint8Array(buffer, byteOffset, byteLength);
+  const writer = new CarBufferWriter(bytes, headerSize);
+  for (const root of roots) {
+    writer.addRoot(root);
+  }
+  return writer;
+};

package/esm/lib/decoder.js

@@ -2,18 +2,40 @@ import varint from 'varint';
 import { CID } from 'multiformats/cid';
 import * as Digest from 'multiformats/hashes/digest';
 import { decode as decodeDagCbor } from '@ipld/dag-cbor';
+import { CarHeader as headerValidator } from './header-validator.js';
 const CIDV0_BYTES = {
   SHA2_256: 18,
   LENGTH: 32,
   DAG_PB: 112
 };
+const V2_HEADER_LENGTH = 16 + 8 + 8 + 8;
 async function readVarint(reader) {
   const bytes = await reader.upTo(8);
+  if (!bytes.length) {
+    throw new Error('Unexpected end of data');
+  }
   const i = varint.decode(bytes);
   reader.seek(varint.decode.bytes);
   return i;
 }
-export async function readHeader(reader) {
+async function readV2Header(reader) {
+  const bytes = await reader.exactly(V2_HEADER_LENGTH);
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  let offset = 0;
+  const header = {
+    version: 2,
+    characteristics: [
+      dv.getBigUint64(offset, true),
+      dv.getBigUint64(offset += 8, true)
+    ],
+    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
+    dataSize: Number(dv.getBigUint64(offset += 8, true)),
+    indexOffset: Number(dv.getBigUint64(offset += 8, true))
+  };
+  reader.seek(V2_HEADER_LENGTH);
+  return header;
+}
+export async function readHeader(reader, strictVersion) {
   const length = await readVarint(reader);
   if (length === 0) {
     throw new Error('Invalid CAR header (zero length)');
@@ -21,22 +43,23 @@ export async function readHeader(reader) {
   const header = await reader.exactly(length);
   reader.seek(length);
   const block = decodeDagCbor(header);
-  if (block == null || Array.isArray(block) || typeof block !== 'object') {
+  if (!headerValidator(block)) {
     throw new Error('Invalid CAR header format');
   }
-  if (block.version !== 1) {
-    if (typeof block.version === 'string') {
-      throw new Error(`Invalid CAR version: "${ block.version }"`);
-    }
-    throw new Error(`Invalid CAR version: ${ block.version }`);
+  if (block.version !== 1 && block.version !== 2 || strictVersion !== undefined && block.version !== strictVersion) {
+    throw new Error(`Invalid CAR version: ${ block.version }${ strictVersion !== undefined ? ` (expected ${ strictVersion })` : '' }`);
   }
-  if (!Array.isArray(block.roots)) {
+  const hasRoots = Array.isArray(block.roots);
+  if (block.version === 1 && !hasRoots || block.version === 2 && hasRoots) {
     throw new Error('Invalid CAR header format');
   }
-  if (Object.keys(block).filter(p => p !== 'roots' && p !== 'version').length) {
-    throw new Error('Invalid CAR header format');
+  if (block.version === 1) {
+    return block;
   }
-  return block;
+  const v2Header = await readV2Header(reader);
+  reader.seek(v2Header.dataOffset - reader.pos);
+  const v1Header = await readHeader(reader, 1);
+  return Object.assign(v1Header, v2Header);
 }
 async function readMultihash(reader) {
   const bytes = await reader.upTo(8);
@@ -74,7 +97,7 @@ export async function readBlockHead(reader) {
   }
   length += reader.pos - start;
   const cid = await readCid(reader);
-  const blockLength = length - (reader.pos - start);
+  const blockLength = length - Number(reader.pos - start);
   return {
     cid,
     length,
@@ -104,7 +127,14 @@ async function readBlockIndex(reader) {
   return index;
 }
 export function createDecoder(reader) {
-  const headerPromise = readHeader(reader);
+  const headerPromise = (async () => {
+    const header = await readHeader(reader);
+    if (header.version === 2) {
+      const v1length = reader.pos - header.dataOffset;
+      reader = limitReader(reader, header.dataSize - v1length);
+    }
+    return header;
+  })();
   return {
     header: () => headerPromise,
     async *blocks() {
@@ -206,4 +236,30 @@ export function asyncIterableReader(asyncIterable) {
     return next.value;
   }
   return chunkReader(readChunk);
+}
+export function limitReader(reader, byteLimit) {
+  let bytesRead = 0;
+  return {
+    async upTo(length) {
+      let bytes = await reader.upTo(length);
+      if (bytes.length + bytesRead > byteLimit) {
+        bytes = bytes.subarray(0, byteLimit - bytesRead);
+      }
+      return bytes;
+    },
+    async exactly(length) {
+      const bytes = await reader.exactly(length);
+      if (bytes.length + bytesRead > byteLimit) {
+        throw new Error('Unexpected end of data');
+      }
+      return bytes;
+    },
+    seek(length) {
+      bytesRead += length;
+      reader.seek(length);
+    },
+    get pos() {
+      return reader.pos;
+    }
+  };
 }

package/esm/lib/header-validator.js

@@ -0,0 +1,23 @@
+const Kinds = {
+  Null: obj => obj === null,
+  Int: obj => Number.isInteger(obj),
+  Float: obj => typeof obj === 'number' && Number.isFinite(obj),
+  String: obj => typeof obj === 'string',
+  Bool: obj => typeof obj === 'boolean',
+  Bytes: obj => obj instanceof Uint8Array,
+  Link: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
+  List: obj => Array.isArray(obj),
+  Map: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
+};
+const Types = {
+  Int: Kinds.Int,
+  'CarHeader > version': obj => Types.Int(obj),
+  'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
+  'CarHeader > roots (anon)': obj => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
+  'CarHeader > roots': obj => Types['CarHeader > roots (anon)'](obj),
+  CarHeader: obj => {
+    const keys = obj && Object.keys(obj);
+    return Kinds.Map(obj) && ['version'].every(k => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value));
+  }
+};
+export const CarHeader = Types.CarHeader;

package/esm/lib/reader-browser.js

@@ -4,17 +4,16 @@ import {
   createDecoder
 } from './decoder.js';
 export class CarReader {
-  constructor(version, roots, blocks) {
-    this._version = version;
-    this._roots = roots;
+  constructor(header, blocks) {
+    this._header = header;
     this._blocks = blocks;
     this._keys = blocks.map(b => b.cid.toString());
   }
   get version() {
-    return this._version;
+    return this._header.version;
   }
   async getRoots() {
-    return this._roots;
+    return this._header.roots;
   }
   async has(key) {
     return this._keys.indexOf(key.toString()) > -1;
@@ -46,13 +45,13 @@ export class CarReader {
     return decodeReaderComplete(asyncIterableReader(asyncIterable));
   }
 }
-async function decodeReaderComplete(reader) {
+export async function decodeReaderComplete(reader) {
   const decoder = createDecoder(reader);
-  const {version, roots} = await decoder.header();
+  const header = await decoder.header();
   const blocks = [];
   for await (const block of decoder.blocks()) {
     blocks.push(block);
   }
-  return new CarReader(version, roots, blocks);
+  return new CarReader(header, blocks);
 }
 export const __browser = true;

package/esm/lib/writer-browser.js

@@ -63,7 +63,7 @@ export class CarWriter {
     const reader = bytesReader(bytes);
     await readHeader(reader);
     const newHeader = createHeader(roots);
-    if (reader.pos !== newHeader.length) {
+    if (Number(reader.pos) !== newHeader.length) {
       throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${ reader.pos } bytes, new header is ${ newHeader.length } bytes)`);
     }
     bytes.set(newHeader, 0);

package/esm/node-test/common.js

@@ -186,6 +186,77 @@ const goCarIndex = [
     blockLength: 18
   }
 ];
+const goCarV2Bytes = bytes.fromHex('0aa16776657273696f6e02000000000000000000000000000000003300000000000000c001000000000000f30100000000000038a265726f6f747381d82a5823001220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f86776657273696f6e01511220fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f8122d0a221220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6261204f09f8da418a40185011220d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f62612310a221220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f1120962617272656c657965183a122e0a2401551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d1204f09f90a11807581220d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f112340a2401551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d120a666973686d6f6e67657218042801551220b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d666973682b01551220a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d6c6f62737465720100000028000000c800000000000000a2e1c40da1ae335d4dffe729eb4d5ca23b74b9e51fc535f4a804a261080c294d9401000000000000b474a99a2705e23cf905a484ec6d14ef58b56bbe62e9292783466ec363b5072d6b01000000000000d745b7757f5b4593eeab7820306c7bc64eb496a7410a0d07df7a34ffec4b97f11201000000000000d9c0d5376d26f1931f7ad52d7acc00fc1090d2edb0808bf61eeb0a152826f6268b00000000000000fb16f5083412ef1371d031ed4aa239903d84efdadf1ba3cd678e6475b1a232f83900000000000000');
+const goCarV2Roots = [CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z')];
+const goCarV2Index = [
+  {
+    blockLength: 47,
+    blockOffset: 143,
+    cid: CID.parse('QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z'),
+    length: 82,
+    offset: 108
+  },
+  {
+    blockLength: 99,
+    blockOffset: 226,
+    cid: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
+    length: 135,
+    offset: 190
+  },
+  {
+    blockLength: 54,
+    blockOffset: 360,
+    cid: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
+    length: 89,
+    offset: 325
+  },
+  {
+    blockLength: 4,
+    blockOffset: 451,
+    cid: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
+    length: 41,
+    offset: 414
+  },
+  {
+    blockLength: 7,
+    blockOffset: 492,
+    cid: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
+    length: 44,
+    offset: 455
+  }
+];
+const goCarV2Contents = {
+  QmfEoLyB5NndqeKieExd1rtJzTduQUPEV8TwAYcUiy3H5Z: {
+    Links: [{
+        Hash: CID.parse('QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM'),
+        Name: '\uD83C\uDF64',
+        Tsize: 164
+      }]
+  },
+  QmczfirA7VEH7YVvKPTPoU69XM3qY4DC39nnTsWd4K3SkM: {
+    Links: [
+      {
+        Hash: CID.parse('Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE'),
+        Name: 'barreleye',
+        Tsize: 58
+      },
+      {
+        Hash: CID.parse('bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju'),
+        Name: '\uD83D\uDC21',
+        Tsize: 7
+      }
+    ]
+  },
+  Qmcpz2FHJD7VAhg1fxFXdYJKePtkx1BsHuCrAgWVnaHMTE: {
+    Links: [{
+        Hash: CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu'),
+        Name: 'fishmonger',
+        Tsize: 4
+      }]
+  },
+  bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu: 'fish',
+  bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju: 'lobster'
+};
 export {
   toBlock,
   assert,
@@ -195,5 +266,9 @@ export {
   carBytes,
   goCarBytes,
   goCarRoots,
-  goCarIndex
+  goCarIndex,
+  goCarV2Bytes,
+  goCarV2Roots,
+  goCarV2Index,
+  goCarV2Contents
 };