@ipld/car 3.2.4 → 4.1.1

package/lib/buffer-writer.js

@@ -0,0 +1,286 @@
+import varint from 'varint'
+import { Token, Type } from 'cborg'
+import { tokensToLength } from 'cborg/length'
+import * as CBOR from '@ipld/dag-cbor'
+
+/**
+ * @typedef {import('../api').CID} CID
+ * @typedef {import('../api').Block} Block
+ * @typedef {import('../api').CarBufferWriter} Writer
+ * @typedef {import('../api').CarBufferWriterOptions} Options
+ * @typedef {import('./coding').CarEncoder} CarEncoder
+ */
+
+/**
+ * A simple CAR writer that writes to a pre-allocated buffer.
+ *
+ * @class
+ * @name CarBufferWriter
+ * @implements {Writer}
+ */
+class CarBufferWriter {
+  /**
+   * @param {Uint8Array} bytes
+   * @param {number} headerSize
+   */
+  constructor (bytes, headerSize) {
+    /** @readonly */
+    this.bytes = bytes
+    this.byteOffset = headerSize
+
+    /**
+     * @readonly
+     * @type {CID[]}
+     */
+    this.roots = []
+    this.headerSize = headerSize
+  }
+
+  /**
+   * Add a root to this writer, to be used to create a header when the CAR is
+   * finalized with {@link CarBufferWriter.close `close()`}
+   *
+   * @param {CID} root
+   * @param {{resize?:boolean}} [options]
+   * @returns {CarBufferWriter}
+   */
+  addRoot (root, options) {
+    addRoot(this, root, options)
+    return this
+  }
+
+  /**
+   * Write a `Block` (a `{ cid:CID, bytes:Uint8Array }` pair) to the archive.
+   * Throws if there is not enough capacity.
+   *
+   * @param {Block} block A `{ cid:CID, bytes:Uint8Array }` pair.
+   * @returns {CarBufferWriter}
+   */
+  write (block) {
+    addBlock(this, block)
+    return this
+  }
+
+  /**
+   * Finalize the CAR and return it as a `Uint8Array`.
+   *
+   * @param {object} [options]
+   * @param {boolean} [options.resize]
+   * @returns {Uint8Array}
+   */
+  close (options) {
+    return close(this, options)
+  }
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {CID} root
+ * @param {{resize?:boolean}} [options]
+ */
+export const addRoot = (writer, root, { resize = false } = {}) => {
+  const { bytes, headerSize, byteOffset, roots } = writer
+  writer.roots.push(root)
+  const size = headerLength(writer)
+  // If there is not enough space for the new root
+  if (size > headerSize) {
+    // Check if we root would fit if we were to resize the head.
+    if (size - headerSize + byteOffset < bytes.byteLength) {
+      // If resize is enabled resize head
+      if (resize) {
+        resizeHeader(writer, size)
+      // otherwise remove head and throw an error suggesting to resize
+      } else {
+        roots.pop()
+        throw new RangeError(`Header of size ${headerSize} has no capacity for new root ${root}.
+  However there is a space in the buffer and you could call addRoot(root, { resize: root }) to resize header to make a space for this root.`)
+      }
+    // If head would not fit even with resize pop new root and throw error
+    } else {
+      roots.pop()
+      throw new RangeError(`Buffer has no capacity for a new root ${root}`)
+    }
+  }
+}
+
+/**
+ * Calculates number of bytes required for storing given block in CAR. Useful in
+ * estimating size of an `ArrayBuffer` for the `CarBufferWriter`.
+ *
+ * @name CarBufferWriter.blockLength(Block)
+ * @param {Block} block
+ * @returns {number}
+ */
+export const blockLength = ({ cid, bytes }) => {
+  const size = cid.bytes.byteLength + bytes.byteLength
+  return varint.encodingLength(size) + size
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {Block} block
+ */
+export const addBlock = (writer, { cid, bytes }) => {
+  const byteLength = cid.bytes.byteLength + bytes.byteLength
+  const size = varint.encode(byteLength)
+  if (writer.byteOffset + size.length + byteLength > writer.bytes.byteLength) {
+    throw new RangeError('Buffer has no capacity for this block')
+  } else {
+    writeBytes(writer, size)
+    writeBytes(writer, cid.bytes)
+    writeBytes(writer, bytes)
+  }
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {object} [options]
+ * @param {boolean} [options.resize]
+ */
+export const close = (writer, { resize = false } = {}) => {
+  const { roots, bytes, byteOffset, headerSize } = writer
+
+  const headerBytes = CBOR.encode({ version: 1, roots })
+  const varintBytes = varint.encode(headerBytes.length)
+
+  const size = varintBytes.length + headerBytes.byteLength
+  const offset = headerSize - size
+
+  // If header size estimate was accurate we just write header and return
+  // view into buffer.
+  if (offset === 0) {
+    writeHeader(writer, varintBytes, headerBytes)
+    return bytes.subarray(0, byteOffset)
+    // If header was overestimated and `{resize: true}` is passed resize header
+  } else if (resize) {
+    resizeHeader(writer, size)
+    writeHeader(writer, varintBytes, headerBytes)
+    return bytes.subarray(0, writer.byteOffset)
+  } else {
+    throw new RangeError(`Header size was overestimated.
+You can use close({ resize: true }) to resize header`)
+  }
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {number} byteLength
+ */
+export const resizeHeader = (writer, byteLength) => {
+  const { bytes, headerSize } = writer
+  // Move data section to a new offset
+  bytes.set(bytes.subarray(headerSize, writer.byteOffset), byteLength)
+  // Update header size & byteOffset
+  writer.byteOffset += byteLength - headerSize
+  writer.headerSize = byteLength
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {number[]|Uint8Array} bytes
+ */
+
+const writeBytes = (writer, bytes) => {
+  writer.bytes.set(bytes, writer.byteOffset)
+  writer.byteOffset += bytes.length
+}
+/**
+ * @param {{bytes:Uint8Array}} writer
+ * @param {number[]} varint
+ * @param {Uint8Array} header
+ */
+const writeHeader = ({ bytes }, varint, header) => {
+  bytes.set(varint)
+  bytes.set(header, varint.length)
+}
+
+const headerPreludeTokens = [
+  new Token(Type.map, 2),
+  new Token(Type.string, 'version'),
+  new Token(Type.uint, 1),
+  new Token(Type.string, 'roots')
+]
+
+const CID_TAG = new Token(Type.tag, 42)
+
+/**
+ * Calculates header size given the array of byteLength for roots.
+ *
+ * @name CarBufferWriter.calculateHeaderLength(rootLengths)
+ * @param {number[]} rootLengths
+ * @returns {number}
+ */
+export const calculateHeaderLength = (rootLengths) => {
+  const tokens = [...headerPreludeTokens]
+  tokens.push(new Token(Type.array, rootLengths.length))
+  for (const rootLength of rootLengths) {
+    tokens.push(CID_TAG)
+    tokens.push(new Token(Type.bytes, { length: rootLength + 1 }))
+  }
+  const length = tokensToLength(tokens) // no options needed here because we have simple tokens
+  return varint.encodingLength(length) + length
+}
+
+/**
+ * Calculates header size given the array of roots.
+ *
+ * @name CarBufferWriter.headerLength({ roots })
+ * @param {object} options
+ * @param {CID[]} options.roots
+ * @returns {number}
+ */
+export const headerLength = ({ roots }) =>
+  calculateHeaderLength(roots.map(cid => cid.bytes.byteLength))
+
+/**
+ * Estimates header size given a count of the roots and the expected byte length
+ * of the root CIDs. The default length works for a standard CIDv1 with a
+ * single-byte multihash code, such as SHA2-256 (i.e. the most common CIDv1).
+ *
+ * @name CarBufferWriter.estimateHeaderLength(rootCount[, rootByteLength])
+ * @param {number} rootCount
+ * @param {number} [rootByteLength]
+ * @returns {number}
+ */
+export const estimateHeaderLength = (rootCount, rootByteLength = 36) =>
+  calculateHeaderLength(new Array(rootCount).fill(rootByteLength))
+
+/**
+ * Creates synchronous CAR writer that can be used to encode blocks into a given
+ * buffer. Optionally you could pass `byteOffset` and `byteLength` to specify a
+ * range inside buffer to write into. If car file is going to have `roots` you
+ * need to either pass them under `options.roots` (from which header size will
+ * be calculated) or provide `options.headerSize` to allocate required space
+ * in the buffer. You may also provide known `roots` and `headerSize` to
+ * allocate space for the roots that may not be known ahead of time.
+ *
+ * Note: Incorrect `headerSize` may lead to copying bytes inside a buffer
+ * which will have a negative impact on performance.
+ *
+ * @name CarBufferWriter.createWriter(buffer[, options])
+ * @param {ArrayBuffer} buffer
+ * @param {object} [options]
+ * @param {CID[]} [options.roots]
+ * @param {number} [options.byteOffset]
+ * @param {number} [options.byteLength]
+ * @param {number} [options.headerSize]
+ * @returns {CarBufferWriter}
+ */
+export const createWriter = (
+  buffer,
+  {
+    roots = [],
+    byteOffset = 0,
+    byteLength = buffer.byteLength,
+    headerSize = headerLength({ roots })
+  } = {}
+) => {
+  const bytes = new Uint8Array(buffer, byteOffset, byteLength)
+
+  const writer = new CarBufferWriter(bytes, headerSize)
+  for (const root of roots) {
+    writer.addRoot(root)
+  }
+
+  return writer
+}

package/lib/coding.ts

@@ -20,10 +20,25 @@ export interface IteratorChannel<T> {
   iterator: AsyncIterator<T>
 }
 
-export type CarHeader = { version: number, roots: CID[] }
+export interface CarHeader {
+  version: 1,
+  roots: CID[]
+}
+
+export interface CarV2FixedHeader {
+  characteristics: [bigint, bigint],
+  dataOffset: number,
+  dataSize: number,
+  indexOffset: number
+}
+
+export interface CarV2Header extends CarV2FixedHeader {
+  version: 2,
+  roots: CID[],
+}
 
 export interface CarDecoder {
-  header(): Promise<CarHeader>
+  header(): Promise<CarHeader|CarV2Header>
 
   blocks(): AsyncGenerator<Block>
 

package/lib/decoder.js

@@ -2,6 +2,7 @@ import varint from 'varint'
 import { CID } from 'multiformats/cid'
 import * as Digest from 'multiformats/hashes/digest'
 import { decode as decodeDagCbor } from '@ipld/dag-cbor'
+import { CarHeader as headerValidator } from './header-validator.js'
 
 /**
  * @typedef {import('../api').Block} Block
@@ -9,6 +10,8 @@ import { decode as decodeDagCbor } from '@ipld/dag-cbor'
  * @typedef {import('../api').BlockIndex} BlockIndex
  * @typedef {import('./coding').BytesReader} BytesReader
  * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
+ * @typedef {import('./coding').CarV2FixedHeader} CarV2FixedHeader
  * @typedef {import('./coding').CarDecoder} CarDecoder
  */
 
@@ -18,12 +21,17 @@ const CIDV0_BYTES = {
   DAG_PB: 0x70
 }
 
+const V2_HEADER_LENGTH = /* characteristics */ 16 /* v1 offset */ + 8 /* v1 size */ + 8 /* index offset */ + 8
+
 /**
  * @param {BytesReader} reader
  * @returns {Promise<number>}
  */
 async function readVarint (reader) {
   const bytes = await reader.upTo(8)
+  if (!bytes.length) {
+    throw new Error('Unexpected end of data')
+  }
   const i = varint.decode(bytes)
   reader.seek(varint.decode.bytes)
   return i
@@ -33,9 +41,40 @@ async function readVarint (reader) {
 
 /**
  * @param {BytesReader} reader
- * @returns {Promise<CarHeader>}
+ * @returns {Promise<CarV2FixedHeader>}
+ */
+async function readV2Header (reader) {
+  /** @type {Uint8Array} */
+  const bytes = await reader.exactly(V2_HEADER_LENGTH)
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  let offset = 0
+  const header = {
+    version: 2,
+    /** @type {[bigint, bigint]} */
+    characteristics: [
+      dv.getBigUint64(offset, true),
+      dv.getBigUint64(offset += 8, true)
+    ],
+    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
+    dataSize: Number(dv.getBigUint64(offset += 8, true)),
+    indexOffset: Number(dv.getBigUint64(offset += 8, true))
+  }
+  reader.seek(V2_HEADER_LENGTH)
+  return header
+  /* c8 ignore next 2 */
+  // Node.js 12 c8 bug
+}
+
+/**
+ * Reads header data from a `BytesReader`. The header may either be in the form
+ * of a `CarHeader` or `CarV2Header` depending on the CAR being read.
+ *
+ * @name async decoder.readHeader(reader)
+ * @param {BytesReader} reader
+ * @param {number} [strictVersion]
+ * @returns {Promise<CarHeader|CarV2Header>}
  */
-export async function readHeader (reader) {
+export async function readHeader (reader, strictVersion) {
   const length = await readVarint(reader)
   if (length === 0) {
     throw new Error('Invalid CAR header (zero length)')
@@ -43,22 +82,26 @@ export async function readHeader (reader) {
   const header = await reader.exactly(length)
   reader.seek(length)
   const block = decodeDagCbor(header)
-  if (block == null || Array.isArray(block) || typeof block !== 'object') {
+  if (!headerValidator(block)) {
     throw new Error('Invalid CAR header format')
   }
-  if (block.version !== 1) {
-    if (typeof block.version === 'string') {
-      throw new Error(`Invalid CAR version: "${block.version}"`)
-    }
-    throw new Error(`Invalid CAR version: ${block.version}`)
+  if ((block.version !== 1 && block.version !== 2) || (strictVersion !== undefined && block.version !== strictVersion)) {
+    throw new Error(`Invalid CAR version: ${block.version}${strictVersion !== undefined ? ` (expected ${strictVersion})` : ''}`)
   }
-  if (!Array.isArray(block.roots)) {
+  // we've made 'roots' optional in the schema so we can do the version check
+  // before rejecting the block as invalid if there is no version
+  const hasRoots = Array.isArray(block.roots)
+  if ((block.version === 1 && !hasRoots) || (block.version === 2 && hasRoots)) {
     throw new Error('Invalid CAR header format')
   }
-  if (Object.keys(block).filter((p) => p !== 'roots' && p !== 'version').length) {
-    throw new Error('Invalid CAR header format')
+  if (block.version === 1) {
+    return block
   }
-  return block
+  // version 2
+  const v2Header = await readV2Header(reader)
+  reader.seek(v2Header.dataOffset - reader.pos)
+  const v1Header = await readHeader(reader, 1)
+  return Object.assign(v1Header, v2Header)
   /* c8 ignore next 2 */
   // Node.js 12 c8 bug
 }
@@ -112,6 +155,12 @@ async function readCid (reader) {
 }
 
 /**
+ * Reads the leading data of an individual block from CAR data from a
+ * `BytesReader`. Returns a `BlockHeader` object which contains
+ * `{ cid, length, blockLength }` which can be used to either index the block
+ * or read the block binary data.
+ *
+ * @name async decoder.readBlockHead(reader)
  * @param {BytesReader} reader
  * @returns {Promise<BlockHeader>}
  */
@@ -125,7 +174,7 @@ export async function readBlockHead (reader) {
   }
   length += (reader.pos - start)
   const cid = await readCid(reader)
-  const blockLength = length - (reader.pos - start) // subtract CID length
+  const blockLength = length - Number(reader.pos - start) // subtract CID length
 
   return { cid, length, blockLength }
   /* c8 ignore next 2 */
@@ -160,11 +209,25 @@ async function readBlockIndex (reader) {
 }
 
 /**
+ * Creates a `CarDecoder` from a `BytesReader`. The `CarDecoder` is as async
+ * interface that will consume the bytes from the `BytesReader` to yield a
+ * `header()` and either `blocks()` or `blocksIndex()` data.
+ *
+ * @name decoder.createDecoder(reader)
  * @param {BytesReader} reader
  * @returns {CarDecoder}
  */
 export function createDecoder (reader) {
-  const headerPromise = readHeader(reader)
+  const headerPromise = (async () => {
+    const header = await readHeader(reader)
+    if (header.version === 2) {
+      const v1length = reader.pos - header.dataOffset
+      reader = limitReader(reader, header.dataSize - v1length)
+    }
+    return header
+    /* c8 ignore next 2 */
+    // Node.js 12 c8 bug
+  })()
 
   return {
     header: () => headerPromise,
@@ -186,6 +249,9 @@ export function createDecoder (reader) {
 }
 
 /**
+ * Creates a `BytesReader` from a `Uint8Array`.
+ *
+ * @name decoder.bytesReader(bytes)
  * @param {Uint8Array} bytes
  * @returns {BytesReader}
  */
@@ -298,6 +364,10 @@ export function chunkReader (readChunk /*, closer */) {
 }
 
 /**
+ * Creates a `BytesReader` from an `AsyncIterable<Uint8Array>`, which allows for
+ * consumption of CAR data from a streaming source.
+ *
+ * @name decoder.asyncIterableReader(asyncIterable)
  * @param {AsyncIterable<Uint8Array>} asyncIterable
  * @returns {BytesReader}
  */
@@ -316,3 +386,49 @@ export function asyncIterableReader (asyncIterable) {
 
   return chunkReader(readChunk)
 }
+
+/**
+ * Wraps a `BytesReader` in a limiting `BytesReader` which limits maximum read
+ * to `byteLimit` bytes. It _does not_ update `pos` of the original
+ * `BytesReader`.
+ *
+ * @name decoder.limitReader(reader, byteLimit)
+ * @param {BytesReader} reader
+ * @param {number} byteLimit
+ * @returns {BytesReader}
+ */
+export function limitReader (reader, byteLimit) {
+  let bytesRead = 0
+
+  /** @type {BytesReader} */
+  return {
+    async upTo (length) {
+      let bytes = await reader.upTo(length)
+      if (bytes.length + bytesRead > byteLimit) {
+        bytes = bytes.subarray(0, byteLimit - bytesRead)
+      }
+      return bytes
+      /* c8 ignore next 2 */
+      // Node.js 12 c8 bug
+    },
+
+    async exactly (length) {
+      const bytes = await reader.exactly(length)
+      if (bytes.length + bytesRead > byteLimit) {
+        throw new Error('Unexpected end of data')
+      }
+      return bytes
+      /* c8 ignore next 2 */
+      // Node.js 12 c8 bug
+    },
+
+    seek (length) {
+      bytesRead += length
+      reader.seek(length)
+    },
+
+    get pos () {
+      return reader.pos
+    }
+  }
+}

package/lib/header-validator.js

@@ -0,0 +1,33 @@
+/** Auto-generated with ipld-schema-validator@0.0.0-dev at Thu Jun 17 2021 from IPLD Schema:
+ *
+ * type CarHeader struct {
+ *   version Int
+ *   roots optional [&Any]
+ *   # roots is _not_ optional for CarV1 but we defer that check within code to
+ *   # gracefully handle the >V1 case where it's just {version:X}
+ * }
+ *
+ */
+
+const Kinds = {
+  Null: /** @returns {boolean} */ (/** @type {any} */ obj) => obj === null,
+  Int: /** @returns {boolean} */ (/** @type {any} */ obj) => Number.isInteger(obj),
+  Float: /** @returns {boolean} */ (/** @type {any} */ obj) => typeof obj === 'number' && Number.isFinite(obj),
+  String: /** @returns {boolean} */ (/** @type {any} */ obj) => typeof obj === 'string',
+  Bool: /** @returns {boolean} */ (/** @type {any} */ obj) => typeof obj === 'boolean',
+  Bytes: /** @returns {boolean} */ (/** @type {any} */ obj) => obj instanceof Uint8Array,
+  Link: /** @returns {boolean} */ (/** @type {any} */ obj) => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
+  List: /** @returns {boolean} */ (/** @type {any} */ obj) => Array.isArray(obj),
+  Map: /** @returns {boolean} */ (/** @type {any} */ obj) => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
+}
+/** @type {{ [k in string]: (obj:any)=>boolean}} */
+const Types = {
+  Int: Kinds.Int,
+  'CarHeader > version': /** @returns {boolean} */ (/** @type {any} */ obj) => Types.Int(obj),
+  'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
+  'CarHeader > roots (anon)': /** @returns {boolean} */ (/** @type {any} */ obj) => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
+  'CarHeader > roots': /** @returns {boolean} */ (/** @type {any} */ obj) => Types['CarHeader > roots (anon)'](obj),
+  CarHeader: /** @returns {boolean} */ (/** @type {any} */ obj) => { const keys = obj && Object.keys(obj); return Kinds.Map(obj) && ['version'].every((k) => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value)) }
+}
+
+export const CarHeader = Types.CarHeader

package/lib/header.ipldsch

@@ -0,0 +1,6 @@
+type CarHeader struct {
+  version Int
+  roots optional [&Any]
+  # roots is _not_ optional for CarV1 but we defer that check within code to
+  # gracefully handle the >V1 case where it's just {version:X}
+}

package/lib/reader-browser.js

@@ -5,6 +5,8 @@ import { asyncIterableReader, bytesReader, createDecoder } from './decoder.js'
  * @typedef {import('../api').Block} Block
  * @typedef {import('../api').CarReader} CarReaderIface
  * @typedef {import('./coding').BytesReader} BytesReader
+ * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
  */
 
 /**
@@ -26,18 +28,16 @@ import { asyncIterableReader, bytesReader, createDecoder } from './decoder.js'
  * @class
  * @implements {CarReaderIface}
  * @property {number} version The version number of the CAR referenced by this
- * reader (should be `1`).
+ * reader (should be `1` or `2`).
  */
 export class CarReader {
   /**
    * @constructs CarReader
-   * @param {number} version
-   * @param {CID[]} roots
+   * @param {CarHeader|CarV2Header} header
    * @param {Block[]} blocks
    */
-  constructor (version, roots, blocks) {
-    this._version = version
-    this._roots = roots
+  constructor (header, blocks) {
+    this._header = header
     this._blocks = blocks
     this._keys = blocks.map((b) => b.cid.toString())
   }
@@ -48,7 +48,7 @@ export class CarReader {
    * @instance
    */
   get version () {
-    return this._version
+    return this._header.version
   }
 
   /**
@@ -62,7 +62,7 @@ export class CarReader {
    * @returns {Promise<CID[]>}
    */
   async getRoots () {
-    return this._roots
+    return this._header.roots
     /* c8 ignore next 2 */
     // Node.js 12 c8 bug
   }
@@ -190,15 +190,15 @@ export class CarReader {
  * @param {BytesReader} reader
  * @returns {Promise<CarReader>}
  */
-async function decodeReaderComplete (reader) {
+export async function decodeReaderComplete (reader) {
   const decoder = createDecoder(reader)
-  const { version, roots } = await decoder.header()
+  const header = await decoder.header()
   const blocks = []
   for await (const block of decoder.blocks()) {
     blocks.push(block)
   }
 
-  return new CarReader(version, roots, blocks)
+  return new CarReader(header, blocks)
   /* c8 ignore next 2 */
   // Node.js 12 c8 bug
 }

package/lib/writer-browser.js

@@ -176,7 +176,7 @@ export class CarWriter {
     const reader = bytesReader(bytes)
     await readHeader(reader)
     const newHeader = createHeader(roots)
-    if (reader.pos !== newHeader.length) {
+    if (Number(reader.pos) !== newHeader.length) {
       throw new Error(`updateRoots() can only overwrite a header of the same length (old header is ${reader.pos} bytes, new header is ${newHeader.length} bytes)`)
     }
     bytes.set(newHeader, 0)