@ipld/car 5.0.3 → 5.1.1

package/src/buffer-reader-browser.js

@@ -0,0 +1,150 @@
+import * as BufferDecoder from './buffer-decoder.js'
+
+/**
+ * @typedef {import('multiformats').CID} CID
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').CarBufferReader} ICarBufferReader
+ * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
+ */
+
+/**
+ * Provides blockstore-like access to a CAR.
+ *
+ * Implements the `RootsBufferReader` interface:
+ * {@link ICarBufferReader.getRoots `getRoots()`}. And the `BlockBufferReader` interface:
+ * {@link ICarBufferReader.get `get()`}, {@link ICarBufferReader.has `has()`},
+ * {@link ICarBufferReader.blocks `blocks()`} and
+ * {@link ICarBufferReader.cids `cids()`}.
+ *
+ * Load this class with either `import { CarBufferReader } from '@ipld/car/buffer-reader'`
+ * (`const { CarBufferReader } = require('@ipld/car/buffer-reader')`). Or
+ * `import { CarBufferReader } from '@ipld/car'` (`const { CarBufferReader } = require('@ipld/car')`).
+ * The former will likely result in smaller bundle sizes where this is
+ * important.
+ *
+ * @name CarBufferReader
+ * @class
+ * @implements {ICarBufferReader}
+ * @property {number} version The version number of the CAR referenced by this
+ * reader (should be `1` or `2`).
+ */
+export class CarBufferReader {
+  /**
+   * @constructs CarBufferReader
+   * @param {CarHeader|CarV2Header} header
+   * @param {Block[]} blocks
+   */
+  constructor (header, blocks) {
+    this._header = header
+    this._blocks = blocks
+    this._cids = undefined
+  }
+
+  /**
+   * @property version
+   * @memberof CarBufferReader
+   * @instance
+   */
+  get version () {
+    return this._header.version
+  }
+
+  /**
+   * Get the list of roots defined by the CAR referenced by this reader. May be
+   * zero or more `CID`s.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @returns {CID[]}
+   */
+  getRoots () {
+    return this._header.roots
+    /* c8 ignore next 2 */
+    // Node.js 12 c8 bug
+  }
+
+  /**
+   * Check whether a given `CID` exists within the CAR referenced by this
+   * reader.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @param {CID} key
+   * @returns {boolean}
+   */
+  has (key) {
+    return this._blocks.some(b => b.cid.equals(key))
+    /* c8 ignore next 2 */
+    // Node.js 12 c8 bug
+  }
+
+  /**
+   * Fetch a `Block` (a `{ cid:CID, bytes:Uint8Array }` pair) from the CAR
+   * referenced by this reader matching the provided `CID`. In the case where
+   * the provided `CID` doesn't exist within the CAR, `undefined` will be
+   * returned.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @param {CID} key
+   * @returns {Block | undefined}
+   */
+  get (key) {
+    return this._blocks.find(b => b.cid.equals(key))
+    /* c8 ignore next 2 */
+    // Node.js 12 c8 bug
+  }
+
+  /**
+   * Returns a `Block[]` of the `Block`s (`{ cid:CID, bytes:Uint8Array }` pairs) contained within
+   * the CAR referenced by this reader.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @returns {Block[]}
+   */
+  blocks () {
+    return this._blocks
+  }
+
+  /**
+   * Returns a `CID[]` of the `CID`s contained within the CAR referenced by this reader.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @returns {CID[]}
+   */
+  cids () {
+    if (!this._cids) {
+      this._cids = this._blocks.map(b => b.cid)
+    }
+    return this._cids
+  }
+
+  /**
+   * Instantiate a {@link CarBufferReader} from a `Uint8Array` blob. This performs a
+   * decode fully in memory and maintains the decoded state in memory for full
+   * access to the data via the `CarReader` API.
+   *
+   * @static
+   * @memberof CarBufferReader
+   * @param {Uint8Array} bytes
+   * @returns {CarBufferReader}
+   */
+  static fromBytes (bytes) {
+    if (!(bytes instanceof Uint8Array)) {
+      throw new TypeError('fromBytes() requires a Uint8Array')
+    }
+
+    const { header, blocks } = BufferDecoder.fromBytes(bytes)
+    return new CarBufferReader(header, blocks)
+  }
+}
+
+export const __browser = true

package/src/buffer-reader.js

@@ -0,0 +1,53 @@
+import fs from 'fs'
+import { CarBufferReader as BrowserCarBufferReader } from './buffer-reader-browser.js'
+
+/**
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').BlockIndex} BlockIndex
+ * @typedef {import('./api').CarBufferReader} ICarBufferReader
+ */
+
+const fsread = fs.readSync
+
+/**
+ * @class
+ * @implements {ICarBufferReader}
+ */
+export class CarBufferReader extends BrowserCarBufferReader {
+  /**
+   * Reads a block directly from a file descriptor for an open CAR file. This
+   * function is **only available in Node.js** and not a browser environment.
+   *
+   * This function can be used in connection with {@link CarIndexer} which emits
+   * the `BlockIndex` objects that are required by this function.
+   *
+   * The user is responsible for opening and closing the file used in this call.
+   *
+   * @static
+   * @memberof CarBufferReader
+   * @param {number} fd - A file descriptor from the
+   * Node.js `fs` module. An integer, from `fs.open()`.
+   * @param {BlockIndex} blockIndex - An index pointing to the location of the
+   * Block required. This `BlockIndex` should take the form:
+   * `{cid:CID, blockLength:number, blockOffset:number}`.
+   * @returns {Block} A `{ cid:CID, bytes:Uint8Array }` pair.
+   */
+  static readRaw (fd, blockIndex) {
+    const { cid, blockLength, blockOffset } = blockIndex
+    const bytes = new Uint8Array(blockLength)
+    let read
+    if (typeof fd === 'number') {
+      read = fsread(fd, bytes, 0, blockLength, blockOffset)
+    } else {
+      throw new TypeError('Bad fd')
+    }
+    if (read !== blockLength) {
+      throw new Error(`Failed to read entire block (${read} instead of ${blockLength})`)
+    }
+    return { cid, bytes }
+    /* c8 ignore next 2 */
+    // Node.js 12 c8 bug
+  }
+}
+
+export const __browser = false

package/src/coding.ts

@@ -45,12 +45,22 @@ export interface CarDecoder {
   blocksIndex: () => AsyncGenerator<BlockIndex>
 }
 
-export interface BytesReader {
+export interface Seekable {
+  seek: (length: number) => void
+}
+
+export interface BytesReader extends Seekable {
   upTo: (length: number) => Promise<Uint8Array>
 
-  exactly: (length: number) => Promise<Uint8Array>
+  exactly: (length: number, seek?: boolean) => Promise<Uint8Array>
 
-  seek: (length: number) => void
+  pos: number
+}
+
+export interface BytesBufferReader extends Seekable{
+  upTo: (length: number) => Uint8Array
+
+  exactly: (length: number, seek?: boolean) => Uint8Array
 
   pos: number
 }

package/src/decoder-common.js

@@ -0,0 +1,86 @@
+import varint from 'varint'
+
+export const CIDV0_BYTES = {
+  SHA2_256: 0x12,
+  LENGTH: 0x20,
+  DAG_PB: 0x70
+}
+
+export const V2_HEADER_LENGTH = /* characteristics */ 16 /* v1 offset */ + 8 /* v1 size */ + 8 /* index offset */ + 8
+
+/**
+ * Decodes varint and seeks the buffer
+ *
+ * ```js
+ * // needs bytes to be read first
+ * const bytes = reader.upTo(8) // maybe async
+ * ```
+ *
+ * @param {Uint8Array} bytes
+ * @param {import('./coding').Seekable} seeker
+ * @returns {number}
+ */
+export function decodeVarint (bytes, seeker) {
+  if (!bytes.length) {
+    throw new Error('Unexpected end of data')
+  }
+  const i = varint.decode(bytes)
+  seeker.seek(/** @type {number} */(varint.decode.bytes))
+  return i
+  /* c8 ignore next 2 */
+  // Node.js 12 c8 bug
+}
+
+/**
+ * Decode v2 header
+ *
+ * ```js
+ * // needs bytes to be read first
+ * const bytes = reader.exactly(V2_HEADER_LENGTH, true) // maybe async
+ * ```
+ *
+ * @param {Uint8Array} bytes
+ * @returns {import('./coding').CarV2FixedHeader}
+ */
+export function decodeV2Header (bytes) {
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  let offset = 0
+  const header = {
+    version: 2,
+    /** @type {[bigint, bigint]} */
+    characteristics: [
+      dv.getBigUint64(offset, true),
+      dv.getBigUint64(offset += 8, true)
+    ],
+    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
+    dataSize: Number(dv.getBigUint64(offset += 8, true)),
+    indexOffset: Number(dv.getBigUint64(offset += 8, true))
+  }
+  return header
+  /* c8 ignore next 2 */
+  // Node.js 12 c8 bug
+}
+
+/**
+ * Checks the length of the multihash to be read afterwards
+ *
+ * ```js
+ * // needs bytes to be read first
+ * const bytes = reader.upTo(8) // maybe async
+ * ```
+ *
+ * @param {Uint8Array} bytes
+ */
+export function getMultihashLength (bytes) {
+  // | code | length | .... |
+  // where both code and length are varints, so we have to decode
+  // them first before we can know total length
+
+  varint.decode(bytes) // code
+  const codeLength = /** @type {number} */(varint.decode.bytes)
+  const length = varint.decode(bytes.subarray(varint.decode.bytes))
+  const lengthLength = /** @type {number} */(varint.decode.bytes)
+  const mhLength = codeLength + lengthLength + length
+
+  return mhLength
+}

package/src/decoder.js

@@ -1,8 +1,8 @@
-import varint from 'varint'
 import { CID } from 'multiformats/cid'
 import * as Digest from 'multiformats/hashes/digest'
 import { decode as decodeDagCbor } from '@ipld/dag-cbor'
 import { CarHeader as headerValidator } from './header-validator.js'
+import { CIDV0_BYTES, decodeV2Header, decodeVarint, getMultihashLength, V2_HEADER_LENGTH } from './decoder-common.js'
 
 /**
  * @typedef {import('./api').Block} Block
@@ -15,56 +15,6 @@ import { CarHeader as headerValidator } from './header-validator.js'
  * @typedef {import('./coding').CarDecoder} CarDecoder
  */
 
-const CIDV0_BYTES = {
-  SHA2_256: 0x12,
-  LENGTH: 0x20,
-  DAG_PB: 0x70
-}
-
-const V2_HEADER_LENGTH = /* characteristics */ 16 /* v1 offset */ + 8 /* v1 size */ + 8 /* index offset */ + 8
-
-/**
- * @param {BytesReader} reader
- * @returns {Promise<number>}
- */
-async function readVarint (reader) {
-  const bytes = await reader.upTo(8)
-  if (!bytes.length) {
-    throw new Error('Unexpected end of data')
-  }
-  const i = varint.decode(bytes)
-  reader.seek(/** @type {number} */(varint.decode.bytes))
-  return i
-  /* c8 ignore next 2 */
-  // Node.js 12 c8 bug
-}
-
-/**
- * @param {BytesReader} reader
- * @returns {Promise<CarV2FixedHeader>}
- */
-async function readV2Header (reader) {
-  /** @type {Uint8Array} */
-  const bytes = await reader.exactly(V2_HEADER_LENGTH)
-  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
-  let offset = 0
-  const header = {
-    version: 2,
-    /** @type {[bigint, bigint]} */
-    characteristics: [
-      dv.getBigUint64(offset, true),
-      dv.getBigUint64(offset += 8, true)
-    ],
-    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
-    dataSize: Number(dv.getBigUint64(offset += 8, true)),
-    indexOffset: Number(dv.getBigUint64(offset += 8, true))
-  }
-  reader.seek(V2_HEADER_LENGTH)
-  return header
-  /* c8 ignore next 2 */
-  // Node.js 12 c8 bug
-}
-
 /**
  * Reads header data from a `BytesReader`. The header may either be in the form
  * of a `CarHeader` or `CarV2Header` depending on the CAR being read.
@@ -75,12 +25,11 @@ async function readV2Header (reader) {
  * @returns {Promise<CarHeader|CarV2Header>}
  */
 export async function readHeader (reader, strictVersion) {
-  const length = await readVarint(reader)
+  const length = decodeVarint(await reader.upTo(8), reader)
   if (length === 0) {
     throw new Error('Invalid CAR header (zero length)')
   }
-  const header = await reader.exactly(length)
-  reader.seek(length)
+  const header = await reader.exactly(length, true)
   const block = decodeDagCbor(header)
   if (!headerValidator(block)) {
     throw new Error('Invalid CAR header format')
@@ -98,7 +47,7 @@ export async function readHeader (reader, strictVersion) {
     return block
   }
   // version 2
-  const v2Header = await readV2Header(reader)
+  const v2Header = decodeV2Header(await reader.exactly(V2_HEADER_LENGTH, true))
   reader.seek(v2Header.dataOffset - reader.pos)
   const v1Header = await readHeader(reader, 1)
   return Object.assign(v1Header, v2Header)
@@ -106,48 +55,25 @@ export async function readHeader (reader, strictVersion) {
   // Node.js 12 c8 bug
 }
 
-/**
- * @param {BytesReader} reader
- * @returns {Promise<Uint8Array>}
- */
-async function readMultihash (reader) {
-  // | code | length | .... |
-  // where both code and length are varints, so we have to decode
-  // them first before we can know total length
-
-  const bytes = await reader.upTo(8)
-  varint.decode(bytes) // code
-  const codeLength = /** @type {number} */(varint.decode.bytes)
-  const length = varint.decode(bytes.subarray(varint.decode.bytes))
-  const lengthLength = /** @type {number} */(varint.decode.bytes)
-  const mhLength = codeLength + lengthLength + length
-  const multihash = await reader.exactly(mhLength)
-  reader.seek(mhLength)
-  return multihash
-  /* c8 ignore next 2 */
-  // Node.js 12 c8 bug
-}
-
 /**
  * @param {BytesReader} reader
  * @returns {Promise<CID>}
  */
 async function readCid (reader) {
-  const first = await reader.exactly(2)
+  const first = await reader.exactly(2, false)
   if (first[0] === CIDV0_BYTES.SHA2_256 && first[1] === CIDV0_BYTES.LENGTH) {
     // cidv0 32-byte sha2-256
-    const bytes = await reader.exactly(34)
-    reader.seek(34)
+    const bytes = await reader.exactly(34, true)
     const multihash = Digest.decode(bytes)
     return CID.create(0, CIDV0_BYTES.DAG_PB, multihash)
   }
 
-  const version = await readVarint(reader)
+  const version = decodeVarint(await reader.upTo(8), reader)
   if (version !== 1) {
     throw new Error(`Unexpected CID version (${version})`)
   }
-  const codec = await readVarint(reader)
-  const bytes = await readMultihash(reader)
+  const codec = decodeVarint(await reader.upTo(8), reader)
+  const bytes = await reader.exactly(getMultihashLength(await reader.upTo(8)), true)
   const multihash = Digest.decode(bytes)
   return CID.create(version, codec, multihash)
   /* c8 ignore next 2 */
@@ -168,7 +94,7 @@ export async function readBlockHead (reader) {
   // length includes a CID + Binary, where CID has a variable length
   // we have to deal with
   const start = reader.pos
-  let length = await readVarint(reader)
+  let length = decodeVarint(await reader.upTo(8), reader)
   if (length === 0) {
     throw new Error('Invalid CAR section (zero length)')
   }
@@ -187,8 +113,7 @@ export async function readBlockHead (reader) {
  */
 async function readBlock (reader) {
   const { cid, blockLength } = await readBlockHead(reader)
-  const bytes = await reader.exactly(blockLength)
-  reader.seek(blockLength)
+  const bytes = await reader.exactly(blockLength, true)
   return { bytes, cid }
   /* c8 ignore next 2 */
   // Node.js 12 c8 bug
@@ -261,16 +186,21 @@ export function bytesReader (bytes) {
   /** @type {BytesReader} */
   return {
     async upTo (length) {
-      return bytes.subarray(pos, pos + Math.min(length, bytes.length - pos))
+      const out = bytes.subarray(pos, pos + Math.min(length, bytes.length - pos))
       /* c8 ignore next 2 */
+      return out
       // Node.js 12 c8 bug
     },
 
-    async exactly (length) {
+    async exactly (length, seek = false) {
       if (length > bytes.length - pos) {
         throw new Error('Unexpected end of data')
       }
-      return bytes.subarray(pos, pos + length)
+      const out = bytes.subarray(pos, pos + length)
+      if (seek) {
+        pos += length
+      }
+      return out
       /* c8 ignore next 2 */
       // Node.js 12 c8 bug
     },
@@ -340,14 +270,19 @@ export function chunkReader (readChunk /*, closer */) {
       // Node.js 12 c8 bug
     },
 
-    async exactly (length) {
+    async exactly (length, seek = false) {
       if (currentChunk.length - offset < length) {
         await read(length)
       }
       if (currentChunk.length - offset < length) {
         throw new Error('Unexpected end of data')
       }
-      return currentChunk.subarray(offset, offset + length)
+      const out = currentChunk.subarray(offset, offset + length)
+      if (seek) {
+        pos += length
+        offset += length
+      }
+      return out
       /* c8 ignore next 2 */
       // Node.js 12 c8 bug
     },
@@ -412,11 +347,14 @@ export function limitReader (reader, byteLimit) {
       // Node.js 12 c8 bug
     },
 
-    async exactly (length) {
-      const bytes = await reader.exactly(length)
+    async exactly (length, seek = false) {
+      const bytes = await reader.exactly(length, seek)
       if (bytes.length + bytesRead > byteLimit) {
         throw new Error('Unexpected end of data')
       }
+      if (seek) {
+        bytesRead += length
+      }
       return bytes
       /* c8 ignore next 2 */
       // Node.js 12 c8 bug

package/src/index-browser.js

@@ -3,6 +3,8 @@ import { CarIndexer } from './indexer.js'
 import { CarBlockIterator, CarCIDIterator } from './iterator.js'
 import { CarWriter } from './writer-browser.js'
 import { CarIndexedReader } from './indexed-reader-browser.js'
+import { CarBufferReader } from './buffer-reader.js'
+import * as CarBufferWriter from './buffer-writer.js'
 
 export {
   CarReader,
@@ -10,5 +12,7 @@ export {
   CarBlockIterator,
   CarCIDIterator,
   CarWriter,
-  CarIndexedReader
+  CarIndexedReader,
+  CarBufferReader,
+  CarBufferWriter
 }

package/src/index.js

@@ -1,4 +1,5 @@
 import { CarReader } from './reader.js'
+import { CarBufferReader } from './buffer-reader.js'
 import { CarIndexer } from './indexer.js'
 import { CarBlockIterator, CarCIDIterator } from './iterator.js'
 import { CarWriter } from './writer.js'
@@ -7,6 +8,7 @@ import * as CarBufferWriter from './buffer-writer.js'
 
 export {
   CarReader,
+  CarBufferReader,
   CarIndexer,
   CarBlockIterator,
   CarCIDIterator,