@fireproof/vendor 0.0.0-smoke

package/src/@ipld/car/car/src/api.ts

@@ -0,0 +1,90 @@
+import type { CID } from 'multiformats/cid'
+
+/**
+ * Literally any `Iterable` (async or regular).
+ */
+export type AwaitIterable<T> = Iterable<T> | AsyncIterable<T>
+
+export type { CID }
+/* Generic types for interfacing with block storage */
+
+export interface Block {
+  cid: CID
+  bytes: Uint8Array
+}
+
+export interface BlockHeader {
+  cid: CID
+  length: number
+  blockLength: number
+}
+
+export interface BlockIndex extends BlockHeader {
+  offset: number
+  blockOffset: number
+}
+
+export interface RootsReader {
+  version: number
+  getRoots(): Promise<CID[]>
+}
+
+export interface RootsBufferReader {
+  version: number
+  getRoots(): CID[]
+}
+
+export interface BlockIterator extends AsyncIterable<Block> {}
+
+export interface CIDIterator extends AsyncIterable<CID> {}
+
+export interface BlockReader {
+  has(key: CID): Promise<boolean>
+  get(key: CID): Promise<Block | undefined>
+  blocks(): BlockIterator
+  cids(): CIDIterator
+}
+
+export interface BlockBufferReader {
+  has(key: CID): boolean
+  get(key: CID): Block | undefined
+  blocks(): Iterable<Block>
+  cids(): Iterable<CID>
+}
+
+export interface BlockWriter {
+  put(block: Block): Promise<void>
+  close(): Promise<void>
+  version(): number
+}
+
+export interface CarBufferWriter {
+  addRoot(root: CID, options?: { resize?: boolean }): CarBufferWriter
+  write(block: Block): CarBufferWriter
+  close(options?: { resize?: boolean }): Uint8Array
+}
+
+export interface CarBufferWriterOptions {
+  roots?: CID[] // defaults to []
+  byteOffset?: number // defaults to 0
+  byteLength?: number // defaults to buffer.byteLength
+
+  headerSize?: number // defaults to size needed for provided roots
+}
+
+export interface WriterChannel {
+  writer: BlockWriter
+  out: AsyncIterable<Uint8Array>
+}
+
+export interface CarReader extends BlockReader, RootsReader {}
+export interface CarBufferReader extends BlockBufferReader, RootsBufferReader {}
+
+/* Specific implementations for CAR block storage */
+
+/*
+export interface CarBlockIterator extends BlockIterator, RootsReader {}
+export interface CarCIDIterator extends CIDIterator, RootsReader {}
+export interface CarIndexer extends AsyncIterable<BlockIndex>, RootsReader {}
+export interface CarWriter extends BlockWriter {}
+*/

package/src/@ipld/car/car/src/buffer-decoder.js

@@ -0,0 +1,213 @@
+import { decode as decodeDagCbor } from "@fireproof/vendor/@ipld/dag-cbor"
+import { CID } from 'multiformats/cid'
+import * as Digest from 'multiformats/hashes/digest'
+import { CIDV0_BYTES, decodeV2Header, decodeVarint, getMultihashLength, V2_HEADER_LENGTH } from './decoder-common.js'
+import { CarV1HeaderOrV2Pragma } from './header-validator.js'
+
+/**
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').BlockHeader} BlockHeader
+ * @typedef {import('./api').BlockIndex} BlockIndex
+ * @typedef {import('./coding').BytesBufferReader} BytesBufferReader
+ * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
+ * @typedef {import('./coding').CarV2FixedHeader} CarV2FixedHeader
+ */
+
+/**
+ * Reads header data from a `BytesReader`. The header may either be in the form
+ * of a `CarHeader` or `CarV2Header` depending on the CAR being read.
+ *
+ * @name decoder.readHeader(reader)
+ * @param {BytesBufferReader} reader
+ * @param {number} [strictVersion]
+ * @returns {CarHeader | CarV2Header}
+ */
+export function readHeader (reader, strictVersion) {
+  const length = decodeVarint(reader.upTo(8), reader)
+  if (length === 0) {
+    throw new Error('Invalid CAR header (zero length)')
+  }
+  const header = reader.exactly(length, true)
+  const block = decodeDagCbor(header)
+  if (CarV1HeaderOrV2Pragma.toTyped(block) === undefined) {
+    throw new Error('Invalid CAR header format')
+  }
+  if ((block.version !== 1 && block.version !== 2) || (strictVersion !== undefined && block.version !== strictVersion)) {
+    throw new Error(`Invalid CAR version: ${block.version}${strictVersion !== undefined ? ` (expected ${strictVersion})` : ''}`)
+  }
+  if (block.version === 1) {
+    // CarV1HeaderOrV2Pragma makes roots optional, let's make it mandatory
+    if (!Array.isArray(block.roots)) {
+      throw new Error('Invalid CAR header format')
+    }
+    return block
+  }
+  // version 2
+  if (block.roots !== undefined) {
+    throw new Error('Invalid CAR header format')
+  }
+  const v2Header = decodeV2Header(reader.exactly(V2_HEADER_LENGTH, true))
+  reader.seek(v2Header.dataOffset - reader.pos)
+  const v1Header = readHeader(reader, 1)
+  return Object.assign(v1Header, v2Header)
+}
+
+/**
+ * Reads CID sync
+ *
+ * @param {BytesBufferReader} reader
+ * @returns {CID}
+ */
+function readCid (reader) {
+  const first = reader.exactly(2, false)
+  if (first[0] === CIDV0_BYTES.SHA2_256 && first[1] === CIDV0_BYTES.LENGTH) {
+    // cidv0 32-byte sha2-256
+    const bytes = reader.exactly(34, true)
+    const multihash = Digest.decode(bytes)
+    return CID.create(0, CIDV0_BYTES.DAG_PB, multihash)
+  }
+
+  const version = decodeVarint(reader.upTo(8), reader)
+  if (version !== 1) {
+    throw new Error(`Unexpected CID version (${version})`)
+  }
+  const codec = decodeVarint(reader.upTo(8), reader)
+  const bytes = reader.exactly(getMultihashLength(reader.upTo(8)), true)
+  const multihash = Digest.decode(bytes)
+  return CID.create(version, codec, multihash)
+}
+
+/**
+ * Reads the leading data of an individual block from CAR data from a
+ * `BytesBufferReader`. Returns a `BlockHeader` object which contains
+ * `{ cid, length, blockLength }` which can be used to either index the block
+ * or read the block binary data.
+ *
+ * @name async decoder.readBlockHead(reader)
+ * @param {BytesBufferReader} reader
+ * @returns {BlockHeader}
+ */
+export function readBlockHead (reader) {
+  // length includes a CID + Binary, where CID has a variable length
+  // we have to deal with
+  const start = reader.pos
+  let length = decodeVarint(reader.upTo(8), reader)
+  if (length === 0) {
+    throw new Error('Invalid CAR section (zero length)')
+  }
+  length += (reader.pos - start)
+  const cid = readCid(reader)
+  const blockLength = length - Number(reader.pos - start) // subtract CID length
+
+  return { cid, length, blockLength }
+}
+
+/**
+ * Returns Car header and blocks from a Uint8Array
+ *
+ * @param {Uint8Array} bytes
+ * @returns {{ header : CarHeader | CarV2Header , blocks: Block[]}}
+ */
+export function fromBytes (bytes) {
+  let reader = bytesReader(bytes)
+  const header = readHeader(reader)
+  if (header.version === 2) {
+    const v1length = reader.pos - header.dataOffset
+    reader = limitReader(reader, header.dataSize - v1length)
+  }
+
+  const blocks = []
+  while (reader.upTo(8).length > 0) {
+    const { cid, blockLength } = readBlockHead(reader)
+
+    blocks.push({ cid, bytes: reader.exactly(blockLength, true) })
+  }
+
+  return {
+    header, blocks
+  }
+}
+
+/**
+ * Creates a `BytesBufferReader` from a `Uint8Array`.
+ *
+ * @name decoder.bytesReader(bytes)
+ * @param {Uint8Array} bytes
+ * @returns {BytesBufferReader}
+ */
+export function bytesReader (bytes) {
+  let pos = 0
+
+  /** @type {BytesBufferReader} */
+  return {
+    upTo (length) {
+      return bytes.subarray(pos, pos + Math.min(length, bytes.length - pos))
+    },
+
+    exactly (length, seek = false) {
+      if (length > bytes.length - pos) {
+        throw new Error('Unexpected end of data')
+      }
+
+      const out = bytes.subarray(pos, pos + length)
+      if (seek) {
+        pos += length
+      }
+      return out
+    },
+
+    seek (length) {
+      pos += length
+    },
+
+    get pos () {
+      return pos
+    }
+  }
+}
+
+/**
+ * Wraps a `BytesBufferReader` in a limiting `BytesBufferReader` which limits maximum read
+ * to `byteLimit` bytes. It _does not_ update `pos` of the original
+ * `BytesBufferReader`.
+ *
+ * @name decoder.limitReader(reader, byteLimit)
+ * @param {BytesBufferReader} reader
+ * @param {number} byteLimit
+ * @returns {BytesBufferReader}
+ */
+export function limitReader (reader, byteLimit) {
+  let bytesRead = 0
+
+  /** @type {BytesBufferReader} */
+  return {
+    upTo (length) {
+      let bytes = reader.upTo(length)
+      if (bytes.length + bytesRead > byteLimit) {
+        bytes = bytes.subarray(0, byteLimit - bytesRead)
+      }
+      return bytes
+    },
+
+    exactly (length, seek = false) {
+      const bytes = reader.exactly(length, seek)
+      if (bytes.length + bytesRead > byteLimit) {
+        throw new Error('Unexpected end of data')
+      }
+      if (seek) {
+        bytesRead += length
+      }
+      return bytes
+    },
+
+    seek (length) {
+      bytesRead += length
+      reader.seek(length)
+    },
+
+    get pos () {
+      return reader.pos
+    }
+  }
+}

package/src/@ipld/car/car/src/buffer-reader-browser.js

@@ -0,0 +1,144 @@
+import * as BufferDecoder from './buffer-decoder.js'
+
+/**
+ * @typedef {import('multiformats').CID} CID
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').CarBufferReader} ICarBufferReader
+ * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
+ */
+
+/**
+ * Provides blockstore-like access to a CAR.
+ *
+ * Implements the `RootsBufferReader` interface:
+ * {@link ICarBufferReader.getRoots `getRoots()`}. And the `BlockBufferReader` interface:
+ * {@link ICarBufferReader.get `get()`}, {@link ICarBufferReader.has `has()`},
+ * {@link ICarBufferReader.blocks `blocks()`} and
+ * {@link ICarBufferReader.cids `cids()`}.
+ *
+ * Load this class with either `import { CarBufferReader } from '@ipld/car/buffer-reader'`
+ * (`const { CarBufferReader } = require('@ipld/car/buffer-reader')`). Or
+ * `import { CarBufferReader } from '@ipld/car'` (`const { CarBufferReader } = require('@ipld/car')`).
+ * The former will likely result in smaller bundle sizes where this is
+ * important.
+ *
+ * @name CarBufferReader
+ * @class
+ * @implements {ICarBufferReader}
+ * @property {number} version The version number of the CAR referenced by this
+ * reader (should be `1` or `2`).
+ */
+export class CarBufferReader {
+  /**
+   * @constructs CarBufferReader
+   * @param {CarHeader|CarV2Header} header
+   * @param {Block[]} blocks
+   */
+  constructor (header, blocks) {
+    this._header = header
+    this._blocks = blocks
+    this._cids = undefined
+  }
+
+  /**
+   * @property version
+   * @memberof CarBufferReader
+   * @instance
+   */
+  get version () {
+    return this._header.version
+  }
+
+  /**
+   * Get the list of roots defined by the CAR referenced by this reader. May be
+   * zero or more `CID`s.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @returns {CID[]}
+   */
+  getRoots () {
+    return this._header.roots
+  }
+
+  /**
+   * Check whether a given `CID` exists within the CAR referenced by this
+   * reader.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @param {CID} key
+   * @returns {boolean}
+   */
+  has (key) {
+    return this._blocks.some(b => b.cid.equals(key))
+  }
+
+  /**
+   * Fetch a `Block` (a `{ cid:CID, bytes:Uint8Array }` pair) from the CAR
+   * referenced by this reader matching the provided `CID`. In the case where
+   * the provided `CID` doesn't exist within the CAR, `undefined` will be
+   * returned.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @param {CID} key
+   * @returns {Block | undefined}
+   */
+  get (key) {
+    return this._blocks.find(b => b.cid.equals(key))
+  }
+
+  /**
+   * Returns a `Block[]` of the `Block`s (`{ cid:CID, bytes:Uint8Array }` pairs) contained within
+   * the CAR referenced by this reader.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @returns {Block[]}
+   */
+  blocks () {
+    return this._blocks
+  }
+
+  /**
+   * Returns a `CID[]` of the `CID`s contained within the CAR referenced by this reader.
+   *
+   * @function
+   * @memberof CarBufferReader
+   * @instance
+   * @returns {CID[]}
+   */
+  cids () {
+    if (!this._cids) {
+      this._cids = this._blocks.map(b => b.cid)
+    }
+    return this._cids
+  }
+
+  /**
+   * Instantiate a {@link CarBufferReader} from a `Uint8Array` blob. This performs a
+   * decode fully in memory and maintains the decoded state in memory for full
+   * access to the data via the `CarReader` API.
+   *
+   * @static
+   * @memberof CarBufferReader
+   * @param {Uint8Array} bytes
+   * @returns {CarBufferReader}
+   */
+  static fromBytes (bytes) {
+    if (!(bytes instanceof Uint8Array)) {
+      throw new TypeError('fromBytes() requires a Uint8Array')
+    }
+
+    const { header, blocks } = BufferDecoder.fromBytes(bytes)
+    return new CarBufferReader(header, blocks)
+  }
+}
+
+export const __browser = true

package/src/@ipld/car/car/src/buffer-reader.js

@@ -0,0 +1,51 @@
+import fs from 'fs'
+import { CarBufferReader as BrowserCarBufferReader } from './buffer-reader-browser.js'
+
+/**
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').BlockIndex} BlockIndex
+ * @typedef {import('./api').CarBufferReader} ICarBufferReader
+ */
+
+const fsread = fs.readSync
+
+/**
+ * @class
+ * @implements {ICarBufferReader}
+ */
+export class CarBufferReader extends BrowserCarBufferReader {
+  /**
+   * Reads a block directly from a file descriptor for an open CAR file. This
+   * function is **only available in Node.js** and not a browser environment.
+   *
+   * This function can be used in connection with {@link CarIndexer} which emits
+   * the `BlockIndex` objects that are required by this function.
+   *
+   * The user is responsible for opening and closing the file used in this call.
+   *
+   * @static
+   * @memberof CarBufferReader
+   * @param {number} fd - A file descriptor from the
+   * Node.js `fs` module. An integer, from `fs.open()`.
+   * @param {BlockIndex} blockIndex - An index pointing to the location of the
+   * Block required. This `BlockIndex` should take the form:
+   * `{cid:CID, blockLength:number, blockOffset:number}`.
+   * @returns {Block} A `{ cid:CID, bytes:Uint8Array }` pair.
+   */
+  static readRaw (fd, blockIndex) {
+    const { cid, blockLength, blockOffset } = blockIndex
+    const bytes = new Uint8Array(blockLength)
+    let read
+    if (typeof fd === 'number') {
+      read = fsread(fd, bytes, 0, blockLength, blockOffset)
+    } else {
+      throw new TypeError('Bad fd')
+    }
+    if (read !== blockLength) {
+      throw new Error(`Failed to read entire block (${read} instead of ${blockLength})`)
+    }
+    return { cid, bytes }
+  }
+}
+
+export const __browser = false