@fireproof/vendor 1.1.0 → 1.2.0

package/src/@ipld/car/src/iterator-channel.js

@@ -0,0 +1,91 @@
+/**
+ * @template {any} T
+ * @typedef {import('./coding').IteratorChannel<T>} IteratorChannel
+ */
+
+function noop () {}
+
+/**
+ * @template {any} T
+ * @returns {IteratorChannel<T>}
+ */
+export function create () {
+  /** @type {T[]} */
+  const chunkQueue = []
+  /** @type {Promise<void> | null} */
+  let drainer = null
+  let drainerResolver = noop
+  let ended = false
+  /** @type {Promise<IteratorResult<T>> | null} */
+  let outWait = null
+  let outWaitResolver = noop
+
+  const makeDrainer = () => {
+    if (!drainer) {
+      drainer = new Promise((resolve) => {
+        drainerResolver = () => {
+          drainer = null
+          drainerResolver = noop
+          resolve()
+        }
+      })
+    }
+    return drainer
+  }
+
+  /**
+   * @returns {IteratorChannel<T>}
+   */
+  const writer = {
+    /**
+     * @param {T} chunk
+     * @returns {Promise<void>}
+     */
+    write (chunk) {
+      chunkQueue.push(chunk)
+      const drainer = makeDrainer()
+      outWaitResolver()
+      return drainer
+    },
+
+    async end () {
+      ended = true
+      const drainer = makeDrainer()
+      outWaitResolver()
+      await drainer
+    }
+  }
+
+  /** @type {AsyncIterator<T>} */
+  const iterator = {
+    /** @returns {Promise<IteratorResult<T>>} */
+    async next () {
+      const chunk = chunkQueue.shift()
+      if (chunk) {
+        if (chunkQueue.length === 0) {
+          drainerResolver()
+        }
+        return { done: false, value: chunk }
+      }
+
+      if (ended) {
+        drainerResolver()
+        return { done: true, value: undefined }
+      }
+
+      if (!outWait) {
+        outWait = new Promise((resolve) => {
+          outWaitResolver = () => {
+            outWait = null
+            outWaitResolver = noop
+            return resolve(iterator.next())
+          }
+        })
+      }
+
+      return outWait
+    }
+  }
+
+  return { writer, iterator }
+}

package/src/@ipld/car/src/iterator.js

@@ -0,0 +1,267 @@
+import {
+  asyncIterableReader,
+  bytesReader,
+  createDecoder
+} from './decoder.js'
+
+/**
+ * @typedef {import('multiformats').CID} CID
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').RootsReader} RootsReader
+ * @typedef {import('./coding').BytesReader} BytesReader
+ */
+
+/**
+ * @class
+ * @implements {RootsReader}
+ * @property {number} version The version number of the CAR referenced by this reader (should be `1`).
+ */
+export class CarIteratorBase {
+  /**
+   * @param {number} version
+   * @param {CID[]} roots
+   * @param {AsyncIterable<Block>|void} iterable
+   */
+  constructor (version, roots, iterable) {
+    this._version = version
+    this._roots = roots
+    this._iterable = iterable
+    this._decoded = false
+  }
+
+  get version () {
+    return this._version
+  }
+
+  /**
+   * @returns {Promise<CID[]>}
+   */
+  async getRoots () {
+    return this._roots
+  }
+}
+
+/**
+ * Provides an iterator over all of the `Block`s in a CAR. Implements a
+ * `BlockIterator` interface, or `AsyncIterable<Block>`. Where a `Block` is
+ * a `{ cid:CID, bytes:Uint8Array }` pair.
+ *
+ * As an implementer of `AsyncIterable`, this class can be used directly in a
+ * `for await (const block of iterator) {}` loop. Where the `iterator` is
+ * constructed using {@link CarBlockiterator.fromBytes} or
+ * {@link CarBlockiterator.fromIterable}.
+ *
+ * An iteration can only be performce _once_ per instantiation.
+ *
+ * `CarBlockIterator` also implements the `RootsReader` interface and provides
+ * the {@link CarBlockiterator.getRoots `getRoots()`} method.
+ *
+ * Load this class with either
+ * `import { CarBlockIterator } from '@ipld/car/iterator'`
+ * (`const { CarBlockIterator } = require('@ipld/car/iterator')`). Or
+ * `import { CarBlockIterator } from '@ipld/car'`
+ * (`const { CarBlockIterator } = require('@ipld/car')`).
+ *
+ * @name CarBlockIterator
+ * @class
+ * @implements {RootsReader}
+ * @implements {AsyncIterable<Block>}
+ * @property {number} version The version number of the CAR referenced by this
+ * iterator (should be `1`).
+ */
+export class CarBlockIterator extends CarIteratorBase {
+  // inherited method
+  /**
+   * Get the list of roots defined by the CAR referenced by this iterator. May be
+   * zero or more `CID`s.
+   *
+   * @function getRoots
+   * @memberof CarBlockIterator
+   * @instance
+   * @async
+   * @returns {Promise<CID[]>}
+   */
+
+  /**
+   * @returns {AsyncIterator<Block>}
+   */
+  [Symbol.asyncIterator] () {
+    if (this._decoded) {
+      throw new Error('Cannot decode more than once')
+    }
+    /* c8 ignore next 3 */
+    if (!this._iterable) {
+      throw new Error('Block iterable not found')
+    }
+    this._decoded = true
+    return this._iterable[Symbol.asyncIterator]()
+  }
+
+  /**
+   * Instantiate a {@link CarBlockIterator} from a `Uint8Array` blob. Rather
+   * than decoding the entire byte array prior to returning the iterator, as in
+   * {@link CarReader.fromBytes}, only the header is decoded and the remainder
+   * of the CAR is parsed as the `Block`s as yielded.
+   *
+   * @async
+   * @static
+   * @memberof CarBlockIterator
+   * @param {Uint8Array} bytes
+   * @returns {Promise<CarBlockIterator>}
+   */
+  static async fromBytes (bytes) {
+    const { version, roots, iterator } = await fromBytes(bytes)
+    return new CarBlockIterator(version, roots, iterator)
+  }
+
+  /**
+   * Instantiate a {@link CarBlockIterator} from a `AsyncIterable<Uint8Array>`,
+   * such as a [modern Node.js stream](https://nodejs.org/api/stream.html#stream_streams_compatibility_with_async_generators_and_async_iterators).
+   * Rather than decoding the entire byte array prior to returning the iterator,
+   * as in {@link CarReader.fromIterable}, only the header is decoded and the
+   * remainder of the CAR is parsed as the `Block`s as yielded.
+   *
+   * @async
+   * @static
+   * @param {AsyncIterable<Uint8Array>} asyncIterable
+   * @returns {Promise<CarBlockIterator>}
+   */
+  static async fromIterable (asyncIterable) {
+    const { version, roots, iterator } = await fromIterable(asyncIterable)
+    return new CarBlockIterator(version, roots, iterator)
+  }
+}
+
+/**
+ * Provides an iterator over all of the `CID`s in a CAR. Implements a
+ * `CIDIterator` interface, or `AsyncIterable<CID>`. Similar to
+ * {@link CarBlockIterator} but only yields the CIDs in the CAR.
+ *
+ * As an implementer of `AsyncIterable`, this class can be used directly in a
+ * `for await (const cid of iterator) {}` loop. Where the `iterator` is
+ * constructed using {@link CarCIDiterator.fromBytes} or
+ * {@link CarCIDiterator.fromIterable}.
+ *
+ * An iteration can only be performce _once_ per instantiation.
+ *
+ * `CarCIDIterator` also implements the `RootsReader` interface and provides
+ * the {@link CarCIDiterator.getRoots `getRoots()`} method.
+ *
+ * Load this class with either
+ * `import { CarCIDIterator } from '@ipld/car/iterator'`
+ * (`const { CarCIDIterator } = require('@ipld/car/iterator')`). Or
+ * `import { CarCIDIterator } from '@ipld/car'`
+ * (`const { CarCIDIterator } = require('@ipld/car')`).
+ *
+ * @name CarCIDIterator
+ * @class
+ * @implements {RootsReader}
+ * @implements {AsyncIterable<CID>}
+ * @property {number} version The version number of the CAR referenced by this
+ * iterator (should be `1`).
+ */
+export class CarCIDIterator extends CarIteratorBase {
+  // inherited method
+  /**
+   * Get the list of roots defined by the CAR referenced by this iterator. May be
+   * zero or more `CID`s.
+   *
+   * @function getRoots
+   * @memberof CarCIDIterator
+   * @instance
+   * @async
+   * @returns {Promise<CID[]>}
+   */
+
+  /**
+   * @returns {AsyncIterator<CID>}
+   */
+  [Symbol.asyncIterator] () {
+    if (this._decoded) {
+      throw new Error('Cannot decode more than once')
+    }
+    /* c8 ignore next 3 */
+    if (!this._iterable) {
+      throw new Error('Block iterable not found')
+    }
+    this._decoded = true
+    const iterable = this._iterable[Symbol.asyncIterator]()
+    return {
+      async next () {
+        const next = await iterable.next()
+        if (next.done) {
+          return next
+        }
+        return { done: false, value: next.value.cid }
+      }
+    }
+  }
+
+  /**
+   * Instantiate a {@link CarCIDIterator} from a `Uint8Array` blob. Rather
+   * than decoding the entire byte array prior to returning the iterator, as in
+   * {@link CarReader.fromBytes}, only the header is decoded and the remainder
+   * of the CAR is parsed as the `CID`s as yielded.
+   *
+   * @async
+   * @static
+   * @memberof CarCIDIterator
+   * @param {Uint8Array} bytes
+   * @returns {Promise<CarCIDIterator>}
+   */
+  static async fromBytes (bytes) {
+    const { version, roots, iterator } = await fromBytes(bytes)
+    return new CarCIDIterator(version, roots, iterator)
+  }
+
+  /**
+   * Instantiate a {@link CarCIDIterator} from a `AsyncIterable<Uint8Array>`,
+   * such as a [modern Node.js stream](https://nodejs.org/api/stream.html#stream_streams_compatibility_with_async_generators_and_async_iterators).
+   * Rather than decoding the entire byte array prior to returning the iterator,
+   * as in {@link CarReader.fromIterable}, only the header is decoded and the
+   * remainder of the CAR is parsed as the `CID`s as yielded.
+   *
+   * @async
+   * @static
+   * @memberof CarCIDIterator
+   * @param {AsyncIterable<Uint8Array>} asyncIterable
+   * @returns {Promise<CarCIDIterator>}
+   */
+  static async fromIterable (asyncIterable) {
+    const { version, roots, iterator } = await fromIterable(asyncIterable)
+    return new CarCIDIterator(version, roots, iterator)
+  }
+}
+
+/**
+ * @param {Uint8Array} bytes
+ * @returns {Promise<{ version:number, roots:CID[], iterator:AsyncIterable<Block>}>}
+ */
+async function fromBytes (bytes) {
+  if (!(bytes instanceof Uint8Array)) {
+    throw new TypeError('fromBytes() requires a Uint8Array')
+  }
+  return decodeIterator(bytesReader(bytes))
+}
+
+/**
+ * @param {AsyncIterable<Uint8Array>} asyncIterable
+ * @returns {Promise<{ version:number, roots:CID[], iterator:AsyncIterable<Block>}>}
+ */
+async function fromIterable (asyncIterable) {
+  if (!asyncIterable || !(typeof asyncIterable[Symbol.asyncIterator] === 'function')) {
+    throw new TypeError('fromIterable() requires an async iterable')
+  }
+  return decodeIterator(asyncIterableReader(asyncIterable))
+}
+
+/**
+ * @private
+ * @param {BytesReader} reader
+ * @returns {Promise<{ version:number, roots:CID[], iterator:AsyncIterable<Block>}>}
+ */
+async function decodeIterator (reader) {
+  const decoder = createDecoder(reader)
+  const { version, roots } = await decoder.header()
+  return { version, roots, iterator: decoder.blocks() }
+}

package/src/@ipld/car/src/promise-fs-opts.js

@@ -0,0 +1,46 @@
+import fs from 'fs'
+import { promisify } from 'util'
+
+const hasFS = Boolean(fs)
+
+export { hasFS }
+
+/**
+ * @type {any}
+ */
+let _fsReadFn
+/**
+ * @description This function is needed not to initialize the `fs.read` on load time. To run in cf workers without polyfill.
+ * @param {number} fd
+ * @param {Uint8Array} buffer
+ * @param {number} offset
+ * @param {number} length
+ * @param {number} position
+ * @returns {Promise<{ bytesRead: number, buffer: Uint8Array }>}
+ */
+export function fsread (fd, buffer, offset, length, position) {
+  if (!_fsReadFn) {
+    _fsReadFn = promisify(fs.read)
+  }
+  return _fsReadFn(fd, buffer, offset, length, position)
+}
+
+/**
+ * @type {any}
+ */
+let _fsWriteFn
+/**
+ * @description This function is needed not to initialize the `fs.write` on load time. To run in cf workers without polyfill.
+ * @param {number} fd
+ * @param {Uint8Array} buffer
+ * @param {number} offset
+ * @param {number} length
+ * @param {number} position
+ * @returns {Promise<{ bytesRead: number, buffer: Uint8Array }>}
+ */
+export function fswrite (fd, buffer, offset, length, position) {
+  if (!_fsWriteFn) {
+    _fsWriteFn = promisify(fs.write)
+  }
+  return _fsWriteFn(fd, buffer, offset, length, position)
+}

package/src/@ipld/car/src/reader-browser.js

@@ -0,0 +1,194 @@
+import { asyncIterableReader, bytesReader, createDecoder } from './decoder.js'
+
+/**
+ * @typedef {import('multiformats').CID} CID
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').CarReader} CarReaderIface
+ * @typedef {import('./coding').BytesReader} BytesReader
+ * @typedef {import('./coding').CarHeader} CarHeader
+ * @typedef {import('./coding').CarV2Header} CarV2Header
+ */
+
+/**
+ * Provides blockstore-like access to a CAR.
+ *
+ * Implements the `RootsReader` interface:
+ * {@link CarReader.getRoots `getRoots()`}. And the `BlockReader` interface:
+ * {@link CarReader.get `get()`}, {@link CarReader.has `has()`},
+ * {@link CarReader.blocks `blocks()`} (defined as a `BlockIterator`) and
+ * {@link CarReader.cids `cids()`} (defined as a `CIDIterator`).
+ *
+ * Load this class with either `import { CarReader } from '@ipld/car/reader'`
+ * (`const { CarReader } = require('@ipld/car/reader')`). Or
+ * `import { CarReader } from '@ipld/car'` (`const { CarReader } = require('@ipld/car')`).
+ * The former will likely result in smaller bundle sizes where this is
+ * important.
+ *
+ * @name CarReader
+ * @class
+ * @implements {CarReaderIface}
+ * @property {number} version The version number of the CAR referenced by this
+ * reader (should be `1` or `2`).
+ */
+export class CarReader {
+  /**
+   * @constructs CarReader
+   * @param {CarHeader|CarV2Header} header
+   * @param {Block[]} blocks
+   */
+  constructor (header, blocks) {
+    this._header = header
+    this._blocks = blocks
+    this._keys = blocks.map((b) => b.cid.toString())
+  }
+
+  /**
+   * @property
+   * @memberof CarReader
+   * @instance
+   */
+  get version () {
+    return this._header.version
+  }
+
+  /**
+   * Get the list of roots defined by the CAR referenced by this reader. May be
+   * zero or more `CID`s.
+   *
+   * @function
+   * @memberof CarReader
+   * @instance
+   * @async
+   * @returns {Promise<CID[]>}
+   */
+  async getRoots () {
+    return this._header.roots
+  }
+
+  /**
+   * Check whether a given `CID` exists within the CAR referenced by this
+   * reader.
+   *
+   * @function
+   * @memberof CarReader
+   * @instance
+   * @async
+   * @param {CID} key
+   * @returns {Promise<boolean>}
+   */
+  async has (key) {
+    return this._keys.indexOf(key.toString()) > -1
+  }
+
+  /**
+   * Fetch a `Block` (a `{ cid:CID, bytes:Uint8Array }` pair) from the CAR
+   * referenced by this reader matching the provided `CID`. In the case where
+   * the provided `CID` doesn't exist within the CAR, `undefined` will be
+   * returned.
+   *
+   * @function
+   * @memberof CarReader
+   * @instance
+   * @async
+   * @param {CID} key
+   * @returns {Promise<Block | undefined>}
+   */
+  async get (key) {
+    const index = this._keys.indexOf(key.toString())
+    return index > -1 ? this._blocks[index] : undefined
+  }
+
+  /**
+   * Returns a `BlockIterator` (`AsyncIterable<Block>`) that iterates over all
+   * of the `Block`s (`{ cid:CID, bytes:Uint8Array }` pairs) contained within
+   * the CAR referenced by this reader.
+   *
+   * @function
+   * @memberof CarReader
+   * @instance
+   * @async
+   * @generator
+   * @returns {AsyncGenerator<Block>}
+   */
+  async * blocks () {
+    for (const block of this._blocks) {
+      yield block
+    }
+  }
+
+  /**
+   * Returns a `CIDIterator` (`AsyncIterable<CID>`) that iterates over all of
+   * the `CID`s contained within the CAR referenced by this reader.
+   *
+   * @function
+   * @memberof CarReader
+   * @instance
+   * @async
+   * @generator
+   * @returns {AsyncGenerator<CID>}
+   */
+  async * cids () {
+    for (const block of this._blocks) {
+      yield block.cid
+    }
+  }
+
+  /**
+   * Instantiate a {@link CarReader} from a `Uint8Array` blob. This performs a
+   * decode fully in memory and maintains the decoded state in memory for full
+   * access to the data via the `CarReader` API.
+   *
+   * @async
+   * @static
+   * @memberof CarReader
+   * @param {Uint8Array} bytes
+   * @returns {Promise<CarReader>}
+   */
+  static async fromBytes (bytes) {
+    if (!(bytes instanceof Uint8Array)) {
+      throw new TypeError('fromBytes() requires a Uint8Array')
+    }
+    return decodeReaderComplete(bytesReader(bytes))
+  }
+
+  /**
+   * Instantiate a {@link CarReader} from a `AsyncIterable<Uint8Array>`, such as
+   * a [modern Node.js stream](https://nodejs.org/api/stream.html#stream_streams_compatibility_with_async_generators_and_async_iterators).
+   * This performs a decode fully in memory and maintains the decoded state in
+   * memory for full access to the data via the `CarReader` API.
+   *
+   * Care should be taken for large archives; this API may not be appropriate
+   * where memory is a concern or the archive is potentially larger than the
+   * amount of memory that the runtime can handle.
+   *
+   * @async
+   * @static
+   * @memberof CarReader
+   * @param {AsyncIterable<Uint8Array>} asyncIterable
+   * @returns {Promise<CarReader>}
+   */
+  static async fromIterable (asyncIterable) {
+    if (!asyncIterable || !(typeof asyncIterable[Symbol.asyncIterator] === 'function')) {
+      throw new TypeError('fromIterable() requires an async iterable')
+    }
+    return decodeReaderComplete(asyncIterableReader(asyncIterable))
+  }
+}
+
+/**
+ * @private
+ * @param {BytesReader} reader
+ * @returns {Promise<CarReader>}
+ */
+export async function decodeReaderComplete (reader) {
+  const decoder = createDecoder(reader)
+  const header = await decoder.header()
+  const blocks = []
+  for await (const block of decoder.blocks()) {
+    blocks.push(block)
+  }
+
+  return new CarReader(header, blocks)
+}
+
+export const __browser = true

package/src/@ipld/car/src/reader.js

@@ -0,0 +1,54 @@
+import { fsread, hasFS } from './promise-fs-opts.js'
+import { CarReader as BrowserCarReader } from './reader-browser.js'
+
+/**
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').BlockIndex} BlockIndex
+ * @typedef {import('./api').CarReader} CarReaderIface
+ * @typedef {import('fs').promises.FileHandle} FileHandle
+ */
+
+/**
+ * @class
+ * @implements {CarReaderIface}
+ */
+export class CarReader extends BrowserCarReader {
+  /**
+   * Reads a block directly from a file descriptor for an open CAR file. This
+   * function is **only available in Node.js** and not a browser environment.
+   *
+   * This function can be used in connection with {@link CarIndexer} which emits
+   * the `BlockIndex` objects that are required by this function.
+   *
+   * The user is responsible for opening and closing the file used in this call.
+   *
+   * @async
+   * @static
+   * @memberof CarReader
+   * @param {FileHandle | number} fd - A file descriptor from the
+   * Node.js `fs` module. Either an integer, from `fs.open()` or a `FileHandle`
+   * from `fs.promises.open()`.
+   * @param {BlockIndex} blockIndex - An index pointing to the location of the
+   * Block required. This `BlockIndex` should take the form:
+   * `{cid:CID, blockLength:number, blockOffset:number}`.
+   * @returns {Promise<Block>} A `{ cid:CID, bytes:Uint8Array }` pair.
+   */
+  static async readRaw (fd, blockIndex) {
+    const { cid, blockLength, blockOffset } = blockIndex
+    const bytes = new Uint8Array(blockLength)
+    let read
+    if (typeof fd === 'number') {
+      read = (await fsread(fd, bytes, 0, blockLength, blockOffset)).bytesRead
+    } else if (typeof fd === 'object' && typeof fd.read === 'function') { // FileDescriptor
+      read = (await fd.read(bytes, 0, blockLength, blockOffset)).bytesRead
+    } else {
+      throw new TypeError('Bad fd')
+    }
+    if (read !== blockLength) {
+      throw new Error(`Failed to read entire block (${read} instead of ${blockLength})`)
+    }
+    return { cid, bytes }
+  }
+}
+
+export const __browser = !hasFS