@fireproof/vendor 0.0.0-smoke → 0.22.0-dev-preview

package/src/@ipld/car/car/src/indexed-reader.js

@@ -1,211 +0,0 @@
-import fs from 'fs'
-import { Readable } from 'stream'
-import { CID } from 'multiformats/cid'
-import { CarIndexer } from './indexer.js'
-import { CarReader as NodeCarReader } from './reader.js'
-
-/**
- * @typedef {import('fs').promises.FileHandle} FileHandle
- * @typedef {import('./api').Block} Block
- * @typedef {import('./api').BlockIndex} BlockIndex
- * @typedef {import('./api').CarReader} CarReaderIface
- * @typedef {import('./reader-browser').CarReader} CarReader
- * @typedef {{ blockLength:number, blockOffset:number }} RawLocation
- */
-
-/**
- * A form of {@link CarReader} that pre-indexes a CAR archive from a file and
- * provides random access to blocks within the file using the index data. This
- * function is **only available in Node.js** and not a browser environment.
- *
- * For large CAR files, using this form of `CarReader` can be singificantly more
- * efficient in terms of memory. The index consists of a list of `CID`s and
- * their location within the archive (see {@link CarIndexer}). For large numbers
- * of blocks, this index can also occupy a significant amount of memory. In some
- * cases it may be necessary to expand the memory capacity of a Node.js instance
- * to allow this index to fit. (e.g. by running with
- * `NODE_OPTIONS="--max-old-space-size=16384"`).
- *
- * As an `CarIndexedReader` instance maintains an open file descriptor for its
- * CAR file, an additional {@link CarReader#close} method is attached. This
- * _must_ be called to have full clean-up of resources after use.
- *
- * Load this class with either
- * `import { CarIndexedReader } from '@ipld/car/indexed-reader'`
- * (`const { CarIndexedReader } = require('@ipld/car/indexed-reader')`). Or
- * `import { CarIndexedReader } from '@ipld/car'`
- * (`const { CarIndexedReader } = require('@ipld/car')`). The former will likely
- * result in smaller bundle sizes where this is important.
- *
- * @name CarIndexedReader
- * @class
- * @implements {CarReaderIface}
- * @extends {CarReader}
- * @property {number} version The version number of the CAR referenced by this
- * reader (should be `1`).
- */
-export class CarIndexedReader {
-  /**
-   * @param {number} version
-   * @param {string} path
-   * @param {CID[]} roots
-   * @param {Map<string, RawLocation>} index
-   * @param {string[]} order
-   */
-  constructor (version, path, roots, index, order) {
-    this._version = version
-    this._path = path
-    this._roots = roots
-    this._index = index
-    this._order = order
-    this._fd = null
-  }
-
-  get version () {
-    return this._version
-  }
-
-  /**
-   * See {@link CarReader#getRoots}
-   *
-   * @function
-   * @memberof CarIndexedReader
-   * @instance
-   * @async
-   * @returns {Promise<CID[]>}
-   */
-  async getRoots () {
-    return this._roots
-  }
-
-  /**
-   * See {@link CarReader#has}
-   *
-   * @function
-   * @memberof CarIndexedReader
-   * @instance
-   * @async
-   * @param {CID} key
-   * @returns {Promise<boolean>}
-   */
-  async has (key) {
-    return this._index.has(key.toString())
-  }
-
-  /**
-   * See {@link CarReader#get}
-   *
-   * @function
-   * @memberof CarIndexedReader
-   * @instance
-   * @async
-   * @param {CID} key
-   * @returns {Promise<Block | undefined>}
-   */
-  async get (key) {
-    const blockIndex = this._index.get(key.toString())
-    if (!blockIndex) {
-      return undefined
-    }
-    if (!this._fd) {
-      this._fd = await fs.promises.open(this._path, 'r')
-    }
-    const readIndex = {
-      cid: key,
-      length: 0,
-      offset: 0,
-      blockLength: blockIndex.blockLength,
-      blockOffset: blockIndex.blockOffset
-    }
-    return NodeCarReader.readRaw(this._fd, readIndex)
-  }
-
-  /**
-   * See {@link CarReader#blocks}
-   *
-   * @function
-   * @memberof CarIndexedReader
-   * @instance
-   * @async
-   * @generator
-   * @returns {AsyncGenerator<Block>}
-   */
-  async * blocks () {
-    for (const cidStr of this._order) {
-      const block = await this.get(CID.parse(cidStr))
-      /* c8 ignore next 3 */
-      if (!block) {
-        throw new Error('Unexpected internal error')
-      }
-      yield block
-    }
-  }
-
-  /**
-   * See {@link CarReader#cids}
-   *
-   * @function
-   * @memberof CarIndexedReader
-   * @instance
-   * @async
-   * @generator
-   * @returns {AsyncGenerator<CID>}
-   */
-  async * cids () {
-    for (const cidStr of this._order) {
-      yield CID.parse(cidStr)
-    }
-  }
-
-  /**
-   * Close the underlying file descriptor maintained by this `CarIndexedReader`.
-   * This must be called for proper resource clean-up to occur.
-   *
-   * @function
-   * @memberof CarIndexedReader
-   * @instance
-   * @async
-   * @returns {Promise<void>}
-   */
-  async close () {
-    if (this._fd) {
-      return this._fd.close()
-    }
-  }
-
-  /**
-   * Instantiate an {@link CarIndexedReader} from a file with the provided
-   * `path`. The CAR file is first indexed with a full path that collects `CID`s
-   * and block locations. This index is maintained in memory. Subsequent reads
-   * operate on a read-only file descriptor, fetching the block from its in-file
-   * location.
-   *
-   * For large archives, the initial indexing may take some time. The returned
-   * `Promise` will resolve only after this is complete.
-   *
-   * @async
-   * @static
-   * @memberof CarIndexedReader
-   * @param {string} path
-   * @returns {Promise<CarIndexedReader>}
-   */
-  static async fromFile (path) {
-    if (typeof path !== 'string') {
-      throw new TypeError('fromFile() requires a file path string')
-    }
-
-    const iterable = await CarIndexer.fromIterable(Readable.from(fs.createReadStream(path)))
-    /** @type {Map<string, RawLocation>} */
-    const index = new Map()
-    /** @type {string[]} */
-    const order = []
-    for await (const { cid, blockLength, blockOffset } of iterable) {
-      const cidStr = cid.toString()
-      index.set(cidStr, { blockLength, blockOffset })
-      order.push(cidStr)
-    }
-    return new CarIndexedReader(iterable.version, path, await iterable.getRoots(), index, order)
-  }
-}
-
-export const __browser = false

package/src/@ipld/car/car/src/indexer.js

@@ -1,130 +0,0 @@
-import {
-  asyncIterableReader,
-  bytesReader,
-  createDecoder
-} from './decoder.js'
-
-/**
- * @typedef {import('multiformats').CID} CID
- * @typedef {import('./api').Block} Block
- * @typedef {import('./api').RootsReader} RootsReader
- * @typedef {import('./api').BlockIndex} BlockIndex
- * @typedef {import('./coding').BytesReader} BytesReader
- */
-
-/**
- * Provides an iterator over all of the `Block`s in a CAR, returning their CIDs
- * and byte-location information. Implements an `AsyncIterable<BlockIndex>`.
- * Where a `BlockIndex` is a
- * `{ cid:CID, length:number, offset:number, blockLength:number, blockOffset:number }`.
- *
- * As an implementer of `AsyncIterable`, this class can be used directly in a
- * `for await (const blockIndex of iterator) {}` loop. Where the `iterator` is
- * constructed using {@link CarIndexer.fromBytes} or
- * {@link CarIndexer.fromIterable}.
- *
- * An iteration can only be performce _once_ per instantiation.
- *
- * `CarIndexer` also implements the `RootsReader` interface and provides
- * the {@link CarIndexer.getRoots `getRoots()`} method.
- *
- * Load this class with either
- * `import { CarIndexer } from '@ipld/car/indexer'`
- * (`const { CarIndexer } = require('@ipld/car/indexer')`). Or
- * `import { CarIndexer } from '@ipld/car'`
- * (`const { CarIndexer } = require('@ipld/car')`). The former will likely
- * result in smaller bundle sizes where this is important.
- *
- * @name CarIndexer
- * @class
- * @implements {RootsReader}
- * @implements {AsyncIterable<BlockIndex>}
- * @property {number} version The version number of the CAR referenced by this
- * reader (should be `1`).
- */
-export class CarIndexer {
-  /**
-   * @param {number} version
-   * @param {CID[]} roots
-   * @param {AsyncGenerator<BlockIndex>} iterator
-   */
-  constructor (version, roots, iterator) {
-    this._version = version
-    this._roots = roots
-    this._iterator = iterator
-  }
-
-  get version () {
-    return this._version
-  }
-
-  /**
-   * Get the list of roots defined by the CAR referenced by this indexer. May be
-   * zero or more `CID`s.
-   *
-   * @function
-   * @memberof CarIndexer
-   * @instance
-   * @async
-   * @returns {Promise<CID[]>}
-   */
-  async getRoots () {
-    return this._roots
-  }
-
-  /**
-   * @returns {AsyncIterator<BlockIndex>}
-   */
-  [Symbol.asyncIterator] () {
-    return this._iterator
-  }
-
-  /**
-   * Instantiate a {@link CarIndexer} from a `Uint8Array` blob. Only the header
-   * is decoded initially, the remainder is processed and emitted via the
-   * iterator as it is consumed.
-   *
-   * @async
-   * @static
-   * @memberof CarIndexer
-   * @param {Uint8Array} bytes
-   * @returns {Promise<CarIndexer>}
-   */
-  static async fromBytes (bytes) {
-    if (!(bytes instanceof Uint8Array)) {
-      throw new TypeError('fromBytes() requires a Uint8Array')
-    }
-    return decodeIndexerComplete(bytesReader(bytes))
-  }
-
-  /**
-   * Instantiate a {@link CarIndexer} from a `AsyncIterable<Uint8Array>`,
-   * such as a [modern Node.js stream](https://nodejs.org/api/stream.html#stream_streams_compatibility_with_async_generators_and_async_iterators).
-   * is decoded initially, the remainder is processed and emitted via the
-   * iterator as it is consumed.
-   *
-   * @async
-   * @static
-   * @memberof CarIndexer
-   * @param {AsyncIterable<Uint8Array>} asyncIterable
-   * @returns {Promise<CarIndexer>}
-   */
-  static async fromIterable (asyncIterable) {
-    if (!asyncIterable || !(typeof asyncIterable[Symbol.asyncIterator] === 'function')) {
-      throw new TypeError('fromIterable() requires an async iterable')
-    }
-    return decodeIndexerComplete(asyncIterableReader(asyncIterable))
-  }
-}
-
-/**
- * @private
- * @param {BytesReader} reader
- * @returns {Promise<CarIndexer>}
- */
-async function decodeIndexerComplete (reader) {
-  const decoder = createDecoder(reader)
-  const { version, roots } = await decoder.header()
-
-  return new CarIndexer(version, roots, decoder.blocksIndex())
-}

package/src/@ipld/car/car/src/iterator-channel.js

@@ -1,91 +0,0 @@
-/**
- * @template {any} T
- * @typedef {import('./coding').IteratorChannel<T>} IteratorChannel
- */
-
-function noop () {}
-
-/**
- * @template {any} T
- * @returns {IteratorChannel<T>}
- */
-export function create () {
-  /** @type {T[]} */
-  const chunkQueue = []
-  /** @type {Promise<void> | null} */
-  let drainer = null
-  let drainerResolver = noop
-  let ended = false
-  /** @type {Promise<IteratorResult<T>> | null} */
-  let outWait = null
-  let outWaitResolver = noop
-
-  const makeDrainer = () => {
-    if (!drainer) {
-      drainer = new Promise((resolve) => {
-        drainerResolver = () => {
-          drainer = null
-          drainerResolver = noop
-          resolve()
-        }
-      })
-    }
-    return drainer
-  }
-
-  /**
-   * @returns {IteratorChannel<T>}
-   */
-  const writer = {
-    /**
-     * @param {T} chunk
-     * @returns {Promise<void>}
-     */
-    write (chunk) {
-      chunkQueue.push(chunk)
-      const drainer = makeDrainer()
-      outWaitResolver()
-      return drainer
-    },
-
-    async end () {
-      ended = true
-      const drainer = makeDrainer()
-      outWaitResolver()
-      await drainer
-    }
-  }
-
-  /** @type {AsyncIterator<T>} */
-  const iterator = {
-    /** @returns {Promise<IteratorResult<T>>} */
-    async next () {
-      const chunk = chunkQueue.shift()
-      if (chunk) {
-        if (chunkQueue.length === 0) {
-          drainerResolver()
-        }
-        return { done: false, value: chunk }
-      }
-
-      if (ended) {
-        drainerResolver()
-        return { done: true, value: undefined }
-      }
-
-      if (!outWait) {
-        outWait = new Promise((resolve) => {
-          outWaitResolver = () => {
-            outWait = null
-            outWaitResolver = noop
-            return resolve(iterator.next())
-          }
-        })
-      }
-
-      return outWait
-    }
-  }
-
-  return { writer, iterator }
-}

package/src/@ipld/car/car/src/iterator.js

@@ -1,267 +0,0 @@
-import {
-  asyncIterableReader,
-  bytesReader,
-  createDecoder
-} from './decoder.js'
-
-/**
- * @typedef {import('multiformats').CID} CID
- * @typedef {import('./api').Block} Block
- * @typedef {import('./api').RootsReader} RootsReader
- * @typedef {import('./coding').BytesReader} BytesReader
- */
-
-/**
- * @class
- * @implements {RootsReader}
- * @property {number} version The version number of the CAR referenced by this reader (should be `1`).
- */
-export class CarIteratorBase {
-  /**
-   * @param {number} version
-   * @param {CID[]} roots
-   * @param {AsyncIterable<Block>|void} iterable
-   */
-  constructor (version, roots, iterable) {
-    this._version = version
-    this._roots = roots
-    this._iterable = iterable
-    this._decoded = false
-  }
-
-  get version () {
-    return this._version
-  }
-
-  /**
-   * @returns {Promise<CID[]>}
-   */
-  async getRoots () {
-    return this._roots
-  }
-}
-
-/**
- * Provides an iterator over all of the `Block`s in a CAR. Implements a
- * `BlockIterator` interface, or `AsyncIterable<Block>`. Where a `Block` is
- * a `{ cid:CID, bytes:Uint8Array }` pair.
- *
- * As an implementer of `AsyncIterable`, this class can be used directly in a
- * `for await (const block of iterator) {}` loop. Where the `iterator` is
- * constructed using {@link CarBlockiterator.fromBytes} or
- * {@link CarBlockiterator.fromIterable}.
- *
- * An iteration can only be performce _once_ per instantiation.
- *
- * `CarBlockIterator` also implements the `RootsReader` interface and provides
- * the {@link CarBlockiterator.getRoots `getRoots()`} method.
- *
- * Load this class with either
- * `import { CarBlockIterator } from '@ipld/car/iterator'`
- * (`const { CarBlockIterator } = require('@ipld/car/iterator')`). Or
- * `import { CarBlockIterator } from '@ipld/car'`
- * (`const { CarBlockIterator } = require('@ipld/car')`).
- *
- * @name CarBlockIterator
- * @class
- * @implements {RootsReader}
- * @implements {AsyncIterable<Block>}
- * @property {number} version The version number of the CAR referenced by this
- * iterator (should be `1`).
- */
-export class CarBlockIterator extends CarIteratorBase {
-  // inherited method
-  /**
-   * Get the list of roots defined by the CAR referenced by this iterator. May be
-   * zero or more `CID`s.
-   *
-   * @function getRoots
-   * @memberof CarBlockIterator
-   * @instance
-   * @async
-   * @returns {Promise<CID[]>}
-   */
-
-  /**
-   * @returns {AsyncIterator<Block>}
-   */
-  [Symbol.asyncIterator] () {
-    if (this._decoded) {
-      throw new Error('Cannot decode more than once')
-    }
-    /* c8 ignore next 3 */
-    if (!this._iterable) {
-      throw new Error('Block iterable not found')
-    }
-    this._decoded = true
-    return this._iterable[Symbol.asyncIterator]()
-  }
-
-  /**
-   * Instantiate a {@link CarBlockIterator} from a `Uint8Array` blob. Rather
-   * than decoding the entire byte array prior to returning the iterator, as in
-   * {@link CarReader.fromBytes}, only the header is decoded and the remainder
-   * of the CAR is parsed as the `Block`s as yielded.
-   *
-   * @async
-   * @static
-   * @memberof CarBlockIterator
-   * @param {Uint8Array} bytes
-   * @returns {Promise<CarBlockIterator>}
-   */
-  static async fromBytes (bytes) {
-    const { version, roots, iterator } = await fromBytes(bytes)
-    return new CarBlockIterator(version, roots, iterator)
-  }
-
-  /**
-   * Instantiate a {@link CarBlockIterator} from a `AsyncIterable<Uint8Array>`,
-   * such as a [modern Node.js stream](https://nodejs.org/api/stream.html#stream_streams_compatibility_with_async_generators_and_async_iterators).
-   * Rather than decoding the entire byte array prior to returning the iterator,
-   * as in {@link CarReader.fromIterable}, only the header is decoded and the
-   * remainder of the CAR is parsed as the `Block`s as yielded.
-   *
-   * @async
-   * @static
-   * @param {AsyncIterable<Uint8Array>} asyncIterable
-   * @returns {Promise<CarBlockIterator>}
-   */
-  static async fromIterable (asyncIterable) {
-    const { version, roots, iterator } = await fromIterable(asyncIterable)
-    return new CarBlockIterator(version, roots, iterator)
-  }
-}
-
-/**
- * Provides an iterator over all of the `CID`s in a CAR. Implements a
- * `CIDIterator` interface, or `AsyncIterable<CID>`. Similar to
- * {@link CarBlockIterator} but only yields the CIDs in the CAR.
- *
- * As an implementer of `AsyncIterable`, this class can be used directly in a
- * `for await (const cid of iterator) {}` loop. Where the `iterator` is
- * constructed using {@link CarCIDiterator.fromBytes} or
- * {@link CarCIDiterator.fromIterable}.
- *
- * An iteration can only be performce _once_ per instantiation.
- *
- * `CarCIDIterator` also implements the `RootsReader` interface and provides
- * the {@link CarCIDiterator.getRoots `getRoots()`} method.
- *
- * Load this class with either
- * `import { CarCIDIterator } from '@ipld/car/iterator'`
- * (`const { CarCIDIterator } = require('@ipld/car/iterator')`). Or
- * `import { CarCIDIterator } from '@ipld/car'`
- * (`const { CarCIDIterator } = require('@ipld/car')`).
- *
- * @name CarCIDIterator
- * @class
- * @implements {RootsReader}
- * @implements {AsyncIterable<CID>}
- * @property {number} version The version number of the CAR referenced by this
- * iterator (should be `1`).
- */
-export class CarCIDIterator extends CarIteratorBase {
-  // inherited method
-  /**
-   * Get the list of roots defined by the CAR referenced by this iterator. May be
-   * zero or more `CID`s.
-   *
-   * @function getRoots
-   * @memberof CarCIDIterator
-   * @instance
-   * @async
-   * @returns {Promise<CID[]>}
-   */
-
-  /**
-   * @returns {AsyncIterator<CID>}
-   */
-  [Symbol.asyncIterator] () {
-    if (this._decoded) {
-      throw new Error('Cannot decode more than once')
-    }
-    /* c8 ignore next 3 */
-    if (!this._iterable) {
-      throw new Error('Block iterable not found')
-    }
-    this._decoded = true
-    const iterable = this._iterable[Symbol.asyncIterator]()
-    return {
-      async next () {
-        const next = await iterable.next()
-        if (next.done) {
-          return next
-        }
-        return { done: false, value: next.value.cid }
-      }
-    }
-  }
-
-  /**
-   * Instantiate a {@link CarCIDIterator} from a `Uint8Array` blob. Rather
-   * than decoding the entire byte array prior to returning the iterator, as in
-   * {@link CarReader.fromBytes}, only the header is decoded and the remainder
-   * of the CAR is parsed as the `CID`s as yielded.
-   *
-   * @async
-   * @static
-   * @memberof CarCIDIterator
-   * @param {Uint8Array} bytes
-   * @returns {Promise<CarCIDIterator>}
-   */
-  static async fromBytes (bytes) {
-    const { version, roots, iterator } = await fromBytes(bytes)
-    return new CarCIDIterator(version, roots, iterator)
-  }
-
-  /**
-   * Instantiate a {@link CarCIDIterator} from a `AsyncIterable<Uint8Array>`,
-   * such as a [modern Node.js stream](https://nodejs.org/api/stream.html#stream_streams_compatibility_with_async_generators_and_async_iterators).
-   * Rather than decoding the entire byte array prior to returning the iterator,
-   * as in {@link CarReader.fromIterable}, only the header is decoded and the
-   * remainder of the CAR is parsed as the `CID`s as yielded.
-   *
-   * @async
-   * @static
-   * @memberof CarCIDIterator
-   * @param {AsyncIterable<Uint8Array>} asyncIterable
-   * @returns {Promise<CarCIDIterator>}
-   */
-  static async fromIterable (asyncIterable) {
-    const { version, roots, iterator } = await fromIterable(asyncIterable)
-    return new CarCIDIterator(version, roots, iterator)
-  }
-}
-
-/**
- * @param {Uint8Array} bytes
- * @returns {Promise<{ version:number, roots:CID[], iterator:AsyncIterable<Block>}>}
- */
-async function fromBytes (bytes) {
-  if (!(bytes instanceof Uint8Array)) {
-    throw new TypeError('fromBytes() requires a Uint8Array')
-  }
-  return decodeIterator(bytesReader(bytes))
-}
-
-/**
- * @param {AsyncIterable<Uint8Array>} asyncIterable
- * @returns {Promise<{ version:number, roots:CID[], iterator:AsyncIterable<Block>}>}
- */
-async function fromIterable (asyncIterable) {
-  if (!asyncIterable || !(typeof asyncIterable[Symbol.asyncIterator] === 'function')) {
-    throw new TypeError('fromIterable() requires an async iterable')
-  }
-  return decodeIterator(asyncIterableReader(asyncIterable))
-}
-
-/**
- * @private
- * @param {BytesReader} reader
- * @returns {Promise<{ version:number, roots:CID[], iterator:AsyncIterable<Block>}>}
- */
-async function decodeIterator (reader) {
-  const decoder = createDecoder(reader)
-  const { version, roots } = await decoder.header()
-  return { version, roots, iterator: decoder.blocks() }
-}