@fireproof/vendor 0.0.0-smoke

package/src/@ipld/car/car/src/buffer-writer.js

@@ -0,0 +1,286 @@
+import * as CBOR from "@fireproof/vendor/@ipld/dag-cbor"
+import { Token, Type } from "@fireproof/vendor/cborg"
+import { tokensToLength } from "@fireproof/vendor/cborg/length"
+import varint from 'varint'
+
+/**
+ * @typedef {import('./api').CID} CID
+ * @typedef {import('./api').Block} Block
+ * @typedef {import('./api').CarBufferWriter} Writer
+ * @typedef {import('./api').CarBufferWriterOptions} Options
+ * @typedef {import('./coding').CarEncoder} CarEncoder
+ */
+
+/**
+ * A simple CAR writer that writes to a pre-allocated buffer.
+ *
+ * @class
+ * @name CarBufferWriter
+ * @implements {Writer}
+ */
+class CarBufferWriter {
+  /**
+   * @param {Uint8Array} bytes
+   * @param {number} headerSize
+   */
+  constructor (bytes, headerSize) {
+    /** @readonly */
+    this.bytes = bytes
+    this.byteOffset = headerSize
+
+    /**
+     * @readonly
+     * @type {CID[]}
+     */
+    this.roots = []
+    this.headerSize = headerSize
+  }
+
+  /**
+   * Add a root to this writer, to be used to create a header when the CAR is
+   * finalized with {@link CarBufferWriter.close `close()`}
+   *
+   * @param {CID} root
+   * @param {{resize?:boolean}} [options]
+   * @returns {CarBufferWriter}
+   */
+  addRoot (root, options) {
+    addRoot(this, root, options)
+    return this
+  }
+
+  /**
+   * Write a `Block` (a `{ cid:CID, bytes:Uint8Array }` pair) to the archive.
+   * Throws if there is not enough capacity.
+   *
+   * @param {Block} block - A `{ cid:CID, bytes:Uint8Array }` pair.
+   * @returns {CarBufferWriter}
+   */
+  write (block) {
+    addBlock(this, block)
+    return this
+  }
+
+  /**
+   * Finalize the CAR and return it as a `Uint8Array`.
+   *
+   * @param {object} [options]
+   * @param {boolean} [options.resize]
+   * @returns {Uint8Array}
+   */
+  close (options) {
+    return close(this, options)
+  }
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {CID} root
+ * @param {{resize?:boolean}} [options]
+ */
+export const addRoot = (writer, root, options = {}) => {
+  const { resize = false } = options
+  const { bytes, headerSize, byteOffset, roots } = writer
+  writer.roots.push(root)
+  const size = headerLength(writer)
+  // If there is not enough space for the new root
+  if (size > headerSize) {
+    // Check if we root would fit if we were to resize the head.
+    if (size - headerSize + byteOffset < bytes.byteLength) {
+      // If resize is enabled resize head
+      if (resize) {
+        resizeHeader(writer, size)
+      // otherwise remove head and throw an error suggesting to resize
+      } else {
+        roots.pop()
+        throw new RangeError(`Header of size ${headerSize} has no capacity for new root ${root}.
+  However there is a space in the buffer and you could call addRoot(root, { resize: root }) to resize header to make a space for this root.`)
+      }
+    // If head would not fit even with resize pop new root and throw error
+    } else {
+      roots.pop()
+      throw new RangeError(`Buffer has no capacity for a new root ${root}`)
+    }
+  }
+}
+
+/**
+ * Calculates number of bytes required for storing given block in CAR. Useful in
+ * estimating size of an `ArrayBuffer` for the `CarBufferWriter`.
+ *
+ * @name CarBufferWriter.blockLength(Block)
+ * @param {Block} block
+ * @returns {number}
+ */
+export const blockLength = ({ cid, bytes }) => {
+  const size = cid.bytes.byteLength + bytes.byteLength
+  return varint.encodingLength(size) + size
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {Block} block
+ */
+export const addBlock = (writer, { cid, bytes }) => {
+  const byteLength = cid.bytes.byteLength + bytes.byteLength
+  const size = varint.encode(byteLength)
+  if (writer.byteOffset + size.length + byteLength > writer.bytes.byteLength) {
+    throw new RangeError('Buffer has no capacity for this block')
+  } else {
+    writeBytes(writer, size)
+    writeBytes(writer, cid.bytes)
+    writeBytes(writer, bytes)
+  }
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {object} [options]
+ * @param {boolean} [options.resize]
+ */
+export const close = (writer, options = {}) => {
+  const { resize = false } = options
+  const { roots, bytes, byteOffset, headerSize } = writer
+
+  const headerBytes = CBOR.encode({ version: 1, roots })
+  const varintBytes = varint.encode(headerBytes.length)
+
+  const size = varintBytes.length + headerBytes.byteLength
+  const offset = headerSize - size
+
+  // If header size estimate was accurate we just write header and return
+  // view into buffer.
+  if (offset === 0) {
+    writeHeader(writer, varintBytes, headerBytes)
+    return bytes.subarray(0, byteOffset)
+    // If header was overestimated and `{resize: true}` is passed resize header
+  } else if (resize) {
+    resizeHeader(writer, size)
+    writeHeader(writer, varintBytes, headerBytes)
+    return bytes.subarray(0, writer.byteOffset)
+  } else {
+    throw new RangeError(`Header size was overestimated.
+You can use close({ resize: true }) to resize header`)
+  }
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {number} byteLength
+ */
+export const resizeHeader = (writer, byteLength) => {
+  const { bytes, headerSize } = writer
+  // Move data section to a new offset
+  bytes.set(bytes.subarray(headerSize, writer.byteOffset), byteLength)
+  // Update header size & byteOffset
+  writer.byteOffset += byteLength - headerSize
+  writer.headerSize = byteLength
+}
+
+/**
+ * @param {CarBufferWriter} writer
+ * @param {number[]|Uint8Array} bytes
+ */
+
+const writeBytes = (writer, bytes) => {
+  writer.bytes.set(bytes, writer.byteOffset)
+  writer.byteOffset += bytes.length
+}
+/**
+ * @param {{bytes:Uint8Array}} writer
+ * @param {number[]} varint
+ * @param {Uint8Array} header
+ */
+const writeHeader = ({ bytes }, varint, header) => {
+  bytes.set(varint)
+  bytes.set(header, varint.length)
+}
+
+const headerPreludeTokens = [
+  new Token(Type.map, 2),
+  new Token(Type.string, 'version'),
+  new Token(Type.uint, 1),
+  new Token(Type.string, 'roots')
+]
+
+const CID_TAG = new Token(Type.tag, 42)
+
+/**
+ * Calculates header size given the array of byteLength for roots.
+ *
+ * @name CarBufferWriter.calculateHeaderLength(rootLengths)
+ * @param {number[]} rootLengths
+ * @returns {number}
+ */
+export const calculateHeaderLength = (rootLengths) => {
+  const tokens = [...headerPreludeTokens]
+  tokens.push(new Token(Type.array, rootLengths.length))
+  for (const rootLength of rootLengths) {
+    tokens.push(CID_TAG)
+    tokens.push(new Token(Type.bytes, { length: rootLength + 1 }))
+  }
+  const length = tokensToLength(tokens) // no options needed here because we have simple tokens
+  return varint.encodingLength(length) + length
+}
+
+/**
+ * Calculates header size given the array of roots.
+ *
+ * @name CarBufferWriter.headerLength({ roots })
+ * @param {object} options
+ * @param {CID[]} options.roots
+ * @returns {number}
+ */
+export const headerLength = ({ roots }) =>
+  calculateHeaderLength(roots.map(cid => cid.bytes.byteLength))
+
+/**
+ * Estimates header size given a count of the roots and the expected byte length
+ * of the root CIDs. The default length works for a standard CIDv1 with a
+ * single-byte multihash code, such as SHA2-256 (i.e. the most common CIDv1).
+ *
+ * @name CarBufferWriter.estimateHeaderLength(rootCount[, rootByteLength])
+ * @param {number} rootCount
+ * @param {number} [rootByteLength]
+ * @returns {number}
+ */
+export const estimateHeaderLength = (rootCount, rootByteLength = 36) =>
+  calculateHeaderLength(new Array(rootCount).fill(rootByteLength))
+
+/**
+ * Creates synchronous CAR writer that can be used to encode blocks into a given
+ * buffer. Optionally you could pass `byteOffset` and `byteLength` to specify a
+ * range inside buffer to write into. If car file is going to have `roots` you
+ * need to either pass them under `options.roots` (from which header size will
+ * be calculated) or provide `options.headerSize` to allocate required space
+ * in the buffer. You may also provide known `roots` and `headerSize` to
+ * allocate space for the roots that may not be known ahead of time.
+ *
+ * Note: Incorrect `headerSize` may lead to copying bytes inside a buffer
+ * which will have a negative impact on performance.
+ *
+ * @name CarBufferWriter.createWriter(buffer[, options])
+ * @param {ArrayBuffer} buffer
+ * @param {object} [options]
+ * @param {CID[]} [options.roots]
+ * @param {number} [options.byteOffset]
+ * @param {number} [options.byteLength]
+ * @param {number} [options.headerSize]
+ * @returns {CarBufferWriter}
+ */
+export const createWriter = (buffer, options = {}) => {
+  const {
+    roots = [],
+    byteOffset = 0,
+    byteLength = buffer.byteLength,
+    headerSize = headerLength({ roots })
+  } = options
+  const bytes = new Uint8Array(buffer, byteOffset, byteLength)
+
+  const writer = new CarBufferWriter(bytes, headerSize)
+  for (const root of roots) {
+    writer.addRoot(root)
+  }
+
+  return writer
+}

package/src/@ipld/car/car/src/coding.ts

@@ -0,0 +1,68 @@
+import type { Block, BlockIndex } from './api.js'
+import type { CID } from 'multiformats/cid'
+
+export interface CarEncoder {
+  setRoots(roots: CID[]): Promise<void>
+
+  writeBlock(block: Block): Promise<void>
+
+  close(): Promise<void>
+
+  version(): number
+}
+
+export interface IteratorChannel_Writer<T> {
+  write(chunk: T): Promise<void>
+  end(): Promise<void>
+}
+
+export interface IteratorChannel<T> {
+  writer: IteratorChannel_Writer<T>
+
+  iterator: AsyncIterator<T>
+}
+
+export interface CarHeader {
+  version: 1
+  roots: CID[]
+}
+
+export interface CarV2FixedHeader {
+  characteristics: [bigint, bigint]
+  dataOffset: number
+  dataSize: number
+  indexOffset: number
+}
+
+export interface CarV2Header extends CarV2FixedHeader {
+  version: 2
+  roots: CID[]
+}
+
+export interface CarDecoder {
+  header(): Promise<CarHeader | CarV2Header>
+
+  blocks(): AsyncGenerator<Block>
+
+  blocksIndex(): AsyncGenerator<BlockIndex>
+}
+
+export interface Seekable {
+  seek(length: number): void
+}
+
+export interface BytesReader extends Seekable {
+  upTo(length: number): Promise<Uint8Array>
+
+  exactly(length: number, seek?: boolean): Promise<Uint8Array>
+
+  pos: number
+}
+
+export interface BytesBufferReader extends Seekable {
+  upTo(length: number): Uint8Array
+
+  exactly(length: number, seek?: boolean): Uint8Array
+
+  pos: number
+}

package/src/@ipld/car/car/src/decoder-common.js

@@ -0,0 +1,82 @@
+import varint from 'varint'
+
+export const CIDV0_BYTES = {
+  SHA2_256: 0x12,
+  LENGTH: 0x20,
+  DAG_PB: 0x70
+}
+
+export const V2_HEADER_LENGTH = /* characteristics */ 16 /* v1 offset */ + 8 /* v1 size */ + 8 /* index offset */ + 8
+
+/**
+ * Decodes varint and seeks the buffer
+ *
+ * ```js
+ * // needs bytes to be read first
+ * const bytes = reader.upTo(8) // maybe async
+ * ```
+ *
+ * @param {Uint8Array} bytes
+ * @param {import('./coding').Seekable} seeker
+ * @returns {number}
+ */
+export function decodeVarint (bytes, seeker) {
+  if (!bytes.length) {
+    throw new Error('Unexpected end of data')
+  }
+  const i = varint.decode(bytes)
+  seeker.seek(/** @type {number} */(varint.decode.bytes))
+  return i
+}
+
+/**
+ * Decode v2 header
+ *
+ * ```js
+ * // needs bytes to be read first
+ * const bytes = reader.exactly(V2_HEADER_LENGTH, true) // maybe async
+ * ```
+ *
+ * @param {Uint8Array} bytes
+ * @returns {import('./coding').CarV2FixedHeader}
+ */
+export function decodeV2Header (bytes) {
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  let offset = 0
+  const header = {
+    version: 2,
+    /** @type {[bigint, bigint]} */
+    characteristics: [
+      dv.getBigUint64(offset, true),
+      dv.getBigUint64(offset += 8, true)
+    ],
+    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
+    dataSize: Number(dv.getBigUint64(offset += 8, true)),
+    indexOffset: Number(dv.getBigUint64(offset += 8, true))
+  }
+  return header
+}
+
+/**
+ * Checks the length of the multihash to be read afterwards
+ *
+ * ```js
+ * // needs bytes to be read first
+ * const bytes = reader.upTo(8) // maybe async
+ * ```
+ *
+ * @param {Uint8Array} bytes
+ */
+export function getMultihashLength (bytes) {
+  // | code | length | .... |
+  // where both code and length are varints, so we have to decode
+  // them first before we can know total length
+
+  varint.decode(bytes) // code
+  const codeLength = /** @type {number} */(varint.decode.bytes)
+  const length = varint.decode(bytes.subarray(varint.decode.bytes))
+  const lengthLength = /** @type {number} */(varint.decode.bytes)
+  const mhLength = codeLength + lengthLength + length
+
+  return mhLength
+}