@ipld/car 4.1.6 → 5.0.1

package/esm/car.js

@@ -1,18 +0,0 @@
-import { CarReader } from './lib/reader.js';
-import { CarIndexer } from './lib/indexer.js';
-import {
-  CarBlockIterator,
-  CarCIDIterator
-} from './lib/iterator.js';
-import { CarWriter } from './lib/writer.js';
-import { CarIndexedReader } from './lib/indexed-reader.js';
-import * as CarBufferWriter from './lib/buffer-writer.js';
-export {
-  CarReader,
-  CarIndexer,
-  CarBlockIterator,
-  CarCIDIterator,
-  CarWriter,
-  CarIndexedReader,
-  CarBufferWriter
-};

package/esm/lib/buffer-writer.js

@@ -1,129 +0,0 @@
-import varint from 'varint';
-import {
-  Token,
-  Type
-} from 'cborg';
-import { tokensToLength } from 'cborg/length';
-import * as CBOR from '@ipld/dag-cbor';
-class CarBufferWriter {
-  constructor(bytes, headerSize) {
-    this.bytes = bytes;
-    this.byteOffset = headerSize;
-    this.roots = [];
-    this.headerSize = headerSize;
-  }
-  addRoot(root, options) {
-    addRoot(this, root, options);
-    return this;
-  }
-  write(block) {
-    addBlock(this, block);
-    return this;
-  }
-  close(options) {
-    return close(this, options);
-  }
-}
-export const addRoot = (writer, root, options = {}) => {
-  const {
-    resize = false
-  } = options;
-  const {bytes, headerSize, byteOffset, roots} = writer;
-  writer.roots.push(root);
-  const size = headerLength(writer);
-  if (size > headerSize) {
-    if (size - headerSize + byteOffset < bytes.byteLength) {
-      if (resize) {
-        resizeHeader(writer, size);
-      } else {
-        roots.pop();
-        throw new RangeError(`Header of size ${ headerSize } has no capacity for new root ${ root }.
-  However there is a space in the buffer and you could call addRoot(root, { resize: root }) to resize header to make a space for this root.`);
-      }
-    } else {
-      roots.pop();
-      throw new RangeError(`Buffer has no capacity for a new root ${ root }`);
-    }
-  }
-};
-export const blockLength = ({cid, bytes}) => {
-  const size = cid.bytes.byteLength + bytes.byteLength;
-  return varint.encodingLength(size) + size;
-};
-export const addBlock = (writer, {cid, bytes}) => {
-  const byteLength = cid.bytes.byteLength + bytes.byteLength;
-  const size = varint.encode(byteLength);
-  if (writer.byteOffset + size.length + byteLength > writer.bytes.byteLength) {
-    throw new RangeError('Buffer has no capacity for this block');
-  } else {
-    writeBytes(writer, size);
-    writeBytes(writer, cid.bytes);
-    writeBytes(writer, bytes);
-  }
-};
-export const close = (writer, options = {}) => {
-  const {
-    resize = false
-  } = options;
-  const {roots, bytes, byteOffset, headerSize} = writer;
-  const headerBytes = CBOR.encode({
-    version: 1,
-    roots
-  });
-  const varintBytes = varint.encode(headerBytes.length);
-  const size = varintBytes.length + headerBytes.byteLength;
-  const offset = headerSize - size;
-  if (offset === 0) {
-    writeHeader(writer, varintBytes, headerBytes);
-    return bytes.subarray(0, byteOffset);
-  } else if (resize) {
-    resizeHeader(writer, size);
-    writeHeader(writer, varintBytes, headerBytes);
-    return bytes.subarray(0, writer.byteOffset);
-  } else {
-    throw new RangeError(`Header size was overestimated.
-You can use close({ resize: true }) to resize header`);
-  }
-};
-export const resizeHeader = (writer, byteLength) => {
-  const {bytes, headerSize} = writer;
-  bytes.set(bytes.subarray(headerSize, writer.byteOffset), byteLength);
-  writer.byteOffset += byteLength - headerSize;
-  writer.headerSize = byteLength;
-};
-const writeBytes = (writer, bytes) => {
-  writer.bytes.set(bytes, writer.byteOffset);
-  writer.byteOffset += bytes.length;
-};
-const writeHeader = ({bytes}, varint, header) => {
-  bytes.set(varint);
-  bytes.set(header, varint.length);
-};
-const headerPreludeTokens = [
-  new Token(Type.map, 2),
-  new Token(Type.string, 'version'),
-  new Token(Type.uint, 1),
-  new Token(Type.string, 'roots')
-];
-const CID_TAG = new Token(Type.tag, 42);
-export const calculateHeaderLength = rootLengths => {
-  const tokens = [...headerPreludeTokens];
-  tokens.push(new Token(Type.array, rootLengths.length));
-  for (const rootLength of rootLengths) {
-    tokens.push(CID_TAG);
-    tokens.push(new Token(Type.bytes, { length: rootLength + 1 }));
-  }
-  const length = tokensToLength(tokens);
-  return varint.encodingLength(length) + length;
-};
-export const headerLength = ({roots}) => calculateHeaderLength(roots.map(cid => cid.bytes.byteLength));
-export const estimateHeaderLength = (rootCount, rootByteLength = 36) => calculateHeaderLength(new Array(rootCount).fill(rootByteLength));
-export const createWriter = (buffer, options = {}) => {
-  const {roots = [], byteOffset = 0, byteLength = buffer.byteLength, headerSize = headerLength({ roots })} = options;
-  const bytes = new Uint8Array(buffer, byteOffset, byteLength);
-  const writer = new CarBufferWriter(bytes, headerSize);
-  for (const root of roots) {
-    writer.addRoot(root);
-  }
-  return writer;
-};

package/esm/lib/decoder.js

@@ -1,265 +0,0 @@
-import varint from 'varint';
-import { CID } from 'multiformats/cid';
-import * as Digest from 'multiformats/hashes/digest';
-import { decode as decodeDagCbor } from '@ipld/dag-cbor';
-import { CarHeader as headerValidator } from './header-validator.js';
-const CIDV0_BYTES = {
-  SHA2_256: 18,
-  LENGTH: 32,
-  DAG_PB: 112
-};
-const V2_HEADER_LENGTH = 16 + 8 + 8 + 8;
-async function readVarint(reader) {
-  const bytes = await reader.upTo(8);
-  if (!bytes.length) {
-    throw new Error('Unexpected end of data');
-  }
-  const i = varint.decode(bytes);
-  reader.seek(varint.decode.bytes);
-  return i;
-}
-async function readV2Header(reader) {
-  const bytes = await reader.exactly(V2_HEADER_LENGTH);
-  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-  let offset = 0;
-  const header = {
-    version: 2,
-    characteristics: [
-      dv.getBigUint64(offset, true),
-      dv.getBigUint64(offset += 8, true)
-    ],
-    dataOffset: Number(dv.getBigUint64(offset += 8, true)),
-    dataSize: Number(dv.getBigUint64(offset += 8, true)),
-    indexOffset: Number(dv.getBigUint64(offset += 8, true))
-  };
-  reader.seek(V2_HEADER_LENGTH);
-  return header;
-}
-export async function readHeader(reader, strictVersion) {
-  const length = await readVarint(reader);
-  if (length === 0) {
-    throw new Error('Invalid CAR header (zero length)');
-  }
-  const header = await reader.exactly(length);
-  reader.seek(length);
-  const block = decodeDagCbor(header);
-  if (!headerValidator(block)) {
-    throw new Error('Invalid CAR header format');
-  }
-  if (block.version !== 1 && block.version !== 2 || strictVersion !== undefined && block.version !== strictVersion) {
-    throw new Error(`Invalid CAR version: ${ block.version }${ strictVersion !== undefined ? ` (expected ${ strictVersion })` : '' }`);
-  }
-  const hasRoots = Array.isArray(block.roots);
-  if (block.version === 1 && !hasRoots || block.version === 2 && hasRoots) {
-    throw new Error('Invalid CAR header format');
-  }
-  if (block.version === 1) {
-    return block;
-  }
-  const v2Header = await readV2Header(reader);
-  reader.seek(v2Header.dataOffset - reader.pos);
-  const v1Header = await readHeader(reader, 1);
-  return Object.assign(v1Header, v2Header);
-}
-async function readMultihash(reader) {
-  const bytes = await reader.upTo(8);
-  varint.decode(bytes);
-  const codeLength = varint.decode.bytes;
-  const length = varint.decode(bytes.subarray(varint.decode.bytes));
-  const lengthLength = varint.decode.bytes;
-  const mhLength = codeLength + lengthLength + length;
-  const multihash = await reader.exactly(mhLength);
-  reader.seek(mhLength);
-  return multihash;
-}
-async function readCid(reader) {
-  const first = await reader.exactly(2);
-  if (first[0] === CIDV0_BYTES.SHA2_256 && first[1] === CIDV0_BYTES.LENGTH) {
-    const bytes = await reader.exactly(34);
-    reader.seek(34);
-    const multihash = Digest.decode(bytes);
-    return CID.create(0, CIDV0_BYTES.DAG_PB, multihash);
-  }
-  const version = await readVarint(reader);
-  if (version !== 1) {
-    throw new Error(`Unexpected CID version (${ version })`);
-  }
-  const codec = await readVarint(reader);
-  const bytes = await readMultihash(reader);
-  const multihash = Digest.decode(bytes);
-  return CID.create(version, codec, multihash);
-}
-export async function readBlockHead(reader) {
-  const start = reader.pos;
-  let length = await readVarint(reader);
-  if (length === 0) {
-    throw new Error('Invalid CAR section (zero length)');
-  }
-  length += reader.pos - start;
-  const cid = await readCid(reader);
-  const blockLength = length - Number(reader.pos - start);
-  return {
-    cid,
-    length,
-    blockLength
-  };
-}
-async function readBlock(reader) {
-  const {cid, blockLength} = await readBlockHead(reader);
-  const bytes = await reader.exactly(blockLength);
-  reader.seek(blockLength);
-  return {
-    bytes,
-    cid
-  };
-}
-async function readBlockIndex(reader) {
-  const offset = reader.pos;
-  const {cid, length, blockLength} = await readBlockHead(reader);
-  const index = {
-    cid,
-    length,
-    blockLength,
-    offset,
-    blockOffset: reader.pos
-  };
-  reader.seek(index.blockLength);
-  return index;
-}
-export function createDecoder(reader) {
-  const headerPromise = (async () => {
-    const header = await readHeader(reader);
-    if (header.version === 2) {
-      const v1length = reader.pos - header.dataOffset;
-      reader = limitReader(reader, header.dataSize - v1length);
-    }
-    return header;
-  })();
-  return {
-    header: () => headerPromise,
-    async *blocks() {
-      await headerPromise;
-      while ((await reader.upTo(8)).length > 0) {
-        yield await readBlock(reader);
-      }
-    },
-    async *blocksIndex() {
-      await headerPromise;
-      while ((await reader.upTo(8)).length > 0) {
-        yield await readBlockIndex(reader);
-      }
-    }
-  };
-}
-export function bytesReader(bytes) {
-  let pos = 0;
-  return {
-    async upTo(length) {
-      return bytes.subarray(pos, pos + Math.min(length, bytes.length - pos));
-    },
-    async exactly(length) {
-      if (length > bytes.length - pos) {
-        throw new Error('Unexpected end of data');
-      }
-      return bytes.subarray(pos, pos + length);
-    },
-    seek(length) {
-      pos += length;
-    },
-    get pos() {
-      return pos;
-    }
-  };
-}
-export function chunkReader(readChunk) {
-  let pos = 0;
-  let have = 0;
-  let offset = 0;
-  let currentChunk = new Uint8Array(0);
-  const read = async length => {
-    have = currentChunk.length - offset;
-    const bufa = [currentChunk.subarray(offset)];
-    while (have < length) {
-      const chunk = await readChunk();
-      if (chunk == null) {
-        break;
-      }
-      if (have < 0) {
-        if (chunk.length > have) {
-          bufa.push(chunk.subarray(-have));
-        }
-      } else {
-        bufa.push(chunk);
-      }
-      have += chunk.length;
-    }
-    currentChunk = new Uint8Array(bufa.reduce((p, c) => p + c.length, 0));
-    let off = 0;
-    for (const b of bufa) {
-      currentChunk.set(b, off);
-      off += b.length;
-    }
-    offset = 0;
-  };
-  return {
-    async upTo(length) {
-      if (currentChunk.length - offset < length) {
-        await read(length);
-      }
-      return currentChunk.subarray(offset, offset + Math.min(currentChunk.length - offset, length));
-    },
-    async exactly(length) {
-      if (currentChunk.length - offset < length) {
-        await read(length);
-      }
-      if (currentChunk.length - offset < length) {
-        throw new Error('Unexpected end of data');
-      }
-      return currentChunk.subarray(offset, offset + length);
-    },
-    seek(length) {
-      pos += length;
-      offset += length;
-    },
-    get pos() {
-      return pos;
-    }
-  };
-}
-export function asyncIterableReader(asyncIterable) {
-  const iterator = asyncIterable[Symbol.asyncIterator]();
-  async function readChunk() {
-    const next = await iterator.next();
-    if (next.done) {
-      return null;
-    }
-    return next.value;
-  }
-  return chunkReader(readChunk);
-}
-export function limitReader(reader, byteLimit) {
-  let bytesRead = 0;
-  return {
-    async upTo(length) {
-      let bytes = await reader.upTo(length);
-      if (bytes.length + bytesRead > byteLimit) {
-        bytes = bytes.subarray(0, byteLimit - bytesRead);
-      }
-      return bytes;
-    },
-    async exactly(length) {
-      const bytes = await reader.exactly(length);
-      if (bytes.length + bytesRead > byteLimit) {
-        throw new Error('Unexpected end of data');
-      }
-      return bytes;
-    },
-    seek(length) {
-      bytesRead += length;
-      reader.seek(length);
-    },
-    get pos() {
-      return reader.pos;
-    }
-  };
-}

package/esm/lib/encoder.js

@@ -1,35 +0,0 @@
-import varint from 'varint';
-import { encode as dagCborEncode } from '@ipld/dag-cbor';
-export function createHeader(roots) {
-  const headerBytes = dagCborEncode({
-    version: 1,
-    roots
-  });
-  const varintBytes = varint.encode(headerBytes.length);
-  const header = new Uint8Array(varintBytes.length + headerBytes.length);
-  header.set(varintBytes, 0);
-  header.set(headerBytes, varintBytes.length);
-  return header;
-}
-function createEncoder(writer) {
-  return {
-    async setRoots(roots) {
-      const bytes = createHeader(roots);
-      await writer.write(bytes);
-    },
-    async writeBlock(block) {
-      const {cid, bytes} = block;
-      await writer.write(new Uint8Array(varint.encode(cid.bytes.length + bytes.length)));
-      await writer.write(cid.bytes);
-      if (bytes.length) {
-        await writer.write(bytes);
-      }
-    },
-    async close() {
-      await writer.end();
-    }
-  };
-}
-export {
-  createEncoder
-};

package/esm/lib/header-validator.js

@@ -1,23 +0,0 @@
-const Kinds = {
-  Null: obj => obj === null,
-  Int: obj => Number.isInteger(obj),
-  Float: obj => typeof obj === 'number' && Number.isFinite(obj),
-  String: obj => typeof obj === 'string',
-  Bool: obj => typeof obj === 'boolean',
-  Bytes: obj => obj instanceof Uint8Array,
-  Link: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID === obj,
-  List: obj => Array.isArray(obj),
-  Map: obj => !Kinds.Null(obj) && typeof obj === 'object' && obj.asCID !== obj && !Kinds.List(obj) && !Kinds.Bytes(obj)
-};
-const Types = {
-  Int: Kinds.Int,
-  'CarHeader > version': obj => Types.Int(obj),
-  'CarHeader > roots (anon) > valueType (anon)': Kinds.Link,
-  'CarHeader > roots (anon)': obj => Kinds.List(obj) && Array.prototype.every.call(obj, Types['CarHeader > roots (anon) > valueType (anon)']),
-  'CarHeader > roots': obj => Types['CarHeader > roots (anon)'](obj),
-  CarHeader: obj => {
-    const keys = obj && Object.keys(obj);
-    return Kinds.Map(obj) && ['version'].every(k => keys.includes(k)) && Object.entries(obj).every(([name, value]) => Types['CarHeader > ' + name] && Types['CarHeader > ' + name](value));
-  }
-};
-export const CarHeader = Types.CarHeader;

package/esm/lib/indexed-reader-browser.js

@@ -1,6 +0,0 @@
-export class CarIndexedReader {
-  static async fromFile() {
-    throw new Error('Unsupported in this environment');
-  }
-}
-export const __browser = true;

package/esm/lib/indexed-reader.js

@@ -1,78 +0,0 @@
-import fs from 'fs';
-import { Readable } from 'stream';
-import { CID } from 'multiformats/cid';
-import { CarIndexer } from './indexer.js';
-import { CarReader as NodeCarReader } from './reader.js';
-export class CarIndexedReader {
-  constructor(version, path, roots, index, order) {
-    this._version = version;
-    this._path = path;
-    this._roots = roots;
-    this._index = index;
-    this._order = order;
-    this._fd = null;
-  }
-  get version() {
-    return this._version;
-  }
-  async getRoots() {
-    return this._roots;
-  }
-  async has(key) {
-    return this._index.has(key.toString());
-  }
-  async get(key) {
-    const blockIndex = this._index.get(key.toString());
-    if (!blockIndex) {
-      return undefined;
-    }
-    if (!this._fd) {
-      this._fd = await fs.promises.open(this._path, 'r');
-    }
-    const readIndex = {
-      cid: key,
-      length: 0,
-      offset: 0,
-      blockLength: blockIndex.blockLength,
-      blockOffset: blockIndex.blockOffset
-    };
-    return NodeCarReader.readRaw(this._fd, readIndex);
-  }
-  async *blocks() {
-    for (const cidStr of this._order) {
-      const block = await this.get(CID.parse(cidStr));
-      if (!block) {
-        throw new Error('Unexpected internal error');
-      }
-      yield block;
-    }
-  }
-  async *cids() {
-    for (const cidStr of this._order) {
-      yield CID.parse(cidStr);
-    }
-  }
-  async close() {
-    if (this._fd) {
-      return this._fd.close();
-    }
-  }
-  static async fromFile(path) {
-    if (typeof path !== 'string') {
-      throw new TypeError('fromFile() requires a file path string');
-    }
-    const iterable = await CarIndexer.fromIterable(Readable.from(fs.createReadStream(path)));
-    const index = new Map();
-    const order = [];
-    for await (const {cid, blockLength, blockOffset} of iterable) {
-      const cidStr = cid.toString();
-      index.set(cidStr, {
-        blockLength,
-        blockOffset
-      });
-      order.push(cidStr);
-    }
-    return new CarIndexedReader(iterable.version, path, await iterable.getRoots(), index, order);
-  }
-}
-export const __browser = false;

package/esm/lib/indexer.js

@@ -1,38 +0,0 @@
-import {
-  asyncIterableReader,
-  bytesReader,
-  createDecoder
-} from './decoder.js';
-export class CarIndexer {
-  constructor(version, roots, iterator) {
-    this._version = version;
-    this._roots = roots;
-    this._iterator = iterator;
-  }
-  get version() {
-    return this._version;
-  }
-  async getRoots() {
-    return this._roots;
-  }
-  [Symbol.asyncIterator]() {
-    return this._iterator;
-  }
-  static async fromBytes(bytes) {
-    if (!(bytes instanceof Uint8Array)) {
-      throw new TypeError('fromBytes() requires a Uint8Array');
-    }
-    return decodeIndexerComplete(bytesReader(bytes));
-  }
-  static async fromIterable(asyncIterable) {
-    if (!asyncIterable || !(typeof asyncIterable[Symbol.asyncIterator] === 'function')) {
-      throw new TypeError('fromIterable() requires an async iterable');
-    }
-    return decodeIndexerComplete(asyncIterableReader(asyncIterable));
-  }
-}
-async function decodeIndexerComplete(reader) {
-  const decoder = createDecoder(reader);
-  const {version, roots} = await decoder.header();
-  return new CarIndexer(version, roots, decoder.blocksIndex());
-}

package/esm/lib/iterator-channel.js

@@ -1,71 +0,0 @@
-function noop() {
-}
-export function create() {
-  const chunkQueue = [];
-  let drainer = null;
-  let drainerResolver = noop;
-  let ended = false;
-  let outWait = null;
-  let outWaitResolver = noop;
-  const makeDrainer = () => {
-    if (!drainer) {
-      drainer = new Promise(resolve => {
-        drainerResolver = () => {
-          drainer = null;
-          drainerResolver = noop;
-          resolve();
-        };
-      });
-    }
-    return drainer;
-  };
-  const writer = {
-    write(chunk) {
-      chunkQueue.push(chunk);
-      const drainer = makeDrainer();
-      outWaitResolver();
-      return drainer;
-    },
-    async end() {
-      ended = true;
-      const drainer = makeDrainer();
-      outWaitResolver();
-      await drainer;
-    }
-  };
-  const iterator = {
-    async next() {
-      const chunk = chunkQueue.shift();
-      if (chunk) {
-        if (chunkQueue.length === 0) {
-          drainerResolver();
-        }
-        return {
-          done: false,
-          value: chunk
-        };
-      }
-      if (ended) {
-        drainerResolver();
-        return {
-          done: true,
-          value: undefined
-        };
-      }
-      if (!outWait) {
-        outWait = new Promise(resolve => {
-          outWaitResolver = () => {
-            outWait = null;
-            outWaitResolver = noop;
-            return resolve(iterator.next());
-          };
-        });
-      }
-      return outWait;
-    }
-  };
-  return {
-    writer,
-    iterator
-  };
-}