@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

package/dist/esm/parquetjs/encoder/writer.js

@@ -1,421 +0,0 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Transform } from 'stream';
-import { PARQUET_CODECS } from '../codecs';
-import * as Compression from '../compression';
-import * as Shred from '../schema/shred';
-import { ColumnChunk, ColumnMetaData, CompressionCodec, ConvertedType, DataPageHeader, DataPageHeaderV2, Encoding, FieldRepetitionType, FileMetaData, KeyValue, PageHeader, PageType, RowGroup, SchemaElement, Type } from '../parquet-thrift';
-import { osopen, oswrite, osclose } from '../utils/file-utils';
-import { getBitWidth, serializeThrift } from '../utils/read-utils';
-import Int64 from 'node-int64';
-const PARQUET_MAGIC = 'PAR1';
-const PARQUET_VERSION = 1;
-const PARQUET_DEFAULT_PAGE_SIZE = 8192;
-const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
-const PARQUET_RDLVL_TYPE = 'INT32';
-const PARQUET_RDLVL_ENCODING = 'RLE';
-export class ParquetWriter {
-  static async openFile(schema, path, opts) {
-    const outputStream = await osopen(path, opts);
-    return ParquetWriter.openStream(schema, outputStream, opts);
-  }
-
-  static async openStream(schema, outputStream, opts) {
-    if (!opts) {
-      opts = {};
-    }
-
-    const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
-    return new ParquetWriter(schema, envelopeWriter, opts);
-  }
-
-  constructor(schema, envelopeWriter, opts) {
-    _defineProperty(this, "schema", void 0);
-
-    _defineProperty(this, "envelopeWriter", void 0);
-
-    _defineProperty(this, "rowBuffer", void 0);
-
-    _defineProperty(this, "rowGroupSize", void 0);
-
-    _defineProperty(this, "closed", void 0);
-
-    _defineProperty(this, "userMetadata", void 0);
-
-    this.schema = schema;
-    this.envelopeWriter = envelopeWriter;
-    this.rowBuffer = {};
-    this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
-    this.closed = false;
-    this.userMetadata = {};
-    this.writeHeader();
-  }
-
-  async writeHeader() {
-    try {
-      await this.envelopeWriter.writeHeader();
-    } catch (err) {
-      await this.envelopeWriter.close();
-      throw err;
-    }
-  }
-
-  async appendRow(row) {
-    if (this.closed) {
-      throw new Error('writer was closed');
-    }
-
-    Shred.shredRecord(this.schema, row, this.rowBuffer);
-
-    if (this.rowBuffer.rowCount >= this.rowGroupSize) {
-      this.rowBuffer = {};
-    }
-  }
-
-  async close(callback) {
-    if (this.closed) {
-      throw new Error('writer was closed');
-    }
-
-    this.closed = true;
-
-    if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
-      this.rowBuffer = {};
-    }
-
-    await this.envelopeWriter.writeFooter(this.userMetadata);
-    await this.envelopeWriter.close();
-
-    if (callback) {
-      callback();
-    }
-  }
-
-  setMetadata(key, value) {
-    this.userMetadata[String(key)] = String(value);
-  }
-
-  setRowGroupSize(cnt) {
-    this.rowGroupSize = cnt;
-  }
-
-  setPageSize(cnt) {
-    this.envelopeWriter.setPageSize(cnt);
-  }
-
-}
-export class ParquetEnvelopeWriter {
-  static async openStream(schema, outputStream, opts) {
-    const writeFn = oswrite.bind(undefined, outputStream);
-    const closeFn = osclose.bind(undefined, outputStream);
-    return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);
-  }
-
-  constructor(schema, writeFn, closeFn, fileOffset, opts) {
-    _defineProperty(this, "schema", void 0);
-
-    _defineProperty(this, "write", void 0);
-
-    _defineProperty(this, "close", void 0);
-
-    _defineProperty(this, "offset", void 0);
-
-    _defineProperty(this, "rowCount", void 0);
-
-    _defineProperty(this, "rowGroups", void 0);
-
-    _defineProperty(this, "pageSize", void 0);
-
-    _defineProperty(this, "useDataPageV2", void 0);
-
-    this.schema = schema;
-    this.write = writeFn;
-    this.close = closeFn;
-    this.offset = fileOffset;
-    this.rowCount = 0;
-    this.rowGroups = [];
-    this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
-    this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
-  }
-
-  writeSection(buf) {
-    this.offset += buf.length;
-    return this.write(buf);
-  }
-
-  writeHeader() {
-    return this.writeSection(Buffer.from(PARQUET_MAGIC));
-  }
-
-  async writeRowGroup(records) {
-    const rgroup = await encodeRowGroup(this.schema, records, {
-      baseOffset: this.offset,
-      pageSize: this.pageSize,
-      useDataPageV2: this.useDataPageV2
-    });
-    this.rowCount += records.rowCount;
-    this.rowGroups.push(rgroup.metadata);
-    return await this.writeSection(rgroup.body);
-  }
-
-  writeFooter(userMetadata) {
-    if (!userMetadata) {
-      userMetadata = {};
-    }
-
-    return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
-  }
-
-  setPageSize(cnt) {
-    this.pageSize = cnt;
-  }
-
-}
-export class ParquetTransformer extends Transform {
-  constructor(schema, opts = {}) {
-    super({
-      objectMode: true
-    });
-
-    _defineProperty(this, "writer", void 0);
-
-    const writeProxy = function (t) {
-      return async function (b) {
-        t.push(b);
-      };
-    }(this);
-
-    this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts), opts);
-  }
-
-  _transform(row, encoding, callback) {
-    if (row) {
-      return this.writer.appendRow(row).then(callback);
-    }
-
-    callback();
-    return Promise.resolve();
-  }
-
-  async _flush(callback) {
-    await this.writer.close(callback);
-  }
-
-}
-
-function encodeValues(type, encoding, values, opts) {
-  if (!(encoding in PARQUET_CODECS)) {
-    throw new Error("invalid encoding: ".concat(encoding));
-  }
-
-  return PARQUET_CODECS[encoding].encodeValues(type, values, opts);
-}
-
-async function encodeDataPage(column, data) {
-  let rLevelsBuf = Buffer.alloc(0);
-
-  if (column.rLevelMax > 0) {
-    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
-      bitWidth: getBitWidth(column.rLevelMax)
-    });
-  }
-
-  let dLevelsBuf = Buffer.alloc(0);
-
-  if (column.dLevelMax > 0) {
-    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
-      bitWidth: getBitWidth(column.dLevelMax)
-    });
-  }
-
-  const valuesBuf = encodeValues(column.primitiveType, column.encoding, data.values, {
-    typeLength: column.typeLength,
-    bitWidth: column.typeLength
-  });
-  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
-  const compressedBuf = await Compression.deflate(column.compression, dataBuf);
-  const header = new PageHeader({
-    type: PageType.DATA_PAGE,
-    data_page_header: new DataPageHeader({
-      num_values: data.count,
-      encoding: Encoding[column.encoding],
-      definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING],
-      repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING]
-    }),
-    uncompressed_page_size: dataBuf.length,
-    compressed_page_size: compressedBuf.length
-  });
-  const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, compressedBuf]);
-  return {
-    header,
-    headerSize: headerBuf.length,
-    page
-  };
-}
-
-async function encodeDataPageV2(column, data, rowCount) {
-  const valuesBuf = encodeValues(column.primitiveType, column.encoding, data.values, {
-    typeLength: column.typeLength,
-    bitWidth: column.typeLength
-  });
-  const compressedBuf = await Compression.deflate(column.compression, valuesBuf);
-  let rLevelsBuf = Buffer.alloc(0);
-
-  if (column.rLevelMax > 0) {
-    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
-      bitWidth: getBitWidth(column.rLevelMax),
-      disableEnvelope: true
-    });
-  }
-
-  let dLevelsBuf = Buffer.alloc(0);
-
-  if (column.dLevelMax > 0) {
-    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
-      bitWidth: getBitWidth(column.dLevelMax),
-      disableEnvelope: true
-    });
-  }
-
-  const header = new PageHeader({
-    type: PageType.DATA_PAGE_V2,
-    data_page_header_v2: new DataPageHeaderV2({
-      num_values: data.count,
-      num_nulls: data.count - data.values.length,
-      num_rows: rowCount,
-      encoding: Encoding[column.encoding],
-      definition_levels_byte_length: dLevelsBuf.length,
-      repetition_levels_byte_length: rLevelsBuf.length,
-      is_compressed: column.compression !== 'UNCOMPRESSED'
-    }),
-    uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,
-    compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length
-  });
-  const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
-  return {
-    header,
-    headerSize: headerBuf.length,
-    page
-  };
-}
-
-async function encodeColumnChunk(column, buffer, offset, opts) {
-  const data = buffer.columnData[column.path.join()];
-  const baseOffset = (opts.baseOffset || 0) + offset;
-  let pageBuf;
-  let total_uncompressed_size = 0;
-  let total_compressed_size = 0;
-  {
-    const result = opts.useDataPageV2 ? await encodeDataPageV2(column, data, buffer.rowCount) : await encodeDataPage(column, data);
-    pageBuf = result.page;
-    total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
-    total_compressed_size += result.header.compressed_page_size + result.headerSize;
-  }
-  const metadata = new ColumnMetaData({
-    path_in_schema: column.path,
-    num_values: data.count,
-    data_page_offset: baseOffset,
-    encodings: [],
-    total_uncompressed_size,
-    total_compressed_size,
-    type: Type[column.primitiveType],
-    codec: CompressionCodec[column.compression]
-  });
-  metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);
-  metadata.encodings.push(Encoding[column.encoding]);
-  const metadataOffset = baseOffset + pageBuf.length;
-  const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);
-  return {
-    body,
-    metadata,
-    metadataOffset
-  };
-}
-
-async function encodeRowGroup(schema, data, opts) {
-  const metadata = new RowGroup({
-    num_rows: data.rowCount,
-    columns: [],
-    total_byte_size: 0
-  });
-  let body = Buffer.alloc(0);
-
-  for (const field of schema.fieldList) {
-    if (field.isNested) {
-      continue;
-    }
-
-    const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
-    const cchunk = new ColumnChunk({
-      file_offset: cchunkData.metadataOffset,
-      meta_data: cchunkData.metadata
-    });
-    metadata.columns.push(cchunk);
-    metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
-    body = Buffer.concat([body, cchunkData.body]);
-  }
-
-  return {
-    body,
-    metadata
-  };
-}
-
-function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
-  const metadata = new FileMetaData({
-    version: PARQUET_VERSION,
-    created_by: 'parquets',
-    num_rows: rowCount,
-    row_groups: rowGroups,
-    schema: [],
-    key_value_metadata: []
-  });
-
-  for (const key in userMetadata) {
-    var _metadata$key_value_m, _metadata$key_value_m2;
-
-    const kv = new KeyValue({
-      key,
-      value: userMetadata[key]
-    });
-    (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = _metadata$key_value_m.push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m, kv);
-  }
-
-  {
-    const schemaRoot = new SchemaElement({
-      name: 'root',
-      num_children: Object.keys(schema.fields).length
-    });
-    metadata.schema.push(schemaRoot);
-  }
-
-  for (const field of schema.fieldList) {
-    const relt = FieldRepetitionType[field.repetitionType];
-    const schemaElem = new SchemaElement({
-      name: field.name,
-      repetition_type: relt
-    });
-
-    if (field.isNested) {
-      schemaElem.num_children = field.fieldCount;
-    } else {
-      schemaElem.type = Type[field.primitiveType];
-    }
-
-    if (field.originalType) {
-      schemaElem.converted_type = ConvertedType[field.originalType];
-    }
-
-    schemaElem.type_length = field.typeLength;
-    metadata.schema.push(schemaElem);
-  }
-
-  const metadataEncoded = serializeThrift(metadata);
-  const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
-  metadataEncoded.copy(footerEncoded);
-  footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
-  footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
-  return footerEncoded;
-}
-//# sourceMappingURL=writer.js.map

package/dist/esm/parquetjs/encoder/writer.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/encoder/writer.ts"],"names":["Transform","PARQUET_CODECS","Compression","Shred","ColumnChunk","ColumnMetaData","CompressionCodec","ConvertedType","DataPageHeader","DataPageHeaderV2","Encoding","FieldRepetitionType","FileMetaData","KeyValue","PageHeader","PageType","RowGroup","SchemaElement","Type","osopen","oswrite","osclose","getBitWidth","serializeThrift","Int64","PARQUET_MAGIC","PARQUET_VERSION","PARQUET_DEFAULT_PAGE_SIZE","PARQUET_DEFAULT_ROW_GROUP_SIZE","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","ParquetWriter","openFile","schema","path","opts","outputStream","openStream","envelopeWriter","ParquetEnvelopeWriter","constructor","rowBuffer","rowGroupSize","closed","userMetadata","writeHeader","err","close","appendRow","row","Error","shredRecord","rowCount","callback","writeFooter","setMetadata","key","value","String","setRowGroupSize","cnt","setPageSize","writeFn","bind","undefined","closeFn","fileOffset","write","offset","rowGroups","pageSize","useDataPageV2","Boolean","writeSection","buf","length","Buffer","from","writeRowGroup","records","rgroup","encodeRowGroup","baseOffset","push","metadata","body","encodeFooter","ParquetTransformer","objectMode","writeProxy","t","b","writer","_transform","encoding","then","Promise","resolve","_flush","encodeValues","type","values","encodeDataPage","column","data","rLevelsBuf","alloc","rLevelMax","rlevels","bitWidth","dLevelsBuf","dLevelMax","dlevels","valuesBuf","primitiveType","typeLength","dataBuf","concat","compressedBuf","deflate","compression","header","DATA_PAGE","data_page_header","num_values","count","definition_level_encoding","repetition_level_encoding","uncompressed_page_size","compressed_page_size","headerBuf","page","headerSize","encodeDataPageV2","disableEnvelope","DATA_PAGE_V2","data_page_header_v2","num_nulls","num_rows","definition_levels_byte_length","repetition_levels_byte_length","is_compressed","encodeColumnChunk","buffer","columnData","join","pageBuf","total_uncompressed_size","total_compressed_size","result","path_in_schema","data_page_offset","encodings","codec","metadataOffset","columns","total_byte_size","field","fieldList","isNested","cchunkData","cchunk","file_offset","meta_data","Number","version","created_by","row_groups","key_value_metadata","kv","schemaRoot","name","num_children","Object","keys","fields","relt","repetitionType","schemaElem","repetition_type","fieldCount","originalType","converted_type","type_length","metadataEncoded","footerEncoded","copy","writeUInt32LE"],"mappings":";AAEA,SAAQA,SAAR,QAAkC,QAAlC;AACA,SAA6BC,cAA7B,QAAkD,WAAlD;AACA,OAAO,KAAKC,WAAZ,MAA6B,gBAA7B;AASA,OAAO,KAAKC,KAAZ,MAAuB,iBAAvB;AACA,SACEC,WADF,EAEEC,cAFF,EAGEC,gBAHF,EAIEC,aAJF,EAKEC,cALF,EAMEC,gBANF,EAOEC,QAPF,EAQEC,mBARF,EASEC,YATF,EAUEC,QAVF,EAWEC,UAXF,EAYEC,QAZF,EAaEC,QAbF,EAcEC,aAdF,EAeEC,IAfF,QAgBO,mBAhBP;AAiBA,SAAQC,MAAR,EAAgBC,OAAhB,EAAyBC,OAAzB,QAAuC,qBAAvC;AACA,SAAQC,WAAR,EAAqBC,eAArB,QAA2C,qBAA3C;AACA,OAAOC,KAAP,MAAkB,YAAlB;AAKA,MAAMC,aAAa,GAAG,MAAtB;AAKA,MAAMC,eAAe,GAAG,CAAxB;AAKA,MAAMC,yBAAyB,GAAG,IAAlC;AACA,MAAMC,8BAA8B,GAAG,IAAvC;AAKA,MAAMC,kBAAkB,GAAG,OAA3B;AACA,MAAMC,sBAAsB,GAAG,KAA/B;AAuBA,OAAO,MAAMC,aAAN,CAAuB;AAKP,eAARC,QAAQ,CACnBC,MADmB,EAEnBC,IAFmB,EAGnBC,IAHmB,EAIQ;AAC3B,UAAMC,YAAY,GAAG,MAAMjB,MAAM,CAACe,IAAD,EAAOC,IAAP,CAAjC;AACA,WAAOJ,aAAa,CAACM,UAAd,CAAyBJ,MAAzB,EAAiCG,YAAjC,EAA+CD,IAA/C,CAAP;AACD;;AAMsB,eAAVE,UAAU,CACrBJ,MADqB,EAErBG,YAFqB,EAGrBD,IAHqB,EAIM;AAC3B,QAAI,CAACA,IAAL,EAAW;AAETA,MAAAA,IAAI,GAAG,EAAP;AACD;;AAED,UAAMG,cAAc,GAAG,MAAMC,qBAAqB,CAACF,UAAtB,CAAiCJ,MAAjC,EAAyCG,YAAzC,EAAuDD,IAAvD,CAA7B;AAEA,WAAO,IAAIJ,aAAJ,CAAkBE,MAAlB,EAA0BK,cAA1B,EAA0CH,IAA1C,CAAP;AACD;;AAYDK,EAAAA,WAAW,CACTP,MADS,EAETK,cAFS,EAGTH,IAHS,EAIT;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AACA,SAAKF,MAAL,GAAcA,MAAd;AACA,SAAKK,cAAL,GAAsBA,cAAtB;AAEA,SAAKG,SAAL,GAAiB,EAAjB;AACA,SAAKC,YAAL,GAAoBP,IAAI,CAACO,YAAL,IAAqBd,8BAAzC;AACA,SAAKe,MAAL,GAAc,KAAd;AACA,SAAKC,YAAL,GAAoB,EAApB;AAGA,SAAKC,WAAL;AACD;;AAEgB,QAAXA,WAAW,GAAkB;AAEjC,QAAI;AACF,YAAM,KAAKP,cAAL,CAAoBO,WAApB,EAAN;AACD,KAFD,CAEE,OAAOC,GAAP,EAAY;AACZ,YAAM,KAAKR,cAAL,CAAoBS,KAApB,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAMc,QAATE,SAAS,CAAIC,GAAJ,EAA2B;AACxC,QAAI,KAAKN,MAAT,EAAiB;AACf,YAAM,IAAIO,KAAJ,CAAU,mBAAV,CAAN;AACD;;AACD/C,IAAAA,KAAK,CAACgD,WAAN,CAAkB,KAAKlB,MAAvB,EAA+BgB,GAA/B,EAAoC,KAAKR,SAAzC;;AACA,QAAI,KAAKA,SAAL,CAAeW,QAAf,IAA2B,KAAKV,YAApC,EAAkD;AAEhD,WAAKD,SAAL,GAAiB,EAAjB;AACD;AACF;;AAQU,QAALM,KAAK,CAACM,QAAD,EAAuC;AAChD,QAAI,KAAKV,MAAT,EAAiB;AACf,YAAM,IAAIO,KAAJ,CAAU,mBAAV,CAAN;AACD;;AAED,SAAKP,MAAL,GAAc,IAAd;;AAEA,QAAI,KAAKF,SAAL,CAAeW,QAAf,GAA0B,CAA1B,IAA+B,KAAKX,SAAL,CAAeW,QAAf,IAA2B,KAAKV,YAAnE,EAAiF;AAE/E,WAAKD,SAAL,GAAiB,EAAjB;AACD;;AAED,UAAM,KAAKH,cAAL,CAAoBgB,WAApB,CAAgC,KAAKV,YAArC,CAAN;AACA,UAAM,KAAKN,cAAL,CAAoBS,KAApB,EAAN;;AAGA,QAAIM,QAAJ,EAAc;AACZA,MAAAA,QAAQ;AACT;AACF;;AAKDE,EAAAA,WAAW,CAACC,GAAD,EAAcC,KAAd,EAAmC;AAE5C,SAAKb,YAAL,CAAkBc,MAAM,CAACF,GAAD,CAAxB,IAAiCE,MAAM,CAACD,KAAD,CAAvC;AACD;;AAQDE,EAAAA,eAAe,CAACC,GAAD,EAAoB;AACjC,SAAKlB,YAAL,GAAoBkB,GAApB;AACD;;AAMDC,EAAAA,WAAW,CAACD,GAAD,EAAoB;AAC7B,SAAKtB,cAAL,CAAoBuB,WAApB,CAAgCD,GAAhC;AACD;;AAxI2B;AAiJ9B,OAAO,MAAMrB,qBAAN,CAA4B;AAIV,eAAVF,UAAU,CACrBJ,MADqB,EAErBG,YAFqB,EAGrBD,IAHqB,EAIW;AAChC,UAAM2B,OAAO,GAAG1C,OAAO,CAAC2C,IAAR,CAAaC,SAAb,EAAwB5B,YAAxB,CAAhB;AACA,UAAM6B,OAAO,GAAG5C,OAAO,CAAC0C,IAAR,CAAaC,SAAb,EAAwB5B,YAAxB,CAAhB;AACA,WAAO,IAAIG,qBAAJ,CAA0BN,MAA1B,EAAkC6B,OAAlC,EAA2CG,OAA3C,EAAoD,CAApD,EAAuD9B,IAAvD,CAAP;AACD;;AAWDK,EAAAA,WAAW,CACTP,MADS,EAET6B,OAFS,EAGTG,OAHS,EAITC,UAJS,EAKT/B,IALS,EAMT;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AACA,SAAKF,MAAL,GAAcA,MAAd;AACA,SAAKkC,KAAL,GAAaL,OAAb;AACA,SAAKf,KAAL,GAAakB,OAAb;AACA,SAAKG,MAAL,GAAcF,UAAd;AACA,SAAKd,QAAL,GAAgB,CAAhB;AACA,SAAKiB,SAAL,GAAiB,EAAjB;AACA,SAAKC,QAAL,GAAgBnC,IAAI,CAACmC,QAAL,IAAiB3C,yBAAjC;AACA,SAAK4C,aAAL,GAAqB,mBAAmBpC,IAAnB,GAA0BqC,OAAO,CAACrC,IAAI,CAACoC,aAAN,CAAjC,GAAwD,KAA7E;AACD;;AAEDE,EAAAA,YAAY,CAACC,GAAD,EAA6B;AACvC,SAAKN,MAAL,IAAeM,GAAG,CAACC,MAAnB;AACA,WAAO,KAAKR,KAAL,CAAWO,GAAX,CAAP;AACD;;AAKD7B,EAAAA,WAAW,GAAkB;AAC3B,WAAO,KAAK4B,YAAL,CAAkBG,MAAM,CAACC,IAAP,CAAYpD,aAAZ,CAAlB,CAAP;AACD;;AAMkB,QAAbqD,aAAa,CAACC,OAAD,EAAwC;AACzD,UAAMC,MAAM,GAAG,MAAMC,cAAc,CAAC,KAAKhD,MAAN,EAAc8C,OAAd,EAAuB;AACxDG,MAAAA,UAAU,EAAE,KAAKd,MADuC;AAExDE,MAAAA,QAAQ,EAAE,KAAKA,QAFyC;AAGxDC,MAAAA,aAAa,EAAE,KAAKA;AAHoC,KAAvB,CAAnC;AAMA,SAAKnB,QAAL,IAAiB2B,OAAO,CAAC3B,QAAzB;AACA,SAAKiB,SAAL,CAAec,IAAf,CAAoBH,MAAM,CAACI,QAA3B;AACA,WAAO,MAAM,KAAKX,YAAL,CAAkBO,MAAM,CAACK,IAAzB,CAAb;AACD;;AAKD/B,EAAAA,WAAW,CAACV,YAAD,EAAsD;AAC/D,QAAI,CAACA,YAAL,EAAmB;AAEjBA,MAAAA,YAAY,GAAG,EAAf;AACD;;AAED,WAAO,KAAK6B,YAAL,CACLa,YAAY,CAAC,KAAKrD,MAAN,EAAc,KAAKmB,QAAnB,EAA6B,KAAKiB,SAAlC,EAA6CzB,YAA7C,CADP,CAAP;AAGD;;AAMDiB,EAAAA,WAAW,CAACD,GAAD,EAAoB;AAC7B,SAAKU,QAAL,GAAgBV,GAAhB;AACD;;AAxFgC;AA8FnC,OAAO,MAAM2B,kBAAN,SAAoCvF,SAApC,CAA8C;AAGnDwC,EAAAA,WAAW,CAACP,MAAD,EAAwBE,IAA0B,GAAG,EAArD,EAAyD;AAClE,UAAM;AAACqD,MAAAA,UAAU,EAAE;AAAb,KAAN;;AADkE;;AAGlE,UAAMC,UAAU,GAAI,UAAUC,CAAV,EAAsC;AACxD,aAAO,gBAAgBC,CAAhB,EAAuC;AAC5CD,QAAAA,CAAC,CAACP,IAAF,CAAOQ,CAAP;AACD,OAFD;AAGD,KAJkB,CAIhB,IAJgB,CAAnB;;AAMA,SAAKC,MAAL,GAAc,IAAI7D,aAAJ,CACZE,MADY,EAEZ,IAAIM,qBAAJ,CAA0BN,MAA1B,EAAkCwD,UAAlC,EAA8C,YAAY,CAAE,CAA5D,EAA8D,CAA9D,EAAiEtD,IAAjE,CAFY,EAGZA,IAHY,CAAd;AAKD;;AAGD0D,EAAAA,UAAU,CAAC5C,GAAD,EAAW6C,QAAX,EAA6BzC,QAA7B,EAA2E;AACnF,QAAIJ,GAAJ,EAAS;AACP,aAAO,KAAK2C,MAAL,CAAY5C,SAAZ,CAAsBC,GAAtB,EAA2B8C,IAA3B,CAAgC1C,QAAhC,CAAP;AACD;;AACDA,IAAAA,QAAQ;AACR,WAAO2C,OAAO,CAACC,OAAR,EAAP;AACD;;AAGW,QAANC,MAAM,CAAC7C,QAAD,EAAgC;AAC1C,UAAM,KAAKuC,MAAL,CAAY7C,KAAZ,CAAkBM,QAAlB,CAAN;AACD;;AA/BkD;;AAqCrD,SAAS8C,YAAT,CACEC,IADF,EAEEN,QAFF,EAGEO,MAHF,EAIElE,IAJF,EAKE;AACA,MAAI,EAAE2D,QAAQ,IAAI7F,cAAd,CAAJ,EAAmC;AACjC,UAAM,IAAIiD,KAAJ,6BAA+B4C,QAA/B,EAAN;AACD;;AACD,SAAO7F,cAAc,CAAC6F,QAAD,CAAd,CAAyBK,YAAzB,CAAsCC,IAAtC,EAA4CC,MAA5C,EAAoDlE,IAApD,CAAP;AACD;;AAKD,eAAemE,cAAf,CACEC,MADF,EAEEC,IAFF,EAOG;AAED,MAAIC,UAAU,GAAG7B,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACI,SAAP,GAAmB,CAAvB,EAA0B;AACxBF,IAAAA,UAAU,GAAGN,YAAY,CAACtE,kBAAD,EAAqBC,sBAArB,EAA6C0E,IAAI,CAACI,OAAlD,EAA2D;AAClFC,MAAAA,QAAQ,EAAEvF,WAAW,CAACiF,MAAM,CAACI,SAAR;AAD6D,KAA3D,CAAzB;AAID;;AAED,MAAIG,UAAU,GAAGlC,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACQ,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,UAAU,GAAGX,YAAY,CAACtE,kBAAD,EAAqBC,sBAArB,EAA6C0E,IAAI,CAACQ,OAAlD,EAA2D;AAClFH,MAAAA,QAAQ,EAAEvF,WAAW,CAACiF,MAAM,CAACQ,SAAR;AAD6D,KAA3D,CAAzB;AAID;;AAGD,QAAME,SAAS,GAAGd,YAAY,CAACI,MAAM,CAACW,aAAR,EAAwBX,MAAM,CAACT,QAA/B,EAA0CU,IAAI,CAACH,MAA/C,EAAuD;AACnFc,IAAAA,UAAU,EAAEZ,MAAM,CAACY,UADgE;AAEnFN,IAAAA,QAAQ,EAAEN,MAAM,CAACY;AAFkE,GAAvD,CAA9B;AAKA,QAAMC,OAAO,GAAGxC,MAAM,CAACyC,MAAP,CAAc,CAACZ,UAAD,EAAaK,UAAb,EAAyBG,SAAzB,CAAd,CAAhB;AAGA,QAAMK,aAAa,GAAG,MAAMpH,WAAW,CAACqH,OAAZ,CAAoBhB,MAAM,CAACiB,WAA3B,EAAyCJ,OAAzC,CAA5B;AAGA,QAAMK,MAAM,GAAG,IAAI3G,UAAJ,CAAe;AAC5BsF,IAAAA,IAAI,EAAErF,QAAQ,CAAC2G,SADa;AAE5BC,IAAAA,gBAAgB,EAAE,IAAInH,cAAJ,CAAmB;AACnCoH,MAAAA,UAAU,EAAEpB,IAAI,CAACqB,KADkB;AAEnC/B,MAAAA,QAAQ,EAAEpF,QAAQ,CAAC6F,MAAM,CAACT,QAAR,CAFiB;AAGnCgC,MAAAA,yBAAyB,EAAEpH,QAAQ,CAACoB,sBAAD,CAHA;AAInCiG,MAAAA,yBAAyB,EAAErH,QAAQ,CAACoB,sBAAD;AAJA,KAAnB,CAFU;AAQ5BkG,IAAAA,sBAAsB,EAAEZ,OAAO,CAACzC,MARJ;AAS5BsD,IAAAA,oBAAoB,EAAEX,aAAa,CAAC3C;AATR,GAAf,CAAf;AAaA,QAAMuD,SAAS,GAAG3G,eAAe,CAACkG,MAAD,CAAjC;AACA,QAAMU,IAAI,GAAGvD,MAAM,CAACyC,MAAP,CAAc,CAACa,SAAD,EAAYZ,aAAZ,CAAd,CAAb;AAEA,SAAO;AAACG,IAAAA,MAAD;AAASW,IAAAA,UAAU,EAAEF,SAAS,CAACvD,MAA/B;AAAuCwD,IAAAA;AAAvC,GAAP;AACD;;AAKD,eAAeE,gBAAf,CACE9B,MADF,EAEEC,IAFF,EAGEpD,QAHF,EAQG;AAED,QAAM6D,SAAS,GAAGd,YAAY,CAACI,MAAM,CAACW,aAAR,EAAwBX,MAAM,CAACT,QAA/B,EAA0CU,IAAI,CAACH,MAA/C,EAAuD;AACnFc,IAAAA,UAAU,EAAEZ,MAAM,CAACY,UADgE;AAEnFN,IAAAA,QAAQ,EAAEN,MAAM,CAACY;AAFkE,GAAvD,CAA9B;AAMA,QAAMG,aAAa,GAAG,MAAMpH,WAAW,CAACqH,OAAZ,CAAoBhB,MAAM,CAACiB,WAA3B,EAAyCP,SAAzC,CAA5B;AAGA,MAAIR,UAAU,GAAG7B,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACI,SAAP,GAAmB,CAAvB,EAA0B;AACxBF,IAAAA,UAAU,GAAGN,YAAY,CAACtE,kBAAD,EAAqBC,sBAArB,EAA6C0E,IAAI,CAACI,OAAlD,EAA2D;AAClFC,MAAAA,QAAQ,EAAEvF,WAAW,CAACiF,MAAM,CAACI,SAAR,CAD6D;AAElF2B,MAAAA,eAAe,EAAE;AAFiE,KAA3D,CAAzB;AAID;;AAED,MAAIxB,UAAU,GAAGlC,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACQ,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,UAAU,GAAGX,YAAY,CAACtE,kBAAD,EAAqBC,sBAArB,EAA6C0E,IAAI,CAACQ,OAAlD,EAA2D;AAClFH,MAAAA,QAAQ,EAAEvF,WAAW,CAACiF,MAAM,CAACQ,SAAR,CAD6D;AAElFuB,MAAAA,eAAe,EAAE;AAFiE,KAA3D,CAAzB;AAID;;AAGD,QAAMb,MAAM,GAAG,IAAI3G,UAAJ,CAAe;AAC5BsF,IAAAA,IAAI,EAAErF,QAAQ,CAACwH,YADa;AAE5BC,IAAAA,mBAAmB,EAAE,IAAI/H,gBAAJ,CAAqB;AACxCmH,MAAAA,UAAU,EAAEpB,IAAI,CAACqB,KADuB;AAExCY,MAAAA,SAAS,EAAEjC,IAAI,CAACqB,KAAL,GAAarB,IAAI,CAACH,MAAL,CAAY1B,MAFI;AAGxC+D,MAAAA,QAAQ,EAAEtF,QAH8B;AAIxC0C,MAAAA,QAAQ,EAAEpF,QAAQ,CAAC6F,MAAM,CAACT,QAAR,CAJsB;AAKxC6C,MAAAA,6BAA6B,EAAE7B,UAAU,CAACnC,MALF;AAMxCiE,MAAAA,6BAA6B,EAAEnC,UAAU,CAAC9B,MANF;AAOxCkE,MAAAA,aAAa,EAAEtC,MAAM,CAACiB,WAAP,KAAuB;AAPE,KAArB,CAFO;AAW5BQ,IAAAA,sBAAsB,EAAEvB,UAAU,CAAC9B,MAAX,GAAoBmC,UAAU,CAACnC,MAA/B,GAAwCsC,SAAS,CAACtC,MAX9C;AAY5BsD,IAAAA,oBAAoB,EAAExB,UAAU,CAAC9B,MAAX,GAAoBmC,UAAU,CAACnC,MAA/B,GAAwC2C,aAAa,CAAC3C;AAZhD,GAAf,CAAf;AAgBA,QAAMuD,SAAS,GAAG3G,eAAe,CAACkG,MAAD,CAAjC;AACA,QAAMU,IAAI,GAAGvD,MAAM,CAACyC,MAAP,CAAc,CAACa,SAAD,EAAYzB,UAAZ,EAAwBK,UAAxB,EAAoCQ,aAApC,CAAd,CAAb;AACA,SAAO;AAACG,IAAAA,MAAD;AAASW,IAAAA,UAAU,EAAEF,SAAS,CAACvD,MAA/B;AAAuCwD,IAAAA;AAAvC,GAAP;AACD;;AAKD,eAAeW,iBAAf,CACEvC,MADF,EAEEwC,MAFF,EAGE3E,MAHF,EAIEjC,IAJF,EASG;AACD,QAAMqE,IAAI,GAAGuC,MAAM,CAACC,UAAP,CAAkBzC,MAAM,CAACrE,IAAP,CAAY+G,IAAZ,EAAlB,CAAb;AACA,QAAM/D,UAAU,GAAG,CAAC/C,IAAI,CAAC+C,UAAL,IAAmB,CAApB,IAAyBd,MAA5C;AAGA,MAAI8E,OAAJ;AAEA,MAAIC,uBAAuB,GAAG,CAA9B;AAEA,MAAIC,qBAAqB,GAAG,CAA5B;AACA;AACE,UAAMC,MAAM,GAAGlH,IAAI,CAACoC,aAAL,GACX,MAAM8D,gBAAgB,CAAC9B,MAAD,EAASC,IAAT,EAAeuC,MAAM,CAAC3F,QAAtB,CADX,GAEX,MAAMkD,cAAc,CAACC,MAAD,EAASC,IAAT,CAFxB;AAIA0C,IAAAA,OAAO,GAAGG,MAAM,CAAClB,IAAjB;AACAgB,IAAAA,uBAAuB,IAAIE,MAAM,CAAC5B,MAAP,CAAcO,sBAAd,GAAuCqB,MAAM,CAACjB,UAAzE;AACAgB,IAAAA,qBAAqB,IAAIC,MAAM,CAAC5B,MAAP,CAAcQ,oBAAd,GAAqCoB,MAAM,CAACjB,UAArE;AACD;AAMD,QAAMhD,QAAQ,GAAG,IAAI/E,cAAJ,CAAmB;AAClCiJ,IAAAA,cAAc,EAAE/C,MAAM,CAACrE,IADW;AAElC0F,IAAAA,UAAU,EAAEpB,IAAI,CAACqB,KAFiB;AAGlC0B,IAAAA,gBAAgB,EAAErE,UAHgB;AAIlCsE,IAAAA,SAAS,EAAE,EAJuB;AAKlCL,IAAAA,uBALkC;AAMlCC,IAAAA,qBANkC;AAOlChD,IAAAA,IAAI,EAAElF,IAAI,CAACqF,MAAM,CAACW,aAAR,CAPwB;AAQlCuC,IAAAA,KAAK,EAAEnJ,gBAAgB,CAACiG,MAAM,CAACiB,WAAR;AARW,GAAnB,CAAjB;AAYApC,EAAAA,QAAQ,CAACoE,SAAT,CAAmBrE,IAAnB,CAAwBzE,QAAQ,CAACoB,sBAAD,CAAhC;AACAsD,EAAAA,QAAQ,CAACoE,SAAT,CAAmBrE,IAAnB,CAAwBzE,QAAQ,CAAC6F,MAAM,CAACT,QAAR,CAAhC;AAGA,QAAM4D,cAAc,GAAGxE,UAAU,GAAGgE,OAAO,CAACvE,MAA5C;AACA,QAAMU,IAAI,GAAGT,MAAM,CAACyC,MAAP,CAAc,CAAC6B,OAAD,EAAU3H,eAAe,CAAC6D,QAAD,CAAzB,CAAd,CAAb;AACA,SAAO;AAACC,IAAAA,IAAD;AAAOD,IAAAA,QAAP;AAAiBsE,IAAAA;AAAjB,GAAP;AACD;;AAKD,eAAezE,cAAf,CACEhD,MADF,EAEEuE,IAFF,EAGErE,IAHF,EAOG;AACD,QAAMiD,QAAQ,GAAG,IAAIpE,QAAJ,CAAa;AAC5B0H,IAAAA,QAAQ,EAAElC,IAAI,CAACpD,QADa;AAE5BuG,IAAAA,OAAO,EAAE,EAFmB;AAG5BC,IAAAA,eAAe,EAAE;AAHW,GAAb,CAAjB;AAMA,MAAIvE,IAAI,GAAGT,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAX;;AACA,OAAK,MAAMmD,KAAX,IAAoB5H,MAAM,CAAC6H,SAA3B,EAAsC;AACpC,QAAID,KAAK,CAACE,QAAV,EAAoB;AAClB;AACD;;AAED,UAAMC,UAAU,GAAG,MAAMlB,iBAAiB,CAACe,KAAD,EAAQrD,IAAR,EAAcnB,IAAI,CAACV,MAAnB,EAA2BxC,IAA3B,CAA1C;AAEA,UAAM8H,MAAM,GAAG,IAAI7J,WAAJ,CAAgB;AAC7B8J,MAAAA,WAAW,EAAEF,UAAU,CAACN,cADK;AAE7BS,MAAAA,SAAS,EAAEH,UAAU,CAAC5E;AAFO,KAAhB,CAAf;AAKAA,IAAAA,QAAQ,CAACuE,OAAT,CAAiBxE,IAAjB,CAAsB8E,MAAtB;AACA7E,IAAAA,QAAQ,CAACwE,eAAT,GAA2B,IAAIpI,KAAJ,CAAU4I,MAAM,CAAChF,QAAQ,CAACwE,eAAV,CAAN,GAAmCI,UAAU,CAAC3E,IAAX,CAAgBV,MAA7D,CAA3B;AAEAU,IAAAA,IAAI,GAAGT,MAAM,CAACyC,MAAP,CAAc,CAAChC,IAAD,EAAO2E,UAAU,CAAC3E,IAAlB,CAAd,CAAP;AACD;;AAED,SAAO;AAACA,IAAAA,IAAD;AAAOD,IAAAA;AAAP,GAAP;AACD;;AAKD,SAASE,YAAT,CACErD,MADF,EAEEmB,QAFF,EAGEiB,SAHF,EAIEzB,YAJF,EAKU;AACR,QAAMwC,QAAQ,GAAG,IAAIxE,YAAJ,CAAiB;AAChCyJ,IAAAA,OAAO,EAAE3I,eADuB;AAEhC4I,IAAAA,UAAU,EAAE,UAFoB;AAGhC5B,IAAAA,QAAQ,EAAEtF,QAHsB;AAIhCmH,IAAAA,UAAU,EAAElG,SAJoB;AAKhCpC,IAAAA,MAAM,EAAE,EALwB;AAMhCuI,IAAAA,kBAAkB,EAAE;AANY,GAAjB,CAAjB;;AASA,OAAK,MAAMhH,GAAX,IAAkBZ,YAAlB,EAAgC;AAAA;;AAC9B,UAAM6H,EAAE,GAAG,IAAI5J,QAAJ,CAAa;AACtB2C,MAAAA,GADsB;AAEtBC,MAAAA,KAAK,EAAEb,YAAY,CAACY,GAAD;AAFG,KAAb,CAAX;AAIA,6BAAA4B,QAAQ,CAACoF,kBAAT,0GAA6BrF,IAA7B,8GAAoCsF,EAApC;AACD;;AAED;AACE,UAAMC,UAAU,GAAG,IAAIzJ,aAAJ,CAAkB;AACnC0J,MAAAA,IAAI,EAAE,MAD6B;AAEnCC,MAAAA,YAAY,EAAEC,MAAM,CAACC,IAAP,CAAY7I,MAAM,CAAC8I,MAAnB,EAA2BpG;AAFN,KAAlB,CAAnB;AAIAS,IAAAA,QAAQ,CAACnD,MAAT,CAAgBkD,IAAhB,CAAqBuF,UAArB;AACD;;AAED,OAAK,MAAMb,KAAX,IAAoB5H,MAAM,CAAC6H,SAA3B,EAAsC;AACpC,UAAMkB,IAAI,GAAGrK,mBAAmB,CAACkJ,KAAK,CAACoB,cAAP,CAAhC;AACA,UAAMC,UAAU,GAAG,IAAIjK,aAAJ,CAAkB;AACnC0J,MAAAA,IAAI,EAAEd,KAAK,CAACc,IADuB;AAEnCQ,MAAAA,eAAe,EAAEH;AAFkB,KAAlB,CAAnB;;AAKA,QAAInB,KAAK,CAACE,QAAV,EAAoB;AAClBmB,MAAAA,UAAU,CAACN,YAAX,GAA0Bf,KAAK,CAACuB,UAAhC;AACD,KAFD,MAEO;AACLF,MAAAA,UAAU,CAAC9E,IAAX,GAAkBlF,IAAI,CAAC2I,KAAK,CAAC3C,aAAP,CAAtB;AACD;;AAED,QAAI2C,KAAK,CAACwB,YAAV,EAAwB;AACtBH,MAAAA,UAAU,CAACI,cAAX,GAA4B/K,aAAa,CAACsJ,KAAK,CAACwB,YAAP,CAAzC;AACD;;AAEDH,IAAAA,UAAU,CAACK,WAAX,GAAyB1B,KAAK,CAAC1C,UAA/B;AAEA/B,IAAAA,QAAQ,CAACnD,MAAT,CAAgBkD,IAAhB,CAAqB+F,UAArB;AACD;;AAED,QAAMM,eAAe,GAAGjK,eAAe,CAAC6D,QAAD,CAAvC;AACA,QAAMqG,aAAa,GAAG7G,MAAM,CAAC8B,KAAP,CAAa8E,eAAe,CAAC7G,MAAhB,GAAyB,CAAtC,CAAtB;AACA6G,EAAAA,eAAe,CAACE,IAAhB,CAAqBD,aAArB;AACAA,EAAAA,aAAa,CAACE,aAAd,CAA4BH,eAAe,CAAC7G,MAA5C,EAAoD6G,eAAe,CAAC7G,MAApE;AACA8G,EAAAA,aAAa,CAACtH,KAAd,CAAoB1C,aAApB,EAAmC+J,eAAe,CAAC7G,MAAhB,GAAyB,CAA5D;AACA,SAAO8G,aAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {Transform, Writable} from 'stream';\nimport {ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport * as Compression from '../compression';\nimport {\n  ParquetBuffer,\n  ParquetCodec,\n  ParquetData,\n  ParquetField,\n  PrimitiveType\n} from '../schema/declare';\nimport {ParquetSchema} from '../schema/schema';\nimport * as Shred from '../schema/shred';\nimport {\n  ColumnChunk,\n  ColumnMetaData,\n  CompressionCodec,\n  ConvertedType,\n  DataPageHeader,\n  DataPageHeaderV2,\n  Encoding,\n  FieldRepetitionType,\n  FileMetaData,\n  KeyValue,\n  PageHeader,\n  PageType,\n  RowGroup,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {osopen, oswrite, osclose} from '../utils/file-utils';\nimport {getBitWidth, serializeThrift} from '../utils/read-utils';\nimport Int64 from 'node-int64';\n\n/**\n * Parquet File Magic String\n */\nconst PARQUET_MAGIC = 'PAR1';\n\n/**\n * Parquet File Format Version\n */\nconst PARQUET_VERSION = 1;\n\n/**\n * Default Page and Row Group sizes\n */\nconst PARQUET_DEFAULT_PAGE_SIZE = 8192;\nconst PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;\n\n/**\n * Repetition and Definition Level Encoding\n */\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\nexport interface ParquetWriterOptions {\n  baseOffset?: number;\n  rowGroupSize?: number;\n  pageSize?: number;\n  useDataPageV2?: boolean;\n\n  // Write Stream Options\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\n/**\n * Write a parquet file to an output stream. The ParquetWriter will perform\n * buffering/batching for performance, so close() must be called after all rows\n * are written.\n */\n// eslint-disable-next-line @typescript-eslint/no-unused-vars\nexport class ParquetWriter<T> {\n  /**\n   * Convenience method to create a new buffered parquet writer that writes to\n   * the specified file\n   */\n  static async openFile<T>(\n    schema: ParquetSchema,\n    path: string,\n    opts?: ParquetWriterOptions\n  ): Promise<ParquetWriter<T>> {\n    const outputStream = await osopen(path, opts);\n    return ParquetWriter.openStream(schema, outputStream, opts);\n  }\n\n  /**\n   * Convenience method to create a new buffered parquet writer that writes to\n   * the specified stream\n   */\n  static async openStream<T>(\n    schema: ParquetSchema,\n    outputStream: Writable,\n    opts?: ParquetWriterOptions\n  ): Promise<ParquetWriter<T>> {\n    if (!opts) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      opts = {};\n    }\n\n    const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);\n\n    return new ParquetWriter(schema, envelopeWriter, opts);\n  }\n\n  public schema: ParquetSchema;\n  public envelopeWriter: ParquetEnvelopeWriter;\n  public rowBuffer: ParquetBuffer;\n  public rowGroupSize: number;\n  public closed: boolean;\n  public userMetadata: Record<string, string>;\n\n  /**\n   * Create a new buffered parquet writer for a given envelope writer\n   */\n  constructor(\n    schema: ParquetSchema,\n    envelopeWriter: ParquetEnvelopeWriter,\n    opts: ParquetWriterOptions\n  ) {\n    this.schema = schema;\n    this.envelopeWriter = envelopeWriter;\n    // @ts-ignore Row buffer typings...\n    this.rowBuffer = {};\n    this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;\n    this.closed = false;\n    this.userMetadata = {};\n\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.writeHeader();\n  }\n\n  async writeHeader(): Promise<void> {\n    // TODO - better not mess with promises in the constructor\n    try {\n      await this.envelopeWriter.writeHeader();\n    } catch (err) {\n      await this.envelopeWriter.close();\n      throw err;\n    }\n  }\n\n  /**\n   * Append a single row to the parquet file. Rows are buffered in memory until\n   * rowGroupSize rows are in the buffer or close() is called\n   */\n  async appendRow<T>(row: T): Promise<void> {\n    if (this.closed) {\n      throw new Error('writer was closed');\n    }\n    Shred.shredRecord(this.schema, row, this.rowBuffer);\n    if (this.rowBuffer.rowCount >= this.rowGroupSize) {\n      // @ts-ignore\n      this.rowBuffer = {};\n    }\n  }\n\n  /**\n   * Finish writing the parquet file and commit the footer to disk. This method\n   * MUST be called after you are finished adding rows. You must not call this\n   * method twice on the same object or add any rows after the close() method has\n   * been called\n   */\n  async close(callback?: () => void): Promise<void> {\n    if (this.closed) {\n      throw new Error('writer was closed');\n    }\n\n    this.closed = true;\n\n    if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {\n      // @ts-ignore\n      this.rowBuffer = {};\n    }\n\n    await this.envelopeWriter.writeFooter(this.userMetadata);\n    await this.envelopeWriter.close();\n    // this.envelopeWriter = null;\n\n    if (callback) {\n      callback();\n    }\n  }\n\n  /**\n   * Add key<>value metadata to the file\n   */\n  setMetadata(key: string, value: string): void {\n    // TODO: value to be any, obj -> JSON\n    this.userMetadata[String(key)] = String(value);\n  }\n\n  /**\n   * Set the parquet row group size. This values controls the maximum number\n   * of rows that are buffered in memory at any given time as well as the number\n   * of rows that are co-located on disk. A higher value is generally better for\n   * read-time I/O performance at the tradeoff of write-time memory usage.\n   */\n  setRowGroupSize(cnt: number): void {\n    this.rowGroupSize = cnt;\n  }\n\n  /**\n   * Set the parquet data page size. The data page size controls the maximum\n   * number of column values that are written to disk as a consecutive array\n   */\n  setPageSize(cnt: number): void {\n    this.envelopeWriter.setPageSize(cnt);\n  }\n}\n\n/**\n * Create a parquet file from a schema and a number of row groups. This class\n * performs direct, unbuffered writes to the underlying output stream and is\n * intendend for advanced and internal users; the writeXXX methods must be\n * called in the correct order to produce a valid file.\n */\nexport class ParquetEnvelopeWriter {\n  /**\n   * Create a new parquet envelope writer that writes to the specified stream\n   */\n  static async openStream(\n    schema: ParquetSchema,\n    outputStream: Writable,\n    opts: ParquetWriterOptions\n  ): Promise<ParquetEnvelopeWriter> {\n    const writeFn = oswrite.bind(undefined, outputStream);\n    const closeFn = osclose.bind(undefined, outputStream);\n    return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);\n  }\n\n  public schema: ParquetSchema;\n  public write: (buf: Buffer) => Promise<void>;\n  public close: () => Promise<void>;\n  public offset: number;\n  public rowCount: number;\n  public rowGroups: RowGroup[];\n  public pageSize: number;\n  public useDataPageV2: boolean;\n\n  constructor(\n    schema: ParquetSchema,\n    writeFn: (buf: Buffer) => Promise<void>,\n    closeFn: () => Promise<void>,\n    fileOffset: number,\n    opts: ParquetWriterOptions\n  ) {\n    this.schema = schema;\n    this.write = writeFn;\n    this.close = closeFn;\n    this.offset = fileOffset;\n    this.rowCount = 0;\n    this.rowGroups = [];\n    this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;\n    this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;\n  }\n\n  writeSection(buf: Buffer): Promise<void> {\n    this.offset += buf.length;\n    return this.write(buf);\n  }\n\n  /**\n   * Encode the parquet file header\n   */\n  writeHeader(): Promise<void> {\n    return this.writeSection(Buffer.from(PARQUET_MAGIC));\n  }\n\n  /**\n   * Encode a parquet row group. The records object should be created using the\n   * shredRecord method\n   */\n  async writeRowGroup(records: ParquetBuffer): Promise<void> {\n    const rgroup = await encodeRowGroup(this.schema, records, {\n      baseOffset: this.offset,\n      pageSize: this.pageSize,\n      useDataPageV2: this.useDataPageV2\n    });\n\n    this.rowCount += records.rowCount;\n    this.rowGroups.push(rgroup.metadata);\n    return await this.writeSection(rgroup.body);\n  }\n\n  /**\n   * Write the parquet file footer\n   */\n  writeFooter(userMetadata: Record<string, string>): Promise<void> {\n    if (!userMetadata) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      userMetadata = {};\n    }\n\n    return this.writeSection(\n      encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata)\n    );\n  }\n\n  /**\n   * Set the parquet data page size. The data page size controls the maximum\n   * number of column values that are written to disk as a consecutive array\n   */\n  setPageSize(cnt: number): void {\n    this.pageSize = cnt;\n  }\n}\n\n/**\n * Create a parquet transform stream\n */\nexport class ParquetTransformer<T> extends Transform {\n  public writer: ParquetWriter<T>;\n\n  constructor(schema: ParquetSchema, opts: ParquetWriterOptions = {}) {\n    super({objectMode: true});\n\n    const writeProxy = (function (t: ParquetTransformer<any>) {\n      return async function (b: any): Promise<void> {\n        t.push(b);\n      };\n    })(this);\n\n    this.writer = new ParquetWriter(\n      schema,\n      new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),\n      opts\n    );\n  }\n\n  // tslint:disable-next-line:function-name\n  _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {\n    if (row) {\n      return this.writer.appendRow(row).then(callback);\n    }\n    callback();\n    return Promise.resolve();\n  }\n\n  // tslint:disable-next-line:function-name\n  async _flush(callback: (val?: any) => void) {\n    await this.writer.close(callback);\n  }\n}\n\n/**\n * Encode a consecutive array of data using one of the parquet encodings\n */\nfunction encodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  values: any[],\n  opts: ParquetCodecOptions\n) {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].encodeValues(type, values, opts);\n}\n\n/**\n * Encode a parquet data page\n */\nasync function encodeDataPage(\n  column: ParquetField,\n  data: ParquetData\n): Promise<{\n  header: PageHeader;\n  headerSize: number;\n  page: Buffer;\n}> {\n  /* encode repetition and definition levels */\n  let rLevelsBuf = Buffer.alloc(0);\n  if (column.rLevelMax > 0) {\n    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n      bitWidth: getBitWidth(column.rLevelMax)\n      // disableEnvelope: false\n    });\n  }\n\n  let dLevelsBuf = Buffer.alloc(0);\n  if (column.dLevelMax > 0) {\n    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n      bitWidth: getBitWidth(column.dLevelMax)\n      // disableEnvelope: false\n    });\n  }\n\n  /* encode values */\n  const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);\n\n  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;\n  const compressedBuf = await Compression.deflate(column.compression!, dataBuf);\n\n  /* build page header */\n  const header = new PageHeader({\n    type: PageType.DATA_PAGE,\n    data_page_header: new DataPageHeader({\n      num_values: data.count,\n      encoding: Encoding[column.encoding!] as any,\n      definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],\n      repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]\n    }),\n    uncompressed_page_size: dataBuf.length,\n    compressed_page_size: compressedBuf.length\n  });\n\n  /* concat page header, repetition and definition levels and values */\n  const headerBuf = serializeThrift(header);\n  const page = Buffer.concat([headerBuf, compressedBuf]);\n\n  return {header, headerSize: headerBuf.length, page};\n}\n\n/**\n * Encode a parquet data page (v2)\n */\nasync function encodeDataPageV2(\n  column: ParquetField,\n  data: ParquetData,\n  rowCount: number\n): Promise<{\n  header: PageHeader;\n  headerSize: number;\n  page: Buffer;\n}> {\n  /* encode values */\n  const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;\n  const compressedBuf = await Compression.deflate(column.compression!, valuesBuf);\n\n  /* encode repetition and definition levels */\n  let rLevelsBuf = Buffer.alloc(0);\n  if (column.rLevelMax > 0) {\n    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n      bitWidth: getBitWidth(column.rLevelMax),\n      disableEnvelope: true\n    });\n  }\n\n  let dLevelsBuf = Buffer.alloc(0);\n  if (column.dLevelMax > 0) {\n    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n      bitWidth: getBitWidth(column.dLevelMax),\n      disableEnvelope: true\n    });\n  }\n\n  /* build page header */\n  const header = new PageHeader({\n    type: PageType.DATA_PAGE_V2,\n    data_page_header_v2: new DataPageHeaderV2({\n      num_values: data.count,\n      num_nulls: data.count - data.values.length,\n      num_rows: rowCount,\n      encoding: Encoding[column.encoding!] as any,\n      definition_levels_byte_length: dLevelsBuf.length,\n      repetition_levels_byte_length: rLevelsBuf.length,\n      is_compressed: column.compression !== 'UNCOMPRESSED'\n    }),\n    uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,\n    compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length\n  });\n\n  /* concat page header, repetition and definition levels and values */\n  const headerBuf = serializeThrift(header);\n  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);\n  return {header, headerSize: headerBuf.length, page};\n}\n\n/**\n * Encode an array of values into a parquet column chunk\n */\nasync function encodeColumnChunk(\n  column: ParquetField,\n  buffer: ParquetBuffer,\n  offset: number,\n  opts: ParquetWriterOptions\n): Promise<{\n  body: Buffer;\n  metadata: ColumnMetaData;\n  metadataOffset: number;\n}> {\n  const data = buffer.columnData[column.path.join()];\n  const baseOffset = (opts.baseOffset || 0) + offset;\n  /* encode data page(s) */\n  // const pages: Buffer[] = [];\n  let pageBuf: Buffer;\n  // tslint:disable-next-line:variable-name\n  let total_uncompressed_size = 0;\n  // tslint:disable-next-line:variable-name\n  let total_compressed_size = 0;\n  {\n    const result = opts.useDataPageV2\n      ? await encodeDataPageV2(column, data, buffer.rowCount)\n      : await encodeDataPage(column, data);\n    // pages.push(result.page);\n    pageBuf = result.page;\n    total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;\n    total_compressed_size += result.header.compressed_page_size + result.headerSize;\n  }\n\n  // const pagesBuf = Buffer.concat(pages);\n  // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;\n\n  /* prepare metadata header */\n  const metadata = new ColumnMetaData({\n    path_in_schema: column.path,\n    num_values: data.count,\n    data_page_offset: baseOffset,\n    encodings: [],\n    total_uncompressed_size, //  : pagesBuf.length,\n    total_compressed_size,\n    type: Type[column.primitiveType!],\n    codec: CompressionCodec[column.compression!]\n  });\n\n  /* list encodings */\n  metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);\n  metadata.encodings.push(Encoding[column.encoding!]);\n\n  /* concat metadata header and data pages */\n  const metadataOffset = baseOffset + pageBuf.length;\n  const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);\n  return {body, metadata, metadataOffset};\n}\n\n/**\n * Encode a list of column values into a parquet row group\n */\nasync function encodeRowGroup(\n  schema: ParquetSchema,\n  data: ParquetBuffer,\n  opts: ParquetWriterOptions\n): Promise<{\n  body: Buffer;\n  metadata: RowGroup;\n}> {\n  const metadata = new RowGroup({\n    num_rows: data.rowCount,\n    columns: [],\n    total_byte_size: 0\n  });\n\n  let body = Buffer.alloc(0);\n  for (const field of schema.fieldList) {\n    if (field.isNested) {\n      continue; // eslint-disable-line no-continue\n    }\n\n    const cchunkData = await encodeColumnChunk(field, data, body.length, opts);\n\n    const cchunk = new ColumnChunk({\n      file_offset: cchunkData.metadataOffset,\n      meta_data: cchunkData.metadata\n    });\n\n    metadata.columns.push(cchunk);\n    metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);\n\n    body = Buffer.concat([body, cchunkData.body]);\n  }\n\n  return {body, metadata};\n}\n\n/**\n * Encode a parquet file metadata footer\n */\nfunction encodeFooter(\n  schema: ParquetSchema,\n  rowCount: number,\n  rowGroups: RowGroup[],\n  userMetadata: Record<string, string>\n): Buffer {\n  const metadata = new FileMetaData({\n    version: PARQUET_VERSION,\n    created_by: 'parquets',\n    num_rows: rowCount,\n    row_groups: rowGroups,\n    schema: [],\n    key_value_metadata: []\n  });\n\n  for (const key in userMetadata) {\n    const kv = new KeyValue({\n      key,\n      value: userMetadata[key]\n    });\n    metadata.key_value_metadata?.push?.(kv);\n  }\n\n  {\n    const schemaRoot = new SchemaElement({\n      name: 'root',\n      num_children: Object.keys(schema.fields).length\n    });\n    metadata.schema.push(schemaRoot);\n  }\n\n  for (const field of schema.fieldList) {\n    const relt = FieldRepetitionType[field.repetitionType];\n    const schemaElem = new SchemaElement({\n      name: field.name,\n      repetition_type: relt as any\n    });\n\n    if (field.isNested) {\n      schemaElem.num_children = field.fieldCount;\n    } else {\n      schemaElem.type = Type[field.primitiveType!] as Type;\n    }\n\n    if (field.originalType) {\n      schemaElem.converted_type = ConvertedType[field.originalType] as ConvertedType;\n    }\n\n    schemaElem.type_length = field.typeLength;\n\n    metadata.schema.push(schemaElem);\n  }\n\n  const metadataEncoded = serializeThrift(metadata);\n  const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);\n  metadataEncoded.copy(footerEncoded);\n  footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);\n  footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);\n  return footerEncoded;\n}\n"],"file":"writer.js"}

package/dist/esm/parquetjs/file.js

@@ -1,80 +0,0 @@
-import fs from 'fs';
-export function fopen(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-export function fstat(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-export function fread(fd, position, length) {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-export function fclose(fd) {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(err);
-      }
-    });
-  });
-}
-export function oswrite(os, buf) {
-  return new Promise((resolve, reject) => {
-    os.write(buf, err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-export function osclose(os) {
-  return new Promise((resolve, reject) => {
-    os.close(err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-export function osopen(path, opts) {
-  return new Promise((resolve, reject) => {
-    const outputStream = fs.createWriteStream(path, opts);
-    outputStream.on('open', function (fd) {
-      resolve(outputStream);
-    });
-    outputStream.on('error', function (err) {
-      reject(err);
-    });
-  });
-}
-//# sourceMappingURL=file.js.map

package/dist/esm/parquetjs/file.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/parquetjs/file.ts"],"names":["fs","fopen","filePath","Promise","resolve","reject","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","on"],"mappings":"AACA,OAAOA,EAAP,MAAe,IAAf;AAEA,OAAO,SAASC,KAAT,CAAeC,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCL,IAAAA,EAAE,CAACM,IAAH,CAAQJ,QAAR,EAAkB,GAAlB,EAAuB,CAACK,GAAD,EAAMC,EAAN,KAAa;AAClC,UAAID,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACI,EAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASC,KAAT,CAAeP,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAsB,CAACC,OAAD,EAAUC,MAAV,KAAqB;AAChDL,IAAAA,EAAE,CAACU,IAAH,CAAQR,QAAR,EAAkB,CAACK,GAAD,EAAMG,IAAN,KAAe;AAC/B,UAAIH,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACM,IAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASC,KAAT,CAAeH,EAAf,EAAmBI,QAAnB,EAA6BC,MAA7B,EAAqC;AAC1C,QAAMC,MAAM,GAAGC,MAAM,CAACC,KAAP,CAAaH,MAAb,CAAf;AAEA,SAAO,IAAIV,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCL,IAAAA,EAAE,CAACiB,IAAH,CAAQT,EAAR,EAAYM,MAAZ,EAAoB,CAApB,EAAuBD,MAAvB,EAA+BD,QAA/B,EAAyC,CAACL,GAAD,EAAMW,SAAN,EAAiBC,GAAjB,KAAyB;AAChE,UAAIZ,GAAG,IAAIW,SAAS,KAAKL,MAAzB,EAAiC;AAC/BR,QAAAA,MAAM,CAACE,GAAG,IAAIa,KAAK,CAAC,aAAD,CAAb,CAAN;AACD,OAFD,MAEO;AACLhB,QAAAA,OAAO,CAACe,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASE,MAAT,CAAgBb,EAAhB,EAAoB;AACzB,SAAO,IAAIL,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCL,IAAAA,EAAE,CAACsB,KAAH,CAASd,EAAT,EAAcD,GAAD,IAAS;AACpB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACG,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASgB,OAAT,CAAiBC,EAAjB,EAAqBL,GAArB,EAAyC;AAC9C,SAAO,IAAIhB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCmB,IAAAA,EAAE,CAACC,KAAH,CAASN,GAAT,EAAeZ,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASsB,OAAT,CAAiBF,EAAjB,EAAoC;AACzC,SAAO,IAAIrB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCmB,IAAAA,EAAE,CAACF,KAAH,CAAUf,GAAD,IAAS;AAChB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASuB,MAAT,CAAgBC,IAAhB,EAAsBC,IAAtB,EAA4B;AACjC,SAAO,IAAI1B,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAMyB,YAAY,GAAG9B,EAAE,CAAC+B,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;AAEAC,IAAAA,YAAY,CAACE,EAAb,CAAgB,MAAhB,EAAwB,UAAUxB,EAAV,EAAc;AACpCJ,MAAAA,OAAO,CAAC0B,YAAD,CAAP;AACD,KAFD;AAIAA,IAAAA,YAAY,CAACE,EAAb,CAAgB,OAAhB,EAAyB,UAAUzB,GAAV,EAAe;AACtCF,MAAAA,MAAM,CAACE,GAAD,CAAN;AACD,KAFD;AAGD,GAVM,CAAP;AAWD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\n\nexport function fopen(filePath) {\n  return new Promise((resolve, reject) => {\n    fs.open(filePath, 'r', (err, fd) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(fd);\n      }\n    });\n  });\n}\n\nexport function fstat(filePath) {\n  return new Promise<fs.Stats>((resolve, reject) => {\n    fs.stat(filePath, (err, stat) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(stat);\n      }\n    });\n  });\n}\n\nexport function fread(fd, position, length) {\n  const buffer = Buffer.alloc(length);\n\n  return new Promise((resolve, reject) => {\n    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {\n      if (err || bytesRead !== length) {\n        reject(err || Error('read failed'));\n      } else {\n        resolve(buf);\n      }\n    });\n  });\n}\n\nexport function fclose(fd) {\n  return new Promise((resolve, reject) => {\n    fs.close(fd, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(err);\n      }\n    });\n  });\n}\n\nexport function oswrite(os, buf): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.close((err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path, opts) {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts);\n\n    outputStream.on('open', function (fd) {\n      resolve(outputStream);\n    });\n\n    outputStream.on('error', function (err) {\n      reject(err);\n    });\n  });\n}\n"],"file":"file.js"}

package/dist/esm/parquetjs/modules.d.ts

@@ -1,21 +0,0 @@
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-
-declare module 'int53' {
-  declare function readInt64BE(buffer: Buffer, offset?: number): number;
-  declare function readInt64LE(buffer: Buffer, offset?: number): number;
-  declare function readUInt64BE(buffer: Buffer, offset?: number): number;
-  declare function readUInt64LE(buffer: Buffer, offset?: number): number;
-  declare function writeInt64BE(value: number, buffer: Buffer, offset?: number): void;
-  declare function writeInt64LE(value: number, buffer: Buffer, offset?: number): void;
-  declare function writeUInt64BE(value: number, buffer: Buffer, offset?: number): void;
-  declare function writeUInt64LE(value: number, buffer: Buffer, offset?: number): void;
-}
-
-// declare module 'snappyjs' {
-//   declare function compress(uncompressed: Buffer): Buffer;
-//   declare function compress(uncompressed: ArrayBuffer): ArrayBuffer;
-//   declare function compress(uncompressed: Uint8Array): Uint8Array;
-//   declare function uncompress(compressed: Buffer): Buffer;
-//   declare function uncompress(compressed: ArrayBuffer): ArrayBuffer;
-//   declare function uncompress(compressed: Uint8Array): Uint8Array;
-// }

package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js

@@ -1,8 +0,0 @@
-export let BoundaryOrder;
-
-(function (BoundaryOrder) {
-  BoundaryOrder[BoundaryOrder["UNORDERED"] = 0] = "UNORDERED";
-  BoundaryOrder[BoundaryOrder["ASCENDING"] = 1] = "ASCENDING";
-  BoundaryOrder[BoundaryOrder["DESCENDING"] = 2] = "DESCENDING";
-})(BoundaryOrder || (BoundaryOrder = {}));
-//# sourceMappingURL=BoundaryOrder.js.map

package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/parquet-thrift/BoundaryOrder.ts"],"names":["BoundaryOrder"],"mappings":"AAMA,WAAYA,aAAZ;;WAAYA,a;AAAAA,EAAAA,a,CAAAA,a;AAAAA,EAAAA,a,CAAAA,a;AAAAA,EAAAA,a,CAAAA,a;GAAAA,a,KAAAA,a","sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nexport enum BoundaryOrder {\n  UNORDERED = 0,\n  ASCENDING = 1,\n  DESCENDING = 2\n}\n"],"file":"BoundaryOrder.js"}

package/dist/esm/parquetjs/parquet-thrift/BsonType.js

@@ -1,39 +0,0 @@
-import * as thrift from 'thrift';
-export class BsonType {
-  constructor() {}
-
-  write(output) {
-    output.writeStructBegin('BsonType');
-    output.writeFieldStop();
-    output.writeStructEnd();
-    return;
-  }
-
-  static read(input) {
-    input.readStructBegin();
-
-    while (true) {
-      const ret = input.readFieldBegin();
-      const fieldType = ret.ftype;
-      const fieldId = ret.fid;
-
-      if (fieldType === thrift.Thrift.Type.STOP) {
-        break;
-      }
-
-      switch (fieldId) {
-        default:
-          {
-            input.skip(fieldType);
-          }
-      }
-
-      input.readFieldEnd();
-    }
-
-    input.readStructEnd();
-    return new BsonType();
-  }
-
-}
-//# sourceMappingURL=BsonType.js.map

package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/parquet-thrift/BsonType.ts"],"names":["thrift","BsonType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"mappings":"AAMA,OAAO,KAAKA,MAAZ,MAAwB,QAAxB;AAEA,OAAO,MAAMC,QAAN,CAAe;AACpBC,EAAAA,WAAW,GAAG,CAAE;;AACTC,EAAAA,KAAK,CAACC,MAAD,EAAiC;AAC3CA,IAAAA,MAAM,CAACC,gBAAP,CAAwB,UAAxB;AACAD,IAAAA,MAAM,CAACE,cAAP;AACAF,IAAAA,MAAM,CAACG,cAAP;AACA;AACD;;AACiB,SAAJC,IAAI,CAACC,KAAD,EAAoC;AACpDA,IAAAA,KAAK,CAACC,eAAN;;AACA,WAAO,IAAP,EAAa;AACX,YAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAN,EAA3B;AACA,YAAMC,SAA6B,GAAGF,GAAG,CAACG,KAA1C;AACA,YAAMC,OAAe,GAAGJ,GAAG,CAACK,GAA5B;;AACA,UAAIH,SAAS,KAAKb,MAAM,CAACiB,MAAP,CAAcC,IAAd,CAAmBC,IAArC,EAA2C;AACzC;AACD;;AACD,cAAQJ,OAAR;AACE;AAAS;AACPN,YAAAA,KAAK,CAACW,IAAN,CAAWP,SAAX;AACD;AAHH;;AAKAJ,MAAAA,KAAK,CAACY,YAAN;AACD;;AACDZ,IAAAA,KAAK,CAACa,aAAN;AACA,WAAO,IAAIrB,QAAJ,EAAP;AACD;;AA1BmB","sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nimport * as thrift from 'thrift';\nexport interface IBsonTypeArgs {}\nexport class BsonType {\n  constructor() {}\n  public write(output: thrift.TProtocol): void {\n    output.writeStructBegin('BsonType');\n    output.writeFieldStop();\n    output.writeStructEnd();\n    return;\n  }\n  public static read(input: thrift.TProtocol): BsonType {\n    input.readStructBegin();\n    while (true) {\n      const ret: thrift.TField = input.readFieldBegin();\n      const fieldType: thrift.Thrift.Type = ret.ftype;\n      const fieldId: number = ret.fid;\n      if (fieldType === thrift.Thrift.Type.STOP) {\n        break;\n      }\n      switch (fieldId) {\n        default: {\n          input.skip(fieldType);\n        }\n      }\n      input.readFieldEnd();\n    }\n    input.readStructEnd();\n    return new BsonType();\n  }\n}\n"],"file":"BsonType.js"}