@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.7

package/dist/esm/parquetjs/compression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compression.js","names":["NoCompression","GZipCompression","SnappyCompression","BrotliCompression","LZ4Compression","ZstdCompression","toBuffer","arrayBuffer","Buffer","from","toArrayBuffer","buffer","isBuffer","typedArray","Uint8Array","byteOffset","length","slice","lz4js","modules","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","GZIP","SNAPPY","BROTLI","LZ4","LZ4_RAW","ZSTD","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","concat","inputArrayBuffer","compressedArrayBuffer","compress","decompress","size","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  // LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\n\n/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */\nfunction toBuffer(arrayBuffer: ArrayBuffer): Buffer {\n  return Buffer.from(arrayBuffer);\n}\n\nfunction toArrayBuffer(buffer: Buffer): ArrayBuffer {\n  // TODO - per docs we should just be able to call buffer.buffer, but there are issues\n  if (Buffer.isBuffer(buffer)) {\n    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);\n    return typedArray.slice().buffer;\n  }\n  return buffer;\n}\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\n// import lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js\n  // lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n/**\n * See https://github.com/apache/parquet-format/blob/master/Compression.md\n */\n// @ts-expect-error\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  //\n  // LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":"AAIA,SAEEA,aAAa,EACbC,eAAe,EACfC,iBAAiB,EACjBC,iBAAiB,EAEjBC,cAAc,EACdC,eAAe,QACV,yBAAyB;AAKhC,SAASC,QAAQA,CAACC,WAAwB,EAAU;EAClD,OAAOC,MAAM,CAACC,IAAI,CAACF,WAAW,CAAC;AACjC;AAEA,SAASG,aAAaA,CAACC,MAAc,EAAe;EAElD,IAAIH,MAAM,CAACI,QAAQ,CAACD,MAAM,CAAC,EAAE;IAC3B,MAAME,UAAU,GAAG,IAAIC,UAAU,CAACH,MAAM,CAACA,MAAM,EAAEA,MAAM,CAACI,UAAU,EAAEJ,MAAM,CAACK,MAAM,CAAC;IAClF,OAAOH,UAAU,CAACI,KAAK,CAAC,CAAC,CAACN,MAAM;EAClC;EACA,OAAOA,MAAM;AACf;AAMA,OAAOO,KAAK,MAAM,OAAO;AAKzB,MAAMC,OAAO,GAAG;EAQdD;AAGF,CAAC;AAMD,OAAO,MAAME,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIrB,aAAa,CAAC,CAAC;EACjCsB,IAAI,EAAE,IAAIrB,eAAe,CAAC,CAAC;EAC3BsB,MAAM,EAAE,IAAIrB,iBAAiB,CAAC,CAAC;EAC/BsB,MAAM,EAAE,IAAIrB,iBAAiB,CAAC;IAACgB;EAAO,CAAC,CAAC;EAExCM,GAAG,EAAE,IAAIrB,cAAc,CAAC;IAACe;EAAO,CAAC,CAAC;EAClCO,OAAO,EAAE,IAAItB,cAAc,CAAC;IAACe;EAAO,CAAC,CAAC;EAGtCQ,IAAI,EAAE,IAAItB,eAAe,CAAC;IAACc;EAAO,CAAC;AACrC,CAAC;AAMD,OAAO,eAAeS,mBAAmBA,CAACC,OAAyC,EAAE;EACnF,MAAMC,YAAY,GAAGC,MAAM,CAACC,MAAM,CAACZ,2BAA2B,CAAC;EAC/D,OAAO,MAAMa,OAAO,CAACC,GAAG,CAACJ,YAAY,CAACK,GAAG,CAAEC,WAAW,IAAKA,WAAW,CAACC,OAAO,CAAC,CAAC,CAAC,CAAC;AACpF;AAKA,OAAO,eAAeC,OAAOA,CAACC,MAA0B,EAAEC,KAAa,EAAmB;EACxF,MAAMJ,WAAW,GAAGhB,2BAA2B,CAACmB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,yCAAAC,MAAA,CAAyCH,MAAM,CAAE,CAAC;EACnE;EACA,MAAMI,gBAAgB,GAAGjC,aAAa,CAAC8B,KAAK,CAAC;EAC7C,MAAMI,qBAAqB,GAAG,MAAMR,WAAW,CAACS,QAAQ,CAACF,gBAAgB,CAAC;EAC1E,OAAOrC,QAAQ,CAACsC,qBAAqB,CAAC;AACxC;AAKA,OAAO,eAAeE,UAAUA,CAC9BP,MAA0B,EAC1BC,KAAa,EACbO,IAAY,EACK;EACjB,MAAMX,WAAW,GAAGhB,2BAA2B,CAACmB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,yCAAAC,MAAA,CAAyCH,MAAM,CAAE,CAAC;EACnE;EACA,MAAMI,gBAAgB,GAAGjC,aAAa,CAAC8B,KAAK,CAAC;EAC7C,MAAMI,qBAAqB,GAAG,MAAMR,WAAW,CAACU,UAAU,CAACH,gBAAgB,EAAEI,IAAI,CAAC;EAClF,OAAOzC,QAAQ,CAACsC,qBAAqB,CAAC;AACxC;AAKA,OAAO,SAASI,OAAOA,CAACT,MAA0B,EAAEC,KAAa,EAAEO,IAAY,EAAU;EACvF,IAAI,EAAER,MAAM,IAAInB,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAIqB,KAAK,gCAAAC,MAAA,CAAgCH,MAAM,CAAE,CAAC;EAC1D;EAEA,OAAOnB,2BAA2B,CAACmB,MAAM,CAAC,CAACS,OAAO,CAACR,KAAK,EAAEO,IAAI,CAAC;AACjE"}

package/dist/{parquetjs/encoder/writer.js → esm/parquetjs/encoder/parquet-encoder.js}

@@ -1,5 +1,4 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Transform } from 'stream';
 import { PARQUET_CODECS } from '../codecs';
 import * as Compression from '../compression';
 import * as Shred from '../schema/shred';
@@ -13,34 +12,23 @@ const PARQUET_DEFAULT_PAGE_SIZE = 8192;
 const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
 const PARQUET_RDLVL_TYPE = 'INT32';
 const PARQUET_RDLVL_ENCODING = 'RLE';
-export class ParquetWriter {
+export class ParquetEncoder {
   static async openFile(schema, path, opts) {
     const outputStream = await osopen(path, opts);
-    return ParquetWriter.openStream(schema, outputStream, opts);
+    return ParquetEncoder.openStream(schema, outputStream, opts);
   }
-
-  static async openStream(schema, outputStream, opts) {
-    if (!opts) {
-      opts = {};
-    }
-
+  static async openStream(schema, outputStream) {
+    let opts = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
     const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
-    return new ParquetWriter(schema, envelopeWriter, opts);
+    return new ParquetEncoder(schema, envelopeWriter, opts);
   }
-
   constructor(schema, envelopeWriter, opts) {
     _defineProperty(this, "schema", void 0);
-
     _defineProperty(this, "envelopeWriter", void 0);
-
     _defineProperty(this, "rowBuffer", void 0);
-
     _defineProperty(this, "rowGroupSize", void 0);
-
     _defineProperty(this, "closed", void 0);
-
     _defineProperty(this, "userMetadata", void 0);
-
     this.schema = schema;
     this.envelopeWriter = envelopeWriter;
     this.rowBuffer = {};
@@ -49,7 +37,6 @@ export class ParquetWriter {
     this.userMetadata = {};
     this.writeHeader();
   }
-
   async writeHeader() {
     try {
       await this.envelopeWriter.writeHeader();
@@ -58,50 +45,38 @@ export class ParquetWriter {
       throw err;
     }
   }
-
   async appendRow(row) {
     if (this.closed) {
       throw new Error('writer was closed');
     }
-
     Shred.shredRecord(this.schema, row, this.rowBuffer);
-
     if (this.rowBuffer.rowCount >= this.rowGroupSize) {
       this.rowBuffer = {};
     }
   }
-
   async close(callback) {
     if (this.closed) {
       throw new Error('writer was closed');
     }
-
     this.closed = true;
-
     if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
       this.rowBuffer = {};
     }
-
     await this.envelopeWriter.writeFooter(this.userMetadata);
     await this.envelopeWriter.close();
-
     if (callback) {
       callback();
     }
   }
-
   setMetadata(key, value) {
     this.userMetadata[String(key)] = String(value);
   }
-
   setRowGroupSize(cnt) {
     this.rowGroupSize = cnt;
   }
-
   setPageSize(cnt) {
     this.envelopeWriter.setPageSize(cnt);
   }
-
 }
 export class ParquetEnvelopeWriter {
   static async openStream(schema, outputStream, opts) {
@@ -109,24 +84,15 @@ export class ParquetEnvelopeWriter {
     const closeFn = osclose.bind(undefined, outputStream);
     return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);
   }
-
   constructor(schema, writeFn, closeFn, fileOffset, opts) {
     _defineProperty(this, "schema", void 0);
-
     _defineProperty(this, "write", void 0);
-
     _defineProperty(this, "close", void 0);
-
     _defineProperty(this, "offset", void 0);
-
     _defineProperty(this, "rowCount", void 0);
-
     _defineProperty(this, "rowGroups", void 0);
-
     _defineProperty(this, "pageSize", void 0);
-
     _defineProperty(this, "useDataPageV2", void 0);
-
     this.schema = schema;
     this.write = writeFn;
     this.close = closeFn;
@@ -136,16 +102,13 @@ export class ParquetEnvelopeWriter {
     this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
     this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
   }
-
   writeSection(buf) {
     this.offset += buf.length;
     return this.write(buf);
   }
-
   writeHeader() {
     return this.writeSection(Buffer.from(PARQUET_MAGIC));
   }
-
   async writeRowGroup(records) {
     const rgroup = await encodeRowGroup(this.schema, records, {
       baseOffset: this.offset,
@@ -156,77 +119,35 @@ export class ParquetEnvelopeWriter {
     this.rowGroups.push(rgroup.metadata);
     return await this.writeSection(rgroup.body);
   }
-
   writeFooter(userMetadata) {
     if (!userMetadata) {
       userMetadata = {};
     }
-
     return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
   }
-
   setPageSize(cnt) {
     this.pageSize = cnt;
   }
-
-}
-export class ParquetTransformer extends Transform {
-  constructor(schema, opts = {}) {
-    super({
-      objectMode: true
-    });
-
-    _defineProperty(this, "writer", void 0);
-
-    const writeProxy = function (t) {
-      return async function (b) {
-        t.push(b);
-      };
-    }(this);
-
-    this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts), opts);
-  }
-
-  _transform(row, encoding, callback) {
-    if (row) {
-      return this.writer.appendRow(row).then(callback);
-    }
-
-    callback();
-    return Promise.resolve();
-  }
-
-  async _flush(callback) {
-    await this.writer.close(callback);
-  }
-
 }
-
 function encodeValues(type, encoding, values, opts) {
   if (!(encoding in PARQUET_CODECS)) {
     throw new Error("invalid encoding: ".concat(encoding));
   }
-
   return PARQUET_CODECS[encoding].encodeValues(type, values, opts);
 }
-
 async function encodeDataPage(column, data) {
   let rLevelsBuf = Buffer.alloc(0);
-
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax)
     });
   }
-
   let dLevelsBuf = Buffer.alloc(0);
-
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax)
     });
   }
-
   const valuesBuf = encodeValues(column.primitiveType, column.encoding, data.values, {
     typeLength: column.typeLength,
     bitWidth: column.typeLength
@@ -252,7 +173,6 @@ async function encodeDataPage(column, data) {
     page
   };
 }
-
 async function encodeDataPageV2(column, data, rowCount) {
   const valuesBuf = encodeValues(column.primitiveType, column.encoding, data.values, {
     typeLength: column.typeLength,
@@ -260,23 +180,19 @@ async function encodeDataPageV2(column, data, rowCount) {
   });
   const compressedBuf = await Compression.deflate(column.compression, valuesBuf);
   let rLevelsBuf = Buffer.alloc(0);
-
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax),
       disableEnvelope: true
     });
   }
-
   let dLevelsBuf = Buffer.alloc(0);
-
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax),
       disableEnvelope: true
     });
   }
-
   const header = new PageHeader({
     type: PageType.DATA_PAGE_V2,
     data_page_header_v2: new DataPageHeaderV2({
@@ -299,7 +215,6 @@ async function encodeDataPageV2(column, data, rowCount) {
     page
   };
 }
-
 async function encodeColumnChunk(column, buffer, offset, opts) {
   const data = buffer.columnData[column.path.join()];
   const baseOffset = (opts.baseOffset || 0) + offset;
@@ -332,7 +247,6 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
     metadataOffset
   };
 }
-
 async function encodeRowGroup(schema, data, opts) {
   const metadata = new RowGroup({
     num_rows: data.rowCount,
@@ -340,12 +254,10 @@ async function encodeRowGroup(schema, data, opts) {
     total_byte_size: 0
   });
   let body = Buffer.alloc(0);
-
   for (const field of schema.fieldList) {
     if (field.isNested) {
       continue;
     }
-
     const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
     const cchunk = new ColumnChunk({
       file_offset: cchunkData.metadataOffset,
@@ -355,13 +267,11 @@ async function encodeRowGroup(schema, data, opts) {
     metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
     body = Buffer.concat([body, cchunkData.body]);
   }
-
   return {
     body,
     metadata
   };
 }
-
 function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
   const metadata = new FileMetaData({
     version: PARQUET_VERSION,
@@ -371,17 +281,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
     schema: [],
     key_value_metadata: []
   });
-
   for (const key in userMetadata) {
-    var _metadata$key_value_m, _metadata$key_value_m2;
-
+    var _metadata$key_value_m, _metadata$key_value_m2, _metadata$key_value_m3;
     const kv = new KeyValue({
       key,
       value: userMetadata[key]
     });
-    (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = _metadata$key_value_m.push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m, kv);
+    (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = (_metadata$key_value_m3 = _metadata$key_value_m).push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m3, kv);
   }
-
   {
     const schemaRoot = new SchemaElement({
       name: 'root',
@@ -389,28 +296,23 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
     });
     metadata.schema.push(schemaRoot);
   }
-
   for (const field of schema.fieldList) {
     const relt = FieldRepetitionType[field.repetitionType];
     const schemaElem = new SchemaElement({
       name: field.name,
       repetition_type: relt
     });
-
     if (field.isNested) {
       schemaElem.num_children = field.fieldCount;
     } else {
       schemaElem.type = Type[field.primitiveType];
     }
-
     if (field.originalType) {
       schemaElem.converted_type = ConvertedType[field.originalType];
     }
-
     schemaElem.type_length = field.typeLength;
     metadata.schema.push(schemaElem);
   }
-
   const metadataEncoded = serializeThrift(metadata);
   const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
   metadataEncoded.copy(footerEncoded);
@@ -418,4 +320,4 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
   footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
   return footerEncoded;
 }
-//# sourceMappingURL=writer.js.map
+//# sourceMappingURL=parquet-encoder.js.map

package/dist/esm/parquetjs/encoder/parquet-encoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-encoder.js","names":["PARQUET_CODECS","Compression","Shred","ColumnChunk","ColumnMetaData","CompressionCodec","ConvertedType","DataPageHeader","DataPageHeaderV2","Encoding","FieldRepetitionType","FileMetaData","KeyValue","PageHeader","PageType","RowGroup","SchemaElement","Type","osopen","oswrite","osclose","getBitWidth","serializeThrift","Int64","PARQUET_MAGIC","PARQUET_VERSION","PARQUET_DEFAULT_PAGE_SIZE","PARQUET_DEFAULT_ROW_GROUP_SIZE","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","ParquetEncoder","openFile","schema","path","opts","outputStream","openStream","arguments","length","undefined","envelopeWriter","ParquetEnvelopeWriter","constructor","_defineProperty","rowBuffer","rowGroupSize","closed","userMetadata","writeHeader","err","close","appendRow","row","Error","shredRecord","rowCount","callback","writeFooter","setMetadata","key","value","String","setRowGroupSize","cnt","setPageSize","writeFn","bind","closeFn","fileOffset","write","offset","rowGroups","pageSize","useDataPageV2","Boolean","writeSection","buf","Buffer","from","writeRowGroup","records","rgroup","encodeRowGroup","baseOffset","push","metadata","body","encodeFooter","encodeValues","type","encoding","values","concat","encodeDataPage","column","data","rLevelsBuf","alloc","rLevelMax","rlevels","bitWidth","dLevelsBuf","dLevelMax","dlevels","valuesBuf","primitiveType","typeLength","dataBuf","compressedBuf","deflate","compression","header","DATA_PAGE","data_page_header","num_values","count","definition_level_encoding","repetition_level_encoding","uncompressed_page_size","compressed_page_size","headerBuf","page","headerSize","encodeDataPageV2","disableEnvelope","DATA_PAGE_V2","data_page_header_v2","num_nulls","num_rows","definition_levels_byte_length","repetition_levels_byte_length","is_compressed","encodeColumnChunk","buffer","columnData","join","pageBuf","total_uncompressed_size","total_compressed_size","result","path_in_schema","data_page_offset","encodings","codec","metadataOffset","columns","total_byte_size","field","fieldList","isNested","cchunkData","cchunk","file_offset","meta_data","Number","version","created_by","row_groups","key_value_metadata","_metadata$key_value_m","_metadata$key_value_m2","_metadata$key_value_m3","kv","call","schemaRoot","name","num_children","Object","keys","fields","relt","repetitionType","schemaElem","repetition_type","fieldCount","originalType","converted_type","type_length","metadataEncoded","footerEncoded","copy","writeUInt32LE"],"sources":["../../../../src/parquetjs/encoder/parquet-encoder.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {stream} from '@loaders.gl/loader-utils';\nimport {ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport * as Compression from '../compression';\nimport {\n  ParquetRowGroup,\n  ParquetCodec,\n  ParquetColumnChunk,\n  ParquetField,\n  PrimitiveType,\n  ParquetRow\n} from '../schema/declare';\nimport {ParquetSchema} from '../schema/schema';\nimport * as Shred from '../schema/shred';\nimport {\n  ColumnChunk,\n  ColumnMetaData,\n  CompressionCodec,\n  ConvertedType,\n  DataPageHeader,\n  DataPageHeaderV2,\n  Encoding,\n  FieldRepetitionType,\n  FileMetaData,\n  KeyValue,\n  PageHeader,\n  PageType,\n  RowGroup,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {osopen, oswrite, osclose} from '../utils/file-utils';\nimport {getBitWidth, serializeThrift} from '../utils/read-utils';\nimport Int64 from 'node-int64';\n\n/**\n * Parquet File Magic String\n */\nconst PARQUET_MAGIC = 'PAR1';\n\n/**\n * Parquet File Format Version\n */\nconst PARQUET_VERSION = 1;\n\n/**\n * Default Page and Row Group sizes\n */\nconst PARQUET_DEFAULT_PAGE_SIZE = 8192;\nconst PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;\n\n/**\n * Repetition and Definition Level Encoding\n */\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\nexport interface ParquetEncoderOptions {\n  baseOffset?: number;\n  rowGroupSize?: number;\n  pageSize?: number;\n  useDataPageV2?: boolean;\n\n  // Write Stream Options\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\n/**\n * Write a parquet file to an output stream. The ParquetEncoder will perform\n * buffering/batching for performance, so close() must be called after all rows\n * are written.\n */\n// eslint-disable-next-line @typescript-eslint/no-unused-vars\nexport class ParquetEncoder<T> {\n  /**\n   * Convenience method to create a new buffered parquet writer that writes to\n   * the specified file\n   */\n  static async openFile<T>(\n    schema: ParquetSchema,\n    path: string,\n    opts?: ParquetEncoderOptions\n  ): Promise<ParquetEncoder<T>> {\n    const outputStream = await osopen(path, opts);\n    return ParquetEncoder.openStream(schema, outputStream, opts);\n  }\n\n  /**\n   * Convenience method to create a new buffered parquet writer that writes to\n   * the specified stream\n   */\n  static async openStream<T>(\n    schema: ParquetSchema,\n    outputStream: stream.Writable,\n    opts: ParquetEncoderOptions = {}\n  ): Promise<ParquetEncoder<T>> {\n    const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);\n    return new ParquetEncoder(schema, envelopeWriter, opts);\n  }\n\n  public schema: ParquetSchema;\n  public envelopeWriter: ParquetEnvelopeWriter;\n  public rowBuffer: ParquetRowGroup;\n  public rowGroupSize: number;\n  public closed: boolean;\n  public userMetadata: Record<string, string>;\n\n  /**\n   * Create a new buffered parquet writer for a given envelope writer\n   */\n  constructor(\n    schema: ParquetSchema,\n    envelopeWriter: ParquetEnvelopeWriter,\n    opts: ParquetEncoderOptions\n  ) {\n    this.schema = schema;\n    this.envelopeWriter = envelopeWriter;\n    // @ts-ignore Row buffer typings...\n    this.rowBuffer = {};\n    this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;\n    this.closed = false;\n    this.userMetadata = {};\n\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.writeHeader();\n  }\n\n  async writeHeader(): Promise<void> {\n    // TODO - better not mess with promises in the constructor\n    try {\n      await this.envelopeWriter.writeHeader();\n    } catch (err) {\n      await this.envelopeWriter.close();\n      throw err;\n    }\n  }\n\n  /**\n   * Append a single row to the parquet file. Rows are buffered in memory until\n   * rowGroupSize rows are in the buffer or close() is called\n   */\n  async appendRow<T extends ParquetRow>(row: T): Promise<void> {\n    if (this.closed) {\n      throw new Error('writer was closed');\n    }\n    Shred.shredRecord(this.schema, row, this.rowBuffer);\n    if (this.rowBuffer.rowCount >= this.rowGroupSize) {\n      // @ts-ignore\n      this.rowBuffer = {};\n    }\n  }\n\n  /**\n   * Finish writing the parquet file and commit the footer to disk. This method\n   * MUST be called after you are finished adding rows. You must not call this\n   * method twice on the same object or add any rows after the close() method has\n   * been called\n   */\n  async close(callback?: () => void): Promise<void> {\n    if (this.closed) {\n      throw new Error('writer was closed');\n    }\n\n    this.closed = true;\n\n    if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {\n      // @ts-ignore\n      this.rowBuffer = {};\n    }\n\n    await this.envelopeWriter.writeFooter(this.userMetadata);\n    await this.envelopeWriter.close();\n    // this.envelopeWriter = null;\n\n    if (callback) {\n      callback();\n    }\n  }\n\n  /**\n   * Add key<>value metadata to the file\n   */\n  setMetadata(key: string, value: string): void {\n    // TODO: value to be any, obj -> JSON\n    this.userMetadata[String(key)] = String(value);\n  }\n\n  /**\n   * Set the parquet row group size. This values controls the maximum number\n   * of rows that are buffered in memory at any given time as well as the number\n   * of rows that are co-located on disk. A higher value is generally better for\n   * read-time I/O performance at the tradeoff of write-time memory usage.\n   */\n  setRowGroupSize(cnt: number): void {\n    this.rowGroupSize = cnt;\n  }\n\n  /**\n   * Set the parquet data page size. The data page size controls the maximum\n   * number of column values that are written to disk as a consecutive array\n   */\n  setPageSize(cnt: number): void {\n    this.envelopeWriter.setPageSize(cnt);\n  }\n}\n\n/**\n * Create a parquet file from a schema and a number of row groups. This class\n * performs direct, unbuffered writes to the underlying output stream and is\n * intendend for advanced and internal users; the writeXXX methods must be\n * called in the correct order to produce a valid file.\n */\nexport class ParquetEnvelopeWriter {\n  /**\n   * Create a new parquet envelope writer that writes to the specified stream\n   */\n  static async openStream(\n    schema: ParquetSchema,\n    outputStream: stream.Writable,\n    opts: ParquetEncoderOptions\n  ): Promise<ParquetEnvelopeWriter> {\n    const writeFn = oswrite.bind(undefined, outputStream);\n    const closeFn = osclose.bind(undefined, outputStream);\n    return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);\n  }\n\n  public schema: ParquetSchema;\n  public write: (buf: Buffer) => Promise<void>;\n  public close: () => Promise<void>;\n  public offset: number;\n  public rowCount: number;\n  public rowGroups: RowGroup[];\n  public pageSize: number;\n  public useDataPageV2: boolean;\n\n  constructor(\n    schema: ParquetSchema,\n    writeFn: (buf: Buffer) => Promise<void>,\n    closeFn: () => Promise<void>,\n    fileOffset: number,\n    opts: ParquetEncoderOptions\n  ) {\n    this.schema = schema;\n    this.write = writeFn;\n    this.close = closeFn;\n    this.offset = fileOffset;\n    this.rowCount = 0;\n    this.rowGroups = [];\n    this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;\n    this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;\n  }\n\n  writeSection(buf: Buffer): Promise<void> {\n    this.offset += buf.length;\n    return this.write(buf);\n  }\n\n  /**\n   * Encode the parquet file header\n   */\n  writeHeader(): Promise<void> {\n    return this.writeSection(Buffer.from(PARQUET_MAGIC));\n  }\n\n  /**\n   * Encode a parquet row group. The records object should be created using the\n   * shredRecord method\n   */\n  async writeRowGroup(records: ParquetRowGroup): Promise<void> {\n    const rgroup = await encodeRowGroup(this.schema, records, {\n      baseOffset: this.offset,\n      pageSize: this.pageSize,\n      useDataPageV2: this.useDataPageV2\n    });\n\n    this.rowCount += records.rowCount;\n    this.rowGroups.push(rgroup.metadata);\n    return await this.writeSection(rgroup.body);\n  }\n\n  /**\n   * Write the parquet file footer\n   */\n  writeFooter(userMetadata: Record<string, string>): Promise<void> {\n    if (!userMetadata) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      userMetadata = {};\n    }\n\n    return this.writeSection(\n      encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata)\n    );\n  }\n\n  /**\n   * Set the parquet data page size. The data page size controls the maximum\n   * number of column values that are written to disk as a consecutive array\n   */\n  setPageSize(cnt: number): void {\n    this.pageSize = cnt;\n  }\n}\n\n/**\n * Create a parquet transform stream\nexport class ParquetTransformer<T> extends stream.Transform {\n  public writer: ParquetEncoder<T>;\n\n  constructor(schema: ParquetSchema, opts: ParquetEncoderOptions = {}) {\n    super({objectMode: true});\n\n    const writeProxy = (function (t: ParquetTransformer<any>) {\n      return async function (b: any): Promise<void> {\n        t.push(b);\n      };\n    })(this);\n\n    this.writer = new ParquetEncoder(\n      schema,\n      new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),\n      opts\n    );\n  }\n\n  // tslint:disable-next-line:function-name\n  _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {\n    if (row) {\n      return this.writer.appendRow(row).then(callback);\n    }\n    callback();\n    return Promise.resolve();\n  }\n\n  // tslint:disable-next-line:function-name\n  async _flush(callback: (val?: any) => void) {\n    await this.writer.close(callback);\n  }\n}\n */\n\n/**\n * Encode a consecutive array of data using one of the parquet encodings\n */\nfunction encodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  values: any[],\n  opts: ParquetCodecOptions\n) {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].encodeValues(type, values, opts);\n}\n\n/**\n * Encode a parquet data page\n */\nasync function encodeDataPage(\n  column: ParquetField,\n  data: ParquetColumnChunk\n): Promise<{\n  header: PageHeader;\n  headerSize: number;\n  page: Buffer;\n}> {\n  /* encode repetition and definition levels */\n  let rLevelsBuf = Buffer.alloc(0);\n  if (column.rLevelMax > 0) {\n    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n      bitWidth: getBitWidth(column.rLevelMax)\n      // disableEnvelope: false\n    });\n  }\n\n  let dLevelsBuf = Buffer.alloc(0);\n  if (column.dLevelMax > 0) {\n    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n      bitWidth: getBitWidth(column.dLevelMax)\n      // disableEnvelope: false\n    });\n  }\n\n  /* encode values */\n  const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);\n\n  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;\n  const compressedBuf = await Compression.deflate(column.compression!, dataBuf);\n\n  /* build page header */\n  const header = new PageHeader({\n    type: PageType.DATA_PAGE,\n    data_page_header: new DataPageHeader({\n      num_values: data.count,\n      encoding: Encoding[column.encoding!] as any,\n      definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],\n      repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]\n    }),\n    uncompressed_page_size: dataBuf.length,\n    compressed_page_size: compressedBuf.length\n  });\n\n  /* concat page header, repetition and definition levels and values */\n  const headerBuf = serializeThrift(header);\n  const page = Buffer.concat([headerBuf, compressedBuf]);\n\n  return {header, headerSize: headerBuf.length, page};\n}\n\n/**\n * Encode a parquet data page (v2)\n */\nasync function encodeDataPageV2(\n  column: ParquetField,\n  data: ParquetColumnChunk,\n  rowCount: number\n): Promise<{\n  header: PageHeader;\n  headerSize: number;\n  page: Buffer;\n}> {\n  /* encode values */\n  const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;\n  const compressedBuf = await Compression.deflate(column.compression!, valuesBuf);\n\n  /* encode repetition and definition levels */\n  let rLevelsBuf = Buffer.alloc(0);\n  if (column.rLevelMax > 0) {\n    rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n      bitWidth: getBitWidth(column.rLevelMax),\n      disableEnvelope: true\n    });\n  }\n\n  let dLevelsBuf = Buffer.alloc(0);\n  if (column.dLevelMax > 0) {\n    dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n      bitWidth: getBitWidth(column.dLevelMax),\n      disableEnvelope: true\n    });\n  }\n\n  /* build page header */\n  const header = new PageHeader({\n    type: PageType.DATA_PAGE_V2,\n    data_page_header_v2: new DataPageHeaderV2({\n      num_values: data.count,\n      num_nulls: data.count - data.values.length,\n      num_rows: rowCount,\n      encoding: Encoding[column.encoding!] as any,\n      definition_levels_byte_length: dLevelsBuf.length,\n      repetition_levels_byte_length: rLevelsBuf.length,\n      is_compressed: column.compression !== 'UNCOMPRESSED'\n    }),\n    uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,\n    compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length\n  });\n\n  /* concat page header, repetition and definition levels and values */\n  const headerBuf = serializeThrift(header);\n  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);\n  return {header, headerSize: headerBuf.length, page};\n}\n\n/**\n * Encode an array of values into a parquet column chunk\n */\nasync function encodeColumnChunk(\n  column: ParquetField,\n  buffer: ParquetRowGroup,\n  offset: number,\n  opts: ParquetEncoderOptions\n): Promise<{\n  body: Buffer;\n  metadata: ColumnMetaData;\n  metadataOffset: number;\n}> {\n  const data = buffer.columnData[column.path.join()];\n  const baseOffset = (opts.baseOffset || 0) + offset;\n  /* encode data page(s) */\n  // const pages: Buffer[] = [];\n  let pageBuf: Buffer;\n  // tslint:disable-next-line:variable-name\n  let total_uncompressed_size = 0;\n  // tslint:disable-next-line:variable-name\n  let total_compressed_size = 0;\n  {\n    const result = opts.useDataPageV2\n      ? await encodeDataPageV2(column, data, buffer.rowCount)\n      : await encodeDataPage(column, data);\n    // pages.push(result.page);\n    pageBuf = result.page;\n    total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;\n    total_compressed_size += result.header.compressed_page_size + result.headerSize;\n  }\n\n  // const pagesBuf = Buffer.concat(pages);\n  // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;\n\n  /* prepare metadata header */\n  const metadata = new ColumnMetaData({\n    path_in_schema: column.path,\n    num_values: data.count,\n    data_page_offset: baseOffset,\n    encodings: [],\n    total_uncompressed_size, //  : pagesBuf.length,\n    total_compressed_size,\n    type: Type[column.primitiveType!],\n    codec: CompressionCodec[column.compression!]\n  });\n\n  /* list encodings */\n  metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);\n  metadata.encodings.push(Encoding[column.encoding!]);\n\n  /* concat metadata header and data pages */\n  const metadataOffset = baseOffset + pageBuf.length;\n  const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);\n  return {body, metadata, metadataOffset};\n}\n\n/**\n * Encode a list of column values into a parquet row group\n */\nasync function encodeRowGroup(\n  schema: ParquetSchema,\n  data: ParquetRowGroup,\n  opts: ParquetEncoderOptions\n): Promise<{\n  body: Buffer;\n  metadata: RowGroup;\n}> {\n  const metadata = new RowGroup({\n    num_rows: data.rowCount,\n    columns: [],\n    total_byte_size: 0\n  });\n\n  let body = Buffer.alloc(0);\n  for (const field of schema.fieldList) {\n    if (field.isNested) {\n      continue; // eslint-disable-line no-continue\n    }\n\n    const cchunkData = await encodeColumnChunk(field, data, body.length, opts);\n\n    const cchunk = new ColumnChunk({\n      file_offset: cchunkData.metadataOffset,\n      meta_data: cchunkData.metadata\n    });\n\n    metadata.columns.push(cchunk);\n    metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);\n\n    body = Buffer.concat([body, cchunkData.body]);\n  }\n\n  return {body, metadata};\n}\n\n/**\n * Encode a parquet file metadata footer\n */\nfunction encodeFooter(\n  schema: ParquetSchema,\n  rowCount: number,\n  rowGroups: RowGroup[],\n  userMetadata: Record<string, string>\n): Buffer {\n  const metadata = new FileMetaData({\n    version: PARQUET_VERSION,\n    created_by: 'parquets',\n    num_rows: rowCount,\n    row_groups: rowGroups,\n    schema: [],\n    key_value_metadata: []\n  });\n\n  for (const key in userMetadata) {\n    const kv = new KeyValue({\n      key,\n      value: userMetadata[key]\n    });\n    metadata.key_value_metadata?.push?.(kv);\n  }\n\n  {\n    const schemaRoot = new SchemaElement({\n      name: 'root',\n      num_children: Object.keys(schema.fields).length\n    });\n    metadata.schema.push(schemaRoot);\n  }\n\n  for (const field of schema.fieldList) {\n    const relt = FieldRepetitionType[field.repetitionType];\n    const schemaElem = new SchemaElement({\n      name: field.name,\n      repetition_type: relt as any\n    });\n\n    if (field.isNested) {\n      schemaElem.num_children = field.fieldCount;\n    } else {\n      schemaElem.type = Type[field.primitiveType!] as Type;\n    }\n\n    if (field.originalType) {\n      schemaElem.converted_type = ConvertedType[field.originalType] as ConvertedType;\n    }\n\n    schemaElem.type_length = field.typeLength;\n\n    metadata.schema.push(schemaElem);\n  }\n\n  const metadataEncoded = serializeThrift(metadata);\n  const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);\n  metadataEncoded.copy(footerEncoded);\n  footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);\n  footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);\n  return footerEncoded;\n}\n"],"mappings":";AAGA,SAA6BA,cAAc,QAAO,WAAW;AAC7D,OAAO,KAAKC,WAAW,MAAM,gBAAgB;AAU7C,OAAO,KAAKC,KAAK,MAAM,iBAAiB;AACxC,SACEC,WAAW,EACXC,cAAc,EACdC,gBAAgB,EAChBC,aAAa,EACbC,cAAc,EACdC,gBAAgB,EAChBC,QAAQ,EACRC,mBAAmB,EACnBC,YAAY,EACZC,QAAQ,EACRC,UAAU,EACVC,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,MAAM,EAAEC,OAAO,EAAEC,OAAO,QAAO,qBAAqB;AAC5D,SAAQC,WAAW,EAAEC,eAAe,QAAO,qBAAqB;AAChE,OAAOC,KAAK,MAAM,YAAY;AAK9B,MAAMC,aAAa,GAAG,MAAM;AAK5B,MAAMC,eAAe,GAAG,CAAC;AAKzB,MAAMC,yBAAyB,GAAG,IAAI;AACtC,MAAMC,8BAA8B,GAAG,IAAI;AAK3C,MAAMC,kBAAkB,GAAG,OAAO;AAClC,MAAMC,sBAAsB,GAAG,KAAK;AAuBpC,OAAO,MAAMC,cAAc,CAAI;EAK7B,aAAaC,QAAQA,CACnBC,MAAqB,EACrBC,IAAY,EACZC,IAA4B,EACA;IAC5B,MAAMC,YAAY,GAAG,MAAMjB,MAAM,CAACe,IAAI,EAAEC,IAAI,CAAC;IAC7C,OAAOJ,cAAc,CAACM,UAAU,CAACJ,MAAM,EAAEG,YAAY,EAAED,IAAI,CAAC;EAC9D;EAMA,aAAaE,UAAUA,CACrBJ,MAAqB,EACrBG,YAA6B,EAED;IAAA,IAD5BD,IAA2B,GAAAG,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAEhC,MAAMG,cAAc,GAAG,MAAMC,qBAAqB,CAACL,UAAU,CAACJ,MAAM,EAAEG,YAAY,EAAED,IAAI,CAAC;IACzF,OAAO,IAAIJ,cAAc,CAACE,MAAM,EAAEQ,cAAc,EAAEN,IAAI,CAAC;EACzD;EAYAQ,WAAWA,CACTV,MAAqB,EACrBQ,cAAqC,EACrCN,IAA2B,EAC3B;IAAAS,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IACA,IAAI,CAACX,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACQ,cAAc,GAAGA,cAAc;IAEpC,IAAI,CAACI,SAAS,GAAG,CAAC,CAAC;IACnB,IAAI,CAACC,YAAY,GAAGX,IAAI,CAACW,YAAY,IAAIlB,8BAA8B;IACvE,IAAI,CAACmB,MAAM,GAAG,KAAK;IACnB,IAAI,CAACC,YAAY,GAAG,CAAC,CAAC;IAGtB,IAAI,CAACC,WAAW,CAAC,CAAC;EACpB;EAEA,MAAMA,WAAWA,CAAA,EAAkB;IAEjC,IAAI;MACF,MAAM,IAAI,CAACR,cAAc,CAACQ,WAAW,CAAC,CAAC;IACzC,CAAC,CAAC,OAAOC,GAAG,EAAE;MACZ,MAAM,IAAI,CAACT,cAAc,CAACU,KAAK,CAAC,CAAC;MACjC,MAAMD,GAAG;IACX;EACF;EAMA,MAAME,SAASA,CAAuBC,GAAM,EAAiB;IAC3D,IAAI,IAAI,CAACN,MAAM,EAAE;MACf,MAAM,IAAIO,KAAK,CAAC,mBAAmB,CAAC;IACtC;IACAnD,KAAK,CAACoD,WAAW,CAAC,IAAI,CAACtB,MAAM,EAAEoB,GAAG,EAAE,IAAI,CAACR,SAAS,CAAC;IACnD,IAAI,IAAI,CAACA,SAAS,CAACW,QAAQ,IAAI,IAAI,CAACV,YAAY,EAAE;MAEhD,IAAI,CAACD,SAAS,GAAG,CAAC,CAAC;IACrB;EACF;EAQA,MAAMM,KAAKA,CAACM,QAAqB,EAAiB;IAChD,IAAI,IAAI,CAACV,MAAM,EAAE;MACf,MAAM,IAAIO,KAAK,CAAC,mBAAmB,CAAC;IACtC;IAEA,IAAI,CAACP,MAAM,GAAG,IAAI;IAElB,IAAI,IAAI,CAACF,SAAS,CAACW,QAAQ,GAAG,CAAC,IAAI,IAAI,CAACX,SAAS,CAACW,QAAQ,IAAI,IAAI,CAACV,YAAY,EAAE;MAE/E,IAAI,CAACD,SAAS,GAAG,CAAC,CAAC;IACrB;IAEA,MAAM,IAAI,CAACJ,cAAc,CAACiB,WAAW,CAAC,IAAI,CAACV,YAAY,CAAC;IACxD,MAAM,IAAI,CAACP,cAAc,CAACU,KAAK,CAAC,CAAC;IAGjC,IAAIM,QAAQ,EAAE;MACZA,QAAQ,CAAC,CAAC;IACZ;EACF;EAKAE,WAAWA,CAACC,GAAW,EAAEC,KAAa,EAAQ;IAE5C,IAAI,CAACb,YAAY,CAACc,MAAM,CAACF,GAAG,CAAC,CAAC,GAAGE,MAAM,CAACD,KAAK,CAAC;EAChD;EAQAE,eAAeA,CAACC,GAAW,EAAQ;IACjC,IAAI,CAAClB,YAAY,GAAGkB,GAAG;EACzB;EAMAC,WAAWA,CAACD,GAAW,EAAQ;IAC7B,IAAI,CAACvB,cAAc,CAACwB,WAAW,CAACD,GAAG,CAAC;EACtC;AACF;AAQA,OAAO,MAAMtB,qBAAqB,CAAC;EAIjC,aAAaL,UAAUA,CACrBJ,MAAqB,EACrBG,YAA6B,EAC7BD,IAA2B,EACK;IAChC,MAAM+B,OAAO,GAAG9C,OAAO,CAAC+C,IAAI,CAAC3B,SAAS,EAAEJ,YAAY,CAAC;IACrD,MAAMgC,OAAO,GAAG/C,OAAO,CAAC8C,IAAI,CAAC3B,SAAS,EAAEJ,YAAY,CAAC;IACrD,OAAO,IAAIM,qBAAqB,CAACT,MAAM,EAAEiC,OAAO,EAAEE,OAAO,EAAE,CAAC,EAAEjC,IAAI,CAAC;EACrE;EAWAQ,WAAWA,CACTV,MAAqB,EACrBiC,OAAuC,EACvCE,OAA4B,EAC5BC,UAAkB,EAClBlC,IAA2B,EAC3B;IAAAS,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IAAAA,eAAA;IACA,IAAI,CAACX,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACqC,KAAK,GAAGJ,OAAO;IACpB,IAAI,CAACf,KAAK,GAAGiB,OAAO;IACpB,IAAI,CAACG,MAAM,GAAGF,UAAU;IACxB,IAAI,CAACb,QAAQ,GAAG,CAAC;IACjB,IAAI,CAACgB,SAAS,GAAG,EAAE;IACnB,IAAI,CAACC,QAAQ,GAAGtC,IAAI,CAACsC,QAAQ,IAAI9C,yBAAyB;IAC1D,IAAI,CAAC+C,aAAa,GAAG,eAAe,IAAIvC,IAAI,GAAGwC,OAAO,CAACxC,IAAI,CAACuC,aAAa,CAAC,GAAG,KAAK;EACpF;EAEAE,YAAYA,CAACC,GAAW,EAAiB;IACvC,IAAI,CAACN,MAAM,IAAIM,GAAG,CAACtC,MAAM;IACzB,OAAO,IAAI,CAAC+B,KAAK,CAACO,GAAG,CAAC;EACxB;EAKA5B,WAAWA,CAAA,EAAkB;IAC3B,OAAO,IAAI,CAAC2B,YAAY,CAACE,MAAM,CAACC,IAAI,CAACtD,aAAa,CAAC,CAAC;EACtD;EAMA,MAAMuD,aAAaA,CAACC,OAAwB,EAAiB;IAC3D,MAAMC,MAAM,GAAG,MAAMC,cAAc,CAAC,IAAI,CAAClD,MAAM,EAAEgD,OAAO,EAAE;MACxDG,UAAU,EAAE,IAAI,CAACb,MAAM;MACvBE,QAAQ,EAAE,IAAI,CAACA,QAAQ;MACvBC,aAAa,EAAE,IAAI,CAACA;IACtB,CAAC,CAAC;IAEF,IAAI,CAAClB,QAAQ,IAAIyB,OAAO,CAACzB,QAAQ;IACjC,IAAI,CAACgB,SAAS,CAACa,IAAI,CAACH,MAAM,CAACI,QAAQ,CAAC;IACpC,OAAO,MAAM,IAAI,CAACV,YAAY,CAACM,MAAM,CAACK,IAAI,CAAC;EAC7C;EAKA7B,WAAWA,CAACV,YAAoC,EAAiB;IAC/D,IAAI,CAACA,YAAY,EAAE;MAEjBA,YAAY,GAAG,CAAC,CAAC;IACnB;IAEA,OAAO,IAAI,CAAC4B,YAAY,CACtBY,YAAY,CAAC,IAAI,CAACvD,MAAM,EAAE,IAAI,CAACuB,QAAQ,EAAE,IAAI,CAACgB,SAAS,EAAExB,YAAY,CACvE,CAAC;EACH;EAMAiB,WAAWA,CAACD,GAAW,EAAQ;IAC7B,IAAI,CAACS,QAAQ,GAAGT,GAAG;EACrB;AACF;AA0CA,SAASyB,YAAYA,CACnBC,IAAmB,EACnBC,QAAsB,EACtBC,MAAa,EACbzD,IAAyB,EACzB;EACA,IAAI,EAAEwD,QAAQ,IAAI1F,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIqD,KAAK,sBAAAuC,MAAA,CAAsBF,QAAQ,CAAE,CAAC;EAClD;EACA,OAAO1F,cAAc,CAAC0F,QAAQ,CAAC,CAACF,YAAY,CAACC,IAAI,EAAEE,MAAM,EAAEzD,IAAI,CAAC;AAClE;AAKA,eAAe2D,cAAcA,CAC3BC,MAAoB,EACpBC,IAAwB,EAKvB;EAED,IAAIC,UAAU,GAAGnB,MAAM,CAACoB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACI,SAAS,GAAG,CAAC,EAAE;IACxBF,UAAU,GAAGR,YAAY,CAAC5D,kBAAkB,EAAEC,sBAAsB,EAAEkE,IAAI,CAACI,OAAO,EAAE;MAClFC,QAAQ,EAAE/E,WAAW,CAACyE,MAAM,CAACI,SAAS;IAExC,CAAC,CAAC;EACJ;EAEA,IAAIG,UAAU,GAAGxB,MAAM,CAACoB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IACxBD,UAAU,GAAGb,YAAY,CAAC5D,kBAAkB,EAAEC,sBAAsB,EAAEkE,IAAI,CAACQ,OAAO,EAAE;MAClFH,QAAQ,EAAE/E,WAAW,CAACyE,MAAM,CAACQ,SAAS;IAExC,CAAC,CAAC;EACJ;EAGA,MAAME,SAAS,GAAGhB,YAAY,CAACM,MAAM,CAACW,aAAa,EAAGX,MAAM,CAACJ,QAAQ,EAAGK,IAAI,CAACJ,MAAM,EAAE;IACnFe,UAAU,EAAEZ,MAAM,CAACY,UAAU;IAC7BN,QAAQ,EAAEN,MAAM,CAACY;EACnB,CAAC,CAAC;EAEF,MAAMC,OAAO,GAAG9B,MAAM,CAACe,MAAM,CAAC,CAACI,UAAU,EAAEK,UAAU,EAAEG,SAAS,CAAC,CAAC;EAGlE,MAAMI,aAAa,GAAG,MAAM3G,WAAW,CAAC4G,OAAO,CAACf,MAAM,CAACgB,WAAW,EAAGH,OAAO,CAAC;EAG7E,MAAMI,MAAM,GAAG,IAAIlG,UAAU,CAAC;IAC5B4E,IAAI,EAAE3E,QAAQ,CAACkG,SAAS;IACxBC,gBAAgB,EAAE,IAAI1G,cAAc,CAAC;MACnC2G,UAAU,EAAEnB,IAAI,CAACoB,KAAK;MACtBzB,QAAQ,EAAEjF,QAAQ,CAACqF,MAAM,CAACJ,QAAQ,CAAS;MAC3C0B,yBAAyB,EAAE3G,QAAQ,CAACoB,sBAAsB,CAAC;MAC3DwF,yBAAyB,EAAE5G,QAAQ,CAACoB,sBAAsB;IAC5D,CAAC,CAAC;IACFyF,sBAAsB,EAAEX,OAAO,CAACrE,MAAM;IACtCiF,oBAAoB,EAAEX,aAAa,CAACtE;EACtC,CAAC,CAAC;EAGF,MAAMkF,SAAS,GAAGlG,eAAe,CAACyF,MAAM,CAAC;EACzC,MAAMU,IAAI,GAAG5C,MAAM,CAACe,MAAM,CAAC,CAAC4B,SAAS,EAAEZ,aAAa,CAAC,CAAC;EAEtD,OAAO;IAACG,MAAM;IAAEW,UAAU,EAAEF,SAAS,CAAClF,MAAM;IAAEmF;EAAI,CAAC;AACrD;AAKA,eAAeE,gBAAgBA,CAC7B7B,MAAoB,EACpBC,IAAwB,EACxBxC,QAAgB,EAKf;EAED,MAAMiD,SAAS,GAAGhB,YAAY,CAACM,MAAM,CAACW,aAAa,EAAGX,MAAM,CAACJ,QAAQ,EAAGK,IAAI,CAACJ,MAAM,EAAE;IACnFe,UAAU,EAAEZ,MAAM,CAACY,UAAU;IAC7BN,QAAQ,EAAEN,MAAM,CAACY;EACnB,CAAC,CAAC;EAGF,MAAME,aAAa,GAAG,MAAM3G,WAAW,CAAC4G,OAAO,CAACf,MAAM,CAACgB,WAAW,EAAGN,SAAS,CAAC;EAG/E,IAAIR,UAAU,GAAGnB,MAAM,CAACoB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACI,SAAS,GAAG,CAAC,EAAE;IACxBF,UAAU,GAAGR,YAAY,CAAC5D,kBAAkB,EAAEC,sBAAsB,EAAEkE,IAAI,CAACI,OAAO,EAAE;MAClFC,QAAQ,EAAE/E,WAAW,CAACyE,MAAM,CAACI,SAAS,CAAC;MACvC0B,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ;EAEA,IAAIvB,UAAU,GAAGxB,MAAM,CAACoB,KAAK,CAAC,CAAC,CAAC;EAChC,IAAIH,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IACxBD,UAAU,GAAGb,YAAY,CAAC5D,kBAAkB,EAAEC,sBAAsB,EAAEkE,IAAI,CAACQ,OAAO,EAAE;MAClFH,QAAQ,EAAE/E,WAAW,CAACyE,MAAM,CAACQ,SAAS,CAAC;MACvCsB,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ;EAGA,MAAMb,MAAM,GAAG,IAAIlG,UAAU,CAAC;IAC5B4E,IAAI,EAAE3E,QAAQ,CAAC+G,YAAY;IAC3BC,mBAAmB,EAAE,IAAItH,gBAAgB,CAAC;MACxC0G,UAAU,EAAEnB,IAAI,CAACoB,KAAK;MACtBY,SAAS,EAAEhC,IAAI,CAACoB,KAAK,GAAGpB,IAAI,CAACJ,MAAM,CAACrD,MAAM;MAC1C0F,QAAQ,EAAEzE,QAAQ;MAClBmC,QAAQ,EAAEjF,QAAQ,CAACqF,MAAM,CAACJ,QAAQ,CAAS;MAC3CuC,6BAA6B,EAAE5B,UAAU,CAAC/D,MAAM;MAChD4F,6BAA6B,EAAElC,UAAU,CAAC1D,MAAM;MAChD6F,aAAa,EAAErC,MAAM,CAACgB,WAAW,KAAK;IACxC,CAAC,CAAC;IACFQ,sBAAsB,EAAEtB,UAAU,CAAC1D,MAAM,GAAG+D,UAAU,CAAC/D,MAAM,GAAGkE,SAAS,CAAClE,MAAM;IAChFiF,oBAAoB,EAAEvB,UAAU,CAAC1D,MAAM,GAAG+D,UAAU,CAAC/D,MAAM,GAAGsE,aAAa,CAACtE;EAC9E,CAAC,CAAC;EAGF,MAAMkF,SAAS,GAAGlG,eAAe,CAACyF,MAAM,CAAC;EACzC,MAAMU,IAAI,GAAG5C,MAAM,CAACe,MAAM,CAAC,CAAC4B,SAAS,EAAExB,UAAU,EAAEK,UAAU,EAAEO,aAAa,CAAC,CAAC;EAC9E,OAAO;IAACG,MAAM;IAAEW,UAAU,EAAEF,SAAS,CAAClF,MAAM;IAAEmF;EAAI,CAAC;AACrD;AAKA,eAAeW,iBAAiBA,CAC9BtC,MAAoB,EACpBuC,MAAuB,EACvB/D,MAAc,EACdpC,IAA2B,EAK1B;EACD,MAAM6D,IAAI,GAAGsC,MAAM,CAACC,UAAU,CAACxC,MAAM,CAAC7D,IAAI,CAACsG,IAAI,CAAC,CAAC,CAAC;EAClD,MAAMpD,UAAU,GAAG,CAACjD,IAAI,CAACiD,UAAU,IAAI,CAAC,IAAIb,MAAM;EAGlD,IAAIkE,OAAe;EAEnB,IAAIC,uBAAuB,GAAG,CAAC;EAE/B,IAAIC,qBAAqB,GAAG,CAAC;EAC7B;IACE,MAAMC,MAAM,GAAGzG,IAAI,CAACuC,aAAa,GAC7B,MAAMkD,gBAAgB,CAAC7B,MAAM,EAAEC,IAAI,EAAEsC,MAAM,CAAC9E,QAAQ,CAAC,GACrD,MAAMsC,cAAc,CAACC,MAAM,EAAEC,IAAI,CAAC;IAEtCyC,OAAO,GAAGG,MAAM,CAAClB,IAAI;IACrBgB,uBAAuB,IAAIE,MAAM,CAAC5B,MAAM,CAACO,sBAAsB,GAAGqB,MAAM,CAACjB,UAAU;IACnFgB,qBAAqB,IAAIC,MAAM,CAAC5B,MAAM,CAACQ,oBAAoB,GAAGoB,MAAM,CAACjB,UAAU;EACjF;EAMA,MAAMrC,QAAQ,GAAG,IAAIjF,cAAc,CAAC;IAClCwI,cAAc,EAAE9C,MAAM,CAAC7D,IAAI;IAC3BiF,UAAU,EAAEnB,IAAI,CAACoB,KAAK;IACtB0B,gBAAgB,EAAE1D,UAAU;IAC5B2D,SAAS,EAAE,EAAE;IACbL,uBAAuB;IACvBC,qBAAqB;IACrBjD,IAAI,EAAExE,IAAI,CAAC6E,MAAM,CAACW,aAAa,CAAE;IACjCsC,KAAK,EAAE1I,gBAAgB,CAACyF,MAAM,CAACgB,WAAW;EAC5C,CAAC,CAAC;EAGFzB,QAAQ,CAACyD,SAAS,CAAC1D,IAAI,CAAC3E,QAAQ,CAACoB,sBAAsB,CAAC,CAAC;EACzDwD,QAAQ,CAACyD,SAAS,CAAC1D,IAAI,CAAC3E,QAAQ,CAACqF,MAAM,CAACJ,QAAQ,CAAE,CAAC;EAGnD,MAAMsD,cAAc,GAAG7D,UAAU,GAAGqD,OAAO,CAAClG,MAAM;EAClD,MAAMgD,IAAI,GAAGT,MAAM,CAACe,MAAM,CAAC,CAAC4C,OAAO,EAAElH,eAAe,CAAC+D,QAAQ,CAAC,CAAC,CAAC;EAChE,OAAO;IAACC,IAAI;IAAED,QAAQ;IAAE2D;EAAc,CAAC;AACzC;AAKA,eAAe9D,cAAcA,CAC3BlD,MAAqB,EACrB+D,IAAqB,EACrB7D,IAA2B,EAI1B;EACD,MAAMmD,QAAQ,GAAG,IAAItE,QAAQ,CAAC;IAC5BiH,QAAQ,EAAEjC,IAAI,CAACxC,QAAQ;IACvB0F,OAAO,EAAE,EAAE;IACXC,eAAe,EAAE;EACnB,CAAC,CAAC;EAEF,IAAI5D,IAAI,GAAGT,MAAM,CAACoB,KAAK,CAAC,CAAC,CAAC;EAC1B,KAAK,MAAMkD,KAAK,IAAInH,MAAM,CAACoH,SAAS,EAAE;IACpC,IAAID,KAAK,CAACE,QAAQ,EAAE;MAClB;IACF;IAEA,MAAMC,UAAU,GAAG,MAAMlB,iBAAiB,CAACe,KAAK,EAAEpD,IAAI,EAAET,IAAI,CAAChD,MAAM,EAAEJ,IAAI,CAAC;IAE1E,MAAMqH,MAAM,GAAG,IAAIpJ,WAAW,CAAC;MAC7BqJ,WAAW,EAAEF,UAAU,CAACN,cAAc;MACtCS,SAAS,EAAEH,UAAU,CAACjE;IACxB,CAAC,CAAC;IAEFA,QAAQ,CAAC4D,OAAO,CAAC7D,IAAI,CAACmE,MAAM,CAAC;IAC7BlE,QAAQ,CAAC6D,eAAe,GAAG,IAAI3H,KAAK,CAACmI,MAAM,CAACrE,QAAQ,CAAC6D,eAAe,CAAC,GAAGI,UAAU,CAAChE,IAAI,CAAChD,MAAM,CAAC;IAE/FgD,IAAI,GAAGT,MAAM,CAACe,MAAM,CAAC,CAACN,IAAI,EAAEgE,UAAU,CAAChE,IAAI,CAAC,CAAC;EAC/C;EAEA,OAAO;IAACA,IAAI;IAAED;EAAQ,CAAC;AACzB;AAKA,SAASE,YAAYA,CACnBvD,MAAqB,EACrBuB,QAAgB,EAChBgB,SAAqB,EACrBxB,YAAoC,EAC5B;EACR,MAAMsC,QAAQ,GAAG,IAAI1E,YAAY,CAAC;IAChCgJ,OAAO,EAAElI,eAAe;IACxBmI,UAAU,EAAE,UAAU;IACtB5B,QAAQ,EAAEzE,QAAQ;IAClBsG,UAAU,EAAEtF,SAAS;IACrBvC,MAAM,EAAE,EAAE;IACV8H,kBAAkB,EAAE;EACtB,CAAC,CAAC;EAEF,KAAK,MAAMnG,GAAG,IAAIZ,YAAY,EAAE;IAAA,IAAAgH,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA;IAC9B,MAAMC,EAAE,GAAG,IAAItJ,QAAQ,CAAC;MACtB+C,GAAG;MACHC,KAAK,EAAEb,YAAY,CAACY,GAAG;IACzB,CAAC,CAAC;IACF,CAAAoG,qBAAA,GAAA1E,QAAQ,CAACyE,kBAAkB,cAAAC,qBAAA,wBAAAC,sBAAA,GAA3B,CAAAC,sBAAA,GAAAF,qBAAA,EAA6B3E,IAAI,cAAA4E,sBAAA,uBAAjCA,sBAAA,CAAAG,IAAA,CAAAF,sBAAA,EAAoCC,EAAE,CAAC;EACzC;EAEA;IACE,MAAME,UAAU,GAAG,IAAIpJ,aAAa,CAAC;MACnCqJ,IAAI,EAAE,MAAM;MACZC,YAAY,EAAEC,MAAM,CAACC,IAAI,CAACxI,MAAM,CAACyI,MAAM,CAAC,CAACnI;IAC3C,CAAC,CAAC;IACF+C,QAAQ,CAACrD,MAAM,CAACoD,IAAI,CAACgF,UAAU,CAAC;EAClC;EAEA,KAAK,MAAMjB,KAAK,IAAInH,MAAM,CAACoH,SAAS,EAAE;IACpC,MAAMsB,IAAI,GAAGhK,mBAAmB,CAACyI,KAAK,CAACwB,cAAc,CAAC;IACtD,MAAMC,UAAU,GAAG,IAAI5J,aAAa,CAAC;MACnCqJ,IAAI,EAAElB,KAAK,CAACkB,IAAI;MAChBQ,eAAe,EAAEH;IACnB,CAAC,CAAC;IAEF,IAAIvB,KAAK,CAACE,QAAQ,EAAE;MAClBuB,UAAU,CAACN,YAAY,GAAGnB,KAAK,CAAC2B,UAAU;IAC5C,CAAC,MAAM;MACLF,UAAU,CAACnF,IAAI,GAAGxE,IAAI,CAACkI,KAAK,CAAC1C,aAAa,CAAU;IACtD;IAEA,IAAI0C,KAAK,CAAC4B,YAAY,EAAE;MACtBH,UAAU,CAACI,cAAc,GAAG1K,aAAa,CAAC6I,KAAK,CAAC4B,YAAY,CAAkB;IAChF;IAEAH,UAAU,CAACK,WAAW,GAAG9B,KAAK,CAACzC,UAAU;IAEzCrB,QAAQ,CAACrD,MAAM,CAACoD,IAAI,CAACwF,UAAU,CAAC;EAClC;EAEA,MAAMM,eAAe,GAAG5J,eAAe,CAAC+D,QAAQ,CAAC;EACjD,MAAM8F,aAAa,GAAGtG,MAAM,CAACoB,KAAK,CAACiF,eAAe,CAAC5I,MAAM,GAAG,CAAC,CAAC;EAC9D4I,eAAe,CAACE,IAAI,CAACD,aAAa,CAAC;EACnCA,aAAa,CAACE,aAAa,CAACH,eAAe,CAAC5I,MAAM,EAAE4I,eAAe,CAAC5I,MAAM,CAAC;EAC3E6I,aAAa,CAAC9G,KAAK,CAAC7C,aAAa,EAAE0J,eAAe,CAAC5I,MAAM,GAAG,CAAC,CAAC;EAC9D,OAAO6I,aAAa;AACtB"}

package/dist/esm/parquetjs/modules.d.ts

@@ -0,0 +1,21 @@
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+
+declare module 'int53' {
+  declare function readInt64BE(buffer: Buffer, offset?: number): number;
+  declare function readInt64LE(buffer: Buffer, offset?: number): number;
+  declare function readUInt64BE(buffer: Buffer, offset?: number): number;
+  declare function readUInt64LE(buffer: Buffer, offset?: number): number;
+  declare function writeInt64BE(value: number, buffer: Buffer, offset?: number): void;
+  declare function writeInt64LE(value: number, buffer: Buffer, offset?: number): void;
+  declare function writeUInt64BE(value: number, buffer: Buffer, offset?: number): void;
+  declare function writeUInt64LE(value: number, buffer: Buffer, offset?: number): void;
+}
+
+// declare module 'snappyjs' {
+//   declare function compress(uncompressed: Buffer): Buffer;
+//   declare function compress(uncompressed: ArrayBuffer): ArrayBuffer;
+//   declare function compress(uncompressed: Uint8Array): Uint8Array;
+//   declare function uncompress(compressed: Buffer): Buffer;
+//   declare function uncompress(compressed: ArrayBuffer): ArrayBuffer;
+//   declare function uncompress(compressed: Uint8Array): Uint8Array;
+// }

package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js

@@ -0,0 +1,7 @@
+export let BoundaryOrder = function (BoundaryOrder) {
+  BoundaryOrder[BoundaryOrder["UNORDERED"] = 0] = "UNORDERED";
+  BoundaryOrder[BoundaryOrder["ASCENDING"] = 1] = "ASCENDING";
+  BoundaryOrder[BoundaryOrder["DESCENDING"] = 2] = "DESCENDING";
+  return BoundaryOrder;
+}({});
+//# sourceMappingURL=BoundaryOrder.js.map

package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BoundaryOrder.js","names":["BoundaryOrder"],"sources":["../../../../src/parquetjs/parquet-thrift/BoundaryOrder.ts"],"sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nexport enum BoundaryOrder {\n  UNORDERED = 0,\n  ASCENDING = 1,\n  DESCENDING = 2\n}\n"],"mappings":"AAMA,WAAYA,aAAa,aAAbA,aAAa;EAAbA,aAAa,CAAbA,aAAa;EAAbA,aAAa,CAAbA,aAAa;EAAbA,aAAa,CAAbA,aAAa;EAAA,OAAbA,aAAa;AAAA"}

package/dist/esm/parquetjs/parquet-thrift/BsonType.js

@@ -0,0 +1,31 @@
+import * as thrift from 'thrift';
+export class BsonType {
+  constructor() {}
+  write(output) {
+    output.writeStructBegin('BsonType');
+    output.writeFieldStop();
+    output.writeStructEnd();
+    return;
+  }
+  static read(input) {
+    input.readStructBegin();
+    while (true) {
+      const ret = input.readFieldBegin();
+      const fieldType = ret.ftype;
+      const fieldId = ret.fid;
+      if (fieldType === thrift.Thrift.Type.STOP) {
+        break;
+      }
+      switch (fieldId) {
+        default:
+          {
+            input.skip(fieldType);
+          }
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return new BsonType();
+  }
+}
+//# sourceMappingURL=BsonType.js.map

package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BsonType.js","names":["thrift","BsonType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"sources":["../../../../src/parquetjs/parquet-thrift/BsonType.ts"],"sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nimport * as thrift from 'thrift';\nexport interface IBsonTypeArgs {}\nexport class BsonType {\n  constructor() {}\n  public write(output: thrift.TProtocol): void {\n    output.writeStructBegin('BsonType');\n    output.writeFieldStop();\n    output.writeStructEnd();\n    return;\n  }\n  public static read(input: thrift.TProtocol): BsonType {\n    input.readStructBegin();\n    while (true) {\n      const ret: thrift.TField = input.readFieldBegin();\n      const fieldType: thrift.Thrift.Type = ret.ftype;\n      const fieldId: number = ret.fid;\n      if (fieldType === thrift.Thrift.Type.STOP) {\n        break;\n      }\n      switch (fieldId) {\n        default: {\n          input.skip(fieldType);\n        }\n      }\n      input.readFieldEnd();\n    }\n    input.readStructEnd();\n    return new BsonType();\n  }\n}\n"],"mappings":"AAMA,OAAO,KAAKA,MAAM,MAAM,QAAQ;AAEhC,OAAO,MAAMC,QAAQ,CAAC;EACpBC,WAAWA,CAAA,EAAG,CAAC;EACRC,KAAKA,CAACC,MAAwB,EAAQ;IAC3CA,MAAM,CAACC,gBAAgB,CAAC,UAAU,CAAC;IACnCD,MAAM,CAACE,cAAc,CAAC,CAAC;IACvBF,MAAM,CAACG,cAAc,CAAC,CAAC;IACvB;EACF;EACA,OAAcC,IAAIA,CAACC,KAAuB,EAAY;IACpDA,KAAK,CAACC,eAAe,CAAC,CAAC;IACvB,OAAO,IAAI,EAAE;MACX,MAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAc,CAAC,CAAC;MACjD,MAAMC,SAA6B,GAAGF,GAAG,CAACG,KAAK;MAC/C,MAAMC,OAAe,GAAGJ,GAAG,CAACK,GAAG;MAC/B,IAAIH,SAAS,KAAKb,MAAM,CAACiB,MAAM,CAACC,IAAI,CAACC,IAAI,EAAE;QACzC;MACF;MACA,QAAQJ,OAAO;QACb;UAAS;YACPN,KAAK,CAACW,IAAI,CAACP,SAAS,CAAC;UACvB;MACF;MACAJ,KAAK,CAACY,YAAY,CAAC,CAAC;IACtB;IACAZ,KAAK,CAACa,aAAa,CAAC,CAAC;IACrB,OAAO,IAAIrB,QAAQ,CAAC,CAAC;EACvB;AACF"}