@loaders.gl/parquet 3.3.0 → 3.4.0-alpha.1

package/dist/esm/parquetjs/parser/parquet-reader.js

@@ -1,189 +1,103 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-
+let _Symbol$asyncIterator;
+import { ParquetEnvelopeReader } from './parquet-envelope-reader';
 import { ParquetSchema } from '../schema/schema';
+import { ParquetCursor } from './parquet-cursor';
+import { PARQUET_VERSION } from '../../constants';
 import { decodeSchema } from './decoders';
-import { materializeRecords } from '../schema/shred';
-import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from '../../constants';
-import { CompressionCodec, Type } from '../parquet-thrift';
-import { decodeFileMetadata, getThriftEnum, fieldIndexOf } from '../utils/read-utils';
-import { decodeDataPages, decodePage } from './decoders';
-const DEFAULT_PROPS = {
-  defaultDictionarySize: 1e6
-};
 
+_Symbol$asyncIterator = Symbol.asyncIterator;
 export class ParquetReader {
-  constructor(file, props) {
-    _defineProperty(this, "props", void 0);
-    _defineProperty(this, "file", void 0);
-    _defineProperty(this, "metadata", null);
-    this.file = file;
-    this.props = {
-      ...DEFAULT_PROPS,
-      ...props
+  static async openBlob(blob) {
+    const readFn = async (start, length) => {
+      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
+      return Buffer.from(arrayBuffer);
     };
-  }
-  close() {
-    this.file.close();
-  }
-
-  async *rowIterator(props) {
-    for await (const rows of this.rowBatchIterator(props)) {
-      for (const row of rows) {
-        yield row;
-      }
+    const closeFn = async () => {};
+    const size = blob.size;
+    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
     }
   }
 
-  async *rowBatchIterator(props) {
-    const schema = await this.getSchema();
-    for await (const rowGroup of this.rowGroupIterator(props)) {
-      yield materializeRecords(schema, rowGroup);
+  static async openArrayBuffer(arrayBuffer) {
+    const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
+    const closeFn = async () => {};
+    const size = arrayBuffer.byteLength;
+    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
     }
   }
-
-  async *rowGroupIterator(props) {
-    const columnList = ((props === null || props === void 0 ? void 0 : props.columnList) || []).map(x => Array.isArray(x) ? x : [x]);
-    const metadata = await this.getFileMetadata();
-    const schema = await this.getSchema();
-    const rowGroupCount = (metadata === null || metadata === void 0 ? void 0 : metadata.row_groups.length) || 0;
-    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {
-      const rowGroup = await this.readRowGroup(schema, metadata.row_groups[rowGroupIndex], columnList);
-      yield rowGroup;
+  static async openBuffer(buffer) {
+    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
     }
   }
-  async getRowCount() {
-    const metadata = await this.getFileMetadata();
-    return Number(metadata.num_rows);
-  }
-  async getSchema() {
-    const metadata = await this.getFileMetadata();
-    const root = metadata.schema[0];
+  constructor(metadata, envelopeReader) {
+    _defineProperty(this, "metadata", void 0);
+    _defineProperty(this, "envelopeReader", void 0);
+    _defineProperty(this, "schema", void 0);
+    if (metadata.version !== PARQUET_VERSION) {
+      throw new Error('invalid parquet version');
+    }
+    this.metadata = metadata;
+    this.envelopeReader = envelopeReader;
+    const root = this.metadata.schema[0];
     const {
-      schema: schemaDefinition
-    } = decodeSchema(metadata.schema, 1, root.num_children);
-    const schema = new ParquetSchema(schemaDefinition);
-    return schema;
+      schema
+    } = decodeSchema(this.metadata.schema, 1, root.num_children);
+    this.schema = new ParquetSchema(schema);
   }
 
-  async getSchemaMetadata() {
-    const metadata = await this.getFileMetadata();
-    const md = {};
-    for (const kv of metadata.key_value_metadata) {
-      md[kv.key] = kv.value;
-    }
-    return md;
-  }
-  async getFileMetadata() {
-    if (!this.metadata) {
-      await this.readHeader();
-      this.metadata = this.readFooter();
-    }
-    return this.metadata;
+  async close() {
+    await this.envelopeReader.close();
   }
 
-  async readHeader() {
-    const buffer = await this.file.read(0, PARQUET_MAGIC.length);
-    const magic = buffer.toString();
-    switch (magic) {
-      case PARQUET_MAGIC:
-        break;
-      case PARQUET_MAGIC_ENCRYPTED:
-        throw new Error('Encrypted parquet file not supported');
-      default:
-        throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
+  getCursor(columnList) {
+    if (!columnList) {
+      columnList = [];
     }
-  }
 
-  async readFooter() {
-    const trailerLen = PARQUET_MAGIC.length + 4;
-    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
-    const magic = trailerBuf.slice(4).toString();
-    if (magic !== PARQUET_MAGIC) {
-      throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
-    }
-    const metadataSize = trailerBuf.readUInt32LE(0);
-    const metadataOffset = this.file.size - metadataSize - trailerLen;
-    if (metadataOffset < PARQUET_MAGIC.length) {
-      throw new Error("Invalid metadata size ".concat(metadataOffset));
-    }
-    const metadataBuf = await this.file.read(metadataOffset, metadataSize);
-    const {
-      metadata
-    } = decodeFileMetadata(metadataBuf);
-    return metadata;
+    columnList = columnList.map(x => Array.isArray(x) ? x : [x]);
+    return new ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
   }
 
-  async readRowGroup(schema, rowGroup, columnList) {
-    const buffer = {
-      rowCount: Number(rowGroup.num_rows),
-      columnData: {}
-    };
-    for (const colChunk of rowGroup.columns) {
-      const colMetadata = colChunk.meta_data;
-      const colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
-      if (columnList.length > 0 && fieldIndexOf(columnList, colKey) < 0) {
-        continue;
-      }
+  getRowCount() {
+    return Number(this.metadata.num_rows);
+  }
 
-      buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
-    }
-    return buffer;
+  getSchema() {
+    return this.schema;
   }
 
-  async readColumnChunk(schema, colChunk) {
-    var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
-    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
-      throw new Error('external references are not supported');
-    }
-    const field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
-    const type = getThriftEnum(Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
-    if (type !== field.primitiveType) {
-      throw new Error("chunk type not matching schema: ".concat(type));
-    }
-    const compression = getThriftEnum(CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
-    const pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
-    let pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
-    if (!colChunk.file_path) {
-      var _colChunk$meta_data6;
-      pagesSize = Math.min(this.file.size - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
-    }
-    const options = {
-      type,
-      rLevelMax: field.rLevelMax,
-      dLevelMax: field.dLevelMax,
-      compression,
-      column: field,
-      numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
-      dictionary: []
-    };
-    let dictionary;
-    const dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
-    if (dictionaryPageOffset) {
-      const dictionaryOffset = Number(dictionaryPageOffset);
-      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+  getMetadata() {
+    const md = {};
+    for (const kv of this.metadata.key_value_metadata) {
+      md[kv.key] = kv.value;
     }
-    dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
-    const pagesBuf = await this.file.read(pagesOffset, pagesSize);
-    return await decodeDataPages(pagesBuf, {
-      ...options,
-      dictionary
-    });
+    return md;
   }
 
-  async getDictionary(dictionaryPageOffset, options, pagesOffset) {
-    if (dictionaryPageOffset === 0) {
-
-      return [];
-    }
-    const dictionarySize = Math.min(this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize);
-    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
-    const cursor = {
-      buffer: pagesBuf,
-      offset: 0,
-      size: pagesBuf.length
-    };
-    const decodedPage = await decodePage(cursor, options);
-    return decodedPage.dictionary;
+  [_Symbol$asyncIterator]() {
+    return this.getCursor()[Symbol.asyncIterator]();
   }
 }
 //# sourceMappingURL=parquet-reader.js.map

package/dist/esm/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRecords","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRecords(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";;AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,kBAAkB,QAAO,iBAAiB;AAElD,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAYtD,MAAMC,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;;AAQD,OAAO,MAAMC,aAAa,CAAC;EAKzBC,WAAW,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAA;IAAA;IAAA,kCAFnB,IAAI;IAG3C,IAAI,CAACD,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAC,KAAK,GAAS;IAEZ,IAAI,CAACF,IAAI,CAACE,KAAK,EAAE;EACnB;;EAKA,OAAOC,WAAW,CAACF,KAA6B,EAAE;IAChD,WAAW,MAAMG,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACJ,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMK,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;;EAGA,OAAOD,gBAAgB,CAACJ,KAA6B,EAAE;IACrD,MAAMM,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACT,KAAK,CAAC,EAAE;MACzD,MAAMf,kBAAkB,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IAC5C;EACF;;EAGA,OAAOC,gBAAgB,CAACT,KAA6B,EAAE;IAErD,MAAMU,UAAsB,GAAG,CAAC,CAAAV,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEU,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC,CAC3B;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAW,GAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAAS,GAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG1C,YAAY,CAAC+B,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAY,CAAE;IACvF,MAAMrB,MAAM,GAAG,IAAIvB,aAAa,CAAC2C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;;EAMA,MAAMsB,iBAAiB,GAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAe,GAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,EAAE;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,EAAE;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;;EAKA,MAAMmB,UAAU,GAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACrC,IAAI,CAACsC,IAAI,CAAC,CAAC,EAAEnD,aAAa,CAACiC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,EAAE;IAC/B,QAAQD,KAAK;MACX,KAAKpD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIqD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,uCAAgCF,KAAK,OAAI;IAAC;EAE/D;;EAGA,MAAMH,UAAU,GAA0B;IACxC,MAAMM,UAAU,GAAGvD,aAAa,CAACiC,MAAM,GAAG,CAAC;IAC3C,MAAMuB,UAAU,GAAG,MAAM,IAAI,CAAC3C,IAAI,CAACsC,IAAI,CAAC,IAAI,CAACtC,IAAI,CAAC4C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMH,KAAK,GAAGI,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACL,QAAQ,EAAE;IAC5C,IAAID,KAAK,KAAKpD,aAAa,EAAE;MAC3B,MAAM,IAAIsD,KAAK,6CAAqCF,KAAK,OAAI;IAC/D;IAEA,MAAMO,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAChD,IAAI,CAAC4C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG7D,aAAa,CAACiC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,iCAA0BO,cAAc,EAAG;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACjD,IAAI,CAACsC,IAAI,CAACU,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAAC9B;IAAQ,CAAC,GAAGzB,kBAAkB,CAAC0D,WAAW,CAAC;IAClD,OAAOjC,QAAQ;EACjB;;EAGA,MAAMM,YAAY,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACE;IACxB,MAAM0B,MAAqB,GAAG;MAC5Ba,QAAQ,EAAE1B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC0B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI3C,QAAQ,CAAC4C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI9C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI3B,YAAY,CAACkB,UAAU,EAAE6C,MAAM,CAAE,GAAG,CAAC,EAAE;QAClE;MACF;;MACAnB,MAAM,CAACc,UAAU,CAACK,MAAM,CAAEE,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACpD,MAAM,EAAE6C,QAAQ,CAAC;IAClF;IACA,OAAOf,MAAM;EACf;;EAKA,MAAMsB,eAAe,CAACpD,MAAqB,EAAE6C,QAAqB,EAAwB;IAAA;IACxF,IAAIA,QAAQ,CAACQ,SAAS,KAAKC,SAAS,IAAIT,QAAQ,CAACQ,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAInB,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAMqB,KAAK,GAAGvD,MAAM,CAACwD,SAAS,wBAACX,QAAQ,CAACG,SAAS,wDAAlB,oBAAoBE,cAAc,CAAE;IACnE,MAAMO,IAAmB,GAAGxE,aAAa,CAACF,IAAI,0BAAE8D,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBS,IAAI,CAAS;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIxB,KAAK,2CAAoCuB,IAAI,EAAG;IAC5D;IAEA,MAAME,WAA+B,GAAG1E,aAAa,CACnDH,gBAAgB,0BAChB+D,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBY,KAAK,CACnB;IAER,MAAMC,WAAW,GAAG5C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBc,gBAAgB,CAAE;IACjE,IAAIC,SAAS,GAAG9C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAE;IAElE,IAAI,CAACnB,QAAQ,CAACQ,SAAS,EAAE;MAAA;MACvBU,SAAS,GAAGE,IAAI,CAACC,GAAG,CAClB,IAAI,CAACzE,IAAI,CAAC4C,IAAI,GAAGwB,WAAW,EAC5B5C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAC,CAClD;IACH;IAEA,MAAMG,OAAuB,GAAG;MAC9BV,IAAI;MACJW,SAAS,EAAEb,KAAK,CAACa,SAAS;MAC1BC,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BV,WAAW;MACXW,MAAM,EAAEf,KAAK;MACbgB,SAAS,0BAAE1B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBwB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAG7B,QAAQ,aAARA,QAAQ,+CAARA,QAAQ,CAAEG,SAAS,yDAAnB,qBAAqB2B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAG3D,MAAM,CAACyD,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEN,WAAW,CAAC;IAC/E;IAEAY,UAAU,GAAG,uBAAAN,OAAO,CAACM,UAAU,gDAAlB,oBAAoB5D,MAAM,GAAGsD,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAACrF,IAAI,CAACsC,IAAI,CAAC8B,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAM5E,eAAe,CAAC2F,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;;EASA,MAAMI,aAAa,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBN,WAAmB,EACA;IACnB,IAAIa,oBAAoB,KAAK,CAAC,EAAE;;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACzE,IAAI,CAAC4C,IAAI,GAAGqC,oBAAoB,EACrC,IAAI,CAAChF,KAAK,CAACJ,qBAAqB,CACjC;IACD,MAAMwF,QAAQ,GAAG,MAAM,IAAI,CAACrF,IAAI,CAACsC,IAAI,CAAC2C,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAAClD,MAAM,EAAEgD,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAE5C,IAAI,EAAEyC,QAAQ,CAACjE;IAAM,CAAC;IACnE,MAAMqE,WAAW,GAAG,MAAM9F,UAAU,CAAC4F,MAAM,EAAEb,OAAO,CAAC;IAErD,OAAOe,WAAW,CAACT,UAAU;EAC/B;AACF"}
+{"version":3,"file":"parquet-reader.js","names":["ParquetEnvelopeReader","ParquetSchema","ParquetCursor","PARQUET_VERSION","decodeSchema","Symbol","asyncIterator","ParquetReader","openBlob","blob","readFn","start","length","arrayBuffer","slice","Buffer","from","closeFn","size","envelopeReader","readHeader","metadata","readFooter","err","close","openArrayBuffer","byteLength","openBuffer","buffer","constructor","version","Error","root","schema","num_children","getCursor","columnList","map","x","Array","isArray","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","value"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"mappings":";;AACA,SAAQA,qBAAqB,QAAO,2BAA2B;AAE/D,SAAQC,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,eAAe,QAAO,iBAAiB;AAC/C,SAAQC,YAAY,QAAO,YAAY;;AAAC,wBAyJrCC,MAAM,CAACC,aAAa;AAhJvB,OAAO,MAAMC,aAAa,CAAgC;EAIxD,aAAaC,QAAQ,CAAIC,IAAU,EAA6B;IAC9D,MAAMC,MAAM,GAAG,OAAOC,KAAa,EAAEC,MAAc,KAAK;MACtD,MAAMC,WAAW,GAAG,MAAMJ,IAAI,CAACK,KAAK,CAACH,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC,CAACC,WAAW,EAAE;MACzE,OAAOE,MAAM,CAACC,IAAI,CAACH,WAAW,CAAC;IACjC,CAAC;IACD,MAAMI,OAAO,GAAG,YAAY,CAAC,CAAC;IAC9B,MAAMC,IAAI,GAAGT,IAAI,CAACS,IAAI;IACtB,MAAMC,cAAc,GAAG,IAAInB,qBAAqB,CAACU,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;IACvE,IAAI;MACF,MAAMC,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAACc,QAAQ,EAAEF,cAAc,CAAC;IACpD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;;EAKA,aAAaE,eAAe,CAAIZ,WAAwB,EAA6B;IACnF,MAAMH,MAAM,GAAG,OAAOC,KAAa,EAAEC,MAAc,KAAKG,MAAM,CAACC,IAAI,CAACH,WAAW,EAAEF,KAAK,EAAEC,MAAM,CAAC;IAC/F,MAAMK,OAAO,GAAG,YAAY,CAAC,CAAC;IAC9B,MAAMC,IAAI,GAAGL,WAAW,CAACa,UAAU;IACnC,MAAMP,cAAc,GAAG,IAAInB,qBAAqB,CAACU,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;IACvE,IAAI;MACF,MAAMC,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAACc,QAAQ,EAAEF,cAAc,CAAC;IACpD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;EAEA,aAAaI,UAAU,CAAIC,MAAc,EAA6B;IACpE,MAAMT,cAAc,GAAG,MAAMnB,qBAAqB,CAAC2B,UAAU,CAACC,MAAM,CAAC;IACrE,IAAI;MACF,MAAMT,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAAIc,QAAQ,EAAEF,cAAc,CAAC;IACvD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;EAYAM,WAAW,CAACR,QAAsB,EAAEF,cAAqC,EAAE;IAAA;IAAA;IAAA;IACzE,IAAIE,QAAQ,CAACS,OAAO,KAAK3B,eAAe,EAAE;MACxC,MAAM,IAAI4B,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IAEA,IAAI,CAACV,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACF,cAAc,GAAGA,cAAc;IACpC,MAAMa,IAAI,GAAG,IAAI,CAACX,QAAQ,CAACY,MAAM,CAAC,CAAC,CAAC;IACpC,MAAM;MAACA;IAAM,CAAC,GAAG7B,YAAY,CAAC,IAAI,CAACiB,QAAQ,CAACY,MAAM,EAAE,CAAC,EAAED,IAAI,CAACE,YAAY,CAAE;IAC1E,IAAI,CAACD,MAAM,GAAG,IAAIhC,aAAa,CAACgC,MAAM,CAAC;EACzC;;EAMA,MAAMT,KAAK,GAAkB;IAC3B,MAAM,IAAI,CAACL,cAAc,CAACK,KAAK,EAAE;EAGnC;;EAeAW,SAAS,CAACC,UAAkC,EAA6B;IACvE,IAAI,CAACA,UAAU,EAAE;MAEfA,UAAU,GAAG,EAAE;IACjB;;IAGAA,UAAU,GAAGA,UAAU,CAACC,GAAG,CAAEC,CAAC,IAAMC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAE,CAAC;IAEhE,OAAO,IAAIpC,aAAa,CACtB,IAAI,CAACmB,QAAQ,EACb,IAAI,CAACF,cAAc,EACnB,IAAI,CAACc,MAAM,EACXG,UAAU,CACX;EACH;;EAMAK,WAAW,GAAW;IACpB,OAAOC,MAAM,CAAC,IAAI,CAACrB,QAAQ,CAACsB,QAAQ,CAAC;EACvC;;EAKAC,SAAS,GAAkB;IACzB,OAAO,IAAI,CAACX,MAAM;EACpB;;EAKAY,WAAW,GAA2B;IACpC,MAAMC,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAI,IAAI,CAAC1B,QAAQ,CAAC2B,kBAAkB,EAAG;MAClDF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;;EAMA,0BAA2C;IACzC,OAAO,IAAI,CAACX,SAAS,EAAE,CAAC9B,MAAM,CAACC,aAAa,CAAC,EAAE;EACjD;AACF"}

package/dist/esm/parquetjs/schema/declare.js

@@ -1,7 +1,6 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 
 export class ParquetBuffer {
-
   constructor() {
     let rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
     let columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};

package/dist/esm/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetBuffer {\n  /** Number of rows in this page */\n  rowCount: number;\n\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;AAgIA,OAAO,MAAMA,aAAa,CAAC;;EAKzBC,WAAW,GAAqE;IAAA,IAApEC,QAAgB,uEAAG,CAAC;IAAA,IAAEC,UAAuC,uEAAG,CAAC,CAAC;IAAA;IAAA;IAC5E,IAAI,CAACD,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACC,UAAU,GAAGA,UAAU;EAC9B;AACF"}
+{"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  dictionary?: ParquetDictionary;\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"mappings":";;AAmIA,OAAO,MAAMA,aAAa,CAAC;EAGzBC,WAAW,GAAqE;IAAA,IAApEC,QAAgB,uEAAG,CAAC;IAAA,IAAEC,UAAuC,uEAAG,CAAC,CAAC;IAAA;IAAA;IAC5E,IAAI,CAACD,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACC,UAAU,GAAGA,UAAU;EAC9B;AACF"}

package/dist/esm/parquetjs/schema/shred.js

@@ -84,70 +84,62 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
 
 export function materializeRecords(schema, buffer) {
   const records = [];
-  for (let i = 0; i < buffer.rowCount; i++) {
-    records.push({});
-  }
+  for (let i = 0; i < buffer.rowCount; i++) records.push({});
   for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      materializeColumn(schema, columnData, key, records);
-    }
+    materializeColumn(schema, buffer, key, records);
   }
   return records;
 }
 
-function materializeColumn(schema, columnData, key, records) {
+function materializeColumn(schema, buffer, key, records) {
+  const data = buffer.columnData[key];
+  if (!data.count) return;
   const field = schema.findField(key);
   const branch = schema.findFieldBranch(key);
 
   const rLevels = new Array(field.rLevelMax + 1).fill(0);
   let vIndex = 0;
-  for (let i = 0; i < columnData.count; i++) {
-    const dLevel = columnData.dlevels[i];
-    const rLevel = columnData.rlevels[i];
+  for (let i = 0; i < data.count; i++) {
+    const dLevel = data.dlevels[i];
+    const rLevel = data.rlevels[i];
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
     let rIndex = 0;
     let record = records[rLevels[rIndex++] - 1];
 
     for (const step of branch) {
-      if (step === field || dLevel < step.dLevelMax) {
-        break;
-      }
-      switch (step.repetitionType) {
-        case 'REPEATED':
-          if (!(step.name in record)) {
-            record[step.name] = [];
-          }
-          const ix = rLevels[rIndex++];
-          while (record[step.name].length <= ix) {
-            record[step.name].push({});
-          }
-          record = record[step.name][ix];
-          break;
-        default:
-          record[step.name] = record[step.name] || {};
-          record = record[step.name];
+      if (step === field) break;
+      if (dLevel < step.dLevelMax) break;
+      if (step.repetitionType === 'REPEATED') {
+        if (!(step.name in record)) {
+          record[step.name] = [];
+        }
+        const ix = rLevels[rIndex++];
+        while (record[step.name].length <= ix) {
+          record[step.name].push({});
+        }
+        record = record[step.name][ix];
+      } else {
+        record[step.name] = record[step.name] || {};
+        record = record[step.name];
       }
     }
 
     if (dLevel === field.dLevelMax) {
       const value = Types.fromPrimitive(
-      field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      field.originalType || field.primitiveType, data.values[vIndex], field);
       vIndex++;
-      switch (field.repetitionType) {
-        case 'REPEATED':
-          if (!(field.name in record)) {
-            record[field.name] = [];
-          }
-          const ix = rLevels[rIndex];
-          while (record[field.name].length <= ix) {
-            record[field.name].push(null);
-          }
-          record[field.name][ix] = value;
-          break;
-        default:
-          record[field.name] = value;
+      if (field.repetitionType === 'REPEATED') {
+        if (!(field.name in record)) {
+          record[field.name] = [];
+        }
+        const ix = rLevels[rIndex];
+        while (record[field.name].length <= ix) {
+          record[field.name].push(null);
+        }
+        record[field.name][ix] = value;
+      } else {
+        record[field.name] = value;
       }
     }
   }

package/dist/esm/parquetjs/schema/shred.js.map

@@ -1 +1 @@
-{"version":3,"file":"shred.js","names":["ParquetBuffer","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) {\n    records.push({});\n  }\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      materializeColumn(schema, columnData, key, records);\n    }\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  records: ParquetRecord[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n *\nexport function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {\n  const columns: ParquetRecord = {};\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      extractColumn(schema, columnData, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction extractColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  columns: Record<string, unknown> \n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const record = columns;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n\n  let i = 0;\n  const dLevel = columnData.dlevels[i];\n  const rLevel = columnData.rlevels[i];\n  rLevels[rLevel]++;\n  rLevels.fill(0, rLevel + 1);\n\n  let rIndex = 0;\n  let record = records[rLevels[rIndex++] - 1];\n\n  // Internal nodes\n  for (const step of branch) {\n    if (step === field || dLevel < step.dLevelMax) {\n      break;\n    }\n\n    switch (step.repetitionType) {\n      case 'REPEATED':\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n        break;\n\n      default:\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n    }\n  }\n\n  // Leaf node\n  if (dLevel === field.dLevelMax) {\n    const value = Types.fromPrimitive(\n      // @ts-ignore\n      field.originalType || field.primitiveType,\n      columnData.values[vIndex],\n      field\n    );\n    vIndex++;\n\n    switch (field.repetitionType) {\n      case 'REPEATED':\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n        break;\n\n      default:\n        record[field.name] = value;\n    }\n  }\n}\n*/\n"],"mappings":";;AAEA,SAAQA,aAAa,QAAiD,WAAW;AAEjF,OAAO,KAAKC,KAAK,MAAM,SAAS;AAEhC,SAAQD,aAAa;AAErB,OAAO,SAASE,WAAW,CAACC,MAAqB,EAAiB;EAChE,MAAMC,UAAuC,GAAG,CAAC,CAAC;EAClD,KAAK,MAAMC,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCF,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,GAAG;MACtBC,OAAO,EAAE,EAAE;MACXC,OAAO,EAAE,EAAE;MACXC,MAAM,EAAE,EAAE;MACVC,WAAW,EAAE,EAAE;MACfC,KAAK,EAAE;IACT,CAAC;EACH;EACA,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET;EAAU,CAAC;AAClC;;AAwBA,OAAO,SAASU,WAAW,CAACX,MAAqB,EAAEY,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACZ,UAAU,GAAGa,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EACpB,KAAK,MAAMR,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAAC;IACzFW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAAC;IACzFY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAAC;IACvFM,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK;EAC7D;AACF;;AAGA,SAASM,iBAAiB,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,MAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,MAAMd,KAAK,GAAGc,MAAM,CAACO,IAAI,CAAC;;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNV,KAAK,CAACqB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIV,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,mCAA4B3B,KAAK,CAACqB,IAAI,EAAG;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIxB,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,sCAA+B3B,KAAK,CAACqB,IAAI,EAAG;IAC7D;;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIxB,KAAK,CAAC4B,QAAQ,EAAE;QAClBf,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;;IAGA,KAAK,IAAIS,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGxB,MAAM,CAACmB,MAAM,EAAEK,CAAC,EAAE,EAAE;MACtC,MAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGV,MAAM,GAAGnB,KAAK,CAAC+B,SAAS;MAC/C,IAAI/B,KAAK,CAAC4B,QAAQ,EAAE;QAClBf,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAGT,MAAM,CAACwB,CAAC,CAAC,EAAEjB,IAAI,EAAEkB,IAAI,EAAE9B,KAAK,CAACgC,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLpB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACa,IAAI,CAAC;QAClClB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACjB,KAAK,CAACgC,SAAS,CAAC;QAC7CpB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBrB,KAAK,CAACqC,WAAW,CAAEjC,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EAAI9B,MAAM,CAACwB,CAAC,CAAC,CAAC,CAC3E;MACH;IACF;EACF;AACF;;AAqBA,OAAO,SAASO,kBAAkB,CAACtC,MAAqB,EAAEa,MAAqB,EAAmB;EAChG,MAAM0B,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlB,MAAM,CAACH,QAAQ,EAAEqB,CAAC,EAAE,EAAE;IACxCQ,OAAO,CAACpB,IAAI,CAAC,CAAC,CAAC,CAAC;EAClB;EACA,KAAK,MAAMf,GAAG,IAAIS,MAAM,CAACZ,UAAU,EAAE;IACnC,MAAMA,UAAU,GAAGY,MAAM,CAACZ,UAAU,CAACG,GAAG,CAAC;IACzC,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpB+B,iBAAiB,CAACxC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEmC,OAAO,CAAC;IACrD;EACF;EACA,OAAOA,OAAO;AAChB;;AAGA,SAASC,iBAAiB,CACxBxC,MAAqB,EACrBC,UAAuB,EACvBG,GAAW,EACXmC,OAAwB,EAClB;EACN,MAAMrC,KAAK,GAAGF,MAAM,CAACyC,SAAS,CAACrC,GAAG,CAAC;EACnC,MAAMsC,MAAM,GAAG1C,MAAM,CAAC2C,eAAe,CAACvC,GAAG,CAAC;;EAG1C,MAAMwC,OAAiB,GAAG,IAAI3B,KAAK,CAACf,KAAK,CAAC+B,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG9B,UAAU,CAACQ,KAAK,EAAEsB,CAAC,EAAE,EAAE;IACzC,MAAMT,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC0B,CAAC,CAAC;IACpC,MAAMV,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAACyB,CAAC,CAAC;IACpCa,OAAO,CAACvB,MAAM,CAAC,EAAE;IACjBuB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAExB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI0B,MAAM,GAAG,CAAC;IACd,IAAInC,MAAM,GAAG2B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;;IAG3C,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK9C,KAAK,IAAIoB,MAAM,GAAG0B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACpB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEoB,IAAI,CAACzB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM0B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOnC,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,CAACG,MAAM,IAAIuB,EAAE,EAAE;YAErCrC,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,CAAC0B,EAAE,CAAC;UAC9B;QAEF;UACErC,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,GAAGX,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC;MAAC;IAEjC;;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACgC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGpD,KAAK,CAACqD,aAAa;MAE/BjD,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EACzCpC,UAAU,CAACM,MAAM,CAACuC,MAAM,CAAC,EACzB5C,KAAK,CACN;MACD4C,MAAM,EAAE;MAER,QAAQ5C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM0B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOnC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIuB,EAAE,EAAE;YAEtCrC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC0B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG2B,KAAK;MAAC;IAEjC;EACF;AACF"}
+{"version":3,"file":"shred.js","names":["ParquetBuffer","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) records.push({});\n  for (const key in buffer.columnData) {\n    materializeColumn(schema, buffer, key, records);\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  buffer: ParquetBuffer,\n  key: string,\n  records: ParquetRecord[]\n) {\n  const data = buffer.columnData[key];\n  if (!data.count) return;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < data.count; i++) {\n    const dLevel = data.dlevels[i];\n    const rLevel = data.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes\n    for (const step of branch) {\n      if (step === field) break;\n      if (dLevel < step.dLevelMax) break;\n      if (step.repetitionType === 'REPEATED') {\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n      } else {\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n      }\n    }\n\n    // Leaf node\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        data.values[vIndex],\n        field\n      );\n      vIndex++;\n      if (field.repetitionType === 'REPEATED') {\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n      } else {\n        record[field.name] = value;\n      }\n    }\n  }\n}\n"],"mappings":";;AAEA,SAAQA,aAAa,QAAiD,WAAW;AAEjF,OAAO,KAAKC,KAAK,MAAM,SAAS;AAEhC,SAAQD,aAAa;AAErB,OAAO,SAASE,WAAW,CAACC,MAAqB,EAAiB;EAChE,MAAMC,UAAuC,GAAG,CAAC,CAAC;EAClD,KAAK,MAAMC,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCF,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,GAAG;MACtBC,OAAO,EAAE,EAAE;MACXC,OAAO,EAAE,EAAE;MACXC,MAAM,EAAE,EAAE;MACVC,WAAW,EAAE,EAAE;MACfC,KAAK,EAAE;IACT,CAAC;EACH;EACA,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET;EAAU,CAAC;AAClC;;AAwBA,OAAO,SAASU,WAAW,CAACX,MAAqB,EAAEY,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACZ,UAAU,GAAGa,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EACpB,KAAK,MAAMR,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAAC;IACzFW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAAC;IACzFY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAAC;IACvFM,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK;EAC7D;AACF;;AAGA,SAASM,iBAAiB,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,MAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,MAAMd,KAAK,GAAGc,MAAM,CAACO,IAAI,CAAC;;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNV,KAAK,CAACqB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIV,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,mCAA4B3B,KAAK,CAACqB,IAAI,EAAG;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIxB,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,sCAA+B3B,KAAK,CAACqB,IAAI,EAAG;IAC7D;;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIxB,KAAK,CAAC4B,QAAQ,EAAE;QAClBf,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;;IAGA,KAAK,IAAIS,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGxB,MAAM,CAACmB,MAAM,EAAEK,CAAC,EAAE,EAAE;MACtC,MAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGV,MAAM,GAAGnB,KAAK,CAAC+B,SAAS;MAC/C,IAAI/B,KAAK,CAAC4B,QAAQ,EAAE;QAClBf,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAGT,MAAM,CAACwB,CAAC,CAAC,EAAEjB,IAAI,EAAEkB,IAAI,EAAE9B,KAAK,CAACgC,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLpB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACa,IAAI,CAAC;QAClClB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACjB,KAAK,CAACgC,SAAS,CAAC;QAC7CpB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBrB,KAAK,CAACqC,WAAW,CAAEjC,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EAAI9B,MAAM,CAACwB,CAAC,CAAC,CAAC,CAC3E;MACH;IACF;EACF;AACF;;AAqBA,OAAO,SAASO,kBAAkB,CAACtC,MAAqB,EAAEa,MAAqB,EAAmB;EAChG,MAAM0B,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlB,MAAM,CAACH,QAAQ,EAAEqB,CAAC,EAAE,EAAEQ,OAAO,CAACpB,IAAI,CAAC,CAAC,CAAC,CAAC;EAC1D,KAAK,MAAMf,GAAG,IAAIS,MAAM,CAACZ,UAAU,EAAE;IACnCuC,iBAAiB,CAACxC,MAAM,EAAEa,MAAM,EAAET,GAAG,EAAEmC,OAAO,CAAC;EACjD;EACA,OAAOA,OAAO;AAChB;;AAGA,SAASC,iBAAiB,CACxBxC,MAAqB,EACrBa,MAAqB,EACrBT,GAAW,EACXmC,OAAwB,EACxB;EACA,MAAMzB,IAAI,GAAGD,MAAM,CAACZ,UAAU,CAACG,GAAG,CAAC;EACnC,IAAI,CAACU,IAAI,CAACL,KAAK,EAAE;EAEjB,MAAMP,KAAK,GAAGF,MAAM,CAACyC,SAAS,CAACrC,GAAG,CAAC;EACnC,MAAMsC,MAAM,GAAG1C,MAAM,CAAC2C,eAAe,CAACvC,GAAG,CAAC;;EAG1C,MAAMwC,OAAiB,GAAG,IAAI3B,KAAK,CAACf,KAAK,CAAC+B,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGjB,IAAI,CAACL,KAAK,EAAEsB,CAAC,EAAE,EAAE;IACnC,MAAMT,MAAM,GAAGR,IAAI,CAACT,OAAO,CAAC0B,CAAC,CAAC;IAC9B,MAAMV,MAAM,GAAGP,IAAI,CAACR,OAAO,CAACyB,CAAC,CAAC;IAC9Ba,OAAO,CAACvB,MAAM,CAAC,EAAE;IACjBuB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAExB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI0B,MAAM,GAAG,CAAC;IACd,IAAInC,MAAM,GAAG2B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;;IAG3C,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK9C,KAAK,EAAE;MACpB,IAAIoB,MAAM,GAAG0B,IAAI,CAACd,SAAS,EAAE;MAC7B,IAAIc,IAAI,CAACpB,cAAc,KAAK,UAAU,EAAE;QACtC,IAAI,EAAEoB,IAAI,CAACzB,IAAI,IAAIX,MAAM,CAAC,EAAE;UAE1BA,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,GAAG,EAAE;QACxB;QACA,MAAM0B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;QAC5B,OAAOnC,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,CAACG,MAAM,IAAIuB,EAAE,EAAE;UAErCrC,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B;QACAP,MAAM,GAAGA,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,CAAC0B,EAAE,CAAC;MAChC,CAAC,MAAM;QACLrC,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,GAAGX,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3CX,MAAM,GAAGA,MAAM,CAACoC,IAAI,CAACzB,IAAI,CAAC;MAC5B;IACF;;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACgC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGpD,KAAK,CAACqD,aAAa;MAE/BjD,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EACzCvB,IAAI,CAACP,MAAM,CAACuC,MAAM,CAAC,EACnB5C,KAAK,CACN;MACD4C,MAAM,EAAE;MACR,IAAI5C,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;QACvC,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;UAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;QACzB;QACA,MAAM0B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;QAC1B,OAAOnC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIuB,EAAE,EAAE;UAEtCrC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;QAC/B;QACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC0B,EAAE,CAAC,GAAGC,KAAK;MAChC,CAAC,MAAM;QACLtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG2B,KAAK;MAC5B;IACF;EACF;AACF"}