@loaders.gl/parquet 3.4.13 → 3.4.15

package/dist/es5/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.js","names":["_schema","require","_decoders","_shred","_constants","_parquetThrift","_readUtils","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","ownKeys","object","enumerableOnly","keys","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","arguments","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","_asyncIterator","iterable","method","async","sync","retry","asyncIterator","AsyncFromSyncIterator","AsyncFromSyncIteratorContinuation","r","Promise","reject","resolve","then","_return","ret","throw","_throw","thr","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","file","props","_classCallCheck2","_createClass2","close","rowIterator","_this","_wrapAsyncGenerator2","_regenerator","mark","_callee","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","rows","_iterator3","_step3","row","wrap","_callee$","_context","prev","rowBatchIterator","_awaitAsyncGenerator2","sent","t0","finish","t1","stop","_this2","_callee2","schema","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","rowGroup","_callee2$","_context2","getSchema","rowGroupIterator","materializeRecords","_this3","_callee3","columnList","metadata","rowGroupCount","rowGroupIndex","_callee3$","_context3","map","x","getFileMetadata","row_groups","readRowGroup","_getRowCount","_asyncToGenerator2","_callee4","_callee4$","_context4","abrupt","Number","num_rows","getRowCount","_getSchema","_callee5","root","_decodeSchema","schemaDefinition","_callee5$","_context5","decodeSchema","num_children","ParquetSchema","_getSchemaMetadata","_callee6","md","_iterator4","_step4","kv","_callee6$","_context6","key_value_metadata","getSchemaMetadata","_getFileMetadata","_callee7","_callee7$","_context7","readHeader","readFooter","_readHeader","_callee8","buffer","magic","_callee8$","_context8","read","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","Error","concat","_readFooter","_callee9","trailerLen","trailerBuf","metadataSize","metadataOffset","metadataBuf","_decodeFileMetadata","_callee9$","_context9","size","readUInt32LE","decodeFileMetadata","_readRowGroup","_callee10","_iterator5","_step5","colChunk","colMetadata","colKey","_callee10$","_context10","rowCount","columnData","columns","meta_data","path_in_schema","fieldIndexOf","readColumnChunk","join","_x","_x2","_x3","_readColumnChunk","_callee11","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","field","type","compression","pagesOffset","pagesSize","_colChunk$meta_data6","options","dictionary","dictionaryPageOffset","dictionaryOffset","pagesBuf","_callee11$","_context11","file_path","undefined","findField","getThriftEnum","Type","primitiveType","CompressionCodec","codec","data_page_offset","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary_page_offset","getDictionary","decodeDataPages","_x4","_x5","_getDictionary","_callee12","dictionarySize","cursor","decodedPage","_callee12$","_context12","offset","decodePage","_x6","_x7","_x8","exports"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRecords(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAGA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AACA,IAAAE,MAAA,GAAAF,OAAA;AAEA,IAAAG,UAAA,GAAAH,OAAA;AACA,IAAAI,cAAA,GAAAJ,OAAA;AAQA,IAAAK,UAAA,GAAAL,OAAA;AAAoF,SAAAM,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAd,MAAA,CAAAc,IAAA,CAAAF,MAAA,OAAAZ,MAAA,CAAAe,qBAAA,QAAAC,OAAA,GAAAhB,MAAA,CAAAe,qBAAA,CAAAH,MAAA,GAAAC,cAAA,KAAAG,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAlB,MAAA,CAAAmB,wBAAA,CAAAP,MAAA,EAAAM,GAAA,EAAAE,UAAA,OAAAN,IAAA,CAAAO,IAAA,CAAAC,KAAA,CAAAR,IAAA,EAAAE,OAAA,YAAAF,IAAA;AAAA,SAAAS,cAAAC,MAAA,aAAA5C,CAAA,MAAAA,CAAA,GAAA6C,SAAA,CAAA9C,MAAA,EAAAC,CAAA,UAAA8C,MAAA,WAAAD,SAAA,CAAA7C,CAAA,IAAA6C,SAAA,CAAA7C,CAAA,QAAAA,CAAA,OAAA+B,OAAA,CAAAX,MAAA,CAAA0B,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAN,MAAA,EAAAI,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAA5B,MAAA,CAAA+B,yBAAA,GAAA/B,MAAA,CAAAgC,gBAAA,CAAAR,MAAA,EAAAxB,MAAA,CAAA+B,yBAAA,CAAAL,MAAA,KAAAf,OAAA,CAAAX,MAAA,CAAA0B,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAA5B,MAAA,CAAAiC,cAAA,CAAAT,MAAA,EAAAI,GAAA,EAAA5B,MAAA,CAAAmB,wBAAA,CAAAO,MAAA,EAAAE,GAAA,iBAAAJ,MAAA;AAAA,SAAAU,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAjE,MAAA,KAAA+D,KAAA,GAAA/D,MAAA,CAAAkE,aAAA,EAAAF,IAAA,GAAAhE,MAAA,CAAAC,QAAA,GAAAgE,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAG,qBAAA,CAAAL,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAjD,SAAA;AAAA,SAAAoD,sBAAA3D,CAAA,aAAA4D,kCAAAC,CAAA,QAAA3C,MAAA,CAAA2C,CAAA,MAAAA,CAAA,SAAAC,OAAA,CAAAC,MAAA,KAAAxD,SAAA,CAAAsD,CAAA,+BAAA3D,IAAA,GAAA2D,CAAA,CAAA3D,IAAA,SAAA4D,OAAA,CAAAE,OAAA,CAAAH,CAAA,CAAA1D,KAAA,EAAA8D,IAAA,WAAA9D,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAD,IAAA,EAAAA,IAAA,iBAAAyD,qBAAA,YAAAA,sBAAA3D,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAC,CAAA,GAAAD,CAAA,CAAAa,IAAA,KAAA8C,qBAAA,CAAAxC,SAAA,KAAAnB,CAAA,QAAAC,CAAA,QAAAY,IAAA,WAAAA,KAAA,WAAA+C,iCAAA,MAAA3D,CAAA,CAAAuC,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAA5B,MAAA,WAAAmD,QAAA/D,KAAA,QAAAgE,GAAA,QAAAnE,CAAA,CAAAe,MAAA,oBAAAoD,GAAA,GAAAL,OAAA,CAAAE,OAAA,GAAA7D,KAAA,EAAAA,KAAA,EAAAD,IAAA,UAAA0D,iCAAA,CAAAO,GAAA,CAAA3B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAAyB,KAAA,WAAAC,OAAAlE,KAAA,QAAAmE,GAAA,QAAAtE,CAAA,CAAAe,MAAA,oBAAAuD,GAAA,GAAAR,OAAA,CAAAC,MAAA,CAAA5D,KAAA,IAAAyD,iCAAA,CAAAU,GAAA,CAAA9B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,aAAAgB,qBAAA,CAAA3D,CAAA;AAapF,IAAMuE,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAAC,IAQWC,aAAa;EAKxB,SAAAA,cAAYC,IAAkB,EAAEC,KAA0B,EAAE;IAAA,IAAAC,gBAAA,CAAA5B,OAAA,QAAAyB,aAAA;IAAA,IAAA1B,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA,oBAFnB,IAAI;IAG3C,IAAI,CAAC0B,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAAlC,aAAA,CAAAA,aAAA,KAAO8B,aAAa,GAAKI,KAAK,CAAC;EAC3C;EAAC,IAAAE,aAAA,CAAA7B,OAAA,EAAAyB,aAAA;IAAA3B,GAAA;IAAA3C,KAAA,EAED,SAAA2E,MAAA,EAAc;MAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;IACnB;EAAC;IAAAhC,GAAA;IAAA3C,KAAA,EAKD,SAAA4E,YAAmBJ,KAA6B,EAAE;MAAA,IAAAK,KAAA;MAAA,WAAAC,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAC,QAAA;QAAA,IAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,GAAA;QAAA,OAAAX,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAC,SAAAC,QAAA;UAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAnF,IAAA;YAAA;cAAAwE,yBAAA;cAAAC,iBAAA;cAAAU,QAAA,CAAAC,IAAA;cAAAT,SAAA,GAAApC,cAAA,CACvB4B,KAAI,CAACkB,gBAAgB,CAACvB,KAAK,CAAC;YAAA;cAAAqB,QAAA,CAAAnF,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAAwC,SAAA,CAAA3E,IAAA;YAAA;cAAA,MAAAwE,yBAAA,KAAAI,KAAA,GAAAO,QAAA,CAAAI,IAAA,EAAAlG,IAAA;gBAAA8F,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAApC6E,IAAI,GAAAD,KAAA,CAAAtF,KAAA;cAAAwF,UAAA,GAAAvG,0BAAA,CAEDsG,IAAI;cAAAM,QAAA,CAAAC,IAAA;cAAAN,UAAA,CAAA3F,CAAA;YAAA;cAAA,KAAA4F,MAAA,GAAAD,UAAA,CAAA1F,CAAA,IAAAC,IAAA;gBAAA8F,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAXgF,GAAG,GAAAD,MAAA,CAAAzF,KAAA;cAAA6F,QAAA,CAAAnF,IAAA;cACZ,OAAMgF,GAAG;YAAA;cAAAG,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAK,EAAA,GAAAL,QAAA;cAAAL,UAAA,CAAAvF,CAAA,CAAA4F,QAAA,CAAAK,EAAA;YAAA;cAAAL,QAAA,CAAAC,IAAA;cAAAN,UAAA,CAAArF,CAAA;cAAA,OAAA0F,QAAA,CAAAM,MAAA;YAAA;cAAAjB,yBAAA;cAAAW,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAO,EAAA,GAAAP,QAAA;cAAAV,iBAAA;cAAAC,cAAA,GAAAS,QAAA,CAAAO,EAAA;YAAA;cAAAP,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAC,IAAA;cAAA,MAAAZ,yBAAA,IAAAG,SAAA,CAAAzE,MAAA;gBAAAiF,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAAwC,SAAA,CAAAzE,MAAA;YAAA;cAAAiF,QAAA,CAAAC,IAAA;cAAA,KAAAX,iBAAA;gBAAAU,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAA,MAAA0E,cAAA;YAAA;cAAA,OAAAS,QAAA,CAAAM,MAAA;YAAA;cAAA,OAAAN,QAAA,CAAAM,MAAA;YAAA;YAAA;cAAA,OAAAN,QAAA,CAAAQ,IAAA;UAAA;QAAA,GAAApB,OAAA;MAAA;IAGf;EAAC;IAAAtC,GAAA;IAAA3C,KAAA,EAGD,SAAA+F,iBAAwBvB,KAA6B,EAAE;MAAA,IAAA8B,MAAA;MAAA,WAAAxB,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAuB,SAAA;QAAA,IAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,QAAA;QAAA,OAAA/B,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAoB,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAlB,IAAA,GAAAkB,SAAA,CAAAtG,IAAA;YAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAChCyD,MAAI,CAACW,SAAS,CAAC,CAAC;YAAA;cAA/BT,MAAM,GAAAQ,SAAA,CAAAf,IAAA;cAAAQ,0BAAA;cAAAC,kBAAA;cAAAM,SAAA,CAAAlB,IAAA;cAAAc,UAAA,GAAA3D,cAAA,CACiBqD,MAAI,CAACY,gBAAgB,CAAC1C,KAAK,CAAC;YAAA;cAAAwC,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAA+D,UAAA,CAAAlG,IAAA;YAAA;cAAA,MAAA+F,0BAAA,KAAAI,MAAA,GAAAG,SAAA,CAAAf,IAAA,EAAAlG,IAAA;gBAAAiH,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAxCoG,QAAQ,GAAAD,MAAA,CAAA7G,KAAA;cAAAgH,SAAA,CAAAtG,IAAA;cACvB,OAAM,IAAAyG,yBAAkB,EAACX,MAAM,EAAEM,QAAQ,CAAC;YAAA;cAAAL,0BAAA;cAAAO,SAAA,CAAAtG,IAAA;cAAA;YAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA;YAAA;cAAAsG,SAAA,CAAAlB,IAAA;cAAAkB,SAAA,CAAAd,EAAA,GAAAc,SAAA;cAAAN,kBAAA;cAAAC,eAAA,GAAAK,SAAA,CAAAd,EAAA;YAAA;cAAAc,SAAA,CAAAlB,IAAA;cAAAkB,SAAA,CAAAlB,IAAA;cAAA,MAAAW,0BAAA,IAAAG,UAAA,CAAAhG,MAAA;gBAAAoG,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAA+D,UAAA,CAAAhG,MAAA;YAAA;cAAAoG,SAAA,CAAAlB,IAAA;cAAA,KAAAY,kBAAA;gBAAAM,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAA,MAAAiG,eAAA;YAAA;cAAA,OAAAK,SAAA,CAAAb,MAAA;YAAA;cAAA,OAAAa,SAAA,CAAAb,MAAA;YAAA;YAAA;cAAA,OAAAa,SAAA,CAAAX,IAAA;UAAA;QAAA,GAAAE,QAAA;MAAA;IAE9C;EAAC;IAAA5D,GAAA;IAAA3C,KAAA,EAGD,SAAAkH,iBAAwB1C,KAA6B,EAAE;MAAA,IAAA4C,MAAA;MAAA,WAAAtC,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAqC,SAAA;QAAA,IAAAC,UAAA,EAAAC,QAAA,EAAAf,MAAA,EAAAgB,aAAA,EAAAC,aAAA,EAAAX,QAAA;QAAA,OAAA/B,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA+B,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA7B,IAAA,GAAA6B,SAAA,CAAAjH,IAAA;YAAA;cAE/C4G,UAAsB,GAAG,CAAC,CAAA9C,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAE8C,UAAU,KAAI,EAAE,EAAEM,GAAG,CAAC,UAACC,CAAC;gBAAA,OAC7DtI,KAAK,CAACC,OAAO,CAACqI,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;cAAA,CAC5B,CAAC;cAAAF,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAEsBuE,MAAI,CAACU,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAI,SAAA,CAAA1B,IAAA;cAAA0B,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EACOuE,MAAI,CAACH,SAAS,CAAC,CAAC;YAAA;cAA/BT,MAAM,GAAAmB,SAAA,CAAA1B,IAAA;cAENuB,aAAa,GAAG,CAAAD,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEQ,UAAU,CAACrI,MAAM,KAAI,CAAC;cAE7C+H,aAAa,GAAG,CAAC;YAAA;cAAA,MAAEA,aAAa,GAAGD,aAAa;gBAAAG,SAAA,CAAAjH,IAAA;gBAAA;cAAA;cAAAiH,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAChCuE,MAAI,CAACY,YAAY,CACtCxB,MAAM,EACNe,QAAQ,CAACQ,UAAU,CAACN,aAAa,CAAC,EAClCH,UACF,CAAC;YAAA;cAJKR,QAAQ,GAAAa,SAAA,CAAA1B,IAAA;cAAA0B,SAAA,CAAAjH,IAAA;cAKd,OAAMoG,QAAQ;YAAA;cAN2CW,aAAa,EAAE;cAAAE,SAAA,CAAAjH,IAAA;cAAA;YAAA;YAAA;cAAA,OAAAiH,SAAA,CAAAtB,IAAA;UAAA;QAAA,GAAAgB,QAAA;MAAA;IAQ5E;EAAC;IAAA1E,GAAA;IAAA3C,KAAA;MAAA,IAAAiI,YAAA,OAAAC,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAAmD,SAAA;QAAA,IAAAZ,QAAA;QAAA,OAAAxC,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAyC,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAvC,IAAA,GAAAuC,SAAA,CAAA3H,IAAA;YAAA;cAAA2H,SAAA,CAAA3H,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAc,SAAA,CAAApC,IAAA;cAAA,OAAAoC,SAAA,CAAAC,MAAA,WACPC,MAAM,CAAChB,QAAQ,CAACiB,QAAQ,CAAC;YAAA;YAAA;cAAA,OAAAH,SAAA,CAAAhC,IAAA;UAAA;QAAA,GAAA8B,QAAA;MAAA,CACjC;MAAA,SAAAM,YAAA;QAAA,OAAAR,YAAA,CAAA5F,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAiG,WAAA;IAAA;EAAA;IAAA9F,GAAA;IAAA3C,KAAA;MAAA,IAAA0I,UAAA,OAAAR,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAA2D,SAAA;QAAA,IAAApB,QAAA,EAAAqB,IAAA,EAAAC,aAAA,EAAAC,gBAAA,EAAAtC,MAAA;QAAA,OAAAzB,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAoD,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAlD,IAAA,GAAAkD,SAAA,CAAAtI,IAAA;YAAA;cAAAsI,SAAA,CAAAtI,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAyB,SAAA,CAAA/C,IAAA;cACR2C,IAAI,GAAGrB,QAAQ,CAACf,MAAM,CAAC,CAAC,CAAC;cAAAqC,aAAA,GACI,IAAAI,sBAAY,EAAC1B,QAAQ,CAACf,MAAM,EAAE,CAAC,EAAEoC,IAAI,CAACM,YAAa,CAAC,EAAxEJ,gBAAgB,GAAAD,aAAA,CAAxBrC,MAAM;cACPA,MAAM,GAAG,IAAI2C,qBAAa,CAACL,gBAAgB,CAAC;cAAA,OAAAE,SAAA,CAAAV,MAAA,WAC3C9B,MAAM;YAAA;YAAA;cAAA,OAAAwC,SAAA,CAAA3C,IAAA;UAAA;QAAA,GAAAsC,QAAA;MAAA,CACd;MAAA,SAAA1B,UAAA;QAAA,OAAAyB,UAAA,CAAArG,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAyE,SAAA;IAAA;EAAA;IAAAtE,GAAA;IAAA3C,KAAA;MAAA,IAAAoJ,kBAAA,OAAAlB,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAMD,SAAAqE,SAAA;QAAA,IAAA9B,QAAA,EAAA+B,EAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,EAAA;QAAA,OAAA1E,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA+D,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA7D,IAAA,GAAA6D,SAAA,CAAAjJ,IAAA;YAAA;cAAAiJ,SAAA,CAAAjJ,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAoC,SAAA,CAAA1D,IAAA;cACRqD,EAA0B,GAAG,CAAC,CAAC;cAAAC,UAAA,GAAAtK,0BAAA,CACpBsI,QAAQ,CAACqC,kBAAkB;cAAA;gBAA5C,KAAAL,UAAA,CAAA1J,CAAA,MAAA2J,MAAA,GAAAD,UAAA,CAAAzJ,CAAA,IAAAC,IAAA,GAA+C;kBAApC0J,EAAE,GAAAD,MAAA,CAAAxJ,KAAA;kBACXsJ,EAAE,CAACG,EAAE,CAAC9G,GAAG,CAAC,GAAG8G,EAAE,CAACzJ,KAAM;gBACxB;cAAC,SAAAO,GAAA;gBAAAgJ,UAAA,CAAAtJ,CAAA,CAAAM,GAAA;cAAA;gBAAAgJ,UAAA,CAAApJ,CAAA;cAAA;cAAA,OAAAwJ,SAAA,CAAArB,MAAA,WACMgB,EAAE;YAAA;YAAA;cAAA,OAAAK,SAAA,CAAAtD,IAAA;UAAA;QAAA,GAAAgD,QAAA;MAAA,CACV;MAAA,SAAAQ,kBAAA;QAAA,OAAAT,kBAAA,CAAA/G,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAqH,iBAAA;IAAA;EAAA;IAAAlH,GAAA;IAAA3C,KAAA;MAAA,IAAA8J,gBAAA,OAAA5B,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAA+E,SAAA;QAAA,OAAAhF,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAqE,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAnE,IAAA,GAAAmE,SAAA,CAAAvJ,IAAA;YAAA;cAAA,IACO,IAAI,CAAC6G,QAAQ;gBAAA0C,SAAA,CAAAvJ,IAAA;gBAAA;cAAA;cAAAuJ,SAAA,CAAAvJ,IAAA;cAAA,OACV,IAAI,CAACwJ,UAAU,CAAC,CAAC;YAAA;cACvB,IAAI,CAAC3C,QAAQ,GAAG,IAAI,CAAC4C,UAAU,CAAC,CAAC;YAAC;cAAA,OAAAF,SAAA,CAAA3B,MAAA,WAE7B,IAAI,CAACf,QAAQ;YAAA;YAAA;cAAA,OAAA0C,SAAA,CAAA5D,IAAA;UAAA;QAAA,GAAA0D,QAAA;MAAA,CACrB;MAAA,SAAAjC,gBAAA;QAAA,OAAAgC,gBAAA,CAAAzH,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAsF,eAAA;IAAA;EAAA;IAAAnF,GAAA;IAAA3C,KAAA;MAAA,IAAAoK,WAAA,OAAAlC,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAKD,SAAAqF,SAAA;QAAA,IAAAC,MAAA,EAAAC,KAAA;QAAA,OAAAxF,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA6E,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA3E,IAAA,GAAA2E,SAAA,CAAA/J,IAAA;YAAA;cAAA+J,SAAA,CAAA/J,IAAA;cAAA,OACuB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAAC,CAAC,EAAEC,wBAAa,CAACjL,MAAM,CAAC;YAAA;cAAtD4K,MAAM,GAAAG,SAAA,CAAAxE,IAAA;cACNsE,KAAK,GAAGD,MAAM,CAACrJ,QAAQ,CAAC,CAAC;cAAAwJ,SAAA,CAAAvE,EAAA,GACvBqE,KAAK;cAAAE,SAAA,CAAA/J,IAAA,GAAA+J,SAAA,CAAAvE,EAAA,KACNyE,wBAAa,OAAAF,SAAA,CAAAvE,EAAA,KAEb0E,kCAAuB;cAAA;YAAA;cAAA,OAAAH,SAAA,CAAAnC,MAAA;YAAA;cAAA,MACpB,IAAIuC,KAAK,CAAC,sCAAsC,CAAC;YAAA;cAAA,MAEjD,IAAIA,KAAK,gCAAAC,MAAA,CAAgCP,KAAK,MAAG,CAAC;YAAA;YAAA;cAAA,OAAAE,SAAA,CAAApE,IAAA;UAAA;QAAA,GAAAgE,QAAA;MAAA,CAE7D;MAAA,SAAAH,WAAA;QAAA,OAAAE,WAAA,CAAA/H,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAA0H,UAAA;IAAA;EAAA;IAAAvH,GAAA;IAAA3C,KAAA;MAAA,IAAA+K,WAAA,OAAA7C,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAGD,SAAAgG,SAAA;QAAA,IAAAC,UAAA,EAAAC,UAAA,EAAAX,KAAA,EAAAY,YAAA,EAAAC,cAAA,EAAAC,WAAA,EAAAC,mBAAA,EAAA/D,QAAA;QAAA,OAAAxC,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA4F,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA1F,IAAA,GAAA0F,SAAA,CAAA9K,IAAA;YAAA;cACQuK,UAAU,GAAGN,wBAAa,CAACjL,MAAM,GAAG,CAAC;cAAA8L,SAAA,CAAA9K,IAAA;cAAA,OAClB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAAC,IAAI,CAACnG,IAAI,CAACkH,IAAI,GAAGR,UAAU,EAAEA,UAAU,CAAC;YAAA;cAA1EC,UAAU,GAAAM,SAAA,CAAAvF,IAAA;cAEVsE,KAAK,GAAGW,UAAU,CAAChK,KAAK,CAAC,CAAC,CAAC,CAACD,QAAQ,CAAC,CAAC;cAAA,MACxCsJ,KAAK,KAAKI,wBAAa;gBAAAa,SAAA,CAAA9K,IAAA;gBAAA;cAAA;cAAA,MACnB,IAAImK,KAAK,sCAAAC,MAAA,CAAqCP,KAAK,MAAG,CAAC;YAAA;cAGzDY,YAAY,GAAGD,UAAU,CAACQ,YAAY,CAAC,CAAC,CAAC;cACzCN,cAAc,GAAG,IAAI,CAAC7G,IAAI,CAACkH,IAAI,GAAGN,YAAY,GAAGF,UAAU;cAAA,MAC7DG,cAAc,GAAGT,wBAAa,CAACjL,MAAM;gBAAA8L,SAAA,CAAA9K,IAAA;gBAAA;cAAA;cAAA,MACjC,IAAImK,KAAK,0BAAAC,MAAA,CAA0BM,cAAc,CAAE,CAAC;YAAA;cAAAI,SAAA,CAAA9K,IAAA;cAAA,OAGlC,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACU,cAAc,EAAED,YAAY,CAAC;YAAA;cAAhEE,WAAW,GAAAG,SAAA,CAAAvF,IAAA;cAAAqF,mBAAA,GAGE,IAAAK,6BAAkB,EAACN,WAAW,CAAC,EAA3C9D,QAAQ,GAAA+D,mBAAA,CAAR/D,QAAQ;cAAA,OAAAiE,SAAA,CAAAlD,MAAA,WACRf,QAAQ;YAAA;YAAA;cAAA,OAAAiE,SAAA,CAAAnF,IAAA;UAAA;QAAA,GAAA2E,QAAA;MAAA,CAChB;MAAA,SAAAb,WAAA;QAAA,OAAAY,WAAA,CAAA1I,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAA2H,UAAA;IAAA;EAAA;IAAAxH,GAAA;IAAA3C,KAAA;MAAA,IAAA4L,aAAA,OAAA1D,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAGD,SAAA6G,UACErF,MAAqB,EACrBM,QAAkB,EAClBQ,UAAsB;QAAA,IAAAgD,MAAA,EAAAwB,UAAA,EAAAC,MAAA,EAAAC,QAAA,EAAAC,WAAA,EAAAC,MAAA;QAAA,OAAAnH,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAwG,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAAtG,IAAA,GAAAsG,UAAA,CAAA1L,IAAA;YAAA;cAEhB4J,MAAqB,GAAG;gBAC5B+B,QAAQ,EAAE9D,MAAM,CAACzB,QAAQ,CAAC0B,QAAQ,CAAC;gBACnC8D,UAAU,EAAE,CAAC;cACf,CAAC;cAAAR,UAAA,GAAA7M,0BAAA,CACsB6H,QAAQ,CAACyF,OAAO;cAAAH,UAAA,CAAAtG,IAAA;cAAAgG,UAAA,CAAAjM,CAAA;YAAA;cAAA,KAAAkM,MAAA,GAAAD,UAAA,CAAAhM,CAAA,IAAAC,IAAA;gBAAAqM,UAAA,CAAA1L,IAAA;gBAAA;cAAA;cAA5BsL,QAAQ,GAAAD,MAAA,CAAA/L,KAAA;cACXiM,WAAW,GAAGD,QAAQ,CAACQ,SAAS;cAChCN,MAAM,GAAGD,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEQ,cAAc;cAAA,MACtCnF,UAAU,CAAC5H,MAAM,GAAG,CAAC,IAAI,IAAAgN,uBAAY,EAACpF,UAAU,EAAE4E,MAAO,CAAC,GAAG,CAAC;gBAAAE,UAAA,CAAA1L,IAAA;gBAAA;cAAA;cAAA,OAAA0L,UAAA,CAAA9D,MAAA;YAAA;cAAA8D,UAAA,CAAA1L,IAAA;cAAA,OAGxB,IAAI,CAACiM,eAAe,CAACnG,MAAM,EAAEwF,QAAQ,CAAC;YAAA;cAAhF1B,MAAM,CAACgC,UAAU,CAACJ,MAAM,CAAEU,IAAI,CAAC,CAAC,CAAC,GAAAR,UAAA,CAAAnG,IAAA;YAAA;cAAAmG,UAAA,CAAA1L,IAAA;cAAA;YAAA;cAAA0L,UAAA,CAAA1L,IAAA;cAAA;YAAA;cAAA0L,UAAA,CAAAtG,IAAA;cAAAsG,UAAA,CAAAlG,EAAA,GAAAkG,UAAA;cAAAN,UAAA,CAAA7L,CAAA,CAAAmM,UAAA,CAAAlG,EAAA;YAAA;cAAAkG,UAAA,CAAAtG,IAAA;cAAAgG,UAAA,CAAA3L,CAAA;cAAA,OAAAiM,UAAA,CAAAjG,MAAA;YAAA;cAAA,OAAAiG,UAAA,CAAA9D,MAAA,WAE5BgC,MAAM;YAAA;YAAA;cAAA,OAAA8B,UAAA,CAAA/F,IAAA;UAAA;QAAA,GAAAwF,SAAA;MAAA,CACd;MAAA,SAAA7D,aAAA6E,EAAA,EAAAC,GAAA,EAAAC,GAAA;QAAA,OAAAnB,aAAA,CAAAvJ,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAwF,YAAA;IAAA;EAAA;IAAArF,GAAA;IAAA3C,KAAA;MAAA,IAAAgN,gBAAA,OAAA9E,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAKD,SAAAiI,UAAsBzG,MAAqB,EAAEwF,QAAqB;QAAA,IAAAkB,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;QAAA,IAAAC,KAAA,EAAAC,IAAA,EAAAC,WAAA,EAAAC,WAAA,EAAAC,SAAA,EAAAC,oBAAA,EAAAC,OAAA,EAAAC,UAAA,EAAAC,oBAAA,EAAAC,gBAAA,EAAAC,QAAA;QAAA,OAAArJ,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA0I,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAAxI,IAAA,GAAAwI,UAAA,CAAA5N,IAAA;YAAA;cAAA,MAC5DsL,QAAQ,CAACuC,SAAS,KAAKC,SAAS,IAAIxC,QAAQ,CAACuC,SAAS,KAAK,IAAI;gBAAAD,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAAA,MAC3D,IAAImK,KAAK,CAAC,uCAAuC,CAAC;YAAA;cAGpD6C,KAAK,GAAGlH,MAAM,CAACiI,SAAS,EAAAvB,mBAAA,GAAClB,QAAQ,CAACQ,SAAS,cAAAU,mBAAA,uBAAlBA,mBAAA,CAAoBT,cAAe,CAAC;cAC7DkB,IAAmB,GAAG,IAAAe,wBAAa,EAACC,mBAAI,GAAAxB,oBAAA,GAAEnB,QAAQ,CAACQ,SAAS,cAAAW,oBAAA,uBAAlBA,oBAAA,CAAoBQ,IAAK,CAAC;cAAA,MAEtEA,IAAI,KAAKD,KAAK,CAACkB,aAAa;gBAAAN,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAAA,MACxB,IAAImK,KAAK,oCAAAC,MAAA,CAAoC6C,IAAI,CAAE,CAAC;YAAA;cAGtDC,WAA+B,GAAG,IAAAc,wBAAa,EACnDG,+BAAgB,GAAAzB,oBAAA,GAChBpB,QAAQ,CAACQ,SAAS,cAAAY,oBAAA,uBAAlBA,oBAAA,CAAoB0B,KACtB,CAAC;cAEKjB,WAAW,GAAGtF,MAAM,EAAA8E,oBAAA,GAACrB,QAAQ,CAACQ,SAAS,cAAAa,oBAAA,uBAAlBA,oBAAA,CAAoB0B,gBAAiB,CAAC;cAC7DjB,SAAS,GAAGvF,MAAM,EAAA+E,oBAAA,GAACtB,QAAQ,CAACQ,SAAS,cAAAc,oBAAA,uBAAlBA,oBAAA,CAAoB0B,qBAAsB,CAAC;cAElE,IAAI,CAAChD,QAAQ,CAACuC,SAAS,EAAE;gBACvBT,SAAS,GAAGmB,IAAI,CAACC,GAAG,CAClB,IAAI,CAAC3K,IAAI,CAACkH,IAAI,GAAGoC,WAAW,EAC5BtF,MAAM,EAAAwF,oBAAA,GAAC/B,QAAQ,CAACQ,SAAS,cAAAuB,oBAAA,uBAAlBA,oBAAA,CAAoBiB,qBAAqB,CAClD,CAAC;cACH;cAEMhB,OAAuB,GAAG;gBAC9BL,IAAI,EAAJA,IAAI;gBACJwB,SAAS,EAAEzB,KAAK,CAACyB,SAAS;gBAC1BC,SAAS,EAAE1B,KAAK,CAAC0B,SAAS;gBAC1BxB,WAAW,EAAXA,WAAW;gBACXyB,MAAM,EAAE3B,KAAK;gBACb4B,SAAS,GAAA/B,oBAAA,GAAEvB,QAAQ,CAACQ,SAAS,cAAAe,oBAAA,uBAAlBA,oBAAA,CAAoBgC,UAAU;gBACzCtB,UAAU,EAAE;cACd,CAAC;cAIKC,oBAAoB,GAAGlC,QAAQ,aAARA,QAAQ,wBAAAwB,oBAAA,GAARxB,QAAQ,CAAEQ,SAAS,cAAAgB,oBAAA,uBAAnBA,oBAAA,CAAqBgC,sBAAsB;cAAA,KAEpEtB,oBAAoB;gBAAAI,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAChByN,gBAAgB,GAAG5F,MAAM,CAAC2F,oBAAoB,CAAC;cAAAI,UAAA,CAAA5N,IAAA;cAAA,OAElC,IAAI,CAAC+O,aAAa,CAACtB,gBAAgB,EAAEH,OAAO,EAAEH,WAAW,CAAC;YAAA;cAA7EI,UAAU,GAAAK,UAAA,CAAArI,IAAA;YAAA;cAGZgI,UAAU,GAAG,CAAAR,mBAAA,GAAAO,OAAO,CAACC,UAAU,cAAAR,mBAAA,eAAlBA,mBAAA,CAAoB/N,MAAM,GAAGsO,OAAO,CAACC,UAAU,GAAGA,UAAU;cAACK,UAAA,CAAA5N,IAAA;cAAA,OACnD,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACmD,WAAW,EAAEC,SAAS,CAAC;YAAA;cAAvDM,QAAQ,GAAAE,UAAA,CAAArI,IAAA;cAAAqI,UAAA,CAAA5N,IAAA;cAAA,OACD,IAAAgP,yBAAe,EAACtB,QAAQ,EAAA9L,aAAA,CAAAA,aAAA,KAAM0L,OAAO;gBAAEC,UAAU,EAAVA;cAAU,EAAC,CAAC;YAAA;cAAA,OAAAK,UAAA,CAAAhG,MAAA,WAAAgG,UAAA,CAAArI,IAAA;YAAA;YAAA;cAAA,OAAAqI,UAAA,CAAAjI,IAAA;UAAA;QAAA,GAAA4G,SAAA;MAAA,CACjE;MAAA,SAAAN,gBAAAgD,GAAA,EAAAC,GAAA;QAAA,OAAA5C,gBAAA,CAAA3K,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAmK,eAAA;IAAA;EAAA;IAAAhK,GAAA;IAAA3C,KAAA;MAAA,IAAA6P,cAAA,OAAA3H,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CASD,SAAA8K,UACE5B,oBAA4B,EAC5BF,OAAuB,EACvBH,WAAmB;QAAA,IAAAkC,cAAA,EAAA3B,QAAA,EAAA4B,MAAA,EAAAC,WAAA;QAAA,OAAAlL,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAuK,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAArK,IAAA,GAAAqK,UAAA,CAAAzP,IAAA;YAAA;cAAA,MAEfwN,oBAAoB,KAAK,CAAC;gBAAAiC,UAAA,CAAAzP,IAAA;gBAAA;cAAA;cAAA,OAAAyP,UAAA,CAAA7H,MAAA,WAQrB,EAAE;YAAA;cAGLyH,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAAC3K,IAAI,CAACkH,IAAI,GAAGyC,oBAAoB,EACrC,IAAI,CAAC1J,KAAK,CAACH,qBACb,CAAC;cAAA8L,UAAA,CAAAzP,IAAA;cAAA,OACsB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACwD,oBAAoB,EAAE6B,cAAc,CAAC;YAAA;cAArE3B,QAAQ,GAAA+B,UAAA,CAAAlK,IAAA;cAER+J,MAAM,GAAG;gBAAC1F,MAAM,EAAE8D,QAAQ;gBAAEgC,MAAM,EAAE,CAAC;gBAAE3E,IAAI,EAAE2C,QAAQ,CAAC1O;cAAM,CAAC;cAAAyQ,UAAA,CAAAzP,IAAA;cAAA,OACzC,IAAA2P,oBAAU,EAACL,MAAM,EAAEhC,OAAO,CAAC;YAAA;cAA/CiC,WAAW,GAAAE,UAAA,CAAAlK,IAAA;cAAA,OAAAkK,UAAA,CAAA7H,MAAA,WAEV2H,WAAW,CAAChC,UAAU;YAAA;YAAA;cAAA,OAAAkC,UAAA,CAAA9J,IAAA;UAAA;QAAA,GAAAyJ,SAAA;MAAA,CAC9B;MAAA,SAAAL,cAAAa,GAAA,EAAAC,GAAA,EAAAC,GAAA;QAAA,OAAAX,cAAA,CAAAxN,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAiN,aAAA;IAAA;EAAA;EAAA,OAAAnL,aAAA;AAAA;AAAAmM,OAAA,CAAAnM,aAAA,GAAAA,aAAA"}
+{"version":3,"file":"parquet-reader.js","names":["_schema","require","_decoders","_shred","_constants","_parquetThrift","_readUtils","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","_defineProperty2","default","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","materializeRecords","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","decodeSchema","num_children","ParquetSchema","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","PARQUET_MAGIC","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","concat","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","decodeFileMetadata","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","fieldIndexOf","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","file_path","undefined","field","findField","type","getThriftEnum","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","decodeDataPages","dictionarySize","cursor","offset","decodedPage","decodePage","exports"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRecords(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";;;;;;;;AAGA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AACA,IAAAE,MAAA,GAAAF,OAAA;AAEA,IAAAG,UAAA,GAAAH,OAAA;AACA,IAAAI,cAAA,GAAAJ,OAAA;AAQA,IAAAK,UAAA,GAAAL,OAAA;AAaA,MAAMM,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAQM,MAAMC,aAAa,CAAC;EAKzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA,oBAFnB,IAAI;IAG3C,IAAI,CAACH,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAG,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;EACnB;EAKA,OAAOC,WAAWA,CAACJ,KAA6B,EAAE;IAChD,WAAW,MAAMK,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACN,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMO,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACN,KAA6B,EAAE;IACrD,MAAMQ,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACX,KAAK,CAAC,EAAE;MACzD,MAAM,IAAAY,yBAAkB,EAACJ,MAAM,EAAEE,QAAQ,CAAC;IAC5C;EACF;EAGA,OAAOC,gBAAgBA,CAACX,KAA6B,EAAE;IAErD,MAAMa,UAAsB,GAAG,CAAC,CAAAb,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEa,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAC3B,CAAC;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMX,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IAErC,MAAMW,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACc,YAAY,CACtChB,MAAM,EACNU,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UACF,CAAC;MACD,MAAMH,QAAQ;IAChB;EACF;EAEA,MAAMe,WAAWA,CAAA,EAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMlB,SAASA,CAAA,EAA2B;IACxC,MAAMS,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACV,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEqB;IAAgB,CAAC,GAAG,IAAAC,sBAAY,EAACZ,QAAQ,CAACV,MAAM,EAAE,CAAC,EAAEoB,IAAI,CAACG,YAAa,CAAC;IACvF,MAAMvB,MAAM,GAAG,IAAIwB,qBAAa,CAACH,gBAAgB,CAAC;IAClD,OAAOrB,MAAM;EACf;EAMA,MAAMyB,iBAAiBA,CAAA,EAAoC;IACzD,MAAMf,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMe,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIjB,QAAQ,CAACkB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMf,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACqB,UAAU,CAAC,CAAC;MACvB,IAAI,CAACrB,QAAQ,GAAG,IAAI,CAACsB,UAAU,CAAC,CAAC;IACnC;IACA,OAAO,IAAI,CAACtB,QAAQ;EACtB;EAKA,MAAMqB,UAAUA,CAAA,EAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAAC1C,IAAI,CAAC2C,IAAI,CAAC,CAAC,EAAEC,wBAAa,CAACrB,MAAM,CAAC;IAC5D,MAAMsB,KAAK,GAAGH,MAAM,CAACI,QAAQ,CAAC,CAAC;IAC/B,QAAQD,KAAK;MACX,KAAKD,wBAAa;QAChB;MACF,KAAKG,kCAAuB;QAC1B,MAAM,IAAIC,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,gCAAAC,MAAA,CAAgCJ,KAAK,MAAG,CAAC;IAC5D;EACF;EAGA,MAAMJ,UAAUA,CAAA,EAA0B;IACxC,MAAMS,UAAU,GAAGN,wBAAa,CAACrB,MAAM,GAAG,CAAC;IAC3C,MAAM4B,UAAU,GAAG,MAAM,IAAI,CAACnD,IAAI,CAAC2C,IAAI,CAAC,IAAI,CAAC3C,IAAI,CAACoD,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAML,KAAK,GAAGM,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACP,QAAQ,CAAC,CAAC;IAC5C,IAAID,KAAK,KAAKD,wBAAa,EAAE;MAC3B,MAAM,IAAII,KAAK,sCAAAC,MAAA,CAAqCJ,KAAK,MAAG,CAAC;IAC/D;IAEA,MAAMS,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAACxD,IAAI,CAACoD,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAGZ,wBAAa,CAACrB,MAAM,EAAE;MACzC,MAAM,IAAIyB,KAAK,0BAAAC,MAAA,CAA0BO,cAAc,CAAE,CAAC;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACzD,IAAI,CAAC2C,IAAI,CAACa,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAACnC;IAAQ,CAAC,GAAG,IAAAuC,6BAAkB,EAACD,WAAW,CAAC;IAClD,OAAOtC,QAAQ;EACjB;EAGA,MAAMM,YAAYA,CAChBhB,MAAqB,EACrBE,QAAkB,EAClBG,UAAsB,EACE;IACxB,MAAM4B,MAAqB,GAAG;MAC5BiB,QAAQ,EAAEhC,MAAM,CAAChB,QAAQ,CAACiB,QAAQ,CAAC;MACnCgC,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAIlD,QAAQ,CAACmD,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAIpD,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI,IAAA4C,uBAAY,EAACrD,UAAU,EAAEmD,MAAO,CAAC,GAAG,CAAC,EAAE;QAClE;MACF;MACAvB,MAAM,CAACkB,UAAU,CAACK,MAAM,CAAEG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC5D,MAAM,EAAEoD,QAAQ,CAAC;IAClF;IACA,OAAOnB,MAAM;EACf;EAKA,MAAM2B,eAAeA,CAAC5D,MAAqB,EAAEoD,QAAqB,EAAwB;IAAA,IAAAS,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IACxF,IAAIhB,QAAQ,CAACiB,SAAS,KAAKC,SAAS,IAAIlB,QAAQ,CAACiB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI9B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAMgC,KAAK,GAAGvE,MAAM,CAACwE,SAAS,EAAAX,mBAAA,GAACT,QAAQ,CAACG,SAAS,cAAAM,mBAAA,uBAAlBA,mBAAA,CAAoBJ,cAAe,CAAC;IACnE,MAAMgB,IAAmB,GAAG,IAAAC,wBAAa,EAACC,mBAAI,GAAAb,oBAAA,GAAEV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAK,CAAQ;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACK,aAAa,EAAE;MAChC,MAAM,IAAIrC,KAAK,oCAAAC,MAAA,CAAoCiC,IAAI,CAAE,CAAC;IAC5D;IAEA,MAAMI,WAA+B,GAAG,IAAAH,wBAAa,EACnDI,+BAAgB,GAAAf,oBAAA,GAChBX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBgB,KACtB,CAAQ;IAER,MAAMC,WAAW,GAAG9D,MAAM,EAAA8C,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBiB,gBAAiB,CAAC;IACjE,IAAIC,SAAS,GAAGhE,MAAM,EAAA+C,oBAAA,GAACb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBkB,qBAAsB,CAAC;IAElE,IAAI,CAAC/B,QAAQ,CAACiB,SAAS,EAAE;MAAA,IAAAe,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAAC/F,IAAI,CAACoD,IAAI,GAAGqC,WAAW,EAC5B9D,MAAM,EAAAkE,oBAAA,GAAChC,QAAQ,CAACG,SAAS,cAAA6B,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAClD,CAAC;IACH;IAEA,MAAMI,OAAuB,GAAG;MAC9Bd,IAAI;MACJe,SAAS,EAAEjB,KAAK,CAACiB,SAAS;MAC1BC,SAAS,EAAElB,KAAK,CAACkB,SAAS;MAC1BZ,WAAW;MACXa,MAAM,EAAEnB,KAAK;MACboB,SAAS,GAAAzB,oBAAA,GAAEd,QAAQ,CAACG,SAAS,cAAAW,oBAAA,uBAAlBA,oBAAA,CAAoB0B,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAG1C,QAAQ,aAARA,QAAQ,wBAAAe,oBAAA,GAARf,QAAQ,CAAEG,SAAS,cAAAY,oBAAA,uBAAnBA,oBAAA,CAAqB4B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAG9E,MAAM,CAAC4E,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAzB,mBAAA,GAAAmB,OAAO,CAACM,UAAU,cAAAzB,mBAAA,eAAlBA,mBAAA,CAAoBtD,MAAM,GAAGyE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAAC3G,IAAI,CAAC2C,IAAI,CAAC8C,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAM,IAAAiB,yBAAe,EAACD,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMI,aAAaA,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBP,WAAmB,EACA;IACnB,IAAIc,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMM,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAAC/F,IAAI,CAACoD,IAAI,GAAGmD,oBAAoB,EACrC,IAAI,CAACtG,KAAK,CAACJ,qBACb,CAAC;IACD,MAAM8G,QAAQ,GAAG,MAAM,IAAI,CAAC3G,IAAI,CAAC2C,IAAI,CAAC4D,oBAAoB,EAAEM,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAACpE,MAAM,EAAEiE,QAAQ;MAAEI,MAAM,EAAE,CAAC;MAAE3D,IAAI,EAAEuD,QAAQ,CAACpF;IAAM,CAAC;IACnE,MAAMyF,WAAW,GAAG,MAAM,IAAAC,oBAAU,EAACH,MAAM,EAAEd,OAAO,CAAC;IAErD,OAAOgB,WAAW,CAACV,UAAU;EAC/B;AACF;AAACY,OAAA,CAAApH,aAAA,GAAAA,aAAA"}

package/dist/es5/parquetjs/schema/declare.js

@@ -5,17 +5,16 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetBuffer = void 0;
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-var ParquetBuffer = (0, _createClass2.default)(function ParquetBuffer() {
-  var rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
-  var columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
-  (0, _classCallCheck2.default)(this, ParquetBuffer);
-  (0, _defineProperty2.default)(this, "rowCount", void 0);
-  (0, _defineProperty2.default)(this, "columnData", void 0);
-  this.rowCount = rowCount;
-  this.columnData = columnData;
-});
+class ParquetBuffer {
+  constructor() {
+    let rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
+    let columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+    (0, _defineProperty2.default)(this, "rowCount", void 0);
+    (0, _defineProperty2.default)(this, "columnData", void 0);
+    this.rowCount = rowCount;
+    this.columnData = columnData;
+  }
+}
 exports.ParquetBuffer = ParquetBuffer;
 //# sourceMappingURL=declare.js.map

package/dist/es5/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"file":"declare.js","names":["ParquetBuffer","_createClass2","default","rowCount","arguments","length","undefined","columnData","_classCallCheck2","_defineProperty2","exports"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetBuffer {\n  /** Number of rows in this page */\n  rowCount: number;\n\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,aAAa,OAAAC,aAAA,CAAAC,OAAA,EAKxB,SAAAF,cAAA,EAAgF;EAAA,IAApEG,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;EAAA,IAAEG,UAAuC,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAAA,IAAAI,gBAAA,CAAAN,OAAA,QAAAF,aAAA;EAAA,IAAAS,gBAAA,CAAAP,OAAA;EAAA,IAAAO,gBAAA,CAAAP,OAAA;EAC5E,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAAG,OAAA,CAAAV,aAAA,GAAAA,aAAA"}
+{"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","arguments","length","undefined","columnData","_defineProperty2","default","exports"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetBuffer {\n  /** Number of rows in this page */\n  rowCount: number;\n\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;AAgIO,MAAMA,aAAa,CAAC;EAKzBC,WAAWA,CAAA,EAAqE;IAAA,IAApEC,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;IAAA,IAAEG,UAAuC,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAAA,IAAAI,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IAC5E,IAAI,CAACN,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;EAC9B;AACF;AAACG,OAAA,CAAAT,aAAA,GAAAA,aAAA"}

package/dist/es5/parquetjs/schema/schema.js

@@ -5,16 +5,13 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetSchema = void 0;
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 var _codecs = require("../codecs");
 var _compression = require("../compression");
 var _shred = require("./shred");
 var _types = require("./types");
-var ParquetSchema = function () {
-  function ParquetSchema(schema) {
-    (0, _classCallCheck2.default)(this, ParquetSchema);
+class ParquetSchema {
+  constructor(schema) {
     (0, _defineProperty2.default)(this, "schema", void 0);
     (0, _defineProperty2.default)(this, "fields", void 0);
     (0, _defineProperty2.default)(this, "fieldList", void 0);
@@ -22,65 +19,51 @@ var ParquetSchema = function () {
     this.fields = buildFields(schema, 0, 0, []);
     this.fieldList = listFields(this.fields);
   }
-  (0, _createClass2.default)(ParquetSchema, [{
-    key: "findField",
-    value: function findField(path) {
-      if (typeof path === 'string') {
-        path = path.split(',');
-      } else {
-        path = path.slice(0);
-      }
-      var n = this.fields;
-      for (; path.length > 1; path.shift()) {
-        n = n[path[0]].fields;
-      }
-      return n[path[0]];
-    }
-  }, {
-    key: "findFieldBranch",
-    value: function findFieldBranch(path) {
-      if (typeof path === 'string') {
-        path = path.split(',');
-      }
-      var branch = [];
-      var n = this.fields;
-      for (; path.length > 0; path.shift()) {
-        branch.push(n[path[0]]);
-        if (path.length > 1) {
-          n = n[path[0]].fields;
-        }
-      }
-      return branch;
-    }
-  }, {
-    key: "shredRecord",
-    value: function shredRecord(record, buffer) {
-      (0, _shred.shredRecord)(this, record, buffer);
+  findField(path) {
+    if (typeof path === 'string') {
+      path = path.split(',');
+    } else {
+      path = path.slice(0);
     }
-  }, {
-    key: "materializeRecords",
-    value: function materializeRecords(buffer) {
-      return (0, _shred.materializeRecords)(this, buffer);
+    let n = this.fields;
+    for (; path.length > 1; path.shift()) {
+      n = n[path[0]].fields;
     }
-  }, {
-    key: "compress",
-    value: function compress(type) {
-      setCompress(this.schema, type);
-      setCompress(this.fields, type);
-      return this;
+    return n[path[0]];
+  }
+  findFieldBranch(path) {
+    if (typeof path === 'string') {
+      path = path.split(',');
     }
-  }, {
-    key: "buffer",
-    value: function buffer() {
-      return (0, _shred.shredBuffer)(this);
+    const branch = [];
+    let n = this.fields;
+    for (; path.length > 0; path.shift()) {
+      branch.push(n[path[0]]);
+      if (path.length > 1) {
+        n = n[path[0]].fields;
+      }
     }
-  }]);
-  return ParquetSchema;
-}();
+    return branch;
+  }
+  shredRecord(record, buffer) {
+    (0, _shred.shredRecord)(this, record, buffer);
+  }
+  materializeRecords(buffer) {
+    return (0, _shred.materializeRecords)(this, buffer);
+  }
+  compress(type) {
+    setCompress(this.schema, type);
+    setCompress(this.fields, type);
+    return this;
+  }
+  buffer() {
+    return (0, _shred.shredBuffer)(this);
+  }
+}
 exports.ParquetSchema = ParquetSchema;
 function setCompress(schema, type) {
-  for (var name in schema) {
-    var node = schema[name];
+  for (const name in schema) {
+    const node = schema[name];
     if (node.fields) {
       setCompress(node.fields, type);
     } else {
@@ -89,14 +72,14 @@ function setCompress(schema, type) {
   }
 }
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
-  var fieldList = {};
-  for (var name in schema) {
-    var opts = schema[name];
-    var required = !opts.optional;
-    var repeated = Boolean(opts.repeated);
-    var rLevelMax = rLevelParentMax;
-    var dLevelMax = dLevelParentMax;
-    var repetitionType = 'REQUIRED';
+  const fieldList = {};
+  for (const name in schema) {
+    const opts = schema[name];
+    const required = !opts.optional;
+    const repeated = Boolean(opts.repeated);
+    let rLevelMax = rLevelParentMax;
+    let dLevelMax = dLevelParentMax;
+    let repetitionType = 'REQUIRED';
     if (!required) {
       repetitionType = 'OPTIONAL';
       dLevelMax++;
@@ -107,21 +90,21 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       if (required) dLevelMax++;
     }
     if (opts.fields) {
-      var _cpath = path.concat([name]);
+      const cpath = path.concat([name]);
       fieldList[name] = {
-        name: name,
-        path: _cpath,
-        key: _cpath.join(),
-        repetitionType: repetitionType,
-        rLevelMax: rLevelMax,
-        dLevelMax: dLevelMax,
+        name,
+        path: cpath,
+        key: cpath.join(),
+        repetitionType,
+        rLevelMax,
+        dLevelMax,
         isNested: true,
         fieldCount: Object.keys(opts.fields).length,
-        fields: buildFields(opts.fields, rLevelMax, dLevelMax, _cpath)
+        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
       };
       continue;
     }
-    var typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
+    const typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
     if (!typeDef) {
       throw new Error("invalid parquet type: ".concat(opts.type));
     }
@@ -133,28 +116,28 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
     if (!(opts.compression in _compression.PARQUET_COMPRESSION_METHODS)) {
       throw new Error("unsupported compression method: ".concat(opts.compression));
     }
-    var cpath = path.concat([name]);
+    const cpath = path.concat([name]);
     fieldList[name] = {
-      name: name,
+      name,
       primitiveType: typeDef.primitiveType,
       originalType: typeDef.originalType,
       path: cpath,
       key: cpath.join(),
-      repetitionType: repetitionType,
+      repetitionType,
       encoding: opts.encoding,
       compression: opts.compression,
       typeLength: opts.typeLength || typeDef.typeLength,
       presision: opts.presision,
       scale: opts.scale,
-      rLevelMax: rLevelMax,
-      dLevelMax: dLevelMax
+      rLevelMax,
+      dLevelMax
     };
   }
   return fieldList;
 }
 function listFields(fields) {
-  var list = [];
-  for (var k in fields) {
+  let list = [];
+  for (const k in fields) {
     list.push(fields[k]);
     if (fields[k].isNested) {
       list = list.concat(listFields(fields[k].fields));

package/dist/es5/parquetjs/schema/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","names":["_codecs","require","_compression","_shred","_types","ParquetSchema","schema","_classCallCheck2","default","_defineProperty2","fields","buildFields","fieldList","listFields","_createClass2","key","value","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","record","buffer","materializeRecords","compress","type","setCompress","shredBuffer","exports","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAUA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAA8C,IAKjCI,aAAa;EAQxB,SAAAA,cAAYC,MAAwB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA,QAAAH,aAAA;IAAA,IAAAI,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IACpC,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACI,MAAM,GAAGC,WAAW,CAACL,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACM,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAAC,IAAAI,aAAA,CAAAN,OAAA,EAAAH,aAAA;IAAAU,GAAA;IAAAC,KAAA,EAKD,SAAAC,UAAUC,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;MACvD;MAEA,OAAOW,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;EAAC;IAAAH,GAAA;IAAAC,KAAA,EAKD,SAAAQ,gBAAgBN,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMM,MAAsB,GAAG,EAAE;MACjC,IAAIJ,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;QACvD;MACF;MACA,OAAOe,MAAM;IACf;EAAC;IAAAV,GAAA;IAAAC,KAAA,EAED,SAAAW,YAAYC,MAAqB,EAAEC,MAAqB,EAAQ;MAC9D,IAAAF,kBAAW,EAAC,IAAI,EAAEC,MAAM,EAAEC,MAAM,CAAC;IACnC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAc,mBAAmBD,MAAqB,EAAmB;MACzD,OAAO,IAAAC,yBAAkB,EAAC,IAAI,EAAED,MAAM,CAAC;IACzC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAe,SAASC,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAC3B,MAAM,EAAE0B,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACvB,MAAM,EAAEsB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAAjB,GAAA;IAAAC,KAAA,EAED,SAAAa,OAAA,EAAwB;MACtB,OAAO,IAAAK,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA,OAAA7B,aAAA;AAAA;AAAA8B,OAAA,CAAA9B,aAAA,GAAAA,aAAA;AAGH,SAAS4B,WAAWA,CAAC3B,MAAW,EAAE0B,IAAwB,EAAE;EAC1D,KAAK,IAAMI,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAM+B,IAAI,GAAG/B,MAAM,CAAC8B,IAAI,CAAC;IACzB,IAAIC,IAAI,CAAC3B,MAAM,EAAE;MACfuB,WAAW,CAACI,IAAI,CAAC3B,MAAM,EAAEsB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLK,IAAI,CAACC,WAAW,GAAGN,IAAI;IACzB;EACF;AACF;AAGA,SAASrB,WAAWA,CAClBL,MAAwB,EACxBiC,eAAuB,EACvBC,eAAuB,EACvBtB,IAAc,EACgB;EAC9B,IAAMN,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMwB,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAMmC,IAAI,GAAGnC,MAAM,CAAC8B,IAAI,CAAC;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAAC/B,MAAM,EAAE;MACf,IAAMuC,MAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJlB,IAAI,EAAE+B,MAAK;QACXlC,GAAG,EAAEkC,MAAK,CAACE,IAAI,CAAC,CAAC;QACjBH,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTK,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACd,IAAI,CAAC/B,MAAM,CAAC,CAACY,MAAM;QAC3CZ,MAAM,EAAEC,WAAW,CAAC8B,IAAI,CAAC/B,MAAM,EAAEoC,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;IAEA,IAAMO,OAAY,GAAGC,4BAAqB,CAAChB,IAAI,CAACT,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACT,IAAI,CAAE,CAAC;IACvD;IAEAS,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,CAAE,CAAC;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIuB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,IAAMW,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ0B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC7C,IAAI,EAAE+B,KAAK;MACXlC,GAAG,EAAEkC,KAAK,CAACE,IAAI,CAAC,CAAC;MACjBH,cAAc,EAAdA,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B0B,UAAU,EAAEvB,IAAI,CAACuB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAExB,IAAI,CAACwB,SAAS;MACzBC,KAAK,EAAEzB,IAAI,CAACyB,KAAK;MACjBpB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAOnC,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIyD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAI1D,MAAM,EAAE;IACtByD,IAAI,CAACzC,IAAI,CAAChB,MAAM,CAAC0D,CAAC,CAAC,CAAC;IACpB,IAAI1D,MAAM,CAAC0D,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAACjB,MAAM,CAACrC,UAAU,CAACH,MAAM,CAAC0D,CAAC,CAAC,CAAC1D,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOyD,IAAI;AACb"}
+{"version":3,"file":"schema.js","names":["_codecs","require","_compression","_shred","_types","ParquetSchema","constructor","schema","_defineProperty2","default","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","record","buffer","materializeRecords","compress","type","setCompress","shredBuffer","exports","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAUA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAKO,MAAMI,aAAa,CAAC;EAQzBC,WAAWA,CAACC,MAAwB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IACpC,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACG,MAAM,GAAGC,WAAW,CAACJ,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACK,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAKAI,SAASA,CAACC,IAAuB,EAAgB;IAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB,CAAC,MAAM;MAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACtB;IAEA,IAAIC,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;MACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;IACvD;IAEA,OAAOQ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;EACnB;EAKAM,eAAeA,CAACN,IAAuB,EAAkB;IACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB;IACA,MAAMM,MAAsB,GAAG,EAAE;IACjC,IAAIJ,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;MACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;MACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;QACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;MACvD;IACF;IACA,OAAOY,MAAM;EACf;EAEAE,WAAWA,CAACC,MAAqB,EAAEC,MAAqB,EAAQ;IAC9D,IAAAF,kBAAW,EAAC,IAAI,EAAEC,MAAM,EAAEC,MAAM,CAAC;EACnC;EAEAC,kBAAkBA,CAACD,MAAqB,EAAmB;IACzD,OAAO,IAAAC,yBAAkB,EAAC,IAAI,EAAED,MAAM,CAAC;EACzC;EAEAE,QAAQA,CAACC,IAAwB,EAAQ;IACvCC,WAAW,CAAC,IAAI,CAACvB,MAAM,EAAEsB,IAAI,CAAC;IAC9BC,WAAW,CAAC,IAAI,CAACpB,MAAM,EAAEmB,IAAI,CAAC;IAC9B,OAAO,IAAI;EACb;EAEAH,MAAMA,CAAA,EAAkB;IACtB,OAAO,IAAAK,kBAAW,EAAC,IAAI,CAAC;EAC1B;AACF;AAACC,OAAA,CAAA3B,aAAA,GAAAA,aAAA;AAED,SAASyB,WAAWA,CAACvB,MAAW,EAAEsB,IAAwB,EAAE;EAC1D,KAAK,MAAMI,IAAI,IAAI1B,MAAM,EAAE;IACzB,MAAM2B,IAAI,GAAG3B,MAAM,CAAC0B,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACxB,MAAM,EAAE;MACfoB,WAAW,CAACI,IAAI,CAACxB,MAAM,EAAEmB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLK,IAAI,CAACC,WAAW,GAAGN,IAAI;IACzB;EACF;AACF;AAGA,SAASlB,WAAWA,CAClBJ,MAAwB,EACxB6B,eAAuB,EACvBC,eAAuB,EACvBtB,IAAc,EACgB;EAC9B,MAAMH,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,MAAMqB,IAAI,IAAI1B,MAAM,EAAE;IACzB,MAAM+B,IAAI,GAAG/B,MAAM,CAAC0B,IAAI,CAAC;IAGzB,MAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,MAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAAC5B,MAAM,EAAE;MACf,MAAMoC,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCrB,SAAS,CAACqB,IAAI,CAAC,GAAG;QAChBA,IAAI;QACJlB,IAAI,EAAE+B,KAAK;QACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,CAAC,CAAC;QACjBJ,cAAc;QACdF,SAAS;QACTC,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAAC5B,MAAM,CAAC,CAACS,MAAM;QAC3CT,MAAM,EAAEC,WAAW,CAAC2B,IAAI,CAAC5B,MAAM,EAAEiC,SAAS,EAAEC,SAAS,EAAEE,KAAK;MAC9D,CAAC;MACD;IACF;IAEA,MAAMQ,OAAY,GAAGC,4BAAqB,CAACjB,IAAI,CAACT,IAAI,CAAE;IACtD,IAAI,CAACyB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,0BAAAT,MAAA,CAA0BT,IAAI,CAACT,IAAI,CAAE,CAAC;IACvD;IAEAS,IAAI,CAACmB,QAAQ,GAAGnB,IAAI,CAACmB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAEnB,IAAI,CAACmB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,kCAAAT,MAAA,CAAkCT,IAAI,CAACmB,QAAQ,CAAE,CAAC;IACnE;IAEAnB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIwB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,oCAAAT,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,MAAMW,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCrB,SAAS,CAACqB,IAAI,CAAC,GAAG;MAChBA,IAAI;MACJ2B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC9C,IAAI,EAAE+B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,CAAC,CAAC;MACjBJ,cAAc;MACdY,QAAQ,EAAEnB,IAAI,CAACmB,QAAQ;MACvBtB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B2B,UAAU,EAAExB,IAAI,CAACwB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAEzB,IAAI,CAACyB,SAAS;MACzBC,KAAK,EAAE1B,IAAI,CAAC0B,KAAK;MACjBrB,SAAS;MACTC;IACF,CAAC;EACH;EACA,OAAOhC,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIuD,IAAoB,GAAG,EAAE;EAC7B,KAAK,MAAMC,CAAC,IAAIxD,MAAM,EAAE;IACtBuD,IAAI,CAAC1C,IAAI,CAACb,MAAM,CAACwD,CAAC,CAAC,CAAC;IACpB,IAAIxD,MAAM,CAACwD,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAAClB,MAAM,CAAClC,UAAU,CAACH,MAAM,CAACwD,CAAC,CAAC,CAACxD,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOuD,IAAI;AACb"}