@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/es5/parquetjs/parser/parquet-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-reader.js","names":["_schema","require","_decoders","_shred","_constants","_parquetThrift","_readUtils","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","ownKeys","object","enumerableOnly","keys","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","arguments","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","_asyncIterator","iterable","method","async","sync","retry","asyncIterator","AsyncFromSyncIterator","AsyncFromSyncIteratorContinuation","r","Promise","reject","resolve","then","_return","ret","throw","_throw","thr","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","file","props","_classCallCheck2","_createClass2","close","rowIterator","_this","_wrapAsyncGenerator2","_regenerator","mark","_callee","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","rows","_iterator3","_step3","row","wrap","_callee$","_context","prev","rowBatchIterator","_awaitAsyncGenerator2","sent","t0","finish","t1","stop","_this2","_callee2","schema","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","rowGroup","_callee2$","_context2","getSchema","rowGroupIterator","materializeRows","_this3","_callee3","columnList","metadata","rowGroupCount","rowGroupIndex","_callee3$","_context3","map","x","getFileMetadata","row_groups","readRowGroup","_getRowCount","_asyncToGenerator2","_callee4","_callee4$","_context4","abrupt","Number","num_rows","getRowCount","_getSchema","_callee5","root","_decodeSchema","schemaDefinition","_callee5$","_context5","decodeSchema","num_children","ParquetSchema","_getSchemaMetadata","_callee6","md","_iterator4","_step4","kv","_callee6$","_context6","key_value_metadata","getSchemaMetadata","_getFileMetadata","_callee7","_callee7$","_context7","readHeader","readFooter","_readHeader","_callee8","buffer","magic","_callee8$","_context8","read","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","Error","concat","_readFooter","_callee9","trailerLen","trailerBuf","metadataSize","metadataOffset","metadataBuf","_decodeFileMetadata","_callee9$","_context9","size","readUInt32LE","decodeFileMetadata","_readRowGroup","_callee10","_iterator5","_step5","colChunk","colMetadata","colKey","_callee10$","_context10","rowCount","columnData","columns","meta_data","path_in_schema","fieldIndexOf","readColumnChunk","join","_x","_x2","_x3","_readColumnChunk","_callee11","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","field","type","compression","pagesOffset","pagesSize","_colChunk$meta_data6","options","dictionary","dictionaryPageOffset","dictionaryOffset","pagesBuf","_callee11$","_context11","file_path","undefined","findField","getThriftEnum","Type","primitiveType","CompressionCodec","codec","data_page_offset","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary_page_offset","getDictionary","decodeDataPages","_x4","_x5","_getDictionary","_callee12","dictionarySize","cursor","decodedPage","_callee12$","_context12","offset","decodePage","_x6","_x7","_x8","exports"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRows} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetColumnChunk,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRows(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetRowGroup> {\n    const buffer: ParquetRowGroup = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAGA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AACA,IAAAE,MAAA,GAAAF,OAAA;AAEA,IAAAG,UAAA,GAAAH,OAAA;AACA,IAAAI,cAAA,GAAAJ,OAAA;AAQA,IAAAK,UAAA,GAAAL,OAAA;AAAoF,SAAAM,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAd,MAAA,CAAAc,IAAA,CAAAF,MAAA,OAAAZ,MAAA,CAAAe,qBAAA,QAAAC,OAAA,GAAAhB,MAAA,CAAAe,qBAAA,CAAAH,MAAA,GAAAC,cAAA,KAAAG,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAlB,MAAA,CAAAmB,wBAAA,CAAAP,MAAA,EAAAM,GAAA,EAAAE,UAAA,OAAAN,IAAA,CAAAO,IAAA,CAAAC,KAAA,CAAAR,IAAA,EAAAE,OAAA,YAAAF,IAAA;AAAA,SAAAS,cAAAC,MAAA,aAAA5C,CAAA,MAAAA,CAAA,GAAA6C,SAAA,CAAA9C,MAAA,EAAAC,CAAA,UAAA8C,MAAA,WAAAD,SAAA,CAAA7C,CAAA,IAAA6C,SAAA,CAAA7C,CAAA,QAAAA,CAAA,OAAA+B,OAAA,CAAAX,MAAA,CAAA0B,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAN,MAAA,EAAAI,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAA5B,MAAA,CAAA+B,yBAAA,GAAA/B,MAAA,CAAAgC,gBAAA,CAAAR,MAAA,EAAAxB,MAAA,CAAA+B,yBAAA,CAAAL,MAAA,KAAAf,OAAA,CAAAX,MAAA,CAAA0B,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAA5B,MAAA,CAAAiC,cAAA,CAAAT,MAAA,EAAAI,GAAA,EAAA5B,MAAA,CAAAmB,wBAAA,CAAAO,MAAA,EAAAE,GAAA,iBAAAJ,MAAA;AAAA,SAAAU,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAjE,MAAA,KAAA+D,KAAA,GAAA/D,MAAA,CAAAkE,aAAA,EAAAF,IAAA,GAAAhE,MAAA,CAAAC,QAAA,GAAAgE,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAG,qBAAA,CAAAL,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAjD,SAAA;AAAA,SAAAoD,sBAAA3D,CAAA,aAAA4D,kCAAAC,CAAA,QAAA3C,MAAA,CAAA2C,CAAA,MAAAA,CAAA,SAAAC,OAAA,CAAAC,MAAA,KAAAxD,SAAA,CAAAsD,CAAA,+BAAA3D,IAAA,GAAA2D,CAAA,CAAA3D,IAAA,SAAA4D,OAAA,CAAAE,OAAA,CAAAH,CAAA,CAAA1D,KAAA,EAAA8D,IAAA,WAAA9D,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAD,IAAA,EAAAA,IAAA,iBAAAyD,qBAAA,YAAAA,sBAAA3D,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAC,CAAA,GAAAD,CAAA,CAAAa,IAAA,KAAA8C,qBAAA,CAAAxC,SAAA,KAAAnB,CAAA,QAAAC,CAAA,QAAAY,IAAA,WAAAA,KAAA,WAAA+C,iCAAA,MAAA3D,CAAA,CAAAuC,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAA5B,MAAA,WAAAmD,QAAA/D,KAAA,QAAAgE,GAAA,QAAAnE,CAAA,CAAAe,MAAA,oBAAAoD,GAAA,GAAAL,OAAA,CAAAE,OAAA,GAAA7D,KAAA,EAAAA,KAAA,EAAAD,IAAA,UAAA0D,iCAAA,CAAAO,GAAA,CAAA3B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAAyB,KAAA,WAAAC,OAAAlE,KAAA,QAAAmE,GAAA,QAAAtE,CAAA,CAAAe,MAAA,oBAAAuD,GAAA,GAAAR,OAAA,CAAAC,MAAA,CAAA5D,KAAA,IAAAyD,iCAAA,CAAAU,GAAA,CAAA9B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,aAAAgB,qBAAA,CAAA3D,CAAA;AAapF,IAAMuE,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAAC,IAQWC,aAAa;EAKxB,SAAAA,cAAYC,IAAkB,EAAEC,KAA0B,EAAE;IAAA,IAAAC,gBAAA,CAAA5B,OAAA,QAAAyB,aAAA;IAAA,IAAA1B,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA,oBAFnB,IAAI;IAG3C,IAAI,CAAC0B,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAAlC,aAAA,CAAAA,aAAA,KAAO8B,aAAa,GAAKI,KAAK,CAAC;EAC3C;EAAC,IAAAE,aAAA,CAAA7B,OAAA,EAAAyB,aAAA;IAAA3B,GAAA;IAAA3C,KAAA,EAED,SAAA2E,MAAA,EAAc;MAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;IACnB;EAAC;IAAAhC,GAAA;IAAA3C,KAAA,EAKD,SAAA4E,YAAmBJ,KAA6B,EAAE;MAAA,IAAAK,KAAA;MAAA,WAAAC,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAC,QAAA;QAAA,IAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,GAAA;QAAA,OAAAX,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAC,SAAAC,QAAA;UAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAnF,IAAA;YAAA;cAAAwE,yBAAA;cAAAC,iBAAA;cAAAU,QAAA,CAAAC,IAAA;cAAAT,SAAA,GAAApC,cAAA,CACvB4B,KAAI,CAACkB,gBAAgB,CAACvB,KAAK,CAAC;YAAA;cAAAqB,QAAA,CAAAnF,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAAwC,SAAA,CAAA3E,IAAA;YAAA;cAAA,MAAAwE,yBAAA,KAAAI,KAAA,GAAAO,QAAA,CAAAI,IAAA,EAAAlG,IAAA;gBAAA8F,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAApC6E,IAAI,GAAAD,KAAA,CAAAtF,KAAA;cAAAwF,UAAA,GAAAvG,0BAAA,CAEDsG,IAAI;cAAAM,QAAA,CAAAC,IAAA;cAAAN,UAAA,CAAA3F,CAAA;YAAA;cAAA,KAAA4F,MAAA,GAAAD,UAAA,CAAA1F,CAAA,IAAAC,IAAA;gBAAA8F,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAXgF,GAAG,GAAAD,MAAA,CAAAzF,KAAA;cAAA6F,QAAA,CAAAnF,IAAA;cACZ,OAAMgF,GAAG;YAAA;cAAAG,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAK,EAAA,GAAAL,QAAA;cAAAL,UAAA,CAAAvF,CAAA,CAAA4F,QAAA,CAAAK,EAAA;YAAA;cAAAL,QAAA,CAAAC,IAAA;cAAAN,UAAA,CAAArF,CAAA;cAAA,OAAA0F,QAAA,CAAAM,MAAA;YAAA;cAAAjB,yBAAA;cAAAW,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAO,EAAA,GAAAP,QAAA;cAAAV,iBAAA;cAAAC,cAAA,GAAAS,QAAA,CAAAO,EAAA;YAAA;cAAAP,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAC,IAAA;cAAA,MAAAZ,yBAAA,IAAAG,SAAA,CAAAzE,MAAA;gBAAAiF,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAAwC,SAAA,CAAAzE,MAAA;YAAA;cAAAiF,QAAA,CAAAC,IAAA;cAAA,KAAAX,iBAAA;gBAAAU,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAA,MAAA0E,cAAA;YAAA;cAAA,OAAAS,QAAA,CAAAM,MAAA;YAAA;cAAA,OAAAN,QAAA,CAAAM,MAAA;YAAA;YAAA;cAAA,OAAAN,QAAA,CAAAQ,IAAA;UAAA;QAAA,GAAApB,OAAA;MAAA;IAGf;EAAC;IAAAtC,GAAA;IAAA3C,KAAA,EAGD,SAAA+F,iBAAwBvB,KAA6B,EAAE;MAAA,IAAA8B,MAAA;MAAA,WAAAxB,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAuB,SAAA;QAAA,IAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,QAAA;QAAA,OAAA/B,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAoB,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAlB,IAAA,GAAAkB,SAAA,CAAAtG,IAAA;YAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAChCyD,MAAI,CAACW,SAAS,CAAC,CAAC;YAAA;cAA/BT,MAAM,GAAAQ,SAAA,CAAAf,IAAA;cAAAQ,0BAAA;cAAAC,kBAAA;cAAAM,SAAA,CAAAlB,IAAA;cAAAc,UAAA,GAAA3D,cAAA,CACiBqD,MAAI,CAACY,gBAAgB,CAAC1C,KAAK,CAAC;YAAA;cAAAwC,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAA+D,UAAA,CAAAlG,IAAA;YAAA;cAAA,MAAA+F,0BAAA,KAAAI,MAAA,GAAAG,SAAA,CAAAf,IAAA,EAAAlG,IAAA;gBAAAiH,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAxCoG,QAAQ,GAAAD,MAAA,CAAA7G,KAAA;cAAAgH,SAAA,CAAAtG,IAAA;cACvB,OAAM,IAAAyG,sBAAe,EAACX,MAAM,EAAEM,QAAQ,CAAC;YAAA;cAAAL,0BAAA;cAAAO,SAAA,CAAAtG,IAAA;cAAA;YAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA;YAAA;cAAAsG,SAAA,CAAAlB,IAAA;cAAAkB,SAAA,CAAAd,EAAA,GAAAc,SAAA;cAAAN,kBAAA;cAAAC,eAAA,GAAAK,SAAA,CAAAd,EAAA;YAAA;cAAAc,SAAA,CAAAlB,IAAA;cAAAkB,SAAA,CAAAlB,IAAA;cAAA,MAAAW,0BAAA,IAAAG,UAAA,CAAAhG,MAAA;gBAAAoG,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAA+D,UAAA,CAAAhG,MAAA;YAAA;cAAAoG,SAAA,CAAAlB,IAAA;cAAA,KAAAY,kBAAA;gBAAAM,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAA,MAAAiG,eAAA;YAAA;cAAA,OAAAK,SAAA,CAAAb,MAAA;YAAA;cAAA,OAAAa,SAAA,CAAAb,MAAA;YAAA;YAAA;cAAA,OAAAa,SAAA,CAAAX,IAAA;UAAA;QAAA,GAAAE,QAAA;MAAA;IAE3C;EAAC;IAAA5D,GAAA;IAAA3C,KAAA,EAGD,SAAAkH,iBAAwB1C,KAA6B,EAAE;MAAA,IAAA4C,MAAA;MAAA,WAAAtC,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAqC,SAAA;QAAA,IAAAC,UAAA,EAAAC,QAAA,EAAAf,MAAA,EAAAgB,aAAA,EAAAC,aAAA,EAAAX,QAAA;QAAA,OAAA/B,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA+B,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA7B,IAAA,GAAA6B,SAAA,CAAAjH,IAAA;YAAA;cAE/C4G,UAAsB,GAAG,CAAC,CAAA9C,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAE8C,UAAU,KAAI,EAAE,EAAEM,GAAG,CAAC,UAACC,CAAC;gBAAA,OAC7DtI,KAAK,CAACC,OAAO,CAACqI,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;cAAA,CAC5B,CAAC;cAAAF,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAEsBuE,MAAI,CAACU,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAI,SAAA,CAAA1B,IAAA;cAAA0B,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EACOuE,MAAI,CAACH,SAAS,CAAC,CAAC;YAAA;cAA/BT,MAAM,GAAAmB,SAAA,CAAA1B,IAAA;cAENuB,aAAa,GAAG,CAAAD,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEQ,UAAU,CAACrI,MAAM,KAAI,CAAC;cAE7C+H,aAAa,GAAG,CAAC;YAAA;cAAA,MAAEA,aAAa,GAAGD,aAAa;gBAAAG,SAAA,CAAAjH,IAAA;gBAAA;cAAA;cAAAiH,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAChCuE,MAAI,CAACY,YAAY,CACtCxB,MAAM,EACNe,QAAQ,CAACQ,UAAU,CAACN,aAAa,CAAC,EAClCH,UACF,CAAC;YAAA;cAJKR,QAAQ,GAAAa,SAAA,CAAA1B,IAAA;cAAA0B,SAAA,CAAAjH,IAAA;cAKd,OAAMoG,QAAQ;YAAA;cAN2CW,aAAa,EAAE;cAAAE,SAAA,CAAAjH,IAAA;cAAA;YAAA;YAAA;cAAA,OAAAiH,SAAA,CAAAtB,IAAA;UAAA;QAAA,GAAAgB,QAAA;MAAA;IAQ5E;EAAC;IAAA1E,GAAA;IAAA3C,KAAA;MAAA,IAAAiI,YAAA,OAAAC,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAAmD,SAAA;QAAA,IAAAZ,QAAA;QAAA,OAAAxC,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAyC,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAvC,IAAA,GAAAuC,SAAA,CAAA3H,IAAA;YAAA;cAAA2H,SAAA,CAAA3H,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAc,SAAA,CAAApC,IAAA;cAAA,OAAAoC,SAAA,CAAAC,MAAA,WACPC,MAAM,CAAChB,QAAQ,CAACiB,QAAQ,CAAC;YAAA;YAAA;cAAA,OAAAH,SAAA,CAAAhC,IAAA;UAAA;QAAA,GAAA8B,QAAA;MAAA,CACjC;MAAA,SAAAM,YAAA;QAAA,OAAAR,YAAA,CAAA5F,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAiG,WAAA;IAAA;EAAA;IAAA9F,GAAA;IAAA3C,KAAA;MAAA,IAAA0I,UAAA,OAAAR,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAA2D,SAAA;QAAA,IAAApB,QAAA,EAAAqB,IAAA,EAAAC,aAAA,EAAAC,gBAAA,EAAAtC,MAAA;QAAA,OAAAzB,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAoD,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAlD,IAAA,GAAAkD,SAAA,CAAAtI,IAAA;YAAA;cAAAsI,SAAA,CAAAtI,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAyB,SAAA,CAAA/C,IAAA;cACR2C,IAAI,GAAGrB,QAAQ,CAACf,MAAM,CAAC,CAAC,CAAC;cAAAqC,aAAA,GACI,IAAAI,sBAAY,EAAC1B,QAAQ,CAACf,MAAM,EAAE,CAAC,EAAEoC,IAAI,CAACM,YAAa,CAAC,EAAxEJ,gBAAgB,GAAAD,aAAA,CAAxBrC,MAAM;cACPA,MAAM,GAAG,IAAI2C,qBAAa,CAACL,gBAAgB,CAAC;cAAA,OAAAE,SAAA,CAAAV,MAAA,WAC3C9B,MAAM;YAAA;YAAA;cAAA,OAAAwC,SAAA,CAAA3C,IAAA;UAAA;QAAA,GAAAsC,QAAA;MAAA,CACd;MAAA,SAAA1B,UAAA;QAAA,OAAAyB,UAAA,CAAArG,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAyE,SAAA;IAAA;EAAA;IAAAtE,GAAA;IAAA3C,KAAA;MAAA,IAAAoJ,kBAAA,OAAAlB,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAMD,SAAAqE,SAAA;QAAA,IAAA9B,QAAA,EAAA+B,EAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,EAAA;QAAA,OAAA1E,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA+D,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA7D,IAAA,GAAA6D,SAAA,CAAAjJ,IAAA;YAAA;cAAAiJ,SAAA,CAAAjJ,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,CAAC,CAAC;YAAA;cAAvCP,QAAQ,GAAAoC,SAAA,CAAA1D,IAAA;cACRqD,EAA0B,GAAG,CAAC,CAAC;cAAAC,UAAA,GAAAtK,0BAAA,CACpBsI,QAAQ,CAACqC,kBAAkB;cAAA;gBAA5C,KAAAL,UAAA,CAAA1J,CAAA,MAAA2J,MAAA,GAAAD,UAAA,CAAAzJ,CAAA,IAAAC,IAAA,GAA+C;kBAApC0J,EAAE,GAAAD,MAAA,CAAAxJ,KAAA;kBACXsJ,EAAE,CAACG,EAAE,CAAC9G,GAAG,CAAC,GAAG8G,EAAE,CAACzJ,KAAM;gBACxB;cAAC,SAAAO,GAAA;gBAAAgJ,UAAA,CAAAtJ,CAAA,CAAAM,GAAA;cAAA;gBAAAgJ,UAAA,CAAApJ,CAAA;cAAA;cAAA,OAAAwJ,SAAA,CAAArB,MAAA,WACMgB,EAAE;YAAA;YAAA;cAAA,OAAAK,SAAA,CAAAtD,IAAA;UAAA;QAAA,GAAAgD,QAAA;MAAA,CACV;MAAA,SAAAQ,kBAAA;QAAA,OAAAT,kBAAA,CAAA/G,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAqH,iBAAA;IAAA;EAAA;IAAAlH,GAAA;IAAA3C,KAAA;MAAA,IAAA8J,gBAAA,OAAA5B,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAA+E,SAAA;QAAA,OAAAhF,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAqE,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAnE,IAAA,GAAAmE,SAAA,CAAAvJ,IAAA;YAAA;cAAA,IACO,IAAI,CAAC6G,QAAQ;gBAAA0C,SAAA,CAAAvJ,IAAA;gBAAA;cAAA;cAAAuJ,SAAA,CAAAvJ,IAAA;cAAA,OACV,IAAI,CAACwJ,UAAU,CAAC,CAAC;YAAA;cACvB,IAAI,CAAC3C,QAAQ,GAAG,IAAI,CAAC4C,UAAU,CAAC,CAAC;YAAC;cAAA,OAAAF,SAAA,CAAA3B,MAAA,WAE7B,IAAI,CAACf,QAAQ;YAAA;YAAA;cAAA,OAAA0C,SAAA,CAAA5D,IAAA;UAAA;QAAA,GAAA0D,QAAA;MAAA,CACrB;MAAA,SAAAjC,gBAAA;QAAA,OAAAgC,gBAAA,CAAAzH,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAsF,eAAA;IAAA;EAAA;IAAAnF,GAAA;IAAA3C,KAAA;MAAA,IAAAoK,WAAA,OAAAlC,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAKD,SAAAqF,SAAA;QAAA,IAAAC,MAAA,EAAAC,KAAA;QAAA,OAAAxF,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA6E,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA3E,IAAA,GAAA2E,SAAA,CAAA/J,IAAA;YAAA;cAAA+J,SAAA,CAAA/J,IAAA;cAAA,OACuB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAAC,CAAC,EAAEC,wBAAa,CAACjL,MAAM,CAAC;YAAA;cAAtD4K,MAAM,GAAAG,SAAA,CAAAxE,IAAA;cACNsE,KAAK,GAAGD,MAAM,CAACrJ,QAAQ,CAAC,CAAC;cAAAwJ,SAAA,CAAAvE,EAAA,GACvBqE,KAAK;cAAAE,SAAA,CAAA/J,IAAA,GAAA+J,SAAA,CAAAvE,EAAA,KACNyE,wBAAa,OAAAF,SAAA,CAAAvE,EAAA,KAEb0E,kCAAuB;cAAA;YAAA;cAAA,OAAAH,SAAA,CAAAnC,MAAA;YAAA;cAAA,MACpB,IAAIuC,KAAK,CAAC,sCAAsC,CAAC;YAAA;cAAA,MAEjD,IAAIA,KAAK,gCAAAC,MAAA,CAAgCP,KAAK,MAAG,CAAC;YAAA;YAAA;cAAA,OAAAE,SAAA,CAAApE,IAAA;UAAA;QAAA,GAAAgE,QAAA;MAAA,CAE7D;MAAA,SAAAH,WAAA;QAAA,OAAAE,WAAA,CAAA/H,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAA0H,UAAA;IAAA;EAAA;IAAAvH,GAAA;IAAA3C,KAAA;MAAA,IAAA+K,WAAA,OAAA7C,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAGD,SAAAgG,SAAA;QAAA,IAAAC,UAAA,EAAAC,UAAA,EAAAX,KAAA,EAAAY,YAAA,EAAAC,cAAA,EAAAC,WAAA,EAAAC,mBAAA,EAAA/D,QAAA;QAAA,OAAAxC,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA4F,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA1F,IAAA,GAAA0F,SAAA,CAAA9K,IAAA;YAAA;cACQuK,UAAU,GAAGN,wBAAa,CAACjL,MAAM,GAAG,CAAC;cAAA8L,SAAA,CAAA9K,IAAA;cAAA,OAClB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAAC,IAAI,CAACnG,IAAI,CAACkH,IAAI,GAAGR,UAAU,EAAEA,UAAU,CAAC;YAAA;cAA1EC,UAAU,GAAAM,SAAA,CAAAvF,IAAA;cAEVsE,KAAK,GAAGW,UAAU,CAAChK,KAAK,CAAC,CAAC,CAAC,CAACD,QAAQ,CAAC,CAAC;cAAA,MACxCsJ,KAAK,KAAKI,wBAAa;gBAAAa,SAAA,CAAA9K,IAAA;gBAAA;cAAA;cAAA,MACnB,IAAImK,KAAK,sCAAAC,MAAA,CAAqCP,KAAK,MAAG,CAAC;YAAA;cAGzDY,YAAY,GAAGD,UAAU,CAACQ,YAAY,CAAC,CAAC,CAAC;cACzCN,cAAc,GAAG,IAAI,CAAC7G,IAAI,CAACkH,IAAI,GAAGN,YAAY,GAAGF,UAAU;cAAA,MAC7DG,cAAc,GAAGT,wBAAa,CAACjL,MAAM;gBAAA8L,SAAA,CAAA9K,IAAA;gBAAA;cAAA;cAAA,MACjC,IAAImK,KAAK,0BAAAC,MAAA,CAA0BM,cAAc,CAAE,CAAC;YAAA;cAAAI,SAAA,CAAA9K,IAAA;cAAA,OAGlC,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACU,cAAc,EAAED,YAAY,CAAC;YAAA;cAAhEE,WAAW,GAAAG,SAAA,CAAAvF,IAAA;cAAAqF,mBAAA,GAGE,IAAAK,6BAAkB,EAACN,WAAW,CAAC,EAA3C9D,QAAQ,GAAA+D,mBAAA,CAAR/D,QAAQ;cAAA,OAAAiE,SAAA,CAAAlD,MAAA,WACRf,QAAQ;YAAA;YAAA;cAAA,OAAAiE,SAAA,CAAAnF,IAAA;UAAA;QAAA,GAAA2E,QAAA;MAAA,CAChB;MAAA,SAAAb,WAAA;QAAA,OAAAY,WAAA,CAAA1I,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAA2H,UAAA;IAAA;EAAA;IAAAxH,GAAA;IAAA3C,KAAA;MAAA,IAAA4L,aAAA,OAAA1D,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAGD,SAAA6G,UACErF,MAAqB,EACrBM,QAAkB,EAClBQ,UAAsB;QAAA,IAAAgD,MAAA,EAAAwB,UAAA,EAAAC,MAAA,EAAAC,QAAA,EAAAC,WAAA,EAAAC,MAAA;QAAA,OAAAnH,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAwG,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAAtG,IAAA,GAAAsG,UAAA,CAAA1L,IAAA;YAAA;cAEhB4J,MAAuB,GAAG;gBAC9B+B,QAAQ,EAAE9D,MAAM,CAACzB,QAAQ,CAAC0B,QAAQ,CAAC;gBACnC8D,UAAU,EAAE,CAAC;cACf,CAAC;cAAAR,UAAA,GAAA7M,0BAAA,CACsB6H,QAAQ,CAACyF,OAAO;cAAAH,UAAA,CAAAtG,IAAA;cAAAgG,UAAA,CAAAjM,CAAA;YAAA;cAAA,KAAAkM,MAAA,GAAAD,UAAA,CAAAhM,CAAA,IAAAC,IAAA;gBAAAqM,UAAA,CAAA1L,IAAA;gBAAA;cAAA;cAA5BsL,QAAQ,GAAAD,MAAA,CAAA/L,KAAA;cACXiM,WAAW,GAAGD,QAAQ,CAACQ,SAAS;cAChCN,MAAM,GAAGD,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEQ,cAAc;cAAA,MACtCnF,UAAU,CAAC5H,MAAM,GAAG,CAAC,IAAI,IAAAgN,uBAAY,EAACpF,UAAU,EAAE4E,MAAO,CAAC,GAAG,CAAC;gBAAAE,UAAA,CAAA1L,IAAA;gBAAA;cAAA;cAAA,OAAA0L,UAAA,CAAA9D,MAAA;YAAA;cAAA8D,UAAA,CAAA1L,IAAA;cAAA,OAGxB,IAAI,CAACiM,eAAe,CAACnG,MAAM,EAAEwF,QAAQ,CAAC;YAAA;cAAhF1B,MAAM,CAACgC,UAAU,CAACJ,MAAM,CAAEU,IAAI,CAAC,CAAC,CAAC,GAAAR,UAAA,CAAAnG,IAAA;YAAA;cAAAmG,UAAA,CAAA1L,IAAA;cAAA;YAAA;cAAA0L,UAAA,CAAA1L,IAAA;cAAA;YAAA;cAAA0L,UAAA,CAAAtG,IAAA;cAAAsG,UAAA,CAAAlG,EAAA,GAAAkG,UAAA;cAAAN,UAAA,CAAA7L,CAAA,CAAAmM,UAAA,CAAAlG,EAAA;YAAA;cAAAkG,UAAA,CAAAtG,IAAA;cAAAgG,UAAA,CAAA3L,CAAA;cAAA,OAAAiM,UAAA,CAAAjG,MAAA;YAAA;cAAA,OAAAiG,UAAA,CAAA9D,MAAA,WAE5BgC,MAAM;YAAA;YAAA;cAAA,OAAA8B,UAAA,CAAA/F,IAAA;UAAA;QAAA,GAAAwF,SAAA;MAAA,CACd;MAAA,SAAA7D,aAAA6E,EAAA,EAAAC,GAAA,EAAAC,GAAA;QAAA,OAAAnB,aAAA,CAAAvJ,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAwF,YAAA;IAAA;EAAA;IAAArF,GAAA;IAAA3C,KAAA;MAAA,IAAAgN,gBAAA,OAAA9E,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAKD,SAAAiI,UAAsBzG,MAAqB,EAAEwF,QAAqB;QAAA,IAAAkB,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;QAAA,IAAAC,KAAA,EAAAC,IAAA,EAAAC,WAAA,EAAAC,WAAA,EAAAC,SAAA,EAAAC,oBAAA,EAAAC,OAAA,EAAAC,UAAA,EAAAC,oBAAA,EAAAC,gBAAA,EAAAC,QAAA;QAAA,OAAArJ,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA0I,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAAxI,IAAA,GAAAwI,UAAA,CAAA5N,IAAA;YAAA;cAAA,MAC5DsL,QAAQ,CAACuC,SAAS,KAAKC,SAAS,IAAIxC,QAAQ,CAACuC,SAAS,KAAK,IAAI;gBAAAD,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAAA,MAC3D,IAAImK,KAAK,CAAC,uCAAuC,CAAC;YAAA;cAGpD6C,KAAK,GAAGlH,MAAM,CAACiI,SAAS,EAAAvB,mBAAA,GAAClB,QAAQ,CAACQ,SAAS,cAAAU,mBAAA,uBAAlBA,mBAAA,CAAoBT,cAAe,CAAC;cAC7DkB,IAAmB,GAAG,IAAAe,wBAAa,EAACC,mBAAI,GAAAxB,oBAAA,GAAEnB,QAAQ,CAACQ,SAAS,cAAAW,oBAAA,uBAAlBA,oBAAA,CAAoBQ,IAAK,CAAC;cAAA,MAEtEA,IAAI,KAAKD,KAAK,CAACkB,aAAa;gBAAAN,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAAA,MACxB,IAAImK,KAAK,oCAAAC,MAAA,CAAoC6C,IAAI,CAAE,CAAC;YAAA;cAGtDC,WAA+B,GAAG,IAAAc,wBAAa,EACnDG,+BAAgB,GAAAzB,oBAAA,GAChBpB,QAAQ,CAACQ,SAAS,cAAAY,oBAAA,uBAAlBA,oBAAA,CAAoB0B,KACtB,CAAC;cAEKjB,WAAW,GAAGtF,MAAM,EAAA8E,oBAAA,GAACrB,QAAQ,CAACQ,SAAS,cAAAa,oBAAA,uBAAlBA,oBAAA,CAAoB0B,gBAAiB,CAAC;cAC7DjB,SAAS,GAAGvF,MAAM,EAAA+E,oBAAA,GAACtB,QAAQ,CAACQ,SAAS,cAAAc,oBAAA,uBAAlBA,oBAAA,CAAoB0B,qBAAsB,CAAC;cAElE,IAAI,CAAChD,QAAQ,CAACuC,SAAS,EAAE;gBACvBT,SAAS,GAAGmB,IAAI,CAACC,GAAG,CAClB,IAAI,CAAC3K,IAAI,CAACkH,IAAI,GAAGoC,WAAW,EAC5BtF,MAAM,EAAAwF,oBAAA,GAAC/B,QAAQ,CAACQ,SAAS,cAAAuB,oBAAA,uBAAlBA,oBAAA,CAAoBiB,qBAAqB,CAClD,CAAC;cACH;cAEMhB,OAAuB,GAAG;gBAC9BL,IAAI,EAAJA,IAAI;gBACJwB,SAAS,EAAEzB,KAAK,CAACyB,SAAS;gBAC1BC,SAAS,EAAE1B,KAAK,CAAC0B,SAAS;gBAC1BxB,WAAW,EAAXA,WAAW;gBACXyB,MAAM,EAAE3B,KAAK;gBACb4B,SAAS,GAAA/B,oBAAA,GAAEvB,QAAQ,CAACQ,SAAS,cAAAe,oBAAA,uBAAlBA,oBAAA,CAAoBgC,UAAU;gBACzCtB,UAAU,EAAE;cACd,CAAC;cAIKC,oBAAoB,GAAGlC,QAAQ,aAARA,QAAQ,wBAAAwB,oBAAA,GAARxB,QAAQ,CAAEQ,SAAS,cAAAgB,oBAAA,uBAAnBA,oBAAA,CAAqBgC,sBAAsB;cAAA,KAEpEtB,oBAAoB;gBAAAI,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAChByN,gBAAgB,GAAG5F,MAAM,CAAC2F,oBAAoB,CAAC;cAAAI,UAAA,CAAA5N,IAAA;cAAA,OAElC,IAAI,CAAC+O,aAAa,CAACtB,gBAAgB,EAAEH,OAAO,EAAEH,WAAW,CAAC;YAAA;cAA7EI,UAAU,GAAAK,UAAA,CAAArI,IAAA;YAAA;cAGZgI,UAAU,GAAG,CAAAR,mBAAA,GAAAO,OAAO,CAACC,UAAU,cAAAR,mBAAA,eAAlBA,mBAAA,CAAoB/N,MAAM,GAAGsO,OAAO,CAACC,UAAU,GAAGA,UAAU;cAACK,UAAA,CAAA5N,IAAA;cAAA,OACnD,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACmD,WAAW,EAAEC,SAAS,CAAC;YAAA;cAAvDM,QAAQ,GAAAE,UAAA,CAAArI,IAAA;cAAAqI,UAAA,CAAA5N,IAAA;cAAA,OACD,IAAAgP,yBAAe,EAACtB,QAAQ,EAAA9L,aAAA,CAAAA,aAAA,KAAM0L,OAAO;gBAAEC,UAAU,EAAVA;cAAU,EAAC,CAAC;YAAA;cAAA,OAAAK,UAAA,CAAAhG,MAAA,WAAAgG,UAAA,CAAArI,IAAA;YAAA;YAAA;cAAA,OAAAqI,UAAA,CAAAjI,IAAA;UAAA;QAAA,GAAA4G,SAAA;MAAA,CACjE;MAAA,SAAAN,gBAAAgD,GAAA,EAAAC,GAAA;QAAA,OAAA5C,gBAAA,CAAA3K,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAmK,eAAA;IAAA;EAAA;IAAAhK,GAAA;IAAA3C,KAAA;MAAA,IAAA6P,cAAA,OAAA3H,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CASD,SAAA8K,UACE5B,oBAA4B,EAC5BF,OAAuB,EACvBH,WAAmB;QAAA,IAAAkC,cAAA,EAAA3B,QAAA,EAAA4B,MAAA,EAAAC,WAAA;QAAA,OAAAlL,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAuK,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAArK,IAAA,GAAAqK,UAAA,CAAAzP,IAAA;YAAA;cAAA,MAEfwN,oBAAoB,KAAK,CAAC;gBAAAiC,UAAA,CAAAzP,IAAA;gBAAA;cAAA;cAAA,OAAAyP,UAAA,CAAA7H,MAAA,WAQrB,EAAE;YAAA;cAGLyH,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAAC3K,IAAI,CAACkH,IAAI,GAAGyC,oBAAoB,EACrC,IAAI,CAAC1J,KAAK,CAACH,qBACb,CAAC;cAAA8L,UAAA,CAAAzP,IAAA;cAAA,OACsB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACwD,oBAAoB,EAAE6B,cAAc,CAAC;YAAA;cAArE3B,QAAQ,GAAA+B,UAAA,CAAAlK,IAAA;cAER+J,MAAM,GAAG;gBAAC1F,MAAM,EAAE8D,QAAQ;gBAAEgC,MAAM,EAAE,CAAC;gBAAE3E,IAAI,EAAE2C,QAAQ,CAAC1O;cAAM,CAAC;cAAAyQ,UAAA,CAAAzP,IAAA;cAAA,OACzC,IAAA2P,oBAAU,EAACL,MAAM,EAAEhC,OAAO,CAAC;YAAA;cAA/CiC,WAAW,GAAAE,UAAA,CAAAlK,IAAA;cAAA,OAAAkK,UAAA,CAAA7H,MAAA,WAEV2H,WAAW,CAAChC,UAAU;YAAA;YAAA;cAAA,OAAAkC,UAAA,CAAA9J,IAAA;UAAA;QAAA,GAAAyJ,SAAA;MAAA,CAC9B;MAAA,SAAAL,cAAAa,GAAA,EAAAC,GAAA,EAAAC,GAAA;QAAA,OAAAX,cAAA,CAAAxN,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAiN,aAAA;IAAA;EAAA;EAAA,OAAAnL,aAAA;AAAA;AAAAmM,OAAA,CAAAnM,aAAA,GAAAA,aAAA"}

package/dist/es5/parquetjs/schema/declare.js

@@ -0,0 +1,21 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ParquetRowGroup = void 0;
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+var ParquetRowGroup = (0, _createClass2.default)(function ParquetRowGroup() {
+  var rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
+  var columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+  (0, _classCallCheck2.default)(this, ParquetRowGroup);
+  (0, _defineProperty2.default)(this, "rowCount", void 0);
+  (0, _defineProperty2.default)(this, "columnData", void 0);
+  this.rowCount = rowCount;
+  this.columnData = columnData;
+});
+exports.ParquetRowGroup = ParquetRowGroup;
+//# sourceMappingURL=declare.js.map

package/dist/es5/parquetjs/schema/declare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"declare.js","names":["ParquetRowGroup","_createClass2","default","rowCount","arguments","length","undefined","columnData","_classCallCheck2","_defineProperty2","exports"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRow {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetRowGroup {\n  /** Number of rows in this page */\n  rowCount: number;\n  /** Map of Column chunks */\n  columnData: Record<string, ParquetColumnChunk>;\n\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetColumnChunk> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetColumnChunk {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,eAAe,OAAAC,aAAA,CAAAC,OAAA,EAM1B,SAAAF,gBAAA,EAAuF;EAAA,IAA3EG,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;EAAA,IAAEG,UAA8C,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAAA,IAAAI,gBAAA,CAAAN,OAAA,QAAAF,eAAA;EAAA,IAAAS,gBAAA,CAAAP,OAAA;EAAA,IAAAO,gBAAA,CAAAP,OAAA;EACnF,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAAG,OAAA,CAAAV,eAAA,GAAAA,eAAA"}

package/dist/es5/parquetjs/schema/schema.js

@@ -0,0 +1,165 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ParquetSchema = void 0;
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+var _codecs = require("../codecs");
+var _compression = require("../compression");
+var _shred = require("./shred");
+var _types = require("./types");
+var ParquetSchema = function () {
+  function ParquetSchema(schema) {
+    (0, _classCallCheck2.default)(this, ParquetSchema);
+    (0, _defineProperty2.default)(this, "schema", void 0);
+    (0, _defineProperty2.default)(this, "fields", void 0);
+    (0, _defineProperty2.default)(this, "fieldList", void 0);
+    this.schema = schema;
+    this.fields = buildFields(schema, 0, 0, []);
+    this.fieldList = listFields(this.fields);
+  }
+  (0, _createClass2.default)(ParquetSchema, [{
+    key: "findField",
+    value: function findField(path) {
+      if (typeof path === 'string') {
+        path = path.split(',');
+      } else {
+        path = path.slice(0);
+      }
+      var n = this.fields;
+      for (; path.length > 1; path.shift()) {
+        n = n[path[0]].fields;
+      }
+      return n[path[0]];
+    }
+  }, {
+    key: "findFieldBranch",
+    value: function findFieldBranch(path) {
+      if (typeof path === 'string') {
+        path = path.split(',');
+      }
+      var branch = [];
+      var n = this.fields;
+      for (; path.length > 0; path.shift()) {
+        branch.push(n[path[0]]);
+        if (path.length > 1) {
+          n = n[path[0]].fields;
+        }
+      }
+      return branch;
+    }
+  }, {
+    key: "shredRecord",
+    value: function shredRecord(row, rowGroup) {
+      (0, _shred.shredRecord)(this, row, rowGroup);
+    }
+  }, {
+    key: "materializeRows",
+    value: function materializeRows(rowGroup) {
+      return (0, _shred.materializeRows)(this, rowGroup);
+    }
+  }, {
+    key: "compress",
+    value: function compress(type) {
+      setCompress(this.schema, type);
+      setCompress(this.fields, type);
+      return this;
+    }
+  }, {
+    key: "rowGroup",
+    value: function rowGroup() {
+      return (0, _shred.shredBuffer)(this);
+    }
+  }]);
+  return ParquetSchema;
+}();
+exports.ParquetSchema = ParquetSchema;
+function setCompress(schema, type) {
+  for (var name in schema) {
+    var node = schema[name];
+    if (node.fields) {
+      setCompress(node.fields, type);
+    } else {
+      node.compression = type;
+    }
+  }
+}
+function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
+  var fieldList = {};
+  for (var name in schema) {
+    var opts = schema[name];
+    var required = !opts.optional;
+    var repeated = Boolean(opts.repeated);
+    var rLevelMax = rLevelParentMax;
+    var dLevelMax = dLevelParentMax;
+    var repetitionType = 'REQUIRED';
+    if (!required) {
+      repetitionType = 'OPTIONAL';
+      dLevelMax++;
+    }
+    if (repeated) {
+      repetitionType = 'REPEATED';
+      rLevelMax++;
+      if (required) dLevelMax++;
+    }
+    if (opts.fields) {
+      var _cpath = path.concat([name]);
+      fieldList[name] = {
+        name: name,
+        path: _cpath,
+        key: _cpath.join(),
+        repetitionType: repetitionType,
+        rLevelMax: rLevelMax,
+        dLevelMax: dLevelMax,
+        isNested: true,
+        fieldCount: Object.keys(opts.fields).length,
+        fields: buildFields(opts.fields, rLevelMax, dLevelMax, _cpath)
+      };
+      continue;
+    }
+    var typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
+    if (!typeDef) {
+      throw new Error("invalid parquet type: ".concat(opts.type));
+    }
+    opts.encoding = opts.encoding || 'PLAIN';
+    if (!(opts.encoding in _codecs.PARQUET_CODECS)) {
+      throw new Error("unsupported parquet encoding: ".concat(opts.encoding));
+    }
+    opts.compression = opts.compression || 'UNCOMPRESSED';
+    if (!(opts.compression in _compression.PARQUET_COMPRESSION_METHODS)) {
+      throw new Error("unsupported compression method: ".concat(opts.compression));
+    }
+    var cpath = path.concat([name]);
+    fieldList[name] = {
+      name: name,
+      primitiveType: typeDef.primitiveType,
+      originalType: typeDef.originalType,
+      path: cpath,
+      key: cpath.join(),
+      repetitionType: repetitionType,
+      encoding: opts.encoding,
+      compression: opts.compression,
+      typeLength: opts.typeLength || typeDef.typeLength,
+      presision: opts.presision,
+      scale: opts.scale,
+      rLevelMax: rLevelMax,
+      dLevelMax: dLevelMax
+    };
+  }
+  return fieldList;
+}
+function listFields(fields) {
+  var list = [];
+  for (var k in fields) {
+    list.push(fields[k]);
+    if (fields[k].isNested) {
+      list = list.concat(listFields(fields[k].fields));
+    }
+  }
+  return list;
+}
+//# sourceMappingURL=schema.js.map

package/dist/es5/parquetjs/schema/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","names":["_codecs","require","_compression","_shred","_types","ParquetSchema","schema","_classCallCheck2","default","_defineProperty2","fields","buildFields","fieldList","listFields","_createClass2","key","value","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","row","rowGroup","materializeRows","compress","type","setCompress","shredBuffer","exports","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetField,\n  ParquetRow,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRows, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void {\n    shredRecord(this, row, rowGroup);\n  }\n\n  materializeRows(rowGroup: ParquetRowGroup): ParquetRow[] {\n    return materializeRows(this, rowGroup);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  rowGroup(): ParquetRowGroup {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAUA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAA8C,IAKjCI,aAAa;EAQxB,SAAAA,cAAYC,MAAwB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA,QAAAH,aAAA;IAAA,IAAAI,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IACpC,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACI,MAAM,GAAGC,WAAW,CAACL,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACM,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAAC,IAAAI,aAAA,CAAAN,OAAA,EAAAH,aAAA;IAAAU,GAAA;IAAAC,KAAA,EAKD,SAAAC,UAAUC,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;MACvD;MAEA,OAAOW,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;EAAC;IAAAH,GAAA;IAAAC,KAAA,EAKD,SAAAQ,gBAAgBN,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMM,MAAsB,GAAG,EAAE;MACjC,IAAIJ,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;QACvD;MACF;MACA,OAAOe,MAAM;IACf;EAAC;IAAAV,GAAA;IAAAC,KAAA,EAED,SAAAW,YAAYC,GAAe,EAAEC,QAAyB,EAAQ;MAC5D,IAAAF,kBAAW,EAAC,IAAI,EAAEC,GAAG,EAAEC,QAAQ,CAAC;IAClC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAc,gBAAgBD,QAAyB,EAAgB;MACvD,OAAO,IAAAC,sBAAe,EAAC,IAAI,EAAED,QAAQ,CAAC;IACxC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAe,SAASC,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAC3B,MAAM,EAAE0B,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACvB,MAAM,EAAEsB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAAjB,GAAA;IAAAC,KAAA,EAED,SAAAa,SAAA,EAA4B;MAC1B,OAAO,IAAAK,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA,OAAA7B,aAAA;AAAA;AAAA8B,OAAA,CAAA9B,aAAA,GAAAA,aAAA;AAGH,SAAS4B,WAAWA,CAAC3B,MAAW,EAAE0B,IAAwB,EAAE;EAC1D,KAAK,IAAMI,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAM+B,IAAI,GAAG/B,MAAM,CAAC8B,IAAI,CAAC;IACzB,IAAIC,IAAI,CAAC3B,MAAM,EAAE;MACfuB,WAAW,CAACI,IAAI,CAAC3B,MAAM,EAAEsB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLK,IAAI,CAACC,WAAW,GAAGN,IAAI;IACzB;EACF;AACF;AAGA,SAASrB,WAAWA,CAClBL,MAAwB,EACxBiC,eAAuB,EACvBC,eAAuB,EACvBtB,IAAc,EACgB;EAC9B,IAAMN,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMwB,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAMmC,IAAI,GAAGnC,MAAM,CAAC8B,IAAI,CAAC;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAAC/B,MAAM,EAAE;MACf,IAAMuC,MAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJlB,IAAI,EAAE+B,MAAK;QACXlC,GAAG,EAAEkC,MAAK,CAACE,IAAI,CAAC,CAAC;QACjBH,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTK,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACd,IAAI,CAAC/B,MAAM,CAAC,CAACY,MAAM;QAC3CZ,MAAM,EAAEC,WAAW,CAAC8B,IAAI,CAAC/B,MAAM,EAAEoC,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;IAEA,IAAMO,OAAY,GAAGC,4BAAqB,CAAChB,IAAI,CAACT,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACT,IAAI,CAAE,CAAC;IACvD;IAEAS,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,CAAE,CAAC;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIuB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,IAAMW,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ0B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC7C,IAAI,EAAE+B,KAAK;MACXlC,GAAG,EAAEkC,KAAK,CAACE,IAAI,CAAC,CAAC;MACjBH,cAAc,EAAdA,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B0B,UAAU,EAAEvB,IAAI,CAACuB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAExB,IAAI,CAACwB,SAAS;MACzBC,KAAK,EAAEzB,IAAI,CAACyB,KAAK;MACjBpB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAOnC,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIyD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAI1D,MAAM,EAAE;IACtByD,IAAI,CAACzC,IAAI,CAAChB,MAAM,CAAC0D,CAAC,CAAC,CAAC;IACpB,IAAI1D,MAAM,CAAC0D,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAACjB,MAAM,CAACrC,UAAU,CAACH,MAAM,CAAC0D,CAAC,CAAC,CAAC1D,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOyD,IAAI;AACb"}

package/dist/es5/parquetjs/schema/shred.js

@@ -0,0 +1,282 @@
+"use strict";
+
+var _typeof = require("@babel/runtime/helpers/typeof");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "ParquetRowGroup", {
+  enumerable: true,
+  get: function get() {
+    return _declare.ParquetRowGroup;
+  }
+});
+exports.materializeColumns = materializeColumns;
+exports.materializeRows = materializeRows;
+exports.shredBuffer = shredBuffer;
+exports.shredRecord = shredRecord;
+var _declare = require("./declare");
+var Types = _interopRequireWildcard(require("./types"));
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+function shredBuffer(schema) {
+  var columnData = {};
+  var _iterator = _createForOfIteratorHelper(schema.fieldList),
+    _step;
+  try {
+    for (_iterator.s(); !(_step = _iterator.n()).done;) {
+      var field = _step.value;
+      columnData[field.key] = {
+        dlevels: [],
+        rlevels: [],
+        values: [],
+        pageHeaders: [],
+        count: 0
+      };
+    }
+  } catch (err) {
+    _iterator.e(err);
+  } finally {
+    _iterator.f();
+  }
+  return {
+    rowCount: 0,
+    columnData: columnData
+  };
+}
+function shredRecord(schema, record, rowGroup) {
+  var data = shredBuffer(schema).columnData;
+  shredRecordFields(schema.fields, record, data, 0, 0);
+  if (rowGroup.rowCount === 0) {
+    rowGroup.rowCount = 1;
+    rowGroup.columnData = data;
+    return;
+  }
+  rowGroup.rowCount += 1;
+  var _iterator2 = _createForOfIteratorHelper(schema.fieldList),
+    _step2;
+  try {
+    for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
+      var field = _step2.value;
+      Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);
+      Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);
+      Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);
+      rowGroup.columnData[field.key].count += data[field.key].count;
+    }
+  } catch (err) {
+    _iterator2.e(err);
+  } finally {
+    _iterator2.f();
+  }
+}
+function shredRecordFields(fields, record, data, rLevel, dLevel) {
+  for (var name in fields) {
+    var field = fields[name];
+    var values = [];
+    if (record && field.name in record && record[field.name] !== undefined && record[field.name] !== null) {
+      if (record[field.name].constructor === Array) {
+        values = record[field.name];
+      } else {
+        values.push(record[field.name]);
+      }
+    }
+    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {
+      throw new Error("missing required field: ".concat(field.name));
+    }
+    if (values.length > 1 && field.repetitionType !== 'REPEATED') {
+      throw new Error("too many values for field: ".concat(field.name));
+    }
+    if (values.length === 0) {
+      if (field.isNested) {
+        shredRecordFields(field.fields, null, data, rLevel, dLevel);
+      } else {
+        data[field.key].count += 1;
+        data[field.key].rlevels.push(rLevel);
+        data[field.key].dlevels.push(dLevel);
+      }
+      continue;
+    }
+    for (var i = 0; i < values.length; i++) {
+      var rlvl = i === 0 ? rLevel : field.rLevelMax;
+      if (field.isNested) {
+        shredRecordFields(field.fields, values[i], data, rlvl, field.dLevelMax);
+      } else {
+        data[field.key].count += 1;
+        data[field.key].rlevels.push(rlvl);
+        data[field.key].dlevels.push(field.dLevelMax);
+        data[field.key].values.push(Types.toPrimitive(field.originalType || field.primitiveType, values[i]));
+      }
+    }
+  }
+}
+function materializeRows(schema, rowGroup) {
+  var rows = [];
+  for (var i = 0; i < rowGroup.rowCount; i++) {
+    rows.push({});
+  }
+  for (var key in rowGroup.columnData) {
+    var columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsRows(schema, columnData, key, rows);
+    }
+  }
+  return rows;
+}
+function materializeColumnAsRows(schema, columnData, key, rows) {
+  var field = schema.findField(key);
+  var branch = schema.findFieldBranch(key);
+  var rLevels = new Array(field.rLevelMax + 1).fill(0);
+  var vIndex = 0;
+  for (var i = 0; i < columnData.count; i++) {
+    var dLevel = columnData.dlevels[i];
+    var rLevel = columnData.rlevels[i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+    var rIndex = 0;
+    var record = rows[rLevels[rIndex++] - 1];
+    var _iterator3 = _createForOfIteratorHelper(branch),
+      _step3;
+    try {
+      for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
+        var step = _step3.value;
+        if (step === field || dLevel < step.dLevelMax) {
+          break;
+        }
+        switch (step.repetitionType) {
+          case 'REPEATED':
+            if (!(step.name in record)) {
+              record[step.name] = [];
+            }
+            var _ix = rLevels[rIndex++];
+            while (record[step.name].length <= _ix) {
+              record[step.name].push({});
+            }
+            record = record[step.name][_ix];
+            break;
+          default:
+            record[step.name] = record[step.name] || {};
+            record = record[step.name];
+        }
+      }
+    } catch (err) {
+      _iterator3.e(err);
+    } finally {
+      _iterator3.f();
+    }
+    if (dLevel === field.dLevelMax) {
+      var value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      vIndex++;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          var ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+}
+function materializeColumns(schema, rowGroup) {
+  var columns = {};
+  for (var key in rowGroup.columnData) {
+    var columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
+    }
+  }
+  return columns;
+}
+function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
+  if (columnData.count <= 0) {
+    return;
+  }
+  var field = schema.findField(key);
+  var branch = schema.findFieldBranch(key);
+  var columnName = branch[0].name;
+  var column;
+  var values = columnData.values;
+  if (values.length === rowCount && branch[0].primitiveType) {
+    column = values;
+  }
+  if (column) {
+    columns[columnName] = column;
+    return;
+  }
+  column = new Array(rowCount);
+  for (var i = 0; i < rowCount; i++) {
+    column[i] = {};
+  }
+  columns[columnName] = column;
+  var rLevels = new Array(field.rLevelMax + 1).fill(0);
+  var vIndex = 0;
+  for (var _i = 0; _i < columnData.count; _i++) {
+    var dLevel = columnData.dlevels[_i];
+    var rLevel = columnData.rlevels[_i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+    var rIndex = 0;
+    var record = column[rLevels[rIndex++] - 1];
+    var _iterator4 = _createForOfIteratorHelper(branch),
+      _step4;
+    try {
+      for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
+        var step = _step4.value;
+        if (step === field || dLevel < step.dLevelMax) {
+          break;
+        }
+        switch (step.repetitionType) {
+          case 'REPEATED':
+            if (!(step.name in record)) {
+              record[step.name] = [];
+            }
+            var _ix2 = rLevels[rIndex++];
+            while (record[step.name].length <= _ix2) {
+              record[step.name].push({});
+            }
+            record = record[step.name][_ix2];
+            break;
+          default:
+            record[step.name] = record[step.name] || {};
+            record = record[step.name];
+        }
+      }
+    } catch (err) {
+      _iterator4.e(err);
+    } finally {
+      _iterator4.f();
+    }
+    if (dLevel === field.dLevelMax) {
+      var value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      vIndex++;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          var ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+  for (var _i2 = 0; _i2 < rowCount; ++_i2) {
+    if (columnName in column[_i2]) {
+      column[_i2] = column[_i2][columnName];
+    }
+  }
+}
+//# sourceMappingURL=shred.js.map