@loaders.gl/parquet 3.4.13 → 3.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/dist/dist.min.js +15 -24
  2. package/dist/dist.min.js.map +3 -3
  3. package/dist/es5/bundle.js +1 -1
  4. package/dist/es5/bundle.js.map +1 -1
  5. package/dist/es5/constants.js +5 -5
  6. package/dist/es5/constants.js.map +1 -1
  7. package/dist/es5/index.js +24 -24
  8. package/dist/es5/index.js.map +1 -1
  9. package/dist/es5/lib/arrow/convert-row-group-to-columns.js +2 -7
  10. package/dist/es5/lib/arrow/convert-row-group-to-columns.js.map +1 -1
  11. package/dist/es5/lib/arrow/convert-schema-from-parquet.js +22 -33
  12. package/dist/es5/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  13. package/dist/es5/lib/arrow/convert-schema-to-parquet.js +2 -2
  14. package/dist/es5/lib/arrow/convert-schema-to-parquet.js.map +1 -1
  15. package/dist/es5/lib/geo/decode-geo-metadata.js +16 -27
  16. package/dist/es5/lib/geo/decode-geo-metadata.js.map +1 -1
  17. package/dist/es5/lib/parsers/parse-parquet-to-columns.js +20 -151
  18. package/dist/es5/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  19. package/dist/es5/lib/parsers/parse-parquet-to-rows.js +13 -138
  20. package/dist/es5/lib/parsers/parse-parquet-to-rows.js.map +1 -1
  21. package/dist/es5/lib/wasm/encode-parquet-wasm.js +8 -29
  22. package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -1
  23. package/dist/es5/lib/wasm/load-wasm/index.js +1 -1
  24. package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +10 -33
  25. package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
  26. package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +4 -22
  27. package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
  28. package/dist/es5/lib/wasm/parse-parquet-wasm.js +13 -46
  29. package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -1
  30. package/dist/es5/parquet-loader.js +4 -4
  31. package/dist/es5/parquet-loader.js.map +1 -1
  32. package/dist/es5/parquet-wasm-loader.js +4 -4
  33. package/dist/es5/parquet-wasm-loader.js.map +1 -1
  34. package/dist/es5/parquet-wasm-writer.js +3 -3
  35. package/dist/es5/parquet-wasm-writer.js.map +1 -1
  36. package/dist/es5/parquet-writer.js +4 -4
  37. package/dist/es5/parquet-writer.js.map +1 -1
  38. package/dist/es5/parquetjs/codecs/dictionary.js +3 -6
  39. package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
  40. package/dist/es5/parquetjs/codecs/index.js +4 -5
  41. package/dist/es5/parquetjs/codecs/index.js.map +1 -1
  42. package/dist/es5/parquetjs/codecs/plain.js +41 -41
  43. package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
  44. package/dist/es5/parquetjs/codecs/rle.js +25 -30
  45. package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
  46. package/dist/es5/parquetjs/compression.js +26 -90
  47. package/dist/es5/parquetjs/compression.js.map +1 -1
  48. package/dist/es5/parquetjs/encoder/parquet-encoder.js +245 -536
  49. package/dist/es5/parquetjs/encoder/parquet-encoder.js.map +1 -1
  50. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
  51. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +28 -40
  52. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  53. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +123 -133
  54. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  55. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +138 -150
  56. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  57. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +241 -251
  58. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  59. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +58 -70
  60. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  61. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
  62. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
  63. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +97 -107
  64. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  65. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +136 -146
  66. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  67. package/dist/es5/parquetjs/parquet-thrift/DateType.js +28 -40
  68. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
  69. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +58 -68
  70. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  71. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +71 -81
  72. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  73. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
  74. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +28 -40
  75. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  76. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
  77. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +164 -174
  78. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  79. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +28 -40
  80. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  81. package/dist/es5/parquetjs/parquet-thrift/IntType.js +58 -68
  82. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
  83. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +28 -40
  84. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  85. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +58 -68
  86. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  87. package/dist/es5/parquetjs/parquet-thrift/ListType.js +28 -40
  88. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
  89. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +274 -310
  90. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  91. package/dist/es5/parquetjs/parquet-thrift/MapType.js +28 -40
  92. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
  93. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +28 -40
  94. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  95. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +28 -40
  96. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  97. package/dist/es5/parquetjs/parquet-thrift/NullType.js +28 -40
  98. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
  99. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +56 -66
  100. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  101. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +71 -81
  102. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  103. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +136 -146
  104. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  105. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +71 -81
  106. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  107. package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
  108. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +105 -115
  109. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  110. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +162 -172
  111. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  112. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +71 -81
  113. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  114. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +106 -116
  115. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  116. package/dist/es5/parquetjs/parquet-thrift/StringType.js +28 -40
  117. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
  118. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +58 -68
  119. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  120. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +76 -90
  121. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  122. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +58 -68
  123. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  124. package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
  125. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +28 -40
  126. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  127. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +28 -40
  128. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  129. package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
  130. package/dist/es5/parquetjs/parquet-thrift/index.js.map +1 -1
  131. package/dist/es5/parquetjs/parser/decoders.js +195 -327
  132. package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
  133. package/dist/es5/parquetjs/parser/parquet-reader.js +155 -582
  134. package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
  135. package/dist/es5/parquetjs/schema/declare.js +10 -11
  136. package/dist/es5/parquetjs/schema/declare.js.map +1 -1
  137. package/dist/es5/parquetjs/schema/schema.js +65 -82
  138. package/dist/es5/parquetjs/schema/schema.js.map +1 -1
  139. package/dist/es5/parquetjs/schema/shred.js +56 -87
  140. package/dist/es5/parquetjs/schema/shred.js.map +1 -1
  141. package/dist/es5/parquetjs/schema/types.js +40 -40
  142. package/dist/es5/parquetjs/schema/types.js.map +1 -1
  143. package/dist/es5/parquetjs/utils/file-utils.js +8 -12
  144. package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
  145. package/dist/es5/parquetjs/utils/read-utils.js +22 -39
  146. package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
  147. package/dist/esm/parquet-loader.js +1 -1
  148. package/dist/esm/parquet-wasm-loader.js +1 -1
  149. package/dist/esm/parquet-wasm-writer.js +1 -1
  150. package/dist/esm/parquet-writer.js +1 -1
  151. package/dist/parquet-worker.js +15 -24
  152. package/dist/parquet-worker.js.map +3 -3
  153. package/package.json +6 -6
@@ -1 +1 @@
1
- {"version":3,"file":"decoders.js","names":["_codecs","require","_parquetThrift","_compression","_constants","_readUtils","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","decodeDataPages","_x","_x2","_decodeDataPages","apply","arguments","_asyncToGenerator2","default","_regenerator","mark","_callee","buffer","options","cursor","data","dictionary","page","index","wrap","_callee$","_context","prev","offset","size","rlevels","dlevels","values","pageHeaders","count","numValues","Number","decodePage","sent","abrupt","map","push","undefined","pageHeader","stop","_x3","_x4","_decodePage","_callee2","_decodePageHeader","pageType","_callee2$","_context2","decodePageHeader","getThriftEnum","PageType","type","t0","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","t1","t2","Error","concat","decodeSchema","schemaElements","schema","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","num_children","res","fields","Type","logicalType","converted_type","ConvertedType","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","PARQUET_CODECS","_x5","_x6","_x7","_decodeDataPage","_callee3","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","valueCount","dataCursor","valuesBuf","rLevelEncoding","rLevels","dLevelEncoding","dLevels","valueCountNonNull","_iterator","_step","dlvl","valueEncoding","decodeOptions","_callee3$","_context3","compressed_page_size","data_page_header","num_values","compression","decompress","uncompressed_page_size","Encoding","repetition_level_encoding","column","rLevelMax","PARQUET_RDLVL_TYPE","bitWidth","getBitWidth","disableEnvelope","fill","definition_level_encoding","dLevelMax","primitiveType","_x8","_x9","_x10","_decodeDataPageV","_callee4","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","valuesBufCursor","_callee4$","_context4","data_page_header_v2","num_nulls","PARQUET_RDLVL_ENCODING","is_compressed","_x11","_x12","_x13","_decodeDictionaryPage","_callee5","_pageHeader$dictionar","dictCursor","_callee5$","_context5","dictionary_page_header","d"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n ParquetCodec,\n ParquetData,\n ParquetOptions,\n ParquetPageData,\n ParquetType,\n PrimitiveType,\n SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n ConvertedType,\n Encoding,\n FieldRepetitionType,\n PageHeader,\n PageType,\n SchemaElement,\n Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n buffer: Buffer,\n options: ParquetOptions\n): Promise<ParquetData> {\n const cursor: CursorBuffer = {\n buffer,\n offset: 0,\n size: buffer.length\n };\n\n const data: ParquetData = {\n rlevels: [],\n dlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n\n let dictionary = options.dictionary || [];\n\n while (\n // @ts-ignore size can be undefined\n cursor.offset < cursor.size &&\n (!options.numValues || data.dlevels.length < Number(options.numValues))\n ) {\n // Looks like we have to decode these in sequence due to cursor updates?\n const page = await decodePage(cursor, options);\n\n if (page.dictionary) {\n dictionary = page.dictionary;\n // eslint-disable-next-line no-continue\n continue;\n }\n\n if (dictionary.length) {\n // eslint-disable-next-line no-loop-func\n page.values = page.values.map((value) => dictionary[value]);\n }\n\n for (let index = 0; index < page.rlevels.length; index++) {\n data.rlevels.push(page.rlevels[index]);\n data.dlevels.push(page.dlevels[index]);\n const value = page.values[index];\n\n if (value !== undefined) {\n data.values.push(value);\n }\n }\n\n data.count += page.count;\n data.pageHeaders.push(page.pageHeader);\n }\n\n return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param options\n */\nexport async function decodePage(\n cursor: CursorBuffer,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n let page;\n const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n cursor.offset += length;\n\n const pageType = getThriftEnum(PageType, pageHeader.type);\n\n switch (pageType) {\n case 'DATA_PAGE':\n page = await decodeDataPage(cursor, pageHeader, options);\n break;\n case 'DATA_PAGE_V2':\n page = await decodeDataPageV2(cursor, pageHeader, options);\n break;\n case 'DICTIONARY_PAGE':\n page = {\n dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n pageHeader\n };\n break;\n default:\n throw new Error(`invalid page type: ${pageType}`);\n }\n\n return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n * result.next - offset at the end of function\n * result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n schemaElements: SchemaElement[],\n offset: number,\n len: number\n): {\n offset: number;\n next: number;\n schema: SchemaDefinition;\n} {\n const schema: SchemaDefinition = {};\n let next = offset;\n for (let i = 0; i < len; i++) {\n const schemaElement = schemaElements[next];\n\n const repetitionType =\n next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n let optional = false;\n let repeated = false;\n switch (repetitionType) {\n case 'REQUIRED':\n break;\n case 'OPTIONAL':\n optional = true;\n break;\n case 'REPEATED':\n repeated = true;\n break;\n default:\n throw new Error('parquet: unknown repetition type');\n }\n\n if (schemaElement.num_children! > 0) {\n const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n next = res.next;\n schema[schemaElement.name] = {\n // type: undefined,\n optional,\n repeated,\n fields: res.schema\n };\n } else {\n const type = getThriftEnum(Type, schemaElement.type!);\n let logicalType = type;\n\n if (schemaElement.converted_type) {\n logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n }\n\n switch (logicalType) {\n case 'DECIMAL':\n logicalType = `${logicalType}_${type}` as ParquetType;\n break;\n default:\n }\n\n schema[schemaElement.name] = {\n type: logicalType as ParquetType,\n typeLength: schemaElement.type_length,\n presision: schemaElement.precision,\n scale: schemaElement.scale,\n optional,\n repeated\n };\n next++;\n }\n }\n return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n cursor: CursorBuffer,\n header: PageHeader,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n const valueCount = header.data_page_header?.num_values;\n\n /* uncompress page */\n let dataCursor = cursor;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n dataCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n cursor.offset = cursorEnd;\n }\n\n /* read repetition levels */\n const rLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.repetition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n\n if (options.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.rLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n rLevels.fill(0);\n }\n\n /* read definition levels */\n const dLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.definition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (options.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.dLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n dLevels.fill(0);\n }\n let valueCountNonNull = 0;\n for (const dlvl of dLevels) {\n if (dlvl === options.column.dLevelMax) {\n valueCountNonNull++;\n }\n }\n\n /* read values */\n const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n const decodeOptions = {\n typeLength: options.column.typeLength,\n bitWidth: options.column.typeLength\n };\n\n const values = decodeValues(\n options.column.primitiveType!,\n valueEncoding,\n dataCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n cursor: CursorBuffer,\n header: PageHeader,\n opts: any\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n\n const valueCount = header.data_page_header_v2?.num_values;\n // @ts-ignore\n const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n const valueEncoding = getThriftEnum(\n Encoding,\n header.data_page_header_v2?.encoding!\n ) as ParquetCodec;\n\n /* read repetition levels */\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n if (opts.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.rLevelMax),\n disableEnvelope: true\n });\n } else {\n rLevels.fill(0);\n }\n\n /* read definition levels */\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (opts.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.dLevelMax),\n disableEnvelope: true\n });\n } else {\n dLevels.fill(0);\n }\n\n /* read values */\n let valuesBufCursor = cursor;\n\n if (header.data_page_header_v2?.is_compressed) {\n const valuesBuf = await decompress(\n opts.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n\n valuesBufCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const decodeOptions = {\n typeLength: opts.column.typeLength,\n bitWidth: opts.column.typeLength\n };\n\n const values = decodeValues(\n opts.column.primitiveType!,\n valueEncoding,\n valuesBufCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n cursor: CursorBuffer,\n pageHeader: PageHeader,\n options: ParquetOptions\n): Promise<string[]> {\n const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n let dictCursor = {\n offset: 0,\n buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n size: cursorEnd - cursor.offset\n };\n\n cursor.offset = cursorEnd;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n pageHeader.uncompressed_page_size\n );\n\n dictCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n return decodeValues(\n options.column.primitiveType!,\n options.column.encoding!,\n dictCursor,\n numValues,\n options as ParquetCodecOptions\n ).map((d) => d.toString());\n}\n"],"mappings":";;;;;;;;;;;AAUA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,cAAA,GAAAD,OAAA;AASA,IAAAE,YAAA,GAAAF,OAAA;AACA,IAAAG,UAAA,GAAAH,OAAA;AACA,IAAAI,UAAA,GAAAJ,OAAA;AAAiF,SAAAK,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAS3DC,eAAeA,CAAAC,EAAA,EAAAC,GAAA;EAAA,OAAAC,gBAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAF,iBAAA;EAAAA,gBAAA,OAAAG,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAA9B,SAAAC,QACLC,MAAc,EACdC,OAAuB;IAAA,IAAAC,MAAA,EAAAC,IAAA,EAAAC,UAAA,EAAAC,IAAA,EAAAC,KAAA,EAAA3C,KAAA;IAAA,OAAAkC,YAAA,CAAAD,OAAA,CAAAW,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAApC,IAAA;QAAA;UAEjB6B,MAAoB,GAAG;YAC3BF,MAAM,EAANA,MAAM;YACNW,MAAM,EAAE,CAAC;YACTC,IAAI,EAAEZ,MAAM,CAAC3C;UACf,CAAC;UAEK8C,IAAiB,GAAG;YACxBU,OAAO,EAAE,EAAE;YACXC,OAAO,EAAE,EAAE;YACXC,MAAM,EAAE,EAAE;YACVC,WAAW,EAAE,EAAE;YACfC,KAAK,EAAE;UACT,CAAC;UAEGb,UAAU,GAAGH,OAAO,CAACG,UAAU,IAAI,EAAE;QAAA;UAAA,MAIvCF,MAAM,CAACS,MAAM,GAAGT,MAAM,CAACU,IAAI,KAC1B,CAACX,OAAO,CAACiB,SAAS,IAAIf,IAAI,CAACW,OAAO,CAACzD,MAAM,GAAG8D,MAAM,CAAClB,OAAO,CAACiB,SAAS,CAAC,CAAC;YAAAT,QAAA,CAAApC,IAAA;YAAA;UAAA;UAAAoC,QAAA,CAAApC,IAAA;UAAA,OAGpD+C,UAAU,CAAClB,MAAM,EAAED,OAAO,CAAC;QAAA;UAAxCI,IAAI,GAAAI,QAAA,CAAAY,IAAA;UAAA,KAENhB,IAAI,CAACD,UAAU;YAAAK,QAAA,CAAApC,IAAA;YAAA;UAAA;UACjB+B,UAAU,GAAGC,IAAI,CAACD,UAAU;UAAC,OAAAK,QAAA,CAAAa,MAAA;QAAA;UAK/B,IAAIlB,UAAU,CAAC/C,MAAM,EAAE;YAErBgD,IAAI,CAACU,MAAM,GAAGV,IAAI,CAACU,MAAM,CAACQ,GAAG,CAAC,UAAC5D,KAAK;cAAA,OAAKyC,UAAU,CAACzC,KAAK,CAAC;YAAA,EAAC;UAC7D;UAEA,KAAS2C,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGD,IAAI,CAACQ,OAAO,CAACxD,MAAM,EAAEiD,KAAK,EAAE,EAAE;YACxDH,IAAI,CAACU,OAAO,CAACW,IAAI,CAACnB,IAAI,CAACQ,OAAO,CAACP,KAAK,CAAC,CAAC;YACtCH,IAAI,CAACW,OAAO,CAACU,IAAI,CAACnB,IAAI,CAACS,OAAO,CAACR,KAAK,CAAC,CAAC;YAChC3C,KAAK,GAAG0C,IAAI,CAACU,MAAM,CAACT,KAAK,CAAC;YAEhC,IAAI3C,KAAK,KAAK8D,SAAS,EAAE;cACvBtB,IAAI,CAACY,MAAM,CAACS,IAAI,CAAC7D,KAAK,CAAC;YACzB;UACF;UAEAwC,IAAI,CAACc,KAAK,IAAIZ,IAAI,CAACY,KAAK;UACxBd,IAAI,CAACa,WAAW,CAACQ,IAAI,CAACnB,IAAI,CAACqB,UAAU,CAAC;UAACjB,QAAA,CAAApC,IAAA;UAAA;QAAA;UAAA,OAAAoC,QAAA,CAAAa,MAAA,WAGlCnB,IAAI;QAAA;QAAA;UAAA,OAAAM,QAAA,CAAAkB,IAAA;MAAA;IAAA,GAAA5B,OAAA;EAAA,CACZ;EAAA,OAAAP,gBAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAOqB0B,UAAUA,CAAAQ,GAAA,EAAAC,GAAA;EAAA,OAAAC,WAAA,CAAArC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAoC,YAAA;EAAAA,WAAA,OAAAnC,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAzB,SAAAiC,SACL7B,MAAoB,EACpBD,OAAuB;IAAA,IAAAI,IAAA,EAAA2B,iBAAA,EAAAN,UAAA,EAAArE,MAAA,EAAA4E,QAAA;IAAA,OAAApC,YAAA,CAAAD,OAAA,CAAAW,IAAA,UAAA2B,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAzB,IAAA,GAAAyB,SAAA,CAAA9D,IAAA;QAAA;UAAA2D,iBAAA,GAGM,IAAAI,2BAAgB,EAAClC,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACS,MAAM,CAAC,EAApEe,UAAU,GAAAM,iBAAA,CAAVN,UAAU,EAAErE,MAAM,GAAA2E,iBAAA,CAAN3E,MAAM;UACzB6C,MAAM,CAACS,MAAM,IAAItD,MAAM;UAEjB4E,QAAQ,GAAG,IAAAI,wBAAa,EAACC,uBAAQ,EAAEZ,UAAU,CAACa,IAAI,CAAC;UAAAJ,SAAA,CAAAK,EAAA,GAEjDP,QAAQ;UAAAE,SAAA,CAAA9D,IAAA,GAAA8D,SAAA,CAAAK,EAAA,KACT,WAAW,OAAAL,SAAA,CAAAK,EAAA,KAGX,cAAc,QAAAL,SAAA,CAAAK,EAAA,KAGd,iBAAiB;UAAA;QAAA;UAAAL,SAAA,CAAA9D,IAAA;UAAA,OALPoE,cAAc,CAACvC,MAAM,EAAEwB,UAAU,EAAEzB,OAAO,CAAC;QAAA;UAAxDI,IAAI,GAAA8B,SAAA,CAAAd,IAAA;UAAA,OAAAc,SAAA,CAAAb,MAAA;QAAA;UAAAa,SAAA,CAAA9D,IAAA;UAAA,OAGSqE,gBAAgB,CAACxC,MAAM,EAAEwB,UAAU,EAAEzB,OAAO,CAAC;QAAA;UAA1DI,IAAI,GAAA8B,SAAA,CAAAd,IAAA;UAAA,OAAAc,SAAA,CAAAb,MAAA;QAAA;UAAAa,SAAA,CAAA9D,IAAA;UAAA,OAIgBsE,oBAAoB,CAACzC,MAAM,EAAEwB,UAAU,EAAEzB,OAAO,CAAC;QAAA;UAAAkC,SAAA,CAAAS,EAAA,GAAAT,SAAA,CAAAd,IAAA;UAAAc,SAAA,CAAAU,EAAA,GACnEnB,UAAU;UAFZrB,IAAI;YACFD,UAAU,EAAA+B,SAAA,CAAAS,EAAA;YACVlB,UAAU,EAAAS,SAAA,CAAAU;UAAA;UAAA,OAAAV,SAAA,CAAAb,MAAA;QAAA;UAAA,MAIN,IAAIwB,KAAK,uBAAAC,MAAA,CAAuBd,QAAQ,CAAE,CAAC;QAAA;UAAA,OAAAE,SAAA,CAAAb,MAAA,WAG9CjB,IAAI;QAAA;QAAA;UAAA,OAAA8B,SAAA,CAAAR,IAAA;MAAA;IAAA,GAAAI,QAAA;EAAA,CACZ;EAAA,OAAAD,WAAA,CAAArC,KAAA,OAAAC,SAAA;AAAA;AAYM,SAASsD,YAAYA,CAC1BC,cAA+B,EAC/BtC,MAAc,EACdxB,GAAW,EAKX;EACA,IAAM+D,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAI7E,IAAI,GAAGsC,MAAM;EACjB,KAAK,IAAIrD,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG6B,GAAG,EAAE7B,CAAC,EAAE,EAAE;IAC5B,IAAM6F,aAAa,GAAGF,cAAc,CAAC5E,IAAI,CAAC;IAE1C,IAAM+E,cAAc,GAClB/E,IAAI,GAAG,CAAC,GAAG,IAAAgE,wBAAa,EAACgB,kCAAmB,EAAEF,aAAa,CAACG,eAAgB,CAAC,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQJ,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbG,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIV,KAAK,CAAC,kCAAkC,CAAC;IACvD;IAEA,IAAIK,aAAa,CAACM,YAAY,GAAI,CAAC,EAAE;MACnC,IAAMC,GAAG,GAAGV,YAAY,CAACC,cAAc,EAAE5E,IAAI,GAAG,CAAC,EAAE8E,aAAa,CAACM,YAAa,CAAC;MAC/EpF,IAAI,GAAGqF,GAAG,CAACrF,IAAI;MACf6E,MAAM,CAACC,aAAa,CAACpE,IAAI,CAAC,GAAG;QAE3BwE,QAAQ,EAARA,QAAQ;QACRC,QAAQ,EAARA,QAAQ;QACRG,MAAM,EAAED,GAAG,CAACR;MACd,CAAC;IACH,CAAC,MAAM;MACL,IAAMX,IAAI,GAAG,IAAAF,wBAAa,EAACuB,mBAAI,EAAET,aAAa,CAACZ,IAAK,CAAC;MACrD,IAAIsB,WAAW,GAAGtB,IAAI;MAEtB,IAAIY,aAAa,CAACW,cAAc,EAAE;QAChCD,WAAW,GAAG,IAAAxB,wBAAa,EAAC0B,4BAAa,EAAEZ,aAAa,CAACW,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,MAAAd,MAAA,CAAMc,WAAW,OAAAd,MAAA,CAAIR,IAAI,CAAiB;UACrD;QACF;MACF;MAEAW,MAAM,CAACC,aAAa,CAACpE,IAAI,CAAC,GAAG;QAC3BwD,IAAI,EAAEsB,WAA0B;QAChCG,UAAU,EAAEb,aAAa,CAACc,WAAW;QACrCC,SAAS,EAAEf,aAAa,CAACgB,SAAS;QAClCC,KAAK,EAAEjB,aAAa,CAACiB,KAAK;QAC1Bb,QAAQ,EAARA,QAAQ;QACRC,QAAQ,EAARA;MACF,CAAC;MACDnF,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAAC6E,MAAM,EAANA,MAAM;IAAEvC,MAAM,EAANA,MAAM;IAAEtC,IAAI,EAAJA;EAAI,CAAC;AAC/B;AAKA,SAASgG,YAAYA,CACnB9B,IAAmB,EACnB+B,QAAsB,EACtBpE,MAAoB,EACpBe,KAAa,EACbsD,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIE,sBAAc,CAAC,EAAE;IACjC,MAAM,IAAI1B,KAAK,sBAAAC,MAAA,CAAsBuB,QAAQ,CAAE,CAAC;EAClD;EACA,OAAOE,sBAAc,CAACF,QAAQ,CAAC,CAACD,YAAY,CAAC9B,IAAI,EAAErC,MAAM,EAAEe,KAAK,EAAEsD,IAAI,CAAC;AACzE;AAAC,SAQc9B,cAAcA,CAAAgC,GAAA,EAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,eAAA,CAAAnF,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAkF,gBAAA;EAAAA,eAAA,OAAAjF,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAA7B,SAAA+E,SACE3E,MAAoB,EACpB4E,MAAkB,EAClB7E,OAAuB;IAAA,IAAA8E,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;IAAA,IAAAC,SAAA,EAAAC,UAAA,EAAAC,UAAA,EAAAC,SAAA,EAAAC,cAAA,EAAAC,OAAA,EAAAC,cAAA,EAAAC,OAAA,EAAAC,iBAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,aAAA,EAAAC,aAAA,EAAAjF,MAAA;IAAA,OAAAlB,YAAA,CAAAD,OAAA,CAAAW,IAAA,UAAA0F,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAxF,IAAA,GAAAwF,SAAA,CAAA7H,IAAA;QAAA;UAEjB8G,SAAS,GAAGjF,MAAM,CAACS,MAAM,GAAGmE,MAAM,CAACqB,oBAAoB;UACvDf,UAAU,IAAAL,qBAAA,GAAGD,MAAM,CAACsB,gBAAgB,cAAArB,qBAAA,uBAAvBA,qBAAA,CAAyBsB,UAAU;UAGlDhB,UAAU,GAAGnF,MAAM;UAAA,MAEnBD,OAAO,CAACqG,WAAW,KAAK,cAAc;YAAAJ,SAAA,CAAA7H,IAAA;YAAA;UAAA;UAAA6H,SAAA,CAAA7H,IAAA;UAAA,OAChB,IAAAkI,uBAAU,EAChCtG,OAAO,CAACqG,WAAW,EACnBpG,MAAM,CAACF,MAAM,CAACnB,KAAK,CAACqB,MAAM,CAACS,MAAM,EAAEwE,SAAS,CAAC,EAC7CL,MAAM,CAAC0B,sBACT,CAAC;QAAA;UAJKlB,SAAS,GAAAY,SAAA,CAAA7E,IAAA;UAKfgE,UAAU,GAAG;YACXrF,MAAM,EAAEsF,SAAS;YACjB3E,MAAM,EAAE,CAAC;YACTC,IAAI,EAAE0E,SAAS,CAACjI;UAClB,CAAC;UACD6C,MAAM,CAACS,MAAM,GAAGwE,SAAS;QAAC;UAItBI,cAAc,GAAG,IAAAlD,wBAAa,EAClCoE,uBAAQ,GAAAzB,sBAAA,GACRF,MAAM,CAACsB,gBAAgB,cAAApB,sBAAA,uBAAvBA,sBAAA,CAAyB0B,yBAC3B,CAAC;UAEGlB,OAAO,GAAG,IAAItI,KAAK,CAACkI,UAAU,CAAC;UAEnC,IAAInF,OAAO,CAAC0G,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;YAChCpB,OAAO,GAAGnB,YAAY,CAACwC,6BAAkB,EAAEtB,cAAc,EAAEF,UAAU,EAAED,UAAU,EAAG;cAClF0B,QAAQ,EAAE,IAAAC,sBAAW,EAAC9G,OAAO,CAAC0G,MAAM,CAACC,SAAS,CAAC;cAC/CI,eAAe,EAAE;YAEnB,CAAC,CAAC;UACJ,CAAC,MAAM;YACLxB,OAAO,CAACyB,IAAI,CAAC,CAAC,CAAC;UACjB;UAGMxB,cAAc,GAAG,IAAApD,wBAAa,EAClCoE,uBAAQ,GAAAxB,sBAAA,GACRH,MAAM,CAACsB,gBAAgB,cAAAnB,sBAAA,uBAAvBA,sBAAA,CAAyBiC,yBAC3B,CAAC;UAEGxB,OAAO,GAAG,IAAIxI,KAAK,CAACkI,UAAU,CAAC;UACnC,IAAInF,OAAO,CAAC0G,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;YAChCzB,OAAO,GAAGrB,YAAY,CAACwC,6BAAkB,EAAEpB,cAAc,EAAEJ,UAAU,EAAED,UAAU,EAAG;cAClF0B,QAAQ,EAAE,IAAAC,sBAAW,EAAC9G,OAAO,CAAC0G,MAAM,CAACQ,SAAS,CAAC;cAC/CH,eAAe,EAAE;YAEnB,CAAC,CAAC;UACJ,CAAC,MAAM;YACLtB,OAAO,CAACuB,IAAI,CAAC,CAAC,CAAC;UACjB;UACItB,iBAAiB,GAAG,CAAC;UAAAC,SAAA,GAAAhJ,0BAAA,CACN8I,OAAO;UAAA;YAA1B,KAAAE,SAAA,CAAApI,CAAA,MAAAqI,KAAA,GAAAD,SAAA,CAAAnI,CAAA,IAAAC,IAAA,GAA4B;cAAjBoI,IAAI,GAAAD,KAAA,CAAAlI,KAAA;cACb,IAAImI,IAAI,KAAK7F,OAAO,CAAC0G,MAAM,CAACQ,SAAS,EAAE;gBACrCxB,iBAAiB,EAAE;cACrB;YACF;UAAC,SAAAzH,GAAA;YAAA0H,SAAA,CAAAhI,CAAA,CAAAM,GAAA;UAAA;YAAA0H,SAAA,CAAA9H,CAAA;UAAA;UAGKiI,aAAa,GAAG,IAAA1D,wBAAa,EAACoE,uBAAQ,GAAAvB,sBAAA,GAAEJ,MAAM,CAACsB,gBAAgB,cAAAlB,sBAAA,uBAAvBA,sBAAA,CAAyBZ,QAAS,CAAC;UAC3E0B,aAAa,GAAG;YACpBhC,UAAU,EAAE/D,OAAO,CAAC0G,MAAM,CAAC3C,UAAU;YACrC8C,QAAQ,EAAE7G,OAAO,CAAC0G,MAAM,CAAC3C;UAC3B,CAAC;UAEKjD,MAAM,GAAGsD,YAAY,CACzBpE,OAAO,CAAC0G,MAAM,CAACS,aAAa,EAC5BrB,aAAa,EACbV,UAAU,EACVM,iBAAiB,EACjBK,aACF,CAAC;UAAA,OAAAE,SAAA,CAAA5E,MAAA,WAEM;YACLR,OAAO,EAAE4E,OAAO;YAChB7E,OAAO,EAAE2E,OAAO;YAChBzE,MAAM,EAANA,MAAM;YACNE,KAAK,EAAEmE,UAAW;YAClB1D,UAAU,EAAEoD;UACd,CAAC;QAAA;QAAA;UAAA,OAAAoB,SAAA,CAAAvE,IAAA;MAAA;IAAA,GAAAkD,QAAA;EAAA,CACF;EAAA,OAAAD,eAAA,CAAAnF,KAAA,OAAAC,SAAA;AAAA;AAAA,SAScgD,gBAAgBA,CAAA2E,GAAA,EAAAC,GAAA,EAAAC,IAAA;EAAA,OAAAC,gBAAA,CAAA/H,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAA8H,iBAAA;EAAAA,gBAAA,OAAA7H,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAA/B,SAAA2H,SACEvH,MAAoB,EACpB4E,MAAkB,EAClBP,IAAS;IAAA,IAAAmD,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;IAAA,IAAA1C,SAAA,EAAAC,UAAA,EAAAO,iBAAA,EAAAI,aAAA,EAAAP,OAAA,EAAAE,OAAA,EAAAoC,eAAA,EAAAxC,SAAA,EAAAU,aAAA,EAAAjF,MAAA;IAAA,OAAAlB,YAAA,CAAAD,OAAA,CAAAW,IAAA,UAAAwH,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAtH,IAAA,GAAAsH,SAAA,CAAA3J,IAAA;QAAA;UAEH8G,SAAS,GAAGjF,MAAM,CAACS,MAAM,GAAGmE,MAAM,CAACqB,oBAAoB;UAEvDf,UAAU,IAAAsC,sBAAA,GAAG5C,MAAM,CAACmD,mBAAmB,cAAAP,sBAAA,uBAA1BA,sBAAA,CAA4BrB,UAAU;UAEnDV,iBAAiB,GAAGP,UAAU,KAAAuC,sBAAA,GAAG7C,MAAM,CAACmD,mBAAmB,cAAAN,sBAAA,uBAA1BA,sBAAA,CAA4BO,SAAS;UACtEnC,aAAa,GAAG,IAAA1D,wBAAa,EACjCoE,uBAAQ,GAAAmB,sBAAA,GACR9C,MAAM,CAACmD,mBAAmB,cAAAL,sBAAA,uBAA1BA,sBAAA,CAA4BtD,QAC9B,CAAC;UAIGkB,OAAO,GAAG,IAAItI,KAAK,CAACkI,UAAU,CAAC;UACnC,IAAIb,IAAI,CAACoC,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;YAC7BpB,OAAO,GAAGnB,YAAY,CAACwC,6BAAkB,EAAEsB,iCAAsB,EAAEjI,MAAM,EAAEkF,UAAU,EAAG;cACtF0B,QAAQ,EAAE,IAAAC,sBAAW,EAACxC,IAAI,CAACoC,MAAM,CAACC,SAAS,CAAC;cAC5CI,eAAe,EAAE;YACnB,CAAC,CAAC;UACJ,CAAC,MAAM;YACLxB,OAAO,CAACyB,IAAI,CAAC,CAAC,CAAC;UACjB;UAIIvB,OAAO,GAAG,IAAIxI,KAAK,CAACkI,UAAU,CAAC;UACnC,IAAIb,IAAI,CAACoC,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;YAC7BzB,OAAO,GAAGrB,YAAY,CAACwC,6BAAkB,EAAEsB,iCAAsB,EAAEjI,MAAM,EAAEkF,UAAU,EAAG;cACtF0B,QAAQ,EAAE,IAAAC,sBAAW,EAACxC,IAAI,CAACoC,MAAM,CAACQ,SAAS,CAAC;cAC5CH,eAAe,EAAE;YACnB,CAAC,CAAC;UACJ,CAAC,MAAM;YACLtB,OAAO,CAACuB,IAAI,CAAC,CAAC,CAAC;UACjB;UAGIa,eAAe,GAAG5H,MAAM;UAAA,OAAA2H,sBAAA,GAExB/C,MAAM,CAACmD,mBAAmB,cAAAJ,sBAAA,eAA1BA,sBAAA,CAA4BO,aAAa;YAAAJ,SAAA,CAAA3J,IAAA;YAAA;UAAA;UAAA2J,SAAA,CAAA3J,IAAA;UAAA,OACnB,IAAAkI,uBAAU,EAChChC,IAAI,CAAC+B,WAAW,EAChBpG,MAAM,CAACF,MAAM,CAACnB,KAAK,CAACqB,MAAM,CAACS,MAAM,EAAEwE,SAAS,CAAC,EAC7CL,MAAM,CAAC0B,sBACT,CAAC;QAAA;UAJKlB,SAAS,GAAA0C,SAAA,CAAA3G,IAAA;UAMfyG,eAAe,GAAG;YAChB9H,MAAM,EAAEsF,SAAS;YACjB3E,MAAM,EAAE,CAAC;YACTC,IAAI,EAAE0E,SAAS,CAACjI;UAClB,CAAC;UAED6C,MAAM,CAACS,MAAM,GAAGwE,SAAS;QAAC;UAGtBa,aAAa,GAAG;YACpBhC,UAAU,EAAEO,IAAI,CAACoC,MAAM,CAAC3C,UAAU;YAClC8C,QAAQ,EAAEvC,IAAI,CAACoC,MAAM,CAAC3C;UACxB,CAAC;UAEKjD,MAAM,GAAGsD,YAAY,CACzBE,IAAI,CAACoC,MAAM,CAACS,aAAa,EACzBrB,aAAa,EACb+B,eAAe,EACfnC,iBAAiB,EACjBK,aACF,CAAC;UAAA,OAAAgC,SAAA,CAAA1G,MAAA,WAEM;YACLR,OAAO,EAAE4E,OAAO;YAChB7E,OAAO,EAAE2E,OAAO;YAChBzE,MAAM,EAANA,MAAM;YACNE,KAAK,EAAEmE,UAAW;YAClB1D,UAAU,EAAEoD;UACd,CAAC;QAAA;QAAA;UAAA,OAAAkD,SAAA,CAAArG,IAAA;MAAA;IAAA,GAAA8F,QAAA;EAAA,CACF;EAAA,OAAAD,gBAAA,CAAA/H,KAAA,OAAAC,SAAA;AAAA;AAAA,SAQciD,oBAAoBA,CAAA0F,IAAA,EAAAC,IAAA,EAAAC,IAAA;EAAA,OAAAC,qBAAA,CAAA/I,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAA8I,sBAAA;EAAAA,qBAAA,OAAA7I,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAnC,SAAA2I,SACEvI,MAAoB,EACpBwB,UAAsB,EACtBzB,OAAuB;IAAA,IAAAyI,qBAAA;IAAA,IAAAvD,SAAA,EAAAwD,UAAA,EAAArD,SAAA,EAAApE,SAAA;IAAA,OAAArB,YAAA,CAAAD,OAAA,CAAAW,IAAA,UAAAqI,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAnI,IAAA,GAAAmI,SAAA,CAAAxK,IAAA;QAAA;UAEjB8G,SAAS,GAAGjF,MAAM,CAACS,MAAM,GAAGe,UAAU,CAACyE,oBAAoB;UAE7DwC,UAAU,GAAG;YACfhI,MAAM,EAAE,CAAC;YACTX,MAAM,EAAEE,MAAM,CAACF,MAAM,CAACnB,KAAK,CAACqB,MAAM,CAACS,MAAM,EAAEwE,SAAS,CAAC;YACrDvE,IAAI,EAAEuE,SAAS,GAAGjF,MAAM,CAACS;UAC3B,CAAC;UAEDT,MAAM,CAACS,MAAM,GAAGwE,SAAS;UAAC,MAEtBlF,OAAO,CAACqG,WAAW,KAAK,cAAc;YAAAuC,SAAA,CAAAxK,IAAA;YAAA;UAAA;UAAAwK,SAAA,CAAAxK,IAAA;UAAA,OAChB,IAAAkI,uBAAU,EAChCtG,OAAO,CAACqG,WAAW,EACnBqC,UAAU,CAAC3I,MAAM,CAACnB,KAAK,CAAC8J,UAAU,CAAChI,MAAM,EAAEwE,SAAS,CAAC,EACrDzD,UAAU,CAAC8E,sBACb,CAAC;QAAA;UAJKlB,SAAS,GAAAuD,SAAA,CAAAxH,IAAA;UAMfsH,UAAU,GAAG;YACX3I,MAAM,EAAEsF,SAAS;YACjB3E,MAAM,EAAE,CAAC;YACTC,IAAI,EAAE0E,SAAS,CAACjI;UAClB,CAAC;UAED6C,MAAM,CAACS,MAAM,GAAGwE,SAAS;QAAC;UAGtBjE,SAAS,GAAG,CAAAQ,UAAU,aAAVA,UAAU,wBAAAgH,qBAAA,GAAVhH,UAAU,CAAEoH,sBAAsB,cAAAJ,qBAAA,uBAAlCA,qBAAA,CAAoCrC,UAAU,KAAI,CAAC;UAAA,OAAAwC,SAAA,CAAAvH,MAAA,WAE9D+C,YAAY,CACjBpE,OAAO,CAAC0G,MAAM,CAACS,aAAa,EAC5BnH,OAAO,CAAC0G,MAAM,CAACrC,QAAQ,EACvBqE,UAAU,EACVzH,SAAS,EACTjB,OACF,CAAC,CAACsB,GAAG,CAAC,UAACwH,CAAC;YAAA,OAAKA,CAAC,CAACnK,QAAQ,CAAC,CAAC;UAAA,EAAC;QAAA;QAAA;UAAA,OAAAiK,SAAA,CAAAlH,IAAA;MAAA;IAAA,GAAA8G,QAAA;EAAA,CAC3B;EAAA,OAAAD,qBAAA,CAAA/I,KAAA,OAAAC,SAAA;AAAA"}
1
+ {"version":3,"file":"decoders.js","names":["_codecs","require","_parquetThrift","_compression","_constants","_readUtils","decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","decodePageHeader","pageType","getThriftEnum","PageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","concat","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","num_children","res","name","fields","Type","logicalType","converted_type","ConvertedType","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","PARQUET_CODECS","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","decompress","slice","uncompressed_page_size","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","PARQUET_RDLVL_TYPE","bitWidth","getBitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","data_page_header_v2","num_nulls","PARQUET_RDLVL_ENCODING","valuesBufCursor","is_compressed","_pageHeader$dictionar","dictCursor","dictionary_page_header","d","toString"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n ParquetCodec,\n ParquetData,\n ParquetOptions,\n ParquetPageData,\n ParquetType,\n PrimitiveType,\n SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n ConvertedType,\n Encoding,\n FieldRepetitionType,\n PageHeader,\n PageType,\n SchemaElement,\n Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n buffer: Buffer,\n options: ParquetOptions\n): Promise<ParquetData> {\n const cursor: CursorBuffer = {\n buffer,\n offset: 0,\n size: buffer.length\n };\n\n const data: ParquetData = {\n rlevels: [],\n dlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n\n let dictionary = options.dictionary || [];\n\n while (\n // @ts-ignore size can be undefined\n cursor.offset < cursor.size &&\n (!options.numValues || data.dlevels.length < Number(options.numValues))\n ) {\n // Looks like we have to decode these in sequence due to cursor updates?\n const page = await decodePage(cursor, options);\n\n if (page.dictionary) {\n dictionary = page.dictionary;\n // eslint-disable-next-line no-continue\n continue;\n }\n\n if (dictionary.length) {\n // eslint-disable-next-line no-loop-func\n page.values = page.values.map((value) => dictionary[value]);\n }\n\n for (let index = 0; index < page.rlevels.length; index++) {\n data.rlevels.push(page.rlevels[index]);\n data.dlevels.push(page.dlevels[index]);\n const value = page.values[index];\n\n if (value !== undefined) {\n data.values.push(value);\n }\n }\n\n data.count += page.count;\n data.pageHeaders.push(page.pageHeader);\n }\n\n return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param options\n */\nexport async function decodePage(\n cursor: CursorBuffer,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n let page;\n const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n cursor.offset += length;\n\n const pageType = getThriftEnum(PageType, pageHeader.type);\n\n switch (pageType) {\n case 'DATA_PAGE':\n page = await decodeDataPage(cursor, pageHeader, options);\n break;\n case 'DATA_PAGE_V2':\n page = await decodeDataPageV2(cursor, pageHeader, options);\n break;\n case 'DICTIONARY_PAGE':\n page = {\n dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n pageHeader\n };\n break;\n default:\n throw new Error(`invalid page type: ${pageType}`);\n }\n\n return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n * result.next - offset at the end of function\n * result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n schemaElements: SchemaElement[],\n offset: number,\n len: number\n): {\n offset: number;\n next: number;\n schema: SchemaDefinition;\n} {\n const schema: SchemaDefinition = {};\n let next = offset;\n for (let i = 0; i < len; i++) {\n const schemaElement = schemaElements[next];\n\n const repetitionType =\n next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n let optional = false;\n let repeated = false;\n switch (repetitionType) {\n case 'REQUIRED':\n break;\n case 'OPTIONAL':\n optional = true;\n break;\n case 'REPEATED':\n repeated = true;\n break;\n default:\n throw new Error('parquet: unknown repetition type');\n }\n\n if (schemaElement.num_children! > 0) {\n const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n next = res.next;\n schema[schemaElement.name] = {\n // type: undefined,\n optional,\n repeated,\n fields: res.schema\n };\n } else {\n const type = getThriftEnum(Type, schemaElement.type!);\n let logicalType = type;\n\n if (schemaElement.converted_type) {\n logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n }\n\n switch (logicalType) {\n case 'DECIMAL':\n logicalType = `${logicalType}_${type}` as ParquetType;\n break;\n default:\n }\n\n schema[schemaElement.name] = {\n type: logicalType as ParquetType,\n typeLength: schemaElement.type_length,\n presision: schemaElement.precision,\n scale: schemaElement.scale,\n optional,\n repeated\n };\n next++;\n }\n }\n return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n cursor: CursorBuffer,\n header: PageHeader,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n const valueCount = header.data_page_header?.num_values;\n\n /* uncompress page */\n let dataCursor = cursor;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n dataCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n cursor.offset = cursorEnd;\n }\n\n /* read repetition levels */\n const rLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.repetition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n\n if (options.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.rLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n rLevels.fill(0);\n }\n\n /* read definition levels */\n const dLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.definition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (options.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.dLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n dLevels.fill(0);\n }\n let valueCountNonNull = 0;\n for (const dlvl of dLevels) {\n if (dlvl === options.column.dLevelMax) {\n valueCountNonNull++;\n }\n }\n\n /* read values */\n const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n const decodeOptions = {\n typeLength: options.column.typeLength,\n bitWidth: options.column.typeLength\n };\n\n const values = decodeValues(\n options.column.primitiveType!,\n valueEncoding,\n dataCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n cursor: CursorBuffer,\n header: PageHeader,\n opts: any\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n\n const valueCount = header.data_page_header_v2?.num_values;\n // @ts-ignore\n const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n const valueEncoding = getThriftEnum(\n Encoding,\n header.data_page_header_v2?.encoding!\n ) as ParquetCodec;\n\n /* read repetition levels */\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n if (opts.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.rLevelMax),\n disableEnvelope: true\n });\n } else {\n rLevels.fill(0);\n }\n\n /* read definition levels */\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (opts.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.dLevelMax),\n disableEnvelope: true\n });\n } else {\n dLevels.fill(0);\n }\n\n /* read values */\n let valuesBufCursor = cursor;\n\n if (header.data_page_header_v2?.is_compressed) {\n const valuesBuf = await decompress(\n opts.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n\n valuesBufCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const decodeOptions = {\n typeLength: opts.column.typeLength,\n bitWidth: opts.column.typeLength\n };\n\n const values = decodeValues(\n opts.column.primitiveType!,\n valueEncoding,\n valuesBufCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n cursor: CursorBuffer,\n pageHeader: PageHeader,\n options: ParquetOptions\n): Promise<string[]> {\n const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n let dictCursor = {\n offset: 0,\n buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n size: cursorEnd - cursor.offset\n };\n\n cursor.offset = cursorEnd;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n pageHeader.uncompressed_page_size\n );\n\n dictCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n return decodeValues(\n options.column.primitiveType!,\n options.column.encoding!,\n dictCursor,\n numValues,\n options as ParquetCodecOptions\n ).map((d) => d.toString());\n}\n"],"mappings":";;;;;;;;AAUA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,cAAA,GAAAD,OAAA;AASA,IAAAE,YAAA,GAAAF,OAAA;AACA,IAAAG,UAAA,GAAAH,OAAA;AACA,IAAAI,UAAA,GAAAJ,OAAA;AASO,eAAeK,eAAeA,CACnCC,MAAc,EACdC,OAAuB,EACD;EACtB,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAiB,GAAG;IACxBC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC,OAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;AAOO,eAAeU,UAAUA,CAC9Bd,MAAoB,EACpBD,OAAuB,EACG;EAC1B,IAAIc,IAAI;EACR,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAG,IAAAkB,2BAAgB,EAACrB,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMmB,QAAQ,GAAG,IAAAC,wBAAa,EAACC,uBAAQ,EAAEJ,UAAU,CAACK,IAAI,CAAC;EAEzD,QAAQH,QAAQ;IACd,KAAK,WAAW;MACdT,IAAI,GAAG,MAAMa,cAAc,CAAC1B,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMc,gBAAgB,CAAC3B,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMkB,oBAAoB,CAAC5B,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIS,KAAK,uBAAAC,MAAA,CAAuBR,QAAQ,CAAE,CAAC;EACrD;EAEA,OAAOT,IAAI;AACb;AAYO,SAASkB,YAAYA,CAC1BC,cAA+B,EAC/B/B,MAAc,EACdgC,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAGlC,MAAM;EACjB,KAAK,IAAImC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAG,IAAAZ,wBAAa,EAACgB,kCAAmB,EAAEF,aAAa,CAACG,eAAgB,CAAC,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQJ,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbG,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIb,KAAK,CAAC,kCAAkC,CAAC;IACvD;IAEA,IAAIQ,aAAa,CAACM,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGb,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACM,YAAa,CAAC;MAC/ER,IAAI,GAAGS,GAAG,CAACT,IAAI;MACfD,MAAM,CAACG,aAAa,CAACQ,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACV;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMT,IAAI,GAAG,IAAAF,wBAAa,EAACwB,mBAAI,EAAEV,aAAa,CAACZ,IAAK,CAAC;MACrD,IAAIuB,WAAW,GAAGvB,IAAI;MAEtB,IAAIY,aAAa,CAACY,cAAc,EAAE;QAChCD,WAAW,GAAG,IAAAzB,wBAAa,EAAC2B,4BAAa,EAAEb,aAAa,CAACY,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,MAAAlB,MAAA,CAAMkB,WAAW,OAAAlB,MAAA,CAAIL,IAAI,CAAiB;UACrD;QACF;MACF;MAEAS,MAAM,CAACG,aAAa,CAACQ,IAAI,CAAC,GAAG;QAC3BpB,IAAI,EAAEuB,WAA0B;QAChCG,UAAU,EAAEd,aAAa,CAACe,WAAW;QACrCC,SAAS,EAAEhB,aAAa,CAACiB,SAAS;QAClCC,KAAK,EAAElB,aAAa,CAACkB,KAAK;QAC1Bd,QAAQ;QACRC;MACF,CAAC;MACDP,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAEjC,MAAM;IAAEkC;EAAI,CAAC;AAC/B;AAKA,SAASqB,YAAYA,CACnB/B,IAAmB,EACnBgC,QAAsB,EACtBzD,MAAoB,EACpBS,KAAa,EACbiD,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIE,sBAAc,CAAC,EAAE;IACjC,MAAM,IAAI9B,KAAK,sBAAAC,MAAA,CAAsB2B,QAAQ,CAAE,CAAC;EAClD;EACA,OAAOE,sBAAc,CAACF,QAAQ,CAAC,CAACD,YAAY,CAAC/B,IAAI,EAAEzB,MAAM,EAAES,KAAK,EAAEiD,IAAI,CAAC;AACzE;AAQA,eAAehC,cAAcA,CAC3B1B,MAAoB,EACpB4D,MAAkB,EAClB7D,OAAuB,EACG;EAAA,IAAA8D,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMC,SAAS,GAAGjE,MAAM,CAACC,MAAM,GAAG2D,MAAM,CAACM,oBAAoB;EAC7D,MAAMC,UAAU,IAAAN,qBAAA,GAAGD,MAAM,CAACQ,gBAAgB,cAAAP,qBAAA,uBAAvBA,qBAAA,CAAyBQ,UAAU;EAGtD,IAAIC,UAAU,GAAGtE,MAAM;EAEvB,IAAID,OAAO,CAACwE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM,IAAAC,uBAAU,EAChC1E,OAAO,CAACwE,WAAW,EACnBvE,MAAM,CAACF,MAAM,CAAC4E,KAAK,CAAC1E,MAAM,CAACC,MAAM,EAAEgE,SAAS,CAAC,EAC7CL,MAAM,CAACe,sBACT,CAAC;IACDL,UAAU,GAAG;MACXxE,MAAM,EAAE0E,SAAS;MACjBvE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAEsE,SAAS,CAACrE;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGgE,SAAS;EAC3B;EAGA,MAAMW,cAAc,GAAG,IAAArD,wBAAa,EAClCsD,uBAAQ,GAAAf,sBAAA,GACRF,MAAM,CAACQ,gBAAgB,cAAAN,sBAAA,uBAAvBA,sBAAA,CAAyBgB,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACb,UAAU,CAAC;EAEnC,IAAIpE,OAAO,CAACkF,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGvB,YAAY,CAAC2B,6BAAkB,EAAEP,cAAc,EAAEN,UAAU,EAAEH,UAAU,EAAG;MAClFiB,QAAQ,EAAE,IAAAC,sBAAW,EAACtF,OAAO,CAACkF,MAAM,CAACC,SAAS,CAAC;MAC/CI,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLP,OAAO,CAACQ,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,MAAMC,cAAc,GAAG,IAAAjE,wBAAa,EAClCsD,uBAAQ,GAAAd,sBAAA,GACRH,MAAM,CAACQ,gBAAgB,cAAAL,sBAAA,uBAAvBA,sBAAA,CAAyB0B,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIV,KAAK,CAACb,UAAU,CAAC;EACnC,IAAIpE,OAAO,CAACkF,MAAM,CAACU,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAGlC,YAAY,CAAC2B,6BAAkB,EAAEK,cAAc,EAAElB,UAAU,EAAEH,UAAU,EAAG;MAClFiB,QAAQ,EAAE,IAAAC,sBAAW,EAACtF,OAAO,CAACkF,MAAM,CAACU,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAK9F,OAAO,CAACkF,MAAM,CAACU,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;EAGA,MAAME,aAAa,GAAG,IAAAvE,wBAAa,EAACsD,uBAAQ,GAAAb,sBAAA,GAAEJ,MAAM,CAACQ,gBAAgB,cAAAJ,sBAAA,uBAAvBA,sBAAA,CAAyBP,QAAS,CAAiB;EACjG,MAAMsC,aAAa,GAAG;IACpB5C,UAAU,EAAEpD,OAAO,CAACkF,MAAM,CAAC9B,UAAU;IACrCiC,QAAQ,EAAErF,OAAO,CAACkF,MAAM,CAAC9B;EAC3B,CAAC;EAED,MAAM5C,MAAM,GAAGiD,YAAY,CACzBzD,OAAO,CAACkF,MAAM,CAACe,aAAa,EAC5BF,aAAa,EACbxB,UAAU,EACVsB,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACLzF,OAAO,EAAEoF,OAAO;IAChBrF,OAAO,EAAE0E,OAAO;IAChBxE,MAAM;IACNE,KAAK,EAAE0D,UAAW;IAClB/C,UAAU,EAAEwC;EACd,CAAC;AACH;AASA,eAAejC,gBAAgBA,CAC7B3B,MAAoB,EACpB4D,MAAkB,EAClBF,IAAS,EACiB;EAAA,IAAAuC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMnC,SAAS,GAAGjE,MAAM,CAACC,MAAM,GAAG2D,MAAM,CAACM,oBAAoB;EAE7D,MAAMC,UAAU,IAAA8B,sBAAA,GAAGrC,MAAM,CAACyC,mBAAmB,cAAAJ,sBAAA,uBAA1BA,sBAAA,CAA4B5B,UAAU;EAEzD,MAAMuB,iBAAiB,GAAGzB,UAAU,KAAA+B,sBAAA,GAAGtC,MAAM,CAACyC,mBAAmB,cAAAH,sBAAA,uBAA1BA,sBAAA,CAA4BI,SAAS;EAC5E,MAAMR,aAAa,GAAG,IAAAvE,wBAAa,EACjCsD,uBAAQ,GAAAsB,sBAAA,GACRvC,MAAM,CAACyC,mBAAmB,cAAAF,sBAAA,uBAA1BA,sBAAA,CAA4B1C,QAC9B,CAAiB;EAIjB,IAAIsB,OAAO,GAAG,IAAIC,KAAK,CAACb,UAAU,CAAC;EACnC,IAAIT,IAAI,CAACuB,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAC7BH,OAAO,GAAGvB,YAAY,CAAC2B,6BAAkB,EAAEoB,iCAAsB,EAAEvG,MAAM,EAAEmE,UAAU,EAAG;MACtFiB,QAAQ,EAAE,IAAAC,sBAAW,EAAC3B,IAAI,CAACuB,MAAM,CAACC,SAAS,CAAC;MAC5CI,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLP,OAAO,CAACQ,IAAI,CAAC,CAAC,CAAC;EACjB;EAIA,IAAIG,OAAO,GAAG,IAAIV,KAAK,CAACb,UAAU,CAAC;EACnC,IAAIT,IAAI,CAACuB,MAAM,CAACU,SAAS,GAAG,CAAC,EAAE;IAC7BD,OAAO,GAAGlC,YAAY,CAAC2B,6BAAkB,EAAEoB,iCAAsB,EAAEvG,MAAM,EAAEmE,UAAU,EAAG;MACtFiB,QAAQ,EAAE,IAAAC,sBAAW,EAAC3B,IAAI,CAACuB,MAAM,CAACU,SAAS,CAAC;MAC5CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,IAAIiB,eAAe,GAAGxG,MAAM;EAE5B,KAAAoG,sBAAA,GAAIxC,MAAM,CAACyC,mBAAmB,cAAAD,sBAAA,eAA1BA,sBAAA,CAA4BK,aAAa,EAAE;IAC7C,MAAMjC,SAAS,GAAG,MAAM,IAAAC,uBAAU,EAChCf,IAAI,CAACa,WAAW,EAChBvE,MAAM,CAACF,MAAM,CAAC4E,KAAK,CAAC1E,MAAM,CAACC,MAAM,EAAEgE,SAAS,CAAC,EAC7CL,MAAM,CAACe,sBACT,CAAC;IAED6B,eAAe,GAAG;MAChB1G,MAAM,EAAE0E,SAAS;MACjBvE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAEsE,SAAS,CAACrE;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGgE,SAAS;EAC3B;EAEA,MAAM8B,aAAa,GAAG;IACpB5C,UAAU,EAAEO,IAAI,CAACuB,MAAM,CAAC9B,UAAU;IAClCiC,QAAQ,EAAE1B,IAAI,CAACuB,MAAM,CAAC9B;EACxB,CAAC;EAED,MAAM5C,MAAM,GAAGiD,YAAY,CACzBE,IAAI,CAACuB,MAAM,CAACe,aAAa,EACzBF,aAAa,EACbU,eAAe,EACfZ,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACLzF,OAAO,EAAEoF,OAAO;IAChBrF,OAAO,EAAE0E,OAAO;IAChBxE,MAAM;IACNE,KAAK,EAAE0D,UAAW;IAClB/C,UAAU,EAAEwC;EACd,CAAC;AACH;AAQA,eAAehC,oBAAoBA,CACjC5B,MAAoB,EACpBoB,UAAsB,EACtBrB,OAAuB,EACJ;EAAA,IAAA2G,qBAAA;EACnB,MAAMzC,SAAS,GAAGjE,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAAC8C,oBAAoB;EAEjE,IAAIyC,UAAU,GAAG;IACf1G,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAAC4E,KAAK,CAAC1E,MAAM,CAACC,MAAM,EAAEgE,SAAS,CAAC;IACrD/D,IAAI,EAAE+D,SAAS,GAAGjE,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGgE,SAAS;EAEzB,IAAIlE,OAAO,CAACwE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM,IAAAC,uBAAU,EAChC1E,OAAO,CAACwE,WAAW,EACnBoC,UAAU,CAAC7G,MAAM,CAAC4E,KAAK,CAACiC,UAAU,CAAC1G,MAAM,EAAEgE,SAAS,CAAC,EACrD7C,UAAU,CAACuD,sBACb,CAAC;IAEDgC,UAAU,GAAG;MACX7G,MAAM,EAAE0E,SAAS;MACjBvE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAEsE,SAAS,CAACrE;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGgE,SAAS;EAC3B;EAEA,MAAMtD,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,wBAAAsF,qBAAA,GAAVtF,UAAU,CAAEwF,sBAAsB,cAAAF,qBAAA,uBAAlCA,qBAAA,CAAoCrC,UAAU,KAAI,CAAC;EAErE,OAAOb,YAAY,CACjBzD,OAAO,CAACkF,MAAM,CAACe,aAAa,EAC5BjG,OAAO,CAACkF,MAAM,CAACxB,QAAQ,EACvBkD,UAAU,EACVhG,SAAS,EACTZ,OACF,CAAC,CAACgB,GAAG,CAAE8F,CAAC,IAAKA,CAAC,CAACC,QAAQ,CAAC,CAAC,CAAC;AAC5B"}