@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (387) hide show
  1. package/dist/bundle.d.ts +2 -0
  2. package/dist/bundle.d.ts.map +1 -0
  3. package/dist/bundle.js +5 -0
  4. package/dist/constants.d.ts +15 -0
  5. package/dist/constants.d.ts.map +1 -0
  6. package/dist/constants.js +18 -0
  7. package/dist/dist.min.js +27 -13
  8. package/dist/dist.min.js.map +7 -1
  9. package/dist/es5/bundle.js +1 -1
  10. package/dist/es5/bundle.js.map +1 -1
  11. package/dist/es5/constants.js +5 -5
  12. package/dist/es5/constants.js.map +1 -1
  13. package/dist/es5/index.js +16 -45
  14. package/dist/es5/index.js.map +1 -1
  15. package/dist/es5/lib/convert-schema.js +13 -13
  16. package/dist/es5/lib/convert-schema.js.map +1 -1
  17. package/dist/es5/lib/parse-parquet.js +19 -154
  18. package/dist/es5/lib/parse-parquet.js.map +1 -1
  19. package/dist/es5/lib/read-array-buffer.js +6 -43
  20. package/dist/es5/lib/read-array-buffer.js.map +1 -1
  21. package/dist/es5/parquet-loader.js +4 -4
  22. package/dist/es5/parquet-loader.js.map +1 -1
  23. package/dist/es5/parquet-writer.js +4 -4
  24. package/dist/es5/parquet-writer.js.map +1 -1
  25. package/dist/es5/parquetjs/codecs/dictionary.js +2 -10
  26. package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
  27. package/dist/es5/parquetjs/codecs/index.js +4 -6
  28. package/dist/es5/parquetjs/codecs/index.js.map +1 -1
  29. package/dist/es5/parquetjs/codecs/plain.js +41 -43
  30. package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
  31. package/dist/es5/parquetjs/codecs/rle.js +25 -35
  32. package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
  33. package/dist/es5/parquetjs/compression.js +28 -122
  34. package/dist/es5/parquetjs/compression.js.map +1 -1
  35. package/dist/es5/parquetjs/encoder/writer.js +301 -737
  36. package/dist/es5/parquetjs/encoder/writer.js.map +1 -1
  37. package/dist/es5/parquetjs/file.js +15 -15
  38. package/dist/es5/parquetjs/file.js.map +1 -1
  39. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
  40. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +31 -45
  41. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  42. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +141 -152
  43. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  44. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +147 -160
  45. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  46. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +248 -259
  47. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  48. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +67 -79
  49. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  50. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
  51. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
  52. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +113 -124
  53. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  54. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +158 -169
  55. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  56. package/dist/es5/parquetjs/parquet-thrift/DateType.js +31 -45
  57. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
  58. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +68 -79
  59. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  60. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +83 -94
  61. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  62. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
  63. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +31 -45
  64. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  65. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
  66. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +170 -182
  67. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  68. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +31 -45
  69. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  70. package/dist/es5/parquetjs/parquet-thrift/IntType.js +68 -79
  71. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
  72. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +31 -45
  73. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  74. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +68 -79
  75. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  76. package/dist/es5/parquetjs/parquet-thrift/ListType.js +31 -45
  77. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
  78. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +319 -343
  79. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  80. package/dist/es5/parquetjs/parquet-thrift/MapType.js +31 -45
  81. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
  82. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +31 -45
  83. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  84. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +31 -45
  85. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  86. package/dist/es5/parquetjs/parquet-thrift/NullType.js +31 -45
  87. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
  88. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +64 -75
  89. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  90. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +83 -94
  91. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  92. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +158 -169
  93. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  94. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +83 -94
  95. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  96. package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
  97. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +113 -124
  98. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  99. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +188 -199
  100. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  101. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +83 -94
  102. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  103. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +124 -135
  104. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  105. package/dist/es5/parquetjs/parquet-thrift/StringType.js +31 -45
  106. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
  107. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +68 -79
  108. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  109. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +88 -101
  110. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  111. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +68 -79
  112. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  113. package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
  114. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +31 -45
  115. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  116. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +31 -45
  117. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  118. package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
  119. package/dist/es5/parquetjs/parser/decoders.js +218 -397
  120. package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
  121. package/dist/es5/parquetjs/parser/parquet-cursor.js +62 -180
  122. package/dist/es5/parquetjs/parser/parquet-cursor.js.map +1 -1
  123. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +124 -408
  124. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
  125. package/dist/es5/parquetjs/parser/parquet-reader.js +91 -369
  126. package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
  127. package/dist/es5/parquetjs/schema/declare.js +9 -11
  128. package/dist/es5/parquetjs/schema/declare.js.map +1 -1
  129. package/dist/es5/parquetjs/schema/schema.js +73 -87
  130. package/dist/es5/parquetjs/schema/schema.js.map +1 -1
  131. package/dist/es5/parquetjs/schema/shred.js +56 -96
  132. package/dist/es5/parquetjs/schema/shred.js.map +1 -1
  133. package/dist/es5/parquetjs/schema/types.js +39 -40
  134. package/dist/es5/parquetjs/schema/types.js.map +1 -1
  135. package/dist/es5/parquetjs/utils/buffer-utils.js +1 -1
  136. package/dist/es5/parquetjs/utils/buffer-utils.js.map +1 -1
  137. package/dist/es5/parquetjs/utils/file-utils.js +8 -65
  138. package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
  139. package/dist/es5/parquetjs/utils/read-utils.js +22 -50
  140. package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
  141. package/dist/esm/index.js +2 -3
  142. package/dist/esm/index.js.map +1 -1
  143. package/dist/esm/parquet-loader.js +1 -1
  144. package/dist/esm/parquet-loader.js.map +1 -1
  145. package/dist/esm/parquet-writer.js +1 -1
  146. package/dist/esm/parquet-writer.js.map +1 -1
  147. package/dist/esm/parquetjs/codecs/plain.js +3 -3
  148. package/dist/esm/parquetjs/codecs/plain.js.map +1 -1
  149. package/dist/esm/parquetjs/codecs/rle.js +1 -1
  150. package/dist/esm/parquetjs/codecs/rle.js.map +1 -1
  151. package/dist/esm/parquetjs/compression.js +4 -13
  152. package/dist/esm/parquetjs/compression.js.map +1 -1
  153. package/dist/esm/parquetjs/encoder/writer.js +1 -1
  154. package/dist/esm/parquetjs/encoder/writer.js.map +1 -1
  155. package/dist/esm/parquetjs/parser/decoders.js +4 -4
  156. package/dist/esm/parquetjs/parser/decoders.js.map +1 -1
  157. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +4 -13
  158. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
  159. package/dist/esm/parquetjs/parser/parquet-reader.js +0 -13
  160. package/dist/esm/parquetjs/parser/parquet-reader.js.map +1 -1
  161. package/dist/esm/parquetjs/schema/schema.js +3 -3
  162. package/dist/esm/parquetjs/schema/schema.js.map +1 -1
  163. package/dist/esm/parquetjs/schema/shred.js +2 -2
  164. package/dist/esm/parquetjs/schema/shred.js.map +1 -1
  165. package/dist/esm/parquetjs/schema/types.js +20 -20
  166. package/dist/esm/parquetjs/schema/types.js.map +1 -1
  167. package/dist/esm/parquetjs/utils/file-utils.js +0 -45
  168. package/dist/esm/parquetjs/utils/file-utils.js.map +1 -1
  169. package/dist/index.d.ts +28 -0
  170. package/dist/index.d.ts.map +1 -0
  171. package/dist/index.js +30 -0
  172. package/dist/lib/convert-schema.d.ts +8 -0
  173. package/dist/lib/convert-schema.d.ts.map +1 -0
  174. package/dist/lib/convert-schema.js +70 -0
  175. package/dist/lib/parse-parquet.d.ts +4 -0
  176. package/dist/lib/parse-parquet.d.ts.map +1 -0
  177. package/dist/lib/parse-parquet.js +28 -0
  178. package/dist/lib/read-array-buffer.d.ts +19 -0
  179. package/dist/lib/read-array-buffer.d.ts.map +1 -0
  180. package/dist/lib/read-array-buffer.js +29 -0
  181. package/dist/parquet-loader.d.ts +23 -0
  182. package/dist/parquet-loader.d.ts.map +1 -0
  183. package/dist/parquet-loader.js +27 -0
  184. package/dist/parquet-worker.js +27 -13
  185. package/dist/parquet-worker.js.map +7 -1
  186. package/dist/parquet-writer.d.ts +4 -0
  187. package/dist/parquet-writer.d.ts.map +1 -0
  188. package/dist/parquet-writer.js +21 -0
  189. package/dist/parquetjs/codecs/declare.d.ts +17 -0
  190. package/dist/parquetjs/codecs/declare.d.ts.map +1 -0
  191. package/dist/parquetjs/codecs/declare.js +2 -0
  192. package/dist/parquetjs/codecs/dictionary.d.ts +3 -0
  193. package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -0
  194. package/dist/parquetjs/codecs/dictionary.js +14 -0
  195. package/dist/parquetjs/codecs/index.d.ts +5 -0
  196. package/dist/parquetjs/codecs/index.d.ts.map +1 -0
  197. package/dist/parquetjs/codecs/index.js +51 -0
  198. package/dist/parquetjs/codecs/plain.d.ts +6 -0
  199. package/dist/parquetjs/codecs/plain.d.ts.map +1 -0
  200. package/dist/parquetjs/codecs/plain.js +211 -0
  201. package/dist/parquetjs/codecs/rle.d.ts +6 -0
  202. package/dist/parquetjs/codecs/rle.d.ts.map +1 -0
  203. package/dist/parquetjs/codecs/rle.js +145 -0
  204. package/dist/parquetjs/compression.d.ts +23 -0
  205. package/dist/parquetjs/compression.d.ts.map +1 -0
  206. package/dist/parquetjs/compression.js +168 -0
  207. package/dist/parquetjs/encoder/writer.d.ts +123 -0
  208. package/dist/parquetjs/encoder/writer.d.ts.map +1 -0
  209. package/dist/parquetjs/encoder/writer.js +478 -0
  210. package/dist/parquetjs/file.d.ts +10 -0
  211. package/dist/parquetjs/file.d.ts.map +1 -0
  212. package/dist/parquetjs/file.js +99 -0
  213. package/dist/parquetjs/parquet-thrift/BoundaryOrder.d.ts +6 -0
  214. package/dist/parquetjs/parquet-thrift/BoundaryOrder.d.ts.map +1 -0
  215. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +15 -0
  216. package/dist/parquetjs/parquet-thrift/BsonType.d.ts +9 -0
  217. package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -0
  218. package/dist/parquetjs/parquet-thrift/BsonType.js +58 -0
  219. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +25 -0
  220. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -0
  221. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +207 -0
  222. package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +22 -0
  223. package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -0
  224. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +213 -0
  225. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +42 -0
  226. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -0
  227. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +398 -0
  228. package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +13 -0
  229. package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -0
  230. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +104 -0
  231. package/dist/parquetjs/parquet-thrift/CompressionCodec.d.ts +11 -0
  232. package/dist/parquetjs/parquet-thrift/CompressionCodec.d.ts.map +1 -0
  233. package/dist/parquetjs/parquet-thrift/CompressionCodec.js +20 -0
  234. package/dist/parquetjs/parquet-thrift/ConvertedType.d.ts +25 -0
  235. package/dist/parquetjs/parquet-thrift/ConvertedType.d.ts.map +1 -0
  236. package/dist/parquetjs/parquet-thrift/ConvertedType.js +34 -0
  237. package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +21 -0
  238. package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -0
  239. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +166 -0
  240. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +27 -0
  241. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -0
  242. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +226 -0
  243. package/dist/parquetjs/parquet-thrift/DateType.d.ts +9 -0
  244. package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -0
  245. package/dist/parquetjs/parquet-thrift/DateType.js +58 -0
  246. package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +13 -0
  247. package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -0
  248. package/dist/parquetjs/parquet-thrift/DecimalType.js +105 -0
  249. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +16 -0
  250. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -0
  251. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +122 -0
  252. package/dist/parquetjs/parquet-thrift/Encoding.d.ts +11 -0
  253. package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -0
  254. package/dist/parquetjs/parquet-thrift/Encoding.js +20 -0
  255. package/dist/parquetjs/parquet-thrift/EnumType.d.ts +9 -0
  256. package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -0
  257. package/dist/parquetjs/parquet-thrift/EnumType.js +58 -0
  258. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.d.ts +6 -0
  259. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.d.ts.map +1 -0
  260. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +15 -0
  261. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +28 -0
  262. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -0
  263. package/dist/parquetjs/parquet-thrift/FileMetaData.js +256 -0
  264. package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +9 -0
  265. package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -0
  266. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +58 -0
  267. package/dist/parquetjs/parquet-thrift/IntType.d.ts +13 -0
  268. package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -0
  269. package/dist/parquetjs/parquet-thrift/IntType.js +105 -0
  270. package/dist/parquetjs/parquet-thrift/JsonType.d.ts +9 -0
  271. package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -0
  272. package/dist/parquetjs/parquet-thrift/JsonType.js +58 -0
  273. package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +13 -0
  274. package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -0
  275. package/dist/parquetjs/parquet-thrift/KeyValue.js +102 -0
  276. package/dist/parquetjs/parquet-thrift/ListType.d.ts +9 -0
  277. package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -0
  278. package/dist/parquetjs/parquet-thrift/ListType.js +58 -0
  279. package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +61 -0
  280. package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -0
  281. package/dist/parquetjs/parquet-thrift/LogicalType.js +380 -0
  282. package/dist/parquetjs/parquet-thrift/MapType.d.ts +9 -0
  283. package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -0
  284. package/dist/parquetjs/parquet-thrift/MapType.js +58 -0
  285. package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +9 -0
  286. package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -0
  287. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +58 -0
  288. package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +9 -0
  289. package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -0
  290. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +58 -0
  291. package/dist/parquetjs/parquet-thrift/NullType.d.ts +9 -0
  292. package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -0
  293. package/dist/parquetjs/parquet-thrift/NullType.js +58 -0
  294. package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +12 -0
  295. package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -0
  296. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +97 -0
  297. package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +17 -0
  298. package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -0
  299. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +127 -0
  300. package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +30 -0
  301. package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -0
  302. package/dist/parquetjs/parquet-thrift/PageHeader.js +216 -0
  303. package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +16 -0
  304. package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -0
  305. package/dist/parquetjs/parquet-thrift/PageLocation.js +141 -0
  306. package/dist/parquetjs/parquet-thrift/PageType.d.ts +7 -0
  307. package/dist/parquetjs/parquet-thrift/PageType.d.ts.map +1 -0
  308. package/dist/parquetjs/parquet-thrift/PageType.js +16 -0
  309. package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +20 -0
  310. package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -0
  311. package/dist/parquetjs/parquet-thrift/RowGroup.js +182 -0
  312. package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +33 -0
  313. package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -0
  314. package/dist/parquetjs/parquet-thrift/SchemaElement.js +239 -0
  315. package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +15 -0
  316. package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -0
  317. package/dist/parquetjs/parquet-thrift/SortingColumn.js +127 -0
  318. package/dist/parquetjs/parquet-thrift/Statistics.d.ts +23 -0
  319. package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -0
  320. package/dist/parquetjs/parquet-thrift/Statistics.js +176 -0
  321. package/dist/parquetjs/parquet-thrift/StringType.d.ts +9 -0
  322. package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -0
  323. package/dist/parquetjs/parquet-thrift/StringType.js +58 -0
  324. package/dist/parquetjs/parquet-thrift/TimeType.d.ts +14 -0
  325. package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -0
  326. package/dist/parquetjs/parquet-thrift/TimeType.js +106 -0
  327. package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +17 -0
  328. package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -0
  329. package/dist/parquetjs/parquet-thrift/TimeUnit.js +127 -0
  330. package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +14 -0
  331. package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -0
  332. package/dist/parquetjs/parquet-thrift/TimestampType.js +106 -0
  333. package/dist/parquetjs/parquet-thrift/Type.d.ts +11 -0
  334. package/dist/parquetjs/parquet-thrift/Type.d.ts.map +1 -0
  335. package/dist/parquetjs/parquet-thrift/Type.js +20 -0
  336. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +9 -0
  337. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -0
  338. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +58 -0
  339. package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +9 -0
  340. package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -0
  341. package/dist/parquetjs/parquet-thrift/UUIDType.js +58 -0
  342. package/dist/parquetjs/parquet-thrift/index.d.ts +44 -0
  343. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -0
  344. package/dist/parquetjs/parquet-thrift/index.js +61 -0
  345. package/dist/parquetjs/parser/decoders.d.ts +34 -0
  346. package/dist/parquetjs/parser/decoders.d.ts.map +1 -0
  347. package/dist/parquetjs/parser/decoders.js +318 -0
  348. package/dist/parquetjs/parser/parquet-cursor.d.ts +36 -0
  349. package/dist/parquetjs/parser/parquet-cursor.d.ts.map +1 -0
  350. package/dist/parquetjs/parser/parquet-cursor.js +74 -0
  351. package/dist/parquetjs/parser/parquet-envelope-reader.d.ts +40 -0
  352. package/dist/parquetjs/parser/parquet-envelope-reader.d.ts.map +1 -0
  353. package/dist/parquetjs/parser/parquet-envelope-reader.js +136 -0
  354. package/dist/parquetjs/parser/parquet-reader.d.ts +68 -0
  355. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -0
  356. package/dist/parquetjs/parser/parquet-reader.js +134 -0
  357. package/dist/parquetjs/schema/declare.d.ts +80 -0
  358. package/dist/parquetjs/schema/declare.d.ts.map +1 -0
  359. package/dist/parquetjs/schema/declare.js +10 -0
  360. package/dist/parquetjs/schema/schema.d.ts +26 -0
  361. package/dist/parquetjs/schema/schema.d.ts.map +1 -0
  362. package/dist/parquetjs/schema/schema.js +162 -0
  363. package/dist/parquetjs/schema/shred.d.ts +48 -0
  364. package/dist/parquetjs/schema/shred.d.ts.map +1 -0
  365. package/dist/parquetjs/schema/shred.js +225 -0
  366. package/dist/parquetjs/schema/types.d.ts +20 -0
  367. package/dist/parquetjs/schema/types.d.ts.map +1 -0
  368. package/dist/parquetjs/schema/types.js +418 -0
  369. package/dist/parquetjs/utils/buffer-utils.d.ts +10 -0
  370. package/dist/parquetjs/utils/buffer-utils.d.ts.map +1 -0
  371. package/dist/parquetjs/utils/buffer-utils.js +22 -0
  372. package/dist/parquetjs/utils/file-utils.d.ts +16 -0
  373. package/dist/parquetjs/utils/file-utils.d.ts.map +1 -0
  374. package/dist/parquetjs/utils/file-utils.js +46 -0
  375. package/dist/parquetjs/utils/read-utils.d.ts +25 -0
  376. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -0
  377. package/dist/parquetjs/utils/read-utils.js +109 -0
  378. package/dist/workers/parquet-worker.d.ts +2 -0
  379. package/dist/workers/parquet-worker.d.ts.map +1 -0
  380. package/dist/workers/parquet-worker.js +5 -0
  381. package/package.json +8 -8
  382. package/src/index.ts +3 -3
  383. package/src/parquetjs/compression.ts +10 -10
  384. package/src/parquetjs/parser/decoders.ts +1 -1
  385. package/src/parquetjs/parser/parquet-envelope-reader.ts +0 -11
  386. package/src/parquetjs/parser/parquet-reader.ts +0 -16
  387. package/src/parquetjs/utils/file-utils.ts +0 -49
@@ -1,7 +1,5 @@
1
1
  "use strict";
2
2
 
3
- var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
-
5
3
  Object.defineProperty(exports, "__esModule", {
6
4
  value: true
7
5
  });
@@ -9,10 +7,6 @@ exports.decodeDataPages = decodeDataPages;
9
7
  exports.decodePage = decodePage;
10
8
  exports.decodeSchema = decodeSchema;
11
9
 
12
- var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
13
-
14
- var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
15
-
16
10
  var _codecs = require("../codecs");
17
11
 
18
12
  var _parquetThrift = require("../parquet-thrift");
@@ -23,171 +17,91 @@ var _constants = require("../../constants");
23
17
 
24
18
  var _readUtils = require("../utils/read-utils");
25
19
 
26
- function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
20
+ async function decodeDataPages(buffer, options) {
21
+ const cursor = {
22
+ buffer,
23
+ offset: 0,
24
+ size: buffer.length
25
+ };
26
+ const data = {
27
+ rlevels: [],
28
+ dlevels: [],
29
+ values: [],
30
+ pageHeaders: [],
31
+ count: 0
32
+ };
33
+ let dictionary = options.dictionary || [];
27
34
 
28
- function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
35
+ while (cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
36
+ const page = await decodePage(cursor, options);
29
37
 
30
- function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
38
+ if (page.dictionary) {
39
+ dictionary = page.dictionary;
40
+ continue;
41
+ }
31
42
 
32
- function decodeDataPages(_x, _x2) {
33
- return _decodeDataPages.apply(this, arguments);
34
- }
43
+ if (dictionary.length) {
44
+ page.values = page.values.map(value => dictionary[value]);
45
+ }
46
+
47
+ for (let index = 0; index < page.rlevels.length; index++) {
48
+ data.rlevels.push(page.rlevels[index]);
49
+ data.dlevels.push(page.dlevels[index]);
50
+ const value = page.values[index];
35
51
 
36
- function _decodeDataPages() {
37
- _decodeDataPages = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(buffer, options) {
38
- var cursor, data, dictionary, page, index, value;
39
- return _regenerator.default.wrap(function _callee$(_context) {
40
- while (1) {
41
- switch (_context.prev = _context.next) {
42
- case 0:
43
- cursor = {
44
- buffer: buffer,
45
- offset: 0,
46
- size: buffer.length
47
- };
48
- data = {
49
- rlevels: [],
50
- dlevels: [],
51
- values: [],
52
- pageHeaders: [],
53
- count: 0
54
- };
55
- dictionary = options.dictionary || [];
56
-
57
- case 3:
58
- if (!(cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues)))) {
59
- _context.next = 16;
60
- break;
61
- }
62
-
63
- _context.next = 6;
64
- return decodePage(cursor, options);
65
-
66
- case 6:
67
- page = _context.sent;
68
-
69
- if (!page.dictionary) {
70
- _context.next = 10;
71
- break;
72
- }
73
-
74
- dictionary = page.dictionary;
75
- return _context.abrupt("continue", 3);
76
-
77
- case 10:
78
- if (dictionary.length) {
79
- page.values = page.values.map(function (value) {
80
- return dictionary[value];
81
- });
82
- }
83
-
84
- for (index = 0; index < page.rlevels.length; index++) {
85
- data.rlevels.push(page.rlevels[index]);
86
- data.dlevels.push(page.dlevels[index]);
87
- value = page.values[index];
88
-
89
- if (value !== undefined) {
90
- data.values.push(value);
91
- }
92
- }
93
-
94
- data.count += page.count;
95
- data.pageHeaders.push(page.pageHeader);
96
- _context.next = 3;
97
- break;
98
-
99
- case 16:
100
- return _context.abrupt("return", data);
101
-
102
- case 17:
103
- case "end":
104
- return _context.stop();
105
- }
52
+ if (value !== undefined) {
53
+ data.values.push(value);
106
54
  }
107
- }, _callee);
108
- }));
109
- return _decodeDataPages.apply(this, arguments);
110
- }
55
+ }
56
+
57
+ data.count += page.count;
58
+ data.pageHeaders.push(page.pageHeader);
59
+ }
111
60
 
112
- function decodePage(_x3, _x4) {
113
- return _decodePage.apply(this, arguments);
61
+ return data;
114
62
  }
115
63
 
116
- function _decodePage() {
117
- _decodePage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(cursor, options) {
118
- var page, _yield$decodePageHead, pageHeader, length, pageType;
119
-
120
- return _regenerator.default.wrap(function _callee2$(_context2) {
121
- while (1) {
122
- switch (_context2.prev = _context2.next) {
123
- case 0:
124
- _context2.next = 2;
125
- return (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset);
126
-
127
- case 2:
128
- _yield$decodePageHead = _context2.sent;
129
- pageHeader = _yield$decodePageHead.pageHeader;
130
- length = _yield$decodePageHead.length;
131
- cursor.offset += length;
132
- pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
133
- _context2.t0 = pageType;
134
- _context2.next = _context2.t0 === 'DATA_PAGE' ? 10 : _context2.t0 === 'DATA_PAGE_V2' ? 14 : _context2.t0 === 'DICTIONARY_PAGE' ? 18 : 24;
135
- break;
136
-
137
- case 10:
138
- _context2.next = 12;
139
- return decodeDataPage(cursor, pageHeader, options);
140
-
141
- case 12:
142
- page = _context2.sent;
143
- return _context2.abrupt("break", 25);
144
-
145
- case 14:
146
- _context2.next = 16;
147
- return decodeDataPageV2(cursor, pageHeader, options);
148
-
149
- case 16:
150
- page = _context2.sent;
151
- return _context2.abrupt("break", 25);
152
-
153
- case 18:
154
- _context2.next = 20;
155
- return decodeDictionaryPage(cursor, pageHeader, options);
156
-
157
- case 20:
158
- _context2.t1 = _context2.sent;
159
- _context2.t2 = pageHeader;
160
- page = {
161
- dictionary: _context2.t1,
162
- pageHeader: _context2.t2
163
- };
164
- return _context2.abrupt("break", 25);
165
-
166
- case 24:
167
- throw new Error("invalid page type: ".concat(pageType));
168
-
169
- case 25:
170
- return _context2.abrupt("return", page);
171
-
172
- case 26:
173
- case "end":
174
- return _context2.stop();
175
- }
176
- }
177
- }, _callee2);
178
- }));
179
- return _decodePage.apply(this, arguments);
64
+ async function decodePage(cursor, options) {
65
+ let page;
66
+ const {
67
+ pageHeader,
68
+ length
69
+ } = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset);
70
+ cursor.offset += length;
71
+ const pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
72
+
73
+ switch (pageType) {
74
+ case 'DATA_PAGE':
75
+ page = await decodeDataPage(cursor, pageHeader, options);
76
+ break;
77
+
78
+ case 'DATA_PAGE_V2':
79
+ page = await decodeDataPageV2(cursor, pageHeader, options);
80
+ break;
81
+
82
+ case 'DICTIONARY_PAGE':
83
+ page = {
84
+ dictionary: await decodeDictionaryPage(cursor, pageHeader, options),
85
+ pageHeader
86
+ };
87
+ break;
88
+
89
+ default:
90
+ throw new Error("invalid page type: ".concat(pageType));
91
+ }
92
+
93
+ return page;
180
94
  }
181
95
 
182
96
  function decodeSchema(schemaElements, offset, len) {
183
- var schema = {};
184
- var next = offset;
97
+ const schema = {};
98
+ let next = offset;
185
99
 
186
- for (var i = 0; i < len; i++) {
187
- var schemaElement = schemaElements[next];
188
- var repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
189
- var optional = false;
190
- var repeated = false;
100
+ for (let i = 0; i < len; i++) {
101
+ const schemaElement = schemaElements[next];
102
+ const repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
103
+ let optional = false;
104
+ let repeated = false;
191
105
 
192
106
  switch (repetitionType) {
193
107
  case 'REQUIRED':
@@ -206,16 +120,16 @@ function decodeSchema(schemaElements, offset, len) {
206
120
  }
207
121
 
208
122
  if (schemaElement.num_children > 0) {
209
- var res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
123
+ const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
210
124
  next = res.next;
211
125
  schema[schemaElement.name] = {
212
- optional: optional,
213
- repeated: repeated,
126
+ optional,
127
+ repeated,
214
128
  fields: res.schema
215
129
  };
216
130
  } else {
217
- var type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
218
- var logicalType = type;
131
+ const type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
132
+ let logicalType = type;
219
133
 
220
134
  if (schemaElement.converted_type) {
221
135
  logicalType = (0, _readUtils.getThriftEnum)(_parquetThrift.ConvertedType, schemaElement.converted_type);
@@ -234,17 +148,17 @@ function decodeSchema(schemaElements, offset, len) {
234
148
  typeLength: schemaElement.type_length,
235
149
  presision: schemaElement.precision,
236
150
  scale: schemaElement.scale,
237
- optional: optional,
238
- repeated: repeated
151
+ optional,
152
+ repeated
239
153
  };
240
154
  next++;
241
155
  }
242
156
  }
243
157
 
244
158
  return {
245
- schema: schema,
246
- offset: offset,
247
- next: next
159
+ schema,
160
+ offset,
161
+ next
248
162
  };
249
163
  }
250
164
 
@@ -256,240 +170,147 @@ function decodeValues(type, encoding, cursor, count, opts) {
256
170
  return _codecs.PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
257
171
  }
258
172
 
259
- function decodeDataPage(_x5, _x6, _x7) {
260
- return _decodeDataPage.apply(this, arguments);
261
- }
173
+ async function decodeDataPage(cursor, header, options) {
174
+ var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
175
+
176
+ const cursorEnd = cursor.offset + header.compressed_page_size;
177
+ const valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
178
+ let dataCursor = cursor;
179
+
180
+ if (options.compression !== 'UNCOMPRESSED') {
181
+ const valuesBuf = await (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
182
+ dataCursor = {
183
+ buffer: valuesBuf,
184
+ offset: 0,
185
+ size: valuesBuf.length
186
+ };
187
+ cursor.offset = cursorEnd;
188
+ }
262
189
 
263
- function _decodeDataPage() {
264
- _decodeDataPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(cursor, header, options) {
265
- var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
266
-
267
- var cursorEnd, valueCount, dataCursor, valuesBuf, rLevelEncoding, rLevels, dLevelEncoding, dLevels, valueCountNonNull, _iterator, _step, dlvl, valueEncoding, decodeOptions, values;
268
-
269
- return _regenerator.default.wrap(function _callee3$(_context3) {
270
- while (1) {
271
- switch (_context3.prev = _context3.next) {
272
- case 0:
273
- cursorEnd = cursor.offset + header.compressed_page_size;
274
- valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
275
- dataCursor = cursor;
276
-
277
- if (!(options.compression !== 'UNCOMPRESSED')) {
278
- _context3.next = 9;
279
- break;
280
- }
281
-
282
- _context3.next = 6;
283
- return (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
284
-
285
- case 6:
286
- valuesBuf = _context3.sent;
287
- dataCursor = {
288
- buffer: valuesBuf,
289
- offset: 0,
290
- size: valuesBuf.length
291
- };
292
- cursor.offset = cursorEnd;
293
-
294
- case 9:
295
- rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
296
- rLevels = new Array(valueCount);
297
-
298
- if (options.column.rLevelMax > 0) {
299
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
300
- bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
301
- disableEnvelope: false
302
- });
303
- } else {
304
- rLevels.fill(0);
305
- }
306
-
307
- dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
308
- dLevels = new Array(valueCount);
309
-
310
- if (options.column.dLevelMax > 0) {
311
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
312
- bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
313
- disableEnvelope: false
314
- });
315
- } else {
316
- dLevels.fill(0);
317
- }
318
-
319
- valueCountNonNull = 0;
320
- _iterator = _createForOfIteratorHelper(dLevels);
321
-
322
- try {
323
- for (_iterator.s(); !(_step = _iterator.n()).done;) {
324
- dlvl = _step.value;
325
-
326
- if (dlvl === options.column.dLevelMax) {
327
- valueCountNonNull++;
328
- }
329
- }
330
- } catch (err) {
331
- _iterator.e(err);
332
- } finally {
333
- _iterator.f();
334
- }
335
-
336
- valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
337
- decodeOptions = {
338
- typeLength: options.column.typeLength,
339
- bitWidth: options.column.typeLength
340
- };
341
- values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
342
- return _context3.abrupt("return", {
343
- dlevels: dLevels,
344
- rlevels: rLevels,
345
- values: values,
346
- count: valueCount,
347
- pageHeader: header
348
- });
349
-
350
- case 22:
351
- case "end":
352
- return _context3.stop();
353
- }
354
- }
355
- }, _callee3);
356
- }));
357
- return _decodeDataPage.apply(this, arguments);
358
- }
190
+ const rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
191
+ let rLevels = new Array(valueCount);
359
192
 
360
- function decodeDataPageV2(_x8, _x9, _x10) {
361
- return _decodeDataPageV.apply(this, arguments);
362
- }
193
+ if (options.column.rLevelMax > 0) {
194
+ rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
195
+ bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
196
+ disableEnvelope: false
197
+ });
198
+ } else {
199
+ rLevels.fill(0);
200
+ }
363
201
 
364
- function _decodeDataPageV() {
365
- _decodeDataPageV = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4(cursor, header, opts) {
366
- var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
367
-
368
- var cursorEnd, valueCount, valueCountNonNull, valueEncoding, rLevels, dLevels, valuesBufCursor, valuesBuf, decodeOptions, values;
369
- return _regenerator.default.wrap(function _callee4$(_context4) {
370
- while (1) {
371
- switch (_context4.prev = _context4.next) {
372
- case 0:
373
- cursorEnd = cursor.offset + header.compressed_page_size;
374
- valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
375
- valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
376
- valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
377
- rLevels = new Array(valueCount);
378
-
379
- if (opts.column.rLevelMax > 0) {
380
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
381
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
382
- disableEnvelope: true
383
- });
384
- } else {
385
- rLevels.fill(0);
386
- }
387
-
388
- dLevels = new Array(valueCount);
389
-
390
- if (opts.column.dLevelMax > 0) {
391
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
392
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
393
- disableEnvelope: true
394
- });
395
- } else {
396
- dLevels.fill(0);
397
- }
398
-
399
- valuesBufCursor = cursor;
400
-
401
- if (!((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed)) {
402
- _context4.next = 15;
403
- break;
404
- }
405
-
406
- _context4.next = 12;
407
- return (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
408
-
409
- case 12:
410
- valuesBuf = _context4.sent;
411
- valuesBufCursor = {
412
- buffer: valuesBuf,
413
- offset: 0,
414
- size: valuesBuf.length
415
- };
416
- cursor.offset = cursorEnd;
417
-
418
- case 15:
419
- decodeOptions = {
420
- typeLength: opts.column.typeLength,
421
- bitWidth: opts.column.typeLength
422
- };
423
- values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
424
- return _context4.abrupt("return", {
425
- dlevels: dLevels,
426
- rlevels: rLevels,
427
- values: values,
428
- count: valueCount,
429
- pageHeader: header
430
- });
431
-
432
- case 18:
433
- case "end":
434
- return _context4.stop();
435
- }
436
- }
437
- }, _callee4);
438
- }));
439
- return _decodeDataPageV.apply(this, arguments);
202
+ const dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
203
+ let dLevels = new Array(valueCount);
204
+
205
+ if (options.column.dLevelMax > 0) {
206
+ dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
207
+ bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
208
+ disableEnvelope: false
209
+ });
210
+ } else {
211
+ dLevels.fill(0);
212
+ }
213
+
214
+ let valueCountNonNull = 0;
215
+
216
+ for (const dlvl of dLevels) {
217
+ if (dlvl === options.column.dLevelMax) {
218
+ valueCountNonNull++;
219
+ }
220
+ }
221
+
222
+ const valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
223
+ const decodeOptions = {
224
+ typeLength: options.column.typeLength,
225
+ bitWidth: options.column.typeLength
226
+ };
227
+ const values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
228
+ return {
229
+ dlevels: dLevels,
230
+ rlevels: rLevels,
231
+ values,
232
+ count: valueCount,
233
+ pageHeader: header
234
+ };
440
235
  }
441
236
 
442
- function decodeDictionaryPage(_x11, _x12, _x13) {
443
- return _decodeDictionaryPage.apply(this, arguments);
237
+ async function decodeDataPageV2(cursor, header, opts) {
238
+ var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
239
+
240
+ const cursorEnd = cursor.offset + header.compressed_page_size;
241
+ const valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
242
+ const valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
243
+ const valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
244
+ let rLevels = new Array(valueCount);
245
+
246
+ if (opts.column.rLevelMax > 0) {
247
+ rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
248
+ bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
249
+ disableEnvelope: true
250
+ });
251
+ } else {
252
+ rLevels.fill(0);
253
+ }
254
+
255
+ let dLevels = new Array(valueCount);
256
+
257
+ if (opts.column.dLevelMax > 0) {
258
+ dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
259
+ bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
260
+ disableEnvelope: true
261
+ });
262
+ } else {
263
+ dLevels.fill(0);
264
+ }
265
+
266
+ let valuesBufCursor = cursor;
267
+
268
+ if ((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed) {
269
+ const valuesBuf = await (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
270
+ valuesBufCursor = {
271
+ buffer: valuesBuf,
272
+ offset: 0,
273
+ size: valuesBuf.length
274
+ };
275
+ cursor.offset = cursorEnd;
276
+ }
277
+
278
+ const decodeOptions = {
279
+ typeLength: opts.column.typeLength,
280
+ bitWidth: opts.column.typeLength
281
+ };
282
+ const values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
283
+ return {
284
+ dlevels: dLevels,
285
+ rlevels: rLevels,
286
+ values,
287
+ count: valueCount,
288
+ pageHeader: header
289
+ };
444
290
  }
445
291
 
446
- function _decodeDictionaryPage() {
447
- _decodeDictionaryPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5(cursor, pageHeader, options) {
448
- var _pageHeader$dictionar;
449
-
450
- var cursorEnd, dictCursor, valuesBuf, numValues;
451
- return _regenerator.default.wrap(function _callee5$(_context5) {
452
- while (1) {
453
- switch (_context5.prev = _context5.next) {
454
- case 0:
455
- cursorEnd = cursor.offset + pageHeader.compressed_page_size;
456
- dictCursor = {
457
- offset: 0,
458
- buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
459
- size: cursorEnd - cursor.offset
460
- };
461
- cursor.offset = cursorEnd;
462
-
463
- if (!(options.compression !== 'UNCOMPRESSED')) {
464
- _context5.next = 9;
465
- break;
466
- }
467
-
468
- _context5.next = 6;
469
- return (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
470
-
471
- case 6:
472
- valuesBuf = _context5.sent;
473
- dictCursor = {
474
- buffer: valuesBuf,
475
- offset: 0,
476
- size: valuesBuf.length
477
- };
478
- cursor.offset = cursorEnd;
479
-
480
- case 9:
481
- numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
482
- return _context5.abrupt("return", decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(function (d) {
483
- return d.toString();
484
- }));
485
-
486
- case 11:
487
- case "end":
488
- return _context5.stop();
489
- }
490
- }
491
- }, _callee5);
492
- }));
493
- return _decodeDictionaryPage.apply(this, arguments);
292
+ async function decodeDictionaryPage(cursor, pageHeader, options) {
293
+ var _pageHeader$dictionar;
294
+
295
+ const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
296
+ let dictCursor = {
297
+ offset: 0,
298
+ buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
299
+ size: cursorEnd - cursor.offset
300
+ };
301
+ cursor.offset = cursorEnd;
302
+
303
+ if (options.compression !== 'UNCOMPRESSED') {
304
+ const valuesBuf = await (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
305
+ dictCursor = {
306
+ buffer: valuesBuf,
307
+ offset: 0,
308
+ size: valuesBuf.length
309
+ };
310
+ cursor.offset = cursorEnd;
311
+ }
312
+
313
+ const numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
314
+ return decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(d => d.toString());
494
315
  }
495
316
  //# sourceMappingURL=decoders.js.map