@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (438) hide show
  1. package/dist/bundle.js +2 -2
  2. package/dist/bundle.js.map +1 -0
  3. package/dist/constants.js +6 -18
  4. package/dist/constants.js.map +1 -0
  5. package/dist/dist.min.js +17 -8
  6. package/dist/dist.min.js.map +3 -3
  7. package/dist/index.js +14 -29
  8. package/dist/index.js.map +1 -0
  9. package/dist/lib/convert-schema.js +63 -62
  10. package/dist/lib/convert-schema.js.map +1 -0
  11. package/dist/lib/parse-parquet.js +25 -25
  12. package/dist/lib/parse-parquet.js.map +1 -0
  13. package/dist/lib/read-array-buffer.js +8 -28
  14. package/dist/lib/read-array-buffer.js.map +1 -0
  15. package/dist/parquet-loader.js +19 -24
  16. package/dist/parquet-loader.js.map +1 -0
  17. package/dist/parquet-worker.js +18 -9
  18. package/dist/parquet-worker.js.map +3 -3
  19. package/dist/parquet-writer.js +14 -17
  20. package/dist/parquet-writer.js.map +1 -0
  21. package/dist/{es5/parquetjs → parquetjs}/LICENSE +0 -0
  22. package/dist/parquetjs/codecs/declare.js +2 -2
  23. package/dist/{es5/parquetjs → parquetjs}/codecs/declare.js.map +0 -0
  24. package/dist/parquetjs/codecs/dictionary.js +10 -12
  25. package/dist/parquetjs/codecs/dictionary.js.map +1 -0
  26. package/dist/parquetjs/codecs/index.js +22 -50
  27. package/dist/parquetjs/codecs/index.js.map +1 -0
  28. package/dist/parquetjs/codecs/plain.js +232 -173
  29. package/dist/parquetjs/codecs/plain.js.map +1 -0
  30. package/dist/parquetjs/codecs/rle.js +140 -134
  31. package/dist/parquetjs/codecs/rle.js.map +1 -0
  32. package/dist/parquetjs/compression.js +48 -154
  33. package/dist/parquetjs/compression.js.map +1 -0
  34. package/dist/parquetjs/encoder/writer.js +383 -440
  35. package/dist/parquetjs/encoder/writer.js.map +1 -0
  36. package/dist/parquetjs/file.js +66 -85
  37. package/dist/parquetjs/file.js.map +1 -0
  38. package/dist/{es5/parquetjs → parquetjs}/modules.d.ts +0 -0
  39. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +7 -14
  40. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
  41. package/dist/parquetjs/parquet-thrift/BsonType.js +37 -56
  42. package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -0
  43. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +215 -205
  44. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
  45. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +212 -207
  46. package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
  47. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +422 -391
  48. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
  49. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +90 -99
  50. package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
  51. package/dist/parquetjs/parquet-thrift/CompressionCodec.js +12 -19
  52. package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
  53. package/dist/parquetjs/parquet-thrift/ConvertedType.js +26 -33
  54. package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
  55. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +162 -162
  56. package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
  57. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +234 -224
  58. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
  59. package/dist/parquetjs/parquet-thrift/DateType.js +37 -56
  60. package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -0
  61. package/dist/parquetjs/parquet-thrift/DecimalType.js +91 -101
  62. package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
  63. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +113 -118
  64. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
  65. package/dist/parquetjs/parquet-thrift/Encoding.js +12 -19
  66. package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -0
  67. package/dist/parquetjs/parquet-thrift/EnumType.js +37 -56
  68. package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -0
  69. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +7 -14
  70. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
  71. package/dist/parquetjs/parquet-thrift/FileMetaData.js +264 -250
  72. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
  73. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +37 -56
  74. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
  75. package/dist/parquetjs/parquet-thrift/IntType.js +91 -101
  76. package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -0
  77. package/dist/parquetjs/parquet-thrift/JsonType.js +37 -56
  78. package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -0
  79. package/dist/parquetjs/parquet-thrift/KeyValue.js +89 -98
  80. package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
  81. package/dist/parquetjs/parquet-thrift/ListType.js +37 -56
  82. package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -0
  83. package/dist/parquetjs/parquet-thrift/LogicalType.js +450 -363
  84. package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
  85. package/dist/parquetjs/parquet-thrift/MapType.js +37 -56
  86. package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -0
  87. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +37 -56
  88. package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
  89. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +37 -56
  90. package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
  91. package/dist/parquetjs/parquet-thrift/NullType.js +37 -56
  92. package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -0
  93. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +80 -92
  94. package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
  95. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +115 -123
  96. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
  97. package/dist/parquetjs/parquet-thrift/PageHeader.js +231 -214
  98. package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
  99. package/dist/parquetjs/parquet-thrift/PageLocation.js +124 -137
  100. package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
  101. package/dist/parquetjs/parquet-thrift/PageType.js +8 -15
  102. package/dist/parquetjs/parquet-thrift/PageType.js.map +1 -0
  103. package/dist/parquetjs/parquet-thrift/RowGroup.js +172 -176
  104. package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
  105. package/dist/parquetjs/parquet-thrift/SchemaElement.js +268 -237
  106. package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
  107. package/dist/parquetjs/parquet-thrift/SortingColumn.js +115 -123
  108. package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
  109. package/dist/parquetjs/parquet-thrift/Statistics.js +179 -172
  110. package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -0
  111. package/dist/parquetjs/parquet-thrift/StringType.js +37 -56
  112. package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -0
  113. package/dist/parquetjs/parquet-thrift/TimeType.js +92 -102
  114. package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -0
  115. package/dist/parquetjs/parquet-thrift/TimeUnit.js +120 -121
  116. package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
  117. package/dist/parquetjs/parquet-thrift/TimestampType.js +92 -102
  118. package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
  119. package/dist/parquetjs/parquet-thrift/Type.js +12 -19
  120. package/dist/parquetjs/parquet-thrift/Type.js.map +1 -0
  121. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +37 -56
  122. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
  123. package/dist/parquetjs/parquet-thrift/UUIDType.js +37 -56
  124. package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
  125. package/dist/parquetjs/parquet-thrift/index.js +44 -61
  126. package/dist/parquetjs/parquet-thrift/index.js.map +1 -0
  127. package/dist/parquetjs/parser/decoders.js +283 -301
  128. package/dist/{es5/parquetjs → parquetjs}/parser/decoders.js.map +1 -1
  129. package/dist/parquetjs/parser/parquet-cursor.js +85 -69
  130. package/dist/parquetjs/parser/parquet-cursor.js.map +1 -0
  131. package/dist/parquetjs/parser/parquet-envelope-reader.js +146 -127
  132. package/dist/parquetjs/parser/parquet-envelope-reader.js.map +1 -0
  133. package/dist/parquetjs/parser/parquet-reader.js +113 -127
  134. package/dist/parquetjs/parser/parquet-reader.js.map +1 -0
  135. package/dist/parquetjs/schema/declare.js +12 -9
  136. package/dist/parquetjs/schema/declare.js.map +1 -0
  137. package/dist/parquetjs/schema/schema.js +162 -148
  138. package/dist/{es5/parquetjs → parquetjs}/schema/schema.js.map +1 -1
  139. package/dist/parquetjs/schema/shred.js +151 -214
  140. package/dist/parquetjs/schema/shred.js.map +1 -0
  141. package/dist/parquetjs/schema/types.js +415 -357
  142. package/dist/parquetjs/schema/types.js.map +1 -0
  143. package/dist/parquetjs/utils/buffer-utils.js +10 -20
  144. package/dist/parquetjs/utils/buffer-utils.js.map +1 -0
  145. package/dist/parquetjs/utils/file-utils.js +28 -40
  146. package/dist/parquetjs/utils/file-utils.js.map +1 -0
  147. package/dist/parquetjs/utils/read-utils.js +95 -99
  148. package/dist/parquetjs/utils/read-utils.js.map +1 -0
  149. package/dist/workers/parquet-worker.js +4 -5
  150. package/dist/workers/parquet-worker.js.map +1 -0
  151. package/package.json +8 -8
  152. package/dist/es5/bundle.js +0 -7
  153. package/dist/es5/bundle.js.map +0 -1
  154. package/dist/es5/constants.js +0 -17
  155. package/dist/es5/constants.js.map +0 -1
  156. package/dist/es5/index.js +0 -82
  157. package/dist/es5/index.js.map +0 -1
  158. package/dist/es5/lib/convert-schema.js +0 -82
  159. package/dist/es5/lib/convert-schema.js.map +0 -1
  160. package/dist/es5/lib/parse-parquet.js +0 -173
  161. package/dist/es5/lib/parse-parquet.js.map +0 -1
  162. package/dist/es5/lib/read-array-buffer.js +0 -53
  163. package/dist/es5/lib/read-array-buffer.js.map +0 -1
  164. package/dist/es5/parquet-loader.js +0 -30
  165. package/dist/es5/parquet-loader.js.map +0 -1
  166. package/dist/es5/parquet-writer.js +0 -25
  167. package/dist/es5/parquet-writer.js.map +0 -1
  168. package/dist/es5/parquetjs/codecs/declare.js +0 -2
  169. package/dist/es5/parquetjs/codecs/dictionary.js +0 -30
  170. package/dist/es5/parquetjs/codecs/dictionary.js.map +0 -1
  171. package/dist/es5/parquetjs/codecs/index.js +0 -56
  172. package/dist/es5/parquetjs/codecs/index.js.map +0 -1
  173. package/dist/es5/parquetjs/codecs/plain.js +0 -287
  174. package/dist/es5/parquetjs/codecs/plain.js.map +0 -1
  175. package/dist/es5/parquetjs/codecs/rle.js +0 -174
  176. package/dist/es5/parquetjs/codecs/rle.js.map +0 -1
  177. package/dist/es5/parquetjs/compression.js +0 -167
  178. package/dist/es5/parquetjs/compression.js.map +0 -1
  179. package/dist/es5/parquetjs/encoder/writer.js +0 -875
  180. package/dist/es5/parquetjs/encoder/writer.js.map +0 -1
  181. package/dist/es5/parquetjs/file.js +0 -103
  182. package/dist/es5/parquetjs/file.js.map +0 -1
  183. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +0 -15
  184. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
  185. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +0 -67
  186. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +0 -1
  187. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +0 -241
  188. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
  189. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +0 -245
  190. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
  191. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +0 -449
  192. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
  193. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +0 -124
  194. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
  195. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +0 -20
  196. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
  197. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +0 -34
  198. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
  199. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +0 -191
  200. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
  201. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -258
  202. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
  203. package/dist/es5/parquetjs/parquet-thrift/DateType.js +0 -67
  204. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +0 -1
  205. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +0 -122
  206. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
  207. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -143
  208. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
  209. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +0 -20
  210. package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +0 -1
  211. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +0 -67
  212. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +0 -1
  213. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -15
  214. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
  215. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +0 -298
  216. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
  217. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +0 -67
  218. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
  219. package/dist/es5/parquetjs/parquet-thrift/IntType.js +0 -122
  220. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +0 -1
  221. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +0 -67
  222. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +0 -1
  223. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +0 -120
  224. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
  225. package/dist/es5/parquetjs/parquet-thrift/ListType.js +0 -67
  226. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +0 -1
  227. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +0 -508
  228. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
  229. package/dist/es5/parquetjs/parquet-thrift/MapType.js +0 -67
  230. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +0 -1
  231. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +0 -67
  232. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
  233. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +0 -67
  234. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
  235. package/dist/es5/parquetjs/parquet-thrift/NullType.js +0 -67
  236. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +0 -1
  237. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +0 -114
  238. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
  239. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +0 -145
  240. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
  241. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +0 -258
  242. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
  243. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +0 -155
  244. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
  245. package/dist/es5/parquetjs/parquet-thrift/PageType.js +0 -16
  246. package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +0 -1
  247. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +0 -206
  248. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
  249. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +0 -290
  250. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
  251. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +0 -145
  252. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
  253. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +0 -207
  254. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +0 -1
  255. package/dist/es5/parquetjs/parquet-thrift/StringType.js +0 -67
  256. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +0 -1
  257. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +0 -124
  258. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +0 -1
  259. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +0 -156
  260. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
  261. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +0 -124
  262. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
  263. package/dist/es5/parquetjs/parquet-thrift/Type.js +0 -20
  264. package/dist/es5/parquetjs/parquet-thrift/Type.js.map +0 -1
  265. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -67
  266. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
  267. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +0 -67
  268. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
  269. package/dist/es5/parquetjs/parquet-thrift/index.js +0 -565
  270. package/dist/es5/parquetjs/parquet-thrift/index.js.map +0 -1
  271. package/dist/es5/parquetjs/parser/decoders.js +0 -489
  272. package/dist/es5/parquetjs/parser/parquet-cursor.js +0 -215
  273. package/dist/es5/parquetjs/parser/parquet-cursor.js.map +0 -1
  274. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +0 -413
  275. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
  276. package/dist/es5/parquetjs/parser/parquet-reader.js +0 -364
  277. package/dist/es5/parquetjs/parser/parquet-reader.js.map +0 -1
  278. package/dist/es5/parquetjs/schema/declare.js +0 -25
  279. package/dist/es5/parquetjs/schema/declare.js.map +0 -1
  280. package/dist/es5/parquetjs/schema/schema.js +0 -203
  281. package/dist/es5/parquetjs/schema/shred.js +0 -223
  282. package/dist/es5/parquetjs/schema/shred.js.map +0 -1
  283. package/dist/es5/parquetjs/schema/types.js +0 -492
  284. package/dist/es5/parquetjs/schema/types.js.map +0 -1
  285. package/dist/es5/parquetjs/utils/buffer-utils.js +0 -21
  286. package/dist/es5/parquetjs/utils/buffer-utils.js.map +0 -1
  287. package/dist/es5/parquetjs/utils/file-utils.js +0 -55
  288. package/dist/es5/parquetjs/utils/file-utils.js.map +0 -1
  289. package/dist/es5/parquetjs/utils/read-utils.js +0 -159
  290. package/dist/es5/parquetjs/utils/read-utils.js.map +0 -1
  291. package/dist/es5/workers/parquet-worker.js +0 -8
  292. package/dist/es5/workers/parquet-worker.js.map +0 -1
  293. package/dist/esm/bundle.js +0 -5
  294. package/dist/esm/bundle.js.map +0 -1
  295. package/dist/esm/constants.js +0 -6
  296. package/dist/esm/constants.js.map +0 -1
  297. package/dist/esm/index.js +0 -15
  298. package/dist/esm/index.js.map +0 -1
  299. package/dist/esm/lib/convert-schema.js +0 -71
  300. package/dist/esm/lib/convert-schema.js.map +0 -1
  301. package/dist/esm/lib/parse-parquet.js +0 -28
  302. package/dist/esm/lib/parse-parquet.js.map +0 -1
  303. package/dist/esm/lib/read-array-buffer.js +0 -9
  304. package/dist/esm/lib/read-array-buffer.js.map +0 -1
  305. package/dist/esm/parquet-loader.js +0 -22
  306. package/dist/esm/parquet-loader.js.map +0 -1
  307. package/dist/esm/parquet-writer.js +0 -18
  308. package/dist/esm/parquet-writer.js.map +0 -1
  309. package/dist/esm/parquetjs/LICENSE +0 -20
  310. package/dist/esm/parquetjs/codecs/declare.js +0 -2
  311. package/dist/esm/parquetjs/codecs/declare.js.map +0 -1
  312. package/dist/esm/parquetjs/codecs/dictionary.js +0 -12
  313. package/dist/esm/parquetjs/codecs/dictionary.js.map +0 -1
  314. package/dist/esm/parquetjs/codecs/index.js +0 -23
  315. package/dist/esm/parquetjs/codecs/index.js.map +0 -1
  316. package/dist/esm/parquetjs/codecs/plain.js +0 -270
  317. package/dist/esm/parquetjs/codecs/plain.js.map +0 -1
  318. package/dist/esm/parquetjs/codecs/rle.js +0 -151
  319. package/dist/esm/parquetjs/codecs/rle.js.map +0 -1
  320. package/dist/esm/parquetjs/compression.js +0 -62
  321. package/dist/esm/parquetjs/compression.js.map +0 -1
  322. package/dist/esm/parquetjs/encoder/writer.js +0 -421
  323. package/dist/esm/parquetjs/encoder/writer.js.map +0 -1
  324. package/dist/esm/parquetjs/file.js +0 -80
  325. package/dist/esm/parquetjs/file.js.map +0 -1
  326. package/dist/esm/parquetjs/modules.d.ts +0 -21
  327. package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +0 -8
  328. package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
  329. package/dist/esm/parquetjs/parquet-thrift/BsonType.js +0 -39
  330. package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +0 -1
  331. package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +0 -217
  332. package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
  333. package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +0 -218
  334. package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
  335. package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +0 -429
  336. package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
  337. package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +0 -95
  338. package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
  339. package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +0 -13
  340. package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
  341. package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +0 -27
  342. package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
  343. package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +0 -166
  344. package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
  345. package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -236
  346. package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
  347. package/dist/esm/parquetjs/parquet-thrift/DateType.js +0 -39
  348. package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +0 -1
  349. package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +0 -95
  350. package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
  351. package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -117
  352. package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
  353. package/dist/esm/parquetjs/parquet-thrift/Encoding.js +0 -13
  354. package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +0 -1
  355. package/dist/esm/parquetjs/parquet-thrift/EnumType.js +0 -39
  356. package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +0 -1
  357. package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -8
  358. package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
  359. package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +0 -270
  360. package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
  361. package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +0 -39
  362. package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
  363. package/dist/esm/parquetjs/parquet-thrift/IntType.js +0 -95
  364. package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +0 -1
  365. package/dist/esm/parquetjs/parquet-thrift/JsonType.js +0 -39
  366. package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +0 -1
  367. package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +0 -93
  368. package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
  369. package/dist/esm/parquetjs/parquet-thrift/ListType.js +0 -39
  370. package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +0 -1
  371. package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +0 -467
  372. package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
  373. package/dist/esm/parquetjs/parquet-thrift/MapType.js +0 -39
  374. package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +0 -1
  375. package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +0 -39
  376. package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
  377. package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +0 -39
  378. package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
  379. package/dist/esm/parquetjs/parquet-thrift/NullType.js +0 -39
  380. package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +0 -1
  381. package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +0 -85
  382. package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
  383. package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +0 -119
  384. package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
  385. package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +0 -233
  386. package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
  387. package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +0 -128
  388. package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
  389. package/dist/esm/parquetjs/parquet-thrift/PageType.js +0 -9
  390. package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +0 -1
  391. package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +0 -178
  392. package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
  393. package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +0 -270
  394. package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
  395. package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +0 -119
  396. package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
  397. package/dist/esm/parquetjs/parquet-thrift/Statistics.js +0 -183
  398. package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +0 -1
  399. package/dist/esm/parquetjs/parquet-thrift/StringType.js +0 -39
  400. package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +0 -1
  401. package/dist/esm/parquetjs/parquet-thrift/TimeType.js +0 -96
  402. package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +0 -1
  403. package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +0 -126
  404. package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
  405. package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +0 -96
  406. package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
  407. package/dist/esm/parquetjs/parquet-thrift/Type.js +0 -13
  408. package/dist/esm/parquetjs/parquet-thrift/Type.js.map +0 -1
  409. package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -39
  410. package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
  411. package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +0 -39
  412. package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
  413. package/dist/esm/parquetjs/parquet-thrift/index.js +0 -44
  414. package/dist/esm/parquetjs/parquet-thrift/index.js.map +0 -1
  415. package/dist/esm/parquetjs/parser/decoders.js +0 -300
  416. package/dist/esm/parquetjs/parser/decoders.js.map +0 -1
  417. package/dist/esm/parquetjs/parser/parquet-cursor.js +0 -90
  418. package/dist/esm/parquetjs/parser/parquet-cursor.js.map +0 -1
  419. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +0 -155
  420. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
  421. package/dist/esm/parquetjs/parser/parquet-reader.js +0 -120
  422. package/dist/esm/parquetjs/parser/parquet-reader.js.map +0 -1
  423. package/dist/esm/parquetjs/schema/declare.js +0 -13
  424. package/dist/esm/parquetjs/schema/declare.js.map +0 -1
  425. package/dist/esm/parquetjs/schema/schema.js +0 -176
  426. package/dist/esm/parquetjs/schema/schema.js.map +0 -1
  427. package/dist/esm/parquetjs/schema/shred.js +0 -162
  428. package/dist/esm/parquetjs/schema/shred.js.map +0 -1
  429. package/dist/esm/parquetjs/schema/types.js +0 -476
  430. package/dist/esm/parquetjs/schema/types.js.map +0 -1
  431. package/dist/esm/parquetjs/utils/buffer-utils.js +0 -12
  432. package/dist/esm/parquetjs/utils/buffer-utils.js.map +0 -1
  433. package/dist/esm/parquetjs/utils/file-utils.js +0 -34
  434. package/dist/esm/parquetjs/utils/file-utils.js.map +0 -1
  435. package/dist/esm/parquetjs/utils/read-utils.js +0 -105
  436. package/dist/esm/parquetjs/utils/read-utils.js.map +0 -1
  437. package/dist/esm/workers/parquet-worker.js +0 -4
  438. package/dist/esm/workers/parquet-worker.js.map +0 -1
@@ -1,489 +0,0 @@
1
- "use strict";
2
-
3
- var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
-
5
- Object.defineProperty(exports, "__esModule", {
6
- value: true
7
- });
8
- exports.decodeDataPages = decodeDataPages;
9
- exports.decodePage = decodePage;
10
- exports.decodeSchema = decodeSchema;
11
-
12
- var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
13
-
14
- var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
15
-
16
- var _codecs = require("../codecs");
17
-
18
- var _parquetThrift = require("../parquet-thrift");
19
-
20
- var _compression = require("../compression");
21
-
22
- var _constants = require("../../constants");
23
-
24
- var _readUtils = require("../utils/read-utils");
25
-
26
- function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
27
-
28
- function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
29
-
30
- function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
31
-
32
- function decodeDataPages(_x, _x2) {
33
- return _decodeDataPages.apply(this, arguments);
34
- }
35
-
36
- function _decodeDataPages() {
37
- _decodeDataPages = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(buffer, options) {
38
- var cursor, data, dictionary, page, index, value;
39
- return _regenerator.default.wrap(function _callee$(_context) {
40
- while (1) {
41
- switch (_context.prev = _context.next) {
42
- case 0:
43
- cursor = {
44
- buffer: buffer,
45
- offset: 0,
46
- size: buffer.length
47
- };
48
- data = {
49
- rlevels: [],
50
- dlevels: [],
51
- values: [],
52
- pageHeaders: [],
53
- count: 0
54
- };
55
- dictionary = options.dictionary || [];
56
-
57
- case 3:
58
- if (!(cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues)))) {
59
- _context.next = 16;
60
- break;
61
- }
62
-
63
- _context.next = 6;
64
- return decodePage(cursor, options);
65
-
66
- case 6:
67
- page = _context.sent;
68
-
69
- if (!page.dictionary) {
70
- _context.next = 10;
71
- break;
72
- }
73
-
74
- dictionary = page.dictionary;
75
- return _context.abrupt("continue", 3);
76
-
77
- case 10:
78
- if (dictionary.length) {
79
- page.values = page.values.map(function (value) {
80
- return dictionary[value];
81
- });
82
- }
83
-
84
- for (index = 0; index < page.rlevels.length; index++) {
85
- data.rlevels.push(page.rlevels[index]);
86
- data.dlevels.push(page.dlevels[index]);
87
- value = page.values[index];
88
-
89
- if (value !== undefined) {
90
- data.values.push(value);
91
- }
92
- }
93
-
94
- data.count += page.count;
95
- data.pageHeaders.push(page.pageHeader);
96
- _context.next = 3;
97
- break;
98
-
99
- case 16:
100
- return _context.abrupt("return", data);
101
-
102
- case 17:
103
- case "end":
104
- return _context.stop();
105
- }
106
- }
107
- }, _callee);
108
- }));
109
- return _decodeDataPages.apply(this, arguments);
110
- }
111
-
112
- function decodePage(_x3, _x4) {
113
- return _decodePage.apply(this, arguments);
114
- }
115
-
116
- function _decodePage() {
117
- _decodePage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(cursor, options) {
118
- var page, _decodePageHeader, pageHeader, length, pageType;
119
-
120
- return _regenerator.default.wrap(function _callee2$(_context2) {
121
- while (1) {
122
- switch (_context2.prev = _context2.next) {
123
- case 0:
124
- _decodePageHeader = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset), pageHeader = _decodePageHeader.pageHeader, length = _decodePageHeader.length;
125
- cursor.offset += length;
126
- pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
127
- _context2.t0 = pageType;
128
- _context2.next = _context2.t0 === 'DATA_PAGE' ? 6 : _context2.t0 === 'DATA_PAGE_V2' ? 10 : _context2.t0 === 'DICTIONARY_PAGE' ? 14 : 20;
129
- break;
130
-
131
- case 6:
132
- _context2.next = 8;
133
- return decodeDataPage(cursor, pageHeader, options);
134
-
135
- case 8:
136
- page = _context2.sent;
137
- return _context2.abrupt("break", 21);
138
-
139
- case 10:
140
- _context2.next = 12;
141
- return decodeDataPageV2(cursor, pageHeader, options);
142
-
143
- case 12:
144
- page = _context2.sent;
145
- return _context2.abrupt("break", 21);
146
-
147
- case 14:
148
- _context2.next = 16;
149
- return decodeDictionaryPage(cursor, pageHeader, options);
150
-
151
- case 16:
152
- _context2.t1 = _context2.sent;
153
- _context2.t2 = pageHeader;
154
- page = {
155
- dictionary: _context2.t1,
156
- pageHeader: _context2.t2
157
- };
158
- return _context2.abrupt("break", 21);
159
-
160
- case 20:
161
- throw new Error("invalid page type: ".concat(pageType));
162
-
163
- case 21:
164
- return _context2.abrupt("return", page);
165
-
166
- case 22:
167
- case "end":
168
- return _context2.stop();
169
- }
170
- }
171
- }, _callee2);
172
- }));
173
- return _decodePage.apply(this, arguments);
174
- }
175
-
176
- function decodeSchema(schemaElements, offset, len) {
177
- var schema = {};
178
- var next = offset;
179
-
180
- for (var i = 0; i < len; i++) {
181
- var schemaElement = schemaElements[next];
182
- var repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
183
- var optional = false;
184
- var repeated = false;
185
-
186
- switch (repetitionType) {
187
- case 'REQUIRED':
188
- break;
189
-
190
- case 'OPTIONAL':
191
- optional = true;
192
- break;
193
-
194
- case 'REPEATED':
195
- repeated = true;
196
- break;
197
-
198
- default:
199
- throw new Error('parquet: unknown repetition type');
200
- }
201
-
202
- if (schemaElement.num_children > 0) {
203
- var res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
204
- next = res.next;
205
- schema[schemaElement.name] = {
206
- optional: optional,
207
- repeated: repeated,
208
- fields: res.schema
209
- };
210
- } else {
211
- var type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
212
- var logicalType = type;
213
-
214
- if (schemaElement.converted_type) {
215
- logicalType = (0, _readUtils.getThriftEnum)(_parquetThrift.ConvertedType, schemaElement.converted_type);
216
- }
217
-
218
- switch (logicalType) {
219
- case 'DECIMAL':
220
- logicalType = "".concat(logicalType, "_").concat(type);
221
- break;
222
-
223
- default:
224
- }
225
-
226
- schema[schemaElement.name] = {
227
- type: logicalType,
228
- typeLength: schemaElement.type_length,
229
- presision: schemaElement.precision,
230
- scale: schemaElement.scale,
231
- optional: optional,
232
- repeated: repeated
233
- };
234
- next++;
235
- }
236
- }
237
-
238
- return {
239
- schema: schema,
240
- offset: offset,
241
- next: next
242
- };
243
- }
244
-
245
- function decodeValues(type, encoding, cursor, count, opts) {
246
- if (!(encoding in _codecs.PARQUET_CODECS)) {
247
- throw new Error("invalid encoding: ".concat(encoding));
248
- }
249
-
250
- return _codecs.PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
251
- }
252
-
253
- function decodeDataPage(_x5, _x6, _x7) {
254
- return _decodeDataPage.apply(this, arguments);
255
- }
256
-
257
- function _decodeDataPage() {
258
- _decodeDataPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(cursor, header, options) {
259
- var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
260
-
261
- var cursorEnd, valueCount, dataCursor, valuesBuf, rLevelEncoding, rLevels, dLevelEncoding, dLevels, valueCountNonNull, _iterator, _step, dlvl, valueEncoding, decodeOptions, values;
262
-
263
- return _regenerator.default.wrap(function _callee3$(_context3) {
264
- while (1) {
265
- switch (_context3.prev = _context3.next) {
266
- case 0:
267
- cursorEnd = cursor.offset + header.compressed_page_size;
268
- valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
269
- dataCursor = cursor;
270
-
271
- if (!(options.compression !== 'UNCOMPRESSED')) {
272
- _context3.next = 9;
273
- break;
274
- }
275
-
276
- _context3.next = 6;
277
- return (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
278
-
279
- case 6:
280
- valuesBuf = _context3.sent;
281
- dataCursor = {
282
- buffer: valuesBuf,
283
- offset: 0,
284
- size: valuesBuf.length
285
- };
286
- cursor.offset = cursorEnd;
287
-
288
- case 9:
289
- rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
290
- rLevels = new Array(valueCount);
291
-
292
- if (options.column.rLevelMax > 0) {
293
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
294
- bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
295
- disableEnvelope: false
296
- });
297
- } else {
298
- rLevels.fill(0);
299
- }
300
-
301
- dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
302
- dLevels = new Array(valueCount);
303
-
304
- if (options.column.dLevelMax > 0) {
305
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
306
- bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
307
- disableEnvelope: false
308
- });
309
- } else {
310
- dLevels.fill(0);
311
- }
312
-
313
- valueCountNonNull = 0;
314
- _iterator = _createForOfIteratorHelper(dLevels);
315
-
316
- try {
317
- for (_iterator.s(); !(_step = _iterator.n()).done;) {
318
- dlvl = _step.value;
319
-
320
- if (dlvl === options.column.dLevelMax) {
321
- valueCountNonNull++;
322
- }
323
- }
324
- } catch (err) {
325
- _iterator.e(err);
326
- } finally {
327
- _iterator.f();
328
- }
329
-
330
- valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
331
- decodeOptions = {
332
- typeLength: options.column.typeLength,
333
- bitWidth: options.column.typeLength
334
- };
335
- values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
336
- return _context3.abrupt("return", {
337
- dlevels: dLevels,
338
- rlevels: rLevels,
339
- values: values,
340
- count: valueCount,
341
- pageHeader: header
342
- });
343
-
344
- case 22:
345
- case "end":
346
- return _context3.stop();
347
- }
348
- }
349
- }, _callee3);
350
- }));
351
- return _decodeDataPage.apply(this, arguments);
352
- }
353
-
354
- function decodeDataPageV2(_x8, _x9, _x10) {
355
- return _decodeDataPageV.apply(this, arguments);
356
- }
357
-
358
- function _decodeDataPageV() {
359
- _decodeDataPageV = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4(cursor, header, opts) {
360
- var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
361
-
362
- var cursorEnd, valueCount, valueCountNonNull, valueEncoding, rLevels, dLevels, valuesBufCursor, valuesBuf, decodeOptions, values;
363
- return _regenerator.default.wrap(function _callee4$(_context4) {
364
- while (1) {
365
- switch (_context4.prev = _context4.next) {
366
- case 0:
367
- cursorEnd = cursor.offset + header.compressed_page_size;
368
- valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
369
- valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
370
- valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
371
- rLevels = new Array(valueCount);
372
-
373
- if (opts.column.rLevelMax > 0) {
374
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
375
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
376
- disableEnvelope: true
377
- });
378
- } else {
379
- rLevels.fill(0);
380
- }
381
-
382
- dLevels = new Array(valueCount);
383
-
384
- if (opts.column.dLevelMax > 0) {
385
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
386
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
387
- disableEnvelope: true
388
- });
389
- } else {
390
- dLevels.fill(0);
391
- }
392
-
393
- valuesBufCursor = cursor;
394
-
395
- if (!((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed)) {
396
- _context4.next = 15;
397
- break;
398
- }
399
-
400
- _context4.next = 12;
401
- return (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
402
-
403
- case 12:
404
- valuesBuf = _context4.sent;
405
- valuesBufCursor = {
406
- buffer: valuesBuf,
407
- offset: 0,
408
- size: valuesBuf.length
409
- };
410
- cursor.offset = cursorEnd;
411
-
412
- case 15:
413
- decodeOptions = {
414
- typeLength: opts.column.typeLength,
415
- bitWidth: opts.column.typeLength
416
- };
417
- values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
418
- return _context4.abrupt("return", {
419
- dlevels: dLevels,
420
- rlevels: rLevels,
421
- values: values,
422
- count: valueCount,
423
- pageHeader: header
424
- });
425
-
426
- case 18:
427
- case "end":
428
- return _context4.stop();
429
- }
430
- }
431
- }, _callee4);
432
- }));
433
- return _decodeDataPageV.apply(this, arguments);
434
- }
435
-
436
- function decodeDictionaryPage(_x11, _x12, _x13) {
437
- return _decodeDictionaryPage.apply(this, arguments);
438
- }
439
-
440
- function _decodeDictionaryPage() {
441
- _decodeDictionaryPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5(cursor, pageHeader, options) {
442
- var _pageHeader$dictionar;
443
-
444
- var cursorEnd, dictCursor, valuesBuf, numValues;
445
- return _regenerator.default.wrap(function _callee5$(_context5) {
446
- while (1) {
447
- switch (_context5.prev = _context5.next) {
448
- case 0:
449
- cursorEnd = cursor.offset + pageHeader.compressed_page_size;
450
- dictCursor = {
451
- offset: 0,
452
- buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
453
- size: cursorEnd - cursor.offset
454
- };
455
- cursor.offset = cursorEnd;
456
-
457
- if (!(options.compression !== 'UNCOMPRESSED')) {
458
- _context5.next = 9;
459
- break;
460
- }
461
-
462
- _context5.next = 6;
463
- return (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
464
-
465
- case 6:
466
- valuesBuf = _context5.sent;
467
- dictCursor = {
468
- buffer: valuesBuf,
469
- offset: 0,
470
- size: valuesBuf.length
471
- };
472
- cursor.offset = cursorEnd;
473
-
474
- case 9:
475
- numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
476
- return _context5.abrupt("return", decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(function (d) {
477
- return d.toString();
478
- }));
479
-
480
- case 11:
481
- case "end":
482
- return _context5.stop();
483
- }
484
- }
485
- }, _callee5);
486
- }));
487
- return _decodeDictionaryPage.apply(this, arguments);
488
- }
489
- //# sourceMappingURL=decoders.js.map
@@ -1,215 +0,0 @@
1
- "use strict";
2
-
3
- var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
-
5
- Object.defineProperty(exports, "__esModule", {
6
- value: true
7
- });
8
- exports.ParquetCursor = void 0;
9
-
10
- var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
11
-
12
- var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
13
-
14
- var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
15
-
16
- var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
17
-
18
- var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
19
-
20
- var _shred = require("../schema/shred");
21
-
22
- var _Symbol$asyncIterator;
23
-
24
- _Symbol$asyncIterator = Symbol.asyncIterator;
25
-
26
- var ParquetCursor = function () {
27
- function ParquetCursor(metadata, envelopeReader, schema, columnList) {
28
- (0, _classCallCheck2.default)(this, ParquetCursor);
29
- (0, _defineProperty2.default)(this, "metadata", void 0);
30
- (0, _defineProperty2.default)(this, "envelopeReader", void 0);
31
- (0, _defineProperty2.default)(this, "schema", void 0);
32
- (0, _defineProperty2.default)(this, "columnList", void 0);
33
- (0, _defineProperty2.default)(this, "rowGroup", []);
34
- (0, _defineProperty2.default)(this, "rowGroupIndex", void 0);
35
- this.metadata = metadata;
36
- this.envelopeReader = envelopeReader;
37
- this.schema = schema;
38
- this.columnList = columnList;
39
- this.rowGroupIndex = 0;
40
- }
41
-
42
- (0, _createClass2.default)(ParquetCursor, [{
43
- key: "next",
44
- value: function () {
45
- var _next = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee() {
46
- var rowBuffer;
47
- return _regenerator.default.wrap(function _callee$(_context) {
48
- while (1) {
49
- switch (_context.prev = _context.next) {
50
- case 0:
51
- if (!(this.rowGroup.length === 0)) {
52
- _context.next = 8;
53
- break;
54
- }
55
-
56
- if (!(this.rowGroupIndex >= this.metadata.row_groups.length)) {
57
- _context.next = 3;
58
- break;
59
- }
60
-
61
- return _context.abrupt("return", null);
62
-
63
- case 3:
64
- _context.next = 5;
65
- return this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
66
-
67
- case 5:
68
- rowBuffer = _context.sent;
69
- this.rowGroup = (0, _shred.materializeRecords)(this.schema, rowBuffer);
70
- this.rowGroupIndex++;
71
-
72
- case 8:
73
- return _context.abrupt("return", this.rowGroup.shift());
74
-
75
- case 9:
76
- case "end":
77
- return _context.stop();
78
- }
79
- }
80
- }, _callee, this);
81
- }));
82
-
83
- function next() {
84
- return _next.apply(this, arguments);
85
- }
86
-
87
- return next;
88
- }()
89
- }, {
90
- key: "rewind",
91
- value: function rewind() {
92
- this.rowGroup = [];
93
- this.rowGroupIndex = 0;
94
- }
95
- }, {
96
- key: _Symbol$asyncIterator,
97
- value: function value() {
98
- var _this = this;
99
-
100
- var done = false;
101
- return {
102
- next: function () {
103
- var _next2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2() {
104
- var value;
105
- return _regenerator.default.wrap(function _callee2$(_context2) {
106
- while (1) {
107
- switch (_context2.prev = _context2.next) {
108
- case 0:
109
- if (!done) {
110
- _context2.next = 2;
111
- break;
112
- }
113
-
114
- return _context2.abrupt("return", {
115
- done: done,
116
- value: null
117
- });
118
-
119
- case 2:
120
- _context2.next = 4;
121
- return _this.next();
122
-
123
- case 4:
124
- value = _context2.sent;
125
-
126
- if (!(value === null)) {
127
- _context2.next = 7;
128
- break;
129
- }
130
-
131
- return _context2.abrupt("return", {
132
- done: true,
133
- value: value
134
- });
135
-
136
- case 7:
137
- return _context2.abrupt("return", {
138
- done: false,
139
- value: value
140
- });
141
-
142
- case 8:
143
- case "end":
144
- return _context2.stop();
145
- }
146
- }
147
- }, _callee2);
148
- }));
149
-
150
- function next() {
151
- return _next2.apply(this, arguments);
152
- }
153
-
154
- return next;
155
- }(),
156
- return: function () {
157
- var _return2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3() {
158
- return _regenerator.default.wrap(function _callee3$(_context3) {
159
- while (1) {
160
- switch (_context3.prev = _context3.next) {
161
- case 0:
162
- done = true;
163
- return _context3.abrupt("return", {
164
- done: done,
165
- value: null
166
- });
167
-
168
- case 2:
169
- case "end":
170
- return _context3.stop();
171
- }
172
- }
173
- }, _callee3);
174
- }));
175
-
176
- function _return() {
177
- return _return2.apply(this, arguments);
178
- }
179
-
180
- return _return;
181
- }(),
182
- throw: function () {
183
- var _throw2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4() {
184
- return _regenerator.default.wrap(function _callee4$(_context4) {
185
- while (1) {
186
- switch (_context4.prev = _context4.next) {
187
- case 0:
188
- done = true;
189
- return _context4.abrupt("return", {
190
- done: true,
191
- value: null
192
- });
193
-
194
- case 2:
195
- case "end":
196
- return _context4.stop();
197
- }
198
- }
199
- }, _callee4);
200
- }));
201
-
202
- function _throw() {
203
- return _throw2.apply(this, arguments);
204
- }
205
-
206
- return _throw;
207
- }()
208
- };
209
- }
210
- }]);
211
- return ParquetCursor;
212
- }();
213
-
214
- exports.ParquetCursor = ParquetCursor;
215
- //# sourceMappingURL=parquet-cursor.js.map