@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (438) hide show
  1. package/dist/bundle.js +2 -2
  2. package/dist/bundle.js.map +1 -0
  3. package/dist/constants.js +6 -18
  4. package/dist/constants.js.map +1 -0
  5. package/dist/dist.min.js +17 -8
  6. package/dist/dist.min.js.map +3 -3
  7. package/dist/index.js +14 -29
  8. package/dist/index.js.map +1 -0
  9. package/dist/lib/convert-schema.js +63 -62
  10. package/dist/lib/convert-schema.js.map +1 -0
  11. package/dist/lib/parse-parquet.js +25 -25
  12. package/dist/lib/parse-parquet.js.map +1 -0
  13. package/dist/lib/read-array-buffer.js +8 -28
  14. package/dist/lib/read-array-buffer.js.map +1 -0
  15. package/dist/parquet-loader.js +19 -24
  16. package/dist/parquet-loader.js.map +1 -0
  17. package/dist/parquet-worker.js +18 -9
  18. package/dist/parquet-worker.js.map +3 -3
  19. package/dist/parquet-writer.js +14 -17
  20. package/dist/parquet-writer.js.map +1 -0
  21. package/dist/{es5/parquetjs → parquetjs}/LICENSE +0 -0
  22. package/dist/parquetjs/codecs/declare.js +2 -2
  23. package/dist/{es5/parquetjs → parquetjs}/codecs/declare.js.map +0 -0
  24. package/dist/parquetjs/codecs/dictionary.js +10 -12
  25. package/dist/parquetjs/codecs/dictionary.js.map +1 -0
  26. package/dist/parquetjs/codecs/index.js +22 -50
  27. package/dist/parquetjs/codecs/index.js.map +1 -0
  28. package/dist/parquetjs/codecs/plain.js +232 -173
  29. package/dist/parquetjs/codecs/plain.js.map +1 -0
  30. package/dist/parquetjs/codecs/rle.js +140 -134
  31. package/dist/parquetjs/codecs/rle.js.map +1 -0
  32. package/dist/parquetjs/compression.js +48 -154
  33. package/dist/parquetjs/compression.js.map +1 -0
  34. package/dist/parquetjs/encoder/writer.js +383 -440
  35. package/dist/parquetjs/encoder/writer.js.map +1 -0
  36. package/dist/parquetjs/file.js +66 -85
  37. package/dist/parquetjs/file.js.map +1 -0
  38. package/dist/{es5/parquetjs → parquetjs}/modules.d.ts +0 -0
  39. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +7 -14
  40. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
  41. package/dist/parquetjs/parquet-thrift/BsonType.js +37 -56
  42. package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -0
  43. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +215 -205
  44. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
  45. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +212 -207
  46. package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
  47. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +422 -391
  48. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
  49. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +90 -99
  50. package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
  51. package/dist/parquetjs/parquet-thrift/CompressionCodec.js +12 -19
  52. package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
  53. package/dist/parquetjs/parquet-thrift/ConvertedType.js +26 -33
  54. package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
  55. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +162 -162
  56. package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
  57. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +234 -224
  58. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
  59. package/dist/parquetjs/parquet-thrift/DateType.js +37 -56
  60. package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -0
  61. package/dist/parquetjs/parquet-thrift/DecimalType.js +91 -101
  62. package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
  63. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +113 -118
  64. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
  65. package/dist/parquetjs/parquet-thrift/Encoding.js +12 -19
  66. package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -0
  67. package/dist/parquetjs/parquet-thrift/EnumType.js +37 -56
  68. package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -0
  69. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +7 -14
  70. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
  71. package/dist/parquetjs/parquet-thrift/FileMetaData.js +264 -250
  72. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
  73. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +37 -56
  74. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
  75. package/dist/parquetjs/parquet-thrift/IntType.js +91 -101
  76. package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -0
  77. package/dist/parquetjs/parquet-thrift/JsonType.js +37 -56
  78. package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -0
  79. package/dist/parquetjs/parquet-thrift/KeyValue.js +89 -98
  80. package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
  81. package/dist/parquetjs/parquet-thrift/ListType.js +37 -56
  82. package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -0
  83. package/dist/parquetjs/parquet-thrift/LogicalType.js +450 -363
  84. package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
  85. package/dist/parquetjs/parquet-thrift/MapType.js +37 -56
  86. package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -0
  87. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +37 -56
  88. package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
  89. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +37 -56
  90. package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
  91. package/dist/parquetjs/parquet-thrift/NullType.js +37 -56
  92. package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -0
  93. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +80 -92
  94. package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
  95. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +115 -123
  96. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
  97. package/dist/parquetjs/parquet-thrift/PageHeader.js +231 -214
  98. package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
  99. package/dist/parquetjs/parquet-thrift/PageLocation.js +124 -137
  100. package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
  101. package/dist/parquetjs/parquet-thrift/PageType.js +8 -15
  102. package/dist/parquetjs/parquet-thrift/PageType.js.map +1 -0
  103. package/dist/parquetjs/parquet-thrift/RowGroup.js +172 -176
  104. package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
  105. package/dist/parquetjs/parquet-thrift/SchemaElement.js +268 -237
  106. package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
  107. package/dist/parquetjs/parquet-thrift/SortingColumn.js +115 -123
  108. package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
  109. package/dist/parquetjs/parquet-thrift/Statistics.js +179 -172
  110. package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -0
  111. package/dist/parquetjs/parquet-thrift/StringType.js +37 -56
  112. package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -0
  113. package/dist/parquetjs/parquet-thrift/TimeType.js +92 -102
  114. package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -0
  115. package/dist/parquetjs/parquet-thrift/TimeUnit.js +120 -121
  116. package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
  117. package/dist/parquetjs/parquet-thrift/TimestampType.js +92 -102
  118. package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
  119. package/dist/parquetjs/parquet-thrift/Type.js +12 -19
  120. package/dist/parquetjs/parquet-thrift/Type.js.map +1 -0
  121. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +37 -56
  122. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
  123. package/dist/parquetjs/parquet-thrift/UUIDType.js +37 -56
  124. package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
  125. package/dist/parquetjs/parquet-thrift/index.js +44 -61
  126. package/dist/parquetjs/parquet-thrift/index.js.map +1 -0
  127. package/dist/parquetjs/parser/decoders.js +283 -301
  128. package/dist/{es5/parquetjs → parquetjs}/parser/decoders.js.map +1 -1
  129. package/dist/parquetjs/parser/parquet-cursor.js +85 -69
  130. package/dist/parquetjs/parser/parquet-cursor.js.map +1 -0
  131. package/dist/parquetjs/parser/parquet-envelope-reader.js +146 -127
  132. package/dist/parquetjs/parser/parquet-envelope-reader.js.map +1 -0
  133. package/dist/parquetjs/parser/parquet-reader.js +113 -127
  134. package/dist/parquetjs/parser/parquet-reader.js.map +1 -0
  135. package/dist/parquetjs/schema/declare.js +12 -9
  136. package/dist/parquetjs/schema/declare.js.map +1 -0
  137. package/dist/parquetjs/schema/schema.js +162 -148
  138. package/dist/{es5/parquetjs → parquetjs}/schema/schema.js.map +1 -1
  139. package/dist/parquetjs/schema/shred.js +151 -214
  140. package/dist/parquetjs/schema/shred.js.map +1 -0
  141. package/dist/parquetjs/schema/types.js +415 -357
  142. package/dist/parquetjs/schema/types.js.map +1 -0
  143. package/dist/parquetjs/utils/buffer-utils.js +10 -20
  144. package/dist/parquetjs/utils/buffer-utils.js.map +1 -0
  145. package/dist/parquetjs/utils/file-utils.js +28 -40
  146. package/dist/parquetjs/utils/file-utils.js.map +1 -0
  147. package/dist/parquetjs/utils/read-utils.js +95 -99
  148. package/dist/parquetjs/utils/read-utils.js.map +1 -0
  149. package/dist/workers/parquet-worker.js +4 -5
  150. package/dist/workers/parquet-worker.js.map +1 -0
  151. package/package.json +8 -8
  152. package/dist/es5/bundle.js +0 -7
  153. package/dist/es5/bundle.js.map +0 -1
  154. package/dist/es5/constants.js +0 -17
  155. package/dist/es5/constants.js.map +0 -1
  156. package/dist/es5/index.js +0 -82
  157. package/dist/es5/index.js.map +0 -1
  158. package/dist/es5/lib/convert-schema.js +0 -82
  159. package/dist/es5/lib/convert-schema.js.map +0 -1
  160. package/dist/es5/lib/parse-parquet.js +0 -173
  161. package/dist/es5/lib/parse-parquet.js.map +0 -1
  162. package/dist/es5/lib/read-array-buffer.js +0 -53
  163. package/dist/es5/lib/read-array-buffer.js.map +0 -1
  164. package/dist/es5/parquet-loader.js +0 -30
  165. package/dist/es5/parquet-loader.js.map +0 -1
  166. package/dist/es5/parquet-writer.js +0 -25
  167. package/dist/es5/parquet-writer.js.map +0 -1
  168. package/dist/es5/parquetjs/codecs/declare.js +0 -2
  169. package/dist/es5/parquetjs/codecs/dictionary.js +0 -30
  170. package/dist/es5/parquetjs/codecs/dictionary.js.map +0 -1
  171. package/dist/es5/parquetjs/codecs/index.js +0 -56
  172. package/dist/es5/parquetjs/codecs/index.js.map +0 -1
  173. package/dist/es5/parquetjs/codecs/plain.js +0 -287
  174. package/dist/es5/parquetjs/codecs/plain.js.map +0 -1
  175. package/dist/es5/parquetjs/codecs/rle.js +0 -174
  176. package/dist/es5/parquetjs/codecs/rle.js.map +0 -1
  177. package/dist/es5/parquetjs/compression.js +0 -167
  178. package/dist/es5/parquetjs/compression.js.map +0 -1
  179. package/dist/es5/parquetjs/encoder/writer.js +0 -875
  180. package/dist/es5/parquetjs/encoder/writer.js.map +0 -1
  181. package/dist/es5/parquetjs/file.js +0 -103
  182. package/dist/es5/parquetjs/file.js.map +0 -1
  183. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +0 -15
  184. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
  185. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +0 -67
  186. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +0 -1
  187. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +0 -241
  188. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
  189. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +0 -245
  190. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
  191. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +0 -449
  192. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
  193. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +0 -124
  194. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
  195. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +0 -20
  196. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
  197. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +0 -34
  198. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
  199. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +0 -191
  200. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
  201. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -258
  202. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
  203. package/dist/es5/parquetjs/parquet-thrift/DateType.js +0 -67
  204. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +0 -1
  205. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +0 -122
  206. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
  207. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -143
  208. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
  209. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +0 -20
  210. package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +0 -1
  211. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +0 -67
  212. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +0 -1
  213. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -15
  214. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
  215. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +0 -298
  216. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
  217. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +0 -67
  218. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
  219. package/dist/es5/parquetjs/parquet-thrift/IntType.js +0 -122
  220. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +0 -1
  221. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +0 -67
  222. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +0 -1
  223. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +0 -120
  224. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
  225. package/dist/es5/parquetjs/parquet-thrift/ListType.js +0 -67
  226. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +0 -1
  227. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +0 -508
  228. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
  229. package/dist/es5/parquetjs/parquet-thrift/MapType.js +0 -67
  230. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +0 -1
  231. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +0 -67
  232. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
  233. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +0 -67
  234. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
  235. package/dist/es5/parquetjs/parquet-thrift/NullType.js +0 -67
  236. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +0 -1
  237. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +0 -114
  238. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
  239. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +0 -145
  240. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
  241. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +0 -258
  242. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
  243. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +0 -155
  244. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
  245. package/dist/es5/parquetjs/parquet-thrift/PageType.js +0 -16
  246. package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +0 -1
  247. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +0 -206
  248. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
  249. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +0 -290
  250. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
  251. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +0 -145
  252. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
  253. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +0 -207
  254. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +0 -1
  255. package/dist/es5/parquetjs/parquet-thrift/StringType.js +0 -67
  256. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +0 -1
  257. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +0 -124
  258. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +0 -1
  259. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +0 -156
  260. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
  261. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +0 -124
  262. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
  263. package/dist/es5/parquetjs/parquet-thrift/Type.js +0 -20
  264. package/dist/es5/parquetjs/parquet-thrift/Type.js.map +0 -1
  265. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -67
  266. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
  267. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +0 -67
  268. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
  269. package/dist/es5/parquetjs/parquet-thrift/index.js +0 -565
  270. package/dist/es5/parquetjs/parquet-thrift/index.js.map +0 -1
  271. package/dist/es5/parquetjs/parser/decoders.js +0 -489
  272. package/dist/es5/parquetjs/parser/parquet-cursor.js +0 -215
  273. package/dist/es5/parquetjs/parser/parquet-cursor.js.map +0 -1
  274. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +0 -413
  275. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
  276. package/dist/es5/parquetjs/parser/parquet-reader.js +0 -364
  277. package/dist/es5/parquetjs/parser/parquet-reader.js.map +0 -1
  278. package/dist/es5/parquetjs/schema/declare.js +0 -25
  279. package/dist/es5/parquetjs/schema/declare.js.map +0 -1
  280. package/dist/es5/parquetjs/schema/schema.js +0 -203
  281. package/dist/es5/parquetjs/schema/shred.js +0 -223
  282. package/dist/es5/parquetjs/schema/shred.js.map +0 -1
  283. package/dist/es5/parquetjs/schema/types.js +0 -492
  284. package/dist/es5/parquetjs/schema/types.js.map +0 -1
  285. package/dist/es5/parquetjs/utils/buffer-utils.js +0 -21
  286. package/dist/es5/parquetjs/utils/buffer-utils.js.map +0 -1
  287. package/dist/es5/parquetjs/utils/file-utils.js +0 -55
  288. package/dist/es5/parquetjs/utils/file-utils.js.map +0 -1
  289. package/dist/es5/parquetjs/utils/read-utils.js +0 -159
  290. package/dist/es5/parquetjs/utils/read-utils.js.map +0 -1
  291. package/dist/es5/workers/parquet-worker.js +0 -8
  292. package/dist/es5/workers/parquet-worker.js.map +0 -1
  293. package/dist/esm/bundle.js +0 -5
  294. package/dist/esm/bundle.js.map +0 -1
  295. package/dist/esm/constants.js +0 -6
  296. package/dist/esm/constants.js.map +0 -1
  297. package/dist/esm/index.js +0 -15
  298. package/dist/esm/index.js.map +0 -1
  299. package/dist/esm/lib/convert-schema.js +0 -71
  300. package/dist/esm/lib/convert-schema.js.map +0 -1
  301. package/dist/esm/lib/parse-parquet.js +0 -28
  302. package/dist/esm/lib/parse-parquet.js.map +0 -1
  303. package/dist/esm/lib/read-array-buffer.js +0 -9
  304. package/dist/esm/lib/read-array-buffer.js.map +0 -1
  305. package/dist/esm/parquet-loader.js +0 -22
  306. package/dist/esm/parquet-loader.js.map +0 -1
  307. package/dist/esm/parquet-writer.js +0 -18
  308. package/dist/esm/parquet-writer.js.map +0 -1
  309. package/dist/esm/parquetjs/LICENSE +0 -20
  310. package/dist/esm/parquetjs/codecs/declare.js +0 -2
  311. package/dist/esm/parquetjs/codecs/declare.js.map +0 -1
  312. package/dist/esm/parquetjs/codecs/dictionary.js +0 -12
  313. package/dist/esm/parquetjs/codecs/dictionary.js.map +0 -1
  314. package/dist/esm/parquetjs/codecs/index.js +0 -23
  315. package/dist/esm/parquetjs/codecs/index.js.map +0 -1
  316. package/dist/esm/parquetjs/codecs/plain.js +0 -270
  317. package/dist/esm/parquetjs/codecs/plain.js.map +0 -1
  318. package/dist/esm/parquetjs/codecs/rle.js +0 -151
  319. package/dist/esm/parquetjs/codecs/rle.js.map +0 -1
  320. package/dist/esm/parquetjs/compression.js +0 -62
  321. package/dist/esm/parquetjs/compression.js.map +0 -1
  322. package/dist/esm/parquetjs/encoder/writer.js +0 -421
  323. package/dist/esm/parquetjs/encoder/writer.js.map +0 -1
  324. package/dist/esm/parquetjs/file.js +0 -80
  325. package/dist/esm/parquetjs/file.js.map +0 -1
  326. package/dist/esm/parquetjs/modules.d.ts +0 -21
  327. package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +0 -8
  328. package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
  329. package/dist/esm/parquetjs/parquet-thrift/BsonType.js +0 -39
  330. package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +0 -1
  331. package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +0 -217
  332. package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
  333. package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +0 -218
  334. package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
  335. package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +0 -429
  336. package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
  337. package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +0 -95
  338. package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
  339. package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +0 -13
  340. package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
  341. package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +0 -27
  342. package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
  343. package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +0 -166
  344. package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
  345. package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -236
  346. package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
  347. package/dist/esm/parquetjs/parquet-thrift/DateType.js +0 -39
  348. package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +0 -1
  349. package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +0 -95
  350. package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
  351. package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -117
  352. package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
  353. package/dist/esm/parquetjs/parquet-thrift/Encoding.js +0 -13
  354. package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +0 -1
  355. package/dist/esm/parquetjs/parquet-thrift/EnumType.js +0 -39
  356. package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +0 -1
  357. package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -8
  358. package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
  359. package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +0 -270
  360. package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
  361. package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +0 -39
  362. package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
  363. package/dist/esm/parquetjs/parquet-thrift/IntType.js +0 -95
  364. package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +0 -1
  365. package/dist/esm/parquetjs/parquet-thrift/JsonType.js +0 -39
  366. package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +0 -1
  367. package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +0 -93
  368. package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
  369. package/dist/esm/parquetjs/parquet-thrift/ListType.js +0 -39
  370. package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +0 -1
  371. package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +0 -467
  372. package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
  373. package/dist/esm/parquetjs/parquet-thrift/MapType.js +0 -39
  374. package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +0 -1
  375. package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +0 -39
  376. package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
  377. package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +0 -39
  378. package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
  379. package/dist/esm/parquetjs/parquet-thrift/NullType.js +0 -39
  380. package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +0 -1
  381. package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +0 -85
  382. package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
  383. package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +0 -119
  384. package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
  385. package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +0 -233
  386. package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
  387. package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +0 -128
  388. package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
  389. package/dist/esm/parquetjs/parquet-thrift/PageType.js +0 -9
  390. package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +0 -1
  391. package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +0 -178
  392. package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
  393. package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +0 -270
  394. package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
  395. package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +0 -119
  396. package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
  397. package/dist/esm/parquetjs/parquet-thrift/Statistics.js +0 -183
  398. package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +0 -1
  399. package/dist/esm/parquetjs/parquet-thrift/StringType.js +0 -39
  400. package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +0 -1
  401. package/dist/esm/parquetjs/parquet-thrift/TimeType.js +0 -96
  402. package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +0 -1
  403. package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +0 -126
  404. package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
  405. package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +0 -96
  406. package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
  407. package/dist/esm/parquetjs/parquet-thrift/Type.js +0 -13
  408. package/dist/esm/parquetjs/parquet-thrift/Type.js.map +0 -1
  409. package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -39
  410. package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
  411. package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +0 -39
  412. package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
  413. package/dist/esm/parquetjs/parquet-thrift/index.js +0 -44
  414. package/dist/esm/parquetjs/parquet-thrift/index.js.map +0 -1
  415. package/dist/esm/parquetjs/parser/decoders.js +0 -300
  416. package/dist/esm/parquetjs/parser/decoders.js.map +0 -1
  417. package/dist/esm/parquetjs/parser/parquet-cursor.js +0 -90
  418. package/dist/esm/parquetjs/parser/parquet-cursor.js.map +0 -1
  419. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +0 -155
  420. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
  421. package/dist/esm/parquetjs/parser/parquet-reader.js +0 -120
  422. package/dist/esm/parquetjs/parser/parquet-reader.js.map +0 -1
  423. package/dist/esm/parquetjs/schema/declare.js +0 -13
  424. package/dist/esm/parquetjs/schema/declare.js.map +0 -1
  425. package/dist/esm/parquetjs/schema/schema.js +0 -176
  426. package/dist/esm/parquetjs/schema/schema.js.map +0 -1
  427. package/dist/esm/parquetjs/schema/shred.js +0 -162
  428. package/dist/esm/parquetjs/schema/shred.js.map +0 -1
  429. package/dist/esm/parquetjs/schema/types.js +0 -476
  430. package/dist/esm/parquetjs/schema/types.js.map +0 -1
  431. package/dist/esm/parquetjs/utils/buffer-utils.js +0 -12
  432. package/dist/esm/parquetjs/utils/buffer-utils.js.map +0 -1
  433. package/dist/esm/parquetjs/utils/file-utils.js +0 -34
  434. package/dist/esm/parquetjs/utils/file-utils.js.map +0 -1
  435. package/dist/esm/parquetjs/utils/read-utils.js +0 -105
  436. package/dist/esm/parquetjs/utils/read-utils.js.map +0 -1
  437. package/dist/esm/workers/parquet-worker.js +0 -4
  438. package/dist/esm/workers/parquet-worker.js.map +0 -1
@@ -1,159 +0,0 @@
1
- "use strict";
2
-
3
- var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
-
5
- Object.defineProperty(exports, "__esModule", {
6
- value: true
7
- });
8
- exports.serializeThrift = serializeThrift;
9
- exports.decodeThrift = decodeThrift;
10
- exports.getThriftEnum = getThriftEnum;
11
- exports.decodeFileMetadata = decodeFileMetadata;
12
- exports.decodePageHeader = decodePageHeader;
13
- exports.getBitWidth = getBitWidth;
14
- exports.fieldIndexOf = fieldIndexOf;
15
-
16
- var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
17
-
18
- var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
19
-
20
- var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
21
-
22
- var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
23
-
24
- var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
25
-
26
- var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
27
-
28
- var _thrift = require("thrift");
29
-
30
- var _parquetThrift = require("../parquet-thrift");
31
-
32
- function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
33
-
34
- function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
35
-
36
- var UFramedTransport = function (_TFramedTransport) {
37
- (0, _inherits2.default)(UFramedTransport, _TFramedTransport);
38
-
39
- var _super = _createSuper(UFramedTransport);
40
-
41
- function UFramedTransport() {
42
- var _this;
43
-
44
- (0, _classCallCheck2.default)(this, UFramedTransport);
45
-
46
- for (var _len = arguments.length, args = new Array(_len), _key = 0; _key < _len; _key++) {
47
- args[_key] = arguments[_key];
48
- }
49
-
50
- _this = _super.call.apply(_super, [this].concat(args));
51
- (0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "readPos", 0);
52
- return _this;
53
- }
54
-
55
- return UFramedTransport;
56
- }(_thrift.TFramedTransport);
57
-
58
- function serializeThrift(obj) {
59
- var output = [];
60
- var transport = new _thrift.TBufferedTransport(undefined, function (buf) {
61
- output.push(buf);
62
- });
63
- var protocol = new _thrift.TCompactProtocol(transport);
64
- obj.write(protocol);
65
- transport.flush();
66
- return Buffer.concat(output);
67
- }
68
-
69
- function decodeThrift(obj, buf, offset) {
70
- if (!offset) {
71
- offset = 0;
72
- }
73
-
74
- var transport = new UFramedTransport(buf);
75
- transport.readPos = offset;
76
- var protocol = new _thrift.TCompactProtocol(transport);
77
- obj.read(protocol);
78
- return transport.readPos - offset;
79
- }
80
-
81
- function getThriftEnum(klass, value) {
82
- for (var k in klass) {
83
- if (klass[k] === value) {
84
- return k;
85
- }
86
- }
87
-
88
- throw new Error('Invalid ENUM value');
89
- }
90
-
91
- function decodeFileMetadata(buf, offset) {
92
- if (!offset) {
93
- offset = 0;
94
- }
95
-
96
- var transport = new UFramedTransport(buf);
97
- transport.readPos = offset;
98
- var protocol = new _thrift.TCompactProtocol(transport);
99
-
100
- var metadata = _parquetThrift.FileMetaData.read(protocol);
101
-
102
- return {
103
- length: transport.readPos - offset,
104
- metadata: metadata
105
- };
106
- }
107
-
108
- function decodePageHeader(buf, offset) {
109
- if (!offset) {
110
- offset = 0;
111
- }
112
-
113
- var transport = new UFramedTransport(buf);
114
- transport.readPos = offset;
115
- var protocol = new _thrift.TCompactProtocol(transport);
116
-
117
- var pageHeader = _parquetThrift.PageHeader.read(protocol);
118
-
119
- return {
120
- length: transport.readPos - offset,
121
- pageHeader: pageHeader
122
- };
123
- }
124
-
125
- function getBitWidth(val) {
126
- if (val === 0) {
127
- return 0;
128
- }
129
-
130
- return Math.ceil(Math.log2(val + 1));
131
- }
132
-
133
- function fieldIndexOf(arr, elem) {
134
- for (var j = 0; j < arr.length; j++) {
135
- if (arr[j].length > elem.length) {
136
- continue;
137
- }
138
-
139
- var m = true;
140
-
141
- for (var i = 0; i < elem.length; i++) {
142
- if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
143
- continue;
144
- }
145
-
146
- if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
147
- continue;
148
- }
149
-
150
- m = false;
151
- break;
152
- }
153
-
154
- if (m) return j;
155
- }
156
-
157
- return -1;
158
- }
159
- //# sourceMappingURL=read-utils.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../../src/parquetjs/utils/read-utils.ts"],"names":["UFramedTransport","TFramedTransport","serializeThrift","obj","output","transport","TBufferedTransport","undefined","buf","push","protocol","TCompactProtocol","write","flush","Buffer","concat","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","FileMetaData","length","decodePageHeader","pageHeader","PageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;AACA;;;;;;IAEMA,gB;;;;;;;;;;;;;;;0FACqB,C;;;;;EADIC,wB;;AAOxB,SAASC,eAAT,CAAyBC,GAAzB,EAA2C;AAChD,MAAMC,MAAgB,GAAG,EAAzB;AAEA,MAAMC,SAAS,GAAG,IAAIC,0BAAJ,CAAuBC,SAAvB,EAAkC,UAACC,GAAD,EAAS;AAC3DJ,IAAAA,MAAM,CAACK,IAAP,CAAYD,GAAZ;AACD,GAFiB,CAAlB;AAIA,MAAME,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;AACAF,EAAAA,GAAG,CAACS,KAAJ,CAAUF,QAAV;AACAL,EAAAA,SAAS,CAACQ,KAAV;AAEA,SAAOC,MAAM,CAACC,MAAP,CAAcX,MAAd,CAAP;AACD;;AAEM,SAASY,YAAT,CAAsBb,GAAtB,EAAgCK,GAAhC,EAA6CS,MAA7C,EAA8D;AACnE,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,MAAMZ,SAAS,GAAG,IAAIL,gBAAJ,CAAqBQ,GAArB,CAAlB;AACAH,EAAAA,SAAS,CAACa,OAAV,GAAoBD,MAApB;AACA,MAAMP,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;AACAF,EAAAA,GAAG,CAACgB,IAAJ,CAAST,QAAT;AACA,SAAOL,SAAS,CAACa,OAAV,GAAoBD,MAA3B;AACD;;AAKM,SAASG,aAAT,CAAuBC,KAAvB,EAAmCC,KAAnC,EAAmE;AACxE,OAAK,IAAMC,CAAX,IAAgBF,KAAhB,EAAuB;AACrB,QAAIA,KAAK,CAACE,CAAD,CAAL,KAAaD,KAAjB,EAAwB;AACtB,aAAOC,CAAP;AACD;AACF;;AACD,QAAM,IAAIC,KAAJ,CAAU,oBAAV,CAAN;AACD;;AAEM,SAASC,kBAAT,CAA4BjB,GAA5B,EAAyCS,MAAzC,EAA0D;AAC/D,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,MAAMZ,SAAS,GAAG,IAAIL,gBAAJ,CAAqBQ,GAArB,CAAlB;AACAH,EAAAA,SAAS,CAACa,OAAV,GAAoBD,MAApB;AACA,MAAMP,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;;AACA,MAAMqB,QAAQ,GAAGC,4BAAaR,IAAb,CAAkBT,QAAlB,CAAjB;;AACA,SAAO;AAACkB,IAAAA,MAAM,EAAEvB,SAAS,CAACa,OAAV,GAAoBD,MAA7B;AAAqCS,IAAAA,QAAQ,EAARA;AAArC,GAAP;AACD;;AAEM,SAASG,gBAAT,CAA0BrB,GAA1B,EAAuCS,MAAvC,EAAwD;AAC7D,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,MAAMZ,SAAS,GAAG,IAAIL,gBAAJ,CAAqBQ,GAArB,CAAlB;AACAH,EAAAA,SAAS,CAACa,OAAV,GAAoBD,MAApB;AACA,MAAMP,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;;AACA,MAAMyB,UAAU,GAAGC,0BAAWZ,IAAX,CAAgBT,QAAhB,CAAnB;;AACA,SAAO;AAACkB,IAAAA,MAAM,EAAEvB,SAAS,CAACa,OAAV,GAAoBD,MAA7B;AAAqCa,IAAAA,UAAU,EAAVA;AAArC,GAAP;AACD;;AAKM,SAASE,WAAT,CAAqBC,GAArB,EAA0C;AAC/C,MAAIA,GAAG,KAAK,CAAZ,EAAe;AACb,WAAO,CAAP;AAED;;AACD,SAAOC,IAAI,CAACC,IAAL,CAAUD,IAAI,CAACE,IAAL,CAAUH,GAAG,GAAG,CAAhB,CAAV,CAAP;AACD;;AAKM,SAASI,YAAT,CAAsBC,GAAtB,EAAuCC,IAAvC,EAA+D;AACpE,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGF,GAAG,CAACV,MAAxB,EAAgCY,CAAC,EAAjC,EAAqC;AACnC,QAAIF,GAAG,CAACE,CAAD,CAAH,CAAOZ,MAAP,GAAgBW,IAAI,CAACX,MAAzB,EAAiC;AAC/B;AACD;;AACD,QAAIa,CAAC,GAAG,IAAR;;AACA,SAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,IAAI,CAACX,MAAzB,EAAiCc,CAAC,EAAlC,EAAsC;AACpC,UAAIJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAcH,IAAI,CAACG,CAAD,CAAlB,IAAyBJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAc,GAAvC,IAA8CJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAc,GAAhE,EAAqE;AACnE;AACD;;AACD,UAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAD,CAAH,CAAOZ,MAAZ,IAAsBU,GAAG,CAACE,CAAD,CAAH,CAAOF,GAAG,CAACE,CAAD,CAAH,CAAOZ,MAAP,GAAgB,CAAvB,MAA8B,GAAxD,EAA6D;AAC3D;AACD;;AACDa,MAAAA,CAAC,GAAG,KAAJ;AACA;AACD;;AACD,QAAIA,CAAJ,EAAO,OAAOD,CAAP;AACR;;AACD,SAAO,CAAC,CAAR;AACD","sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n const output: Buffer[] = [];\n\n const transport = new TBufferedTransport(undefined, (buf) => {\n output.push(buf as Buffer);\n });\n\n const protocol = new TCompactProtocol(transport);\n obj.write(protocol);\n transport.flush();\n\n return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n obj.read(protocol);\n return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n for (const k in klass) {\n if (klass[k] === value) {\n return k;\n }\n }\n throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n const metadata = FileMetaData.read(protocol);\n return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n const pageHeader = PageHeader.read(protocol);\n return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n if (val === 0) {\n return 0;\n // tslint:disable-next-line:no-else-after-return\n }\n return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n for (let j = 0; j < arr.length; j++) {\n if (arr[j].length > elem.length) {\n continue; // eslint-disable-line no-continue\n }\n let m = true;\n for (let i = 0; i < elem.length; i++) {\n if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n continue; // eslint-disable-line no-continue\n }\n if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n continue; // eslint-disable-line no-continue\n }\n m = false;\n break;\n }\n if (m) return j;\n }\n return -1;\n}\n"],"file":"read-utils.js"}
@@ -1,8 +0,0 @@
1
- "use strict";
2
-
3
- var _loaderUtils = require("@loaders.gl/loader-utils");
4
-
5
- var _index = require("../index");
6
-
7
- (0, _loaderUtils.createLoaderWorker)(_index.ParquetLoader);
8
- //# sourceMappingURL=parquet-worker.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../src/workers/parquet-worker.ts"],"names":["ParquetLoader"],"mappings":";;AAAA;;AACA;;AAEA,qCAAmBA,oBAAnB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ParquetLoader} from '../index';\n\ncreateLoaderWorker(ParquetLoader);\n"],"file":"parquet-worker.js"}
@@ -1,5 +0,0 @@
1
- const moduleExports = require('./index');
2
-
3
- globalThis.loaders = globalThis.loaders || {};
4
- module.exports = Object.assign(globalThis.loaders, moduleExports);
5
- //# sourceMappingURL=bundle.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}
@@ -1,6 +0,0 @@
1
- export const PARQUET_MAGIC = 'PAR1';
2
- export const PARQUET_MAGIC_ENCRYPTED = 'PARE';
3
- export const PARQUET_VERSION = 1;
4
- export const PARQUET_RDLVL_TYPE = 'INT32';
5
- export const PARQUET_RDLVL_ENCODING = 'RLE';
6
- //# sourceMappingURL=constants.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../src/constants.ts"],"names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"mappings":"AAIA,OAAO,MAAMA,aAAa,GAAG,MAAtB;AACP,OAAO,MAAMC,uBAAuB,GAAG,MAAhC;AAKP,OAAO,MAAMC,eAAe,GAAG,CAAxB;AAKP,OAAO,MAAMC,kBAAkB,GAAG,OAA3B;AACP,OAAO,MAAMC,sBAAsB,GAAG,KAA/B","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"file":"constants.js"}
package/dist/esm/index.js DELETED
@@ -1,15 +0,0 @@
1
- import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
2
- import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
3
- export { ParquetWorkerLoader };
4
- export const ParquetLoader = { ...ParquetWorkerLoader,
5
- parse: parseParquet,
6
- parseFileInBatches: parseParquetFileInBatches
7
- };
8
- export { ParquetWriter as _ParquetWriter } from './parquet-writer';
9
- export { preloadCompressions } from './parquetjs/compression';
10
- export { ParquetSchema } from './parquetjs/schema/schema';
11
- export { ParquetReader } from './parquetjs/parser/parquet-reader';
12
- export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
13
- export { convertParquetToArrowSchema } from './lib/convert-schema';
14
- export const _typecheckParquetLoader = ParquetLoader;
15
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEnvelopeReader","convertParquetToArrowSchema","_typecheckParquetLoader"],"mappings":"AAIA,SAAQA,aAAa,IAAIC,mBAAzB,QAAmD,kBAAnD;AACA,SAAQC,YAAR,EAAsBC,yBAAtB,QAAsD,qBAAtD;AAEA,SAAQF,mBAAR;AAGA,OAAO,MAAMD,aAAa,GAAG,EAC3B,GAAGC,mBADwB;AAE3BG,EAAAA,KAAK,EAAEF,YAFoB;AAG3BG,EAAAA,kBAAkB,EAAEF;AAHO,CAAtB;AAQP,SAAQG,aAAa,IAAIC,cAAzB,QAA8C,kBAA9C;AAIA,SAAQC,mBAAR,QAAkC,yBAAlC;AAEA,SAAQC,aAAR,QAA4B,2BAA5B;AACA,SAAQC,aAAR,QAA4B,mCAA5B;AACA,SAAQC,qBAAR,QAAoC,4CAApC;AAEA,SAAQC,2BAAR,QAA0C,sBAA1C;AAGA,OAAO,MAAMC,uBAAyC,GAAGb,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n ...ParquetWorkerLoader,\n parse: parseParquet,\n parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}
@@ -1,71 +0,0 @@
1
- import { Schema, Struct, Field, Bool, Float64, Int32, Float32, Binary, Utf8, Int64, Uint16, Uint32, Uint64, Int8, Int16 } from '@loaders.gl/schema';
2
- export const PARQUET_TYPE_MAPPING = {
3
- BOOLEAN: Bool,
4
- INT32: Int32,
5
- INT64: Float64,
6
- INT96: Float64,
7
- FLOAT: Float32,
8
- DOUBLE: Float64,
9
- BYTE_ARRAY: Binary,
10
- FIXED_LEN_BYTE_ARRAY: Binary,
11
- UTF8: Utf8,
12
- DATE: Int32,
13
- TIME_MILLIS: Int64,
14
- TIME_MICROS: Int64,
15
- TIMESTAMP_MILLIS: Int64,
16
- TIMESTAMP_MICROS: Int64,
17
- UINT_8: Int32,
18
- UINT_16: Uint16,
19
- UINT_32: Uint32,
20
- UINT_64: Uint64,
21
- INT_8: Int8,
22
- INT_16: Int16,
23
- INT_32: Int32,
24
- INT_64: Int64,
25
- JSON: Binary,
26
- BSON: Binary,
27
- INTERVAL: Binary,
28
- DECIMAL_INT32: Float32,
29
- DECIMAL_INT64: Float64,
30
- DECIMAL_BYTE_ARRAY: Float64,
31
- DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
32
- };
33
- export function convertParquetToArrowSchema(parquetSchema) {
34
- const fields = getFields(parquetSchema.schema);
35
- return new Schema(fields);
36
- }
37
-
38
- function getFieldMetadata(field) {
39
- const metadata = new Map();
40
-
41
- for (const key in field) {
42
- if (key !== 'name') {
43
- const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
44
- metadata.set(key, value);
45
- }
46
- }
47
-
48
- return metadata;
49
- }
50
-
51
- function getFields(schema) {
52
- const fields = [];
53
-
54
- for (const name in schema) {
55
- const field = schema[name];
56
-
57
- if (field.fields) {
58
- const childField = getFields(field.fields);
59
- const nestedField = new Field(name, new Struct(childField), field.optional);
60
- fields.push(nestedField);
61
- } else {
62
- const FieldType = PARQUET_TYPE_MAPPING[field.type];
63
- const metadata = getFieldMetadata(field);
64
- const arrowField = new Field(name, new FieldType(), field.optional, metadata);
65
- fields.push(arrowField);
66
- }
67
- }
68
-
69
- return fields;
70
- }
71
- //# sourceMappingURL=convert-schema.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../src/lib/convert-schema.ts"],"names":["Schema","Struct","Field","Bool","Float64","Int32","Float32","Binary","Utf8","Int64","Uint16","Uint32","Uint64","Int8","Int16","PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetToArrowSchema","parquetSchema","fields","getFields","schema","getFieldMetadata","field","metadata","Map","key","value","stringify","set","name","childField","nestedField","optional","push","FieldType","type","arrowField"],"mappings":"AAGA,SACEA,MADF,EAEEC,MAFF,EAGEC,KAHF,EAKEC,IALF,EAMEC,OANF,EAOEC,KAPF,EAQEC,OARF,EASEC,MATF,EAUEC,IAVF,EAWEC,KAXF,EAYEC,MAZF,EAaEC,MAbF,EAcEC,MAdF,EAeEC,IAfF,EAgBEC,KAhBF,QAiBO,oBAjBP;AAmBA,OAAO,MAAMC,oBAA8D,GAAG;AAC5EC,EAAAA,OAAO,EAAEb,IADmE;AAE5Ec,EAAAA,KAAK,EAAEZ,KAFqE;AAG5Ea,EAAAA,KAAK,EAAEd,OAHqE;AAI5Ee,EAAAA,KAAK,EAAEf,OAJqE;AAK5EgB,EAAAA,KAAK,EAAEd,OALqE;AAM5Ee,EAAAA,MAAM,EAAEjB,OANoE;AAO5EkB,EAAAA,UAAU,EAAEf,MAPgE;AAQ5EgB,EAAAA,oBAAoB,EAAEhB,MARsD;AAS5EiB,EAAAA,IAAI,EAAEhB,IATsE;AAU5EiB,EAAAA,IAAI,EAAEpB,KAVsE;AAW5EqB,EAAAA,WAAW,EAAEjB,KAX+D;AAY5EkB,EAAAA,WAAW,EAAElB,KAZ+D;AAa5EmB,EAAAA,gBAAgB,EAAEnB,KAb0D;AAc5EoB,EAAAA,gBAAgB,EAAEpB,KAd0D;AAe5EqB,EAAAA,MAAM,EAAEzB,KAfoE;AAgB5E0B,EAAAA,OAAO,EAAErB,MAhBmE;AAiB5EsB,EAAAA,OAAO,EAAErB,MAjBmE;AAkB5EsB,EAAAA,OAAO,EAAErB,MAlBmE;AAmB5EsB,EAAAA,KAAK,EAAErB,IAnBqE;AAoB5EsB,EAAAA,MAAM,EAAErB,KApBoE;AAqB5EsB,EAAAA,MAAM,EAAE/B,KArBoE;AAsB5EgC,EAAAA,MAAM,EAAE5B,KAtBoE;AAuB5E6B,EAAAA,IAAI,EAAE/B,MAvBsE;AAwB5EgC,EAAAA,IAAI,EAAEhC,MAxBsE;AA0B5EiC,EAAAA,QAAQ,EAAEjC,MA1BkE;AA2B5EkC,EAAAA,aAAa,EAAEnC,OA3B6D;AA4B5EoC,EAAAA,aAAa,EAAEtC,OA5B6D;AA6B5EuC,EAAAA,kBAAkB,EAAEvC,OA7BwD;AA8B5EwC,EAAAA,4BAA4B,EAAExC;AA9B8C,CAAvE;AAiCP,OAAO,SAASyC,2BAAT,CAAqCC,aAArC,EAA2E;AAChF,QAAMC,MAAM,GAAGC,SAAS,CAACF,aAAa,CAACG,MAAf,CAAxB;AAGA,SAAO,IAAIjD,MAAJ,CAAW+C,MAAX,CAAP;AACD;;AAED,SAASG,gBAAT,CAA0BC,KAA1B,EAAoE;AAClE,QAAMC,QAAQ,GAAG,IAAIC,GAAJ,EAAjB;;AAEA,OAAK,MAAMC,GAAX,IAAkBH,KAAlB,EAAyB;AACvB,QAAIG,GAAG,KAAK,MAAZ,EAAoB;AAClB,YAAMC,KAAK,GAAG,OAAOJ,KAAK,CAACG,GAAD,CAAZ,KAAsB,QAAtB,GAAiChB,IAAI,CAACkB,SAAL,CAAeL,KAAK,CAACG,GAAD,CAApB,CAAjC,GAA8DH,KAAK,CAACG,GAAD,CAAjF;AACAF,MAAAA,QAAQ,CAACK,GAAT,CAAaH,GAAb,EAAkBC,KAAlB;AACD;AACF;;AAED,SAAOH,QAAP;AACD;;AAED,SAASJ,SAAT,CAAmBC,MAAnB,EAAqD;AACnD,QAAMF,MAAe,GAAG,EAAxB;;AAEA,OAAK,MAAMW,IAAX,IAAmBT,MAAnB,EAA2B;AACzB,UAAME,KAAK,GAAGF,MAAM,CAACS,IAAD,CAApB;;AAEA,QAAIP,KAAK,CAACJ,MAAV,EAAkB;AAChB,YAAMY,UAAU,GAAGX,SAAS,CAACG,KAAK,CAACJ,MAAP,CAA5B;AACA,YAAMa,WAAW,GAAG,IAAI1D,KAAJ,CAAUwD,IAAV,EAAgB,IAAIzD,MAAJ,CAAW0D,UAAX,CAAhB,EAAwCR,KAAK,CAACU,QAA9C,CAApB;AACAd,MAAAA,MAAM,CAACe,IAAP,CAAYF,WAAZ;AACD,KAJD,MAIO;AACL,YAAMG,SAAS,GAAGhD,oBAAoB,CAACoC,KAAK,CAACa,IAAP,CAAtC;AACA,YAAMZ,QAAQ,GAAGF,gBAAgB,CAACC,KAAD,CAAjC;AACA,YAAMc,UAAU,GAAG,IAAI/D,KAAJ,CAAUwD,IAAV,EAAgB,IAAIK,SAAJ,EAAhB,EAAiCZ,KAAK,CAACU,QAAvC,EAAiDT,QAAjD,CAAnB;AACAL,MAAAA,MAAM,CAACe,IAAP,CAAYG,UAAZ;AACD;AACF;;AAED,SAAOlB,MAAP;AACD","sourcesContent":["import type {ParquetSchema} from '../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';\n\nimport {\n Schema,\n Struct,\n Field,\n DataType,\n Bool,\n Float64,\n Int32,\n Float32,\n Binary,\n Utf8,\n Int64,\n Uint16,\n Uint32,\n Uint64,\n Int8,\n Int16\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {\n BOOLEAN: Bool,\n INT32: Int32,\n INT64: Float64,\n INT96: Float64,\n FLOAT: Float32,\n DOUBLE: Float64,\n BYTE_ARRAY: Binary,\n FIXED_LEN_BYTE_ARRAY: Binary,\n UTF8: Utf8,\n DATE: Int32,\n TIME_MILLIS: Int64,\n TIME_MICROS: Int64,\n TIMESTAMP_MILLIS: Int64,\n TIMESTAMP_MICROS: Int64,\n UINT_8: Int32,\n UINT_16: Uint16,\n UINT_32: Uint32,\n UINT_64: Uint64,\n INT_8: Int8,\n INT_16: Int16,\n INT_32: Int32,\n INT_64: Int64,\n JSON: Binary,\n BSON: Binary,\n // TODO check interal type\n INTERVAL: Binary,\n DECIMAL_INT32: Float32,\n DECIMAL_INT64: Float64,\n DECIMAL_BYTE_ARRAY: Float64,\n DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64\n};\n\nexport function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {\n const fields = getFields(parquetSchema.schema);\n\n // TODO add metadata if needed.\n return new Schema(fields);\n}\n\nfunction getFieldMetadata(field: ParquetField): Map<string, string> {\n const metadata = new Map();\n\n for (const key in field) {\n if (key !== 'name') {\n const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n metadata.set(key, value);\n }\n }\n\n return metadata;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n const fields: Field[] = [];\n\n for (const name in schema) {\n const field = schema[name];\n\n if (field.fields) {\n const childField = getFields(field.fields);\n const nestedField = new Field(name, new Struct(childField), field.optional);\n fields.push(nestedField);\n } else {\n const FieldType = PARQUET_TYPE_MAPPING[field.type];\n const metadata = getFieldMetadata(field);\n const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n fields.push(arrowField);\n }\n }\n\n return fields;\n}\n"],"file":"convert-schema.js"}
@@ -1,28 +0,0 @@
1
- import { ParquetReader } from '../parquetjs/parser/parquet-reader';
2
- export async function parseParquet(arrayBuffer, options) {
3
- const blob = new Blob([arrayBuffer]);
4
-
5
- for await (const batch of parseParquetFileInBatches(blob, options)) {
6
- return batch;
7
- }
8
-
9
- return null;
10
- }
11
- export async function* parseParquetFileInBatches(blob, options) {
12
- const reader = await ParquetReader.openBlob(blob);
13
- const rows = [];
14
-
15
- try {
16
- const cursor = reader.getCursor();
17
- let record;
18
-
19
- while (record = await cursor.next()) {
20
- rows.push(record);
21
- }
22
- } finally {
23
- await reader.close();
24
- }
25
-
26
- yield rows;
27
- }
28
- //# sourceMappingURL=parse-parquet.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../src/lib/parse-parquet.ts"],"names":["ParquetReader","parseParquet","arrayBuffer","options","blob","Blob","batch","parseParquetFileInBatches","reader","openBlob","rows","cursor","getCursor","record","next","push","close"],"mappings":"AAGA,SAAQA,aAAR,QAA4B,oCAA5B;AAEA,OAAO,eAAeC,YAAf,CAA4BC,WAA5B,EAAsDC,OAAtD,EAAsF;AAC3F,QAAMC,IAAI,GAAG,IAAIC,IAAJ,CAAS,CAACH,WAAD,CAAT,CAAb;;AACA,aAAW,MAAMI,KAAjB,IAA0BC,yBAAyB,CAACH,IAAD,EAAOD,OAAP,CAAnD,EAAoE;AAClE,WAAOG,KAAP;AACD;;AACD,SAAO,IAAP;AACD;AAED,OAAO,gBAAgBC,yBAAhB,CAA0CH,IAA1C,EAAsDD,OAAtD,EAAsF;AAC3F,QAAMK,MAAM,GAAG,MAAMR,aAAa,CAACS,QAAd,CAAuBL,IAAvB,CAArB;AACA,QAAMM,IAAa,GAAG,EAAtB;;AACA,MAAI;AACF,UAAMC,MAAM,GAAGH,MAAM,CAACI,SAAP,EAAf;AACA,QAAIC,MAAJ;;AACA,WAAQA,MAAM,GAAG,MAAMF,MAAM,CAACG,IAAP,EAAvB,EAAuC;AACrCJ,MAAAA,IAAI,CAACK,IAAL,CAAUF,MAAV;AACD;AACF,GAND,SAMU;AACR,UAAML,MAAM,CAACQ,KAAP,EAAN;AACD;;AACD,QAAMN,IAAN;AACD","sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n const blob = new Blob([arrayBuffer]);\n for await (const batch of parseParquetFileInBatches(blob, options)) {\n return batch;\n }\n return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n const reader = await ParquetReader.openBlob(blob);\n const rows: any[][] = [];\n try {\n const cursor = reader.getCursor();\n let record: any[] | null;\n while ((record = await cursor.next())) {\n rows.push(record);\n }\n } finally {\n await reader.close();\n }\n yield rows;\n}\n"],"file":"parse-parquet.js"}
@@ -1,9 +0,0 @@
1
- export async function readArrayBuffer(file, start, length) {
2
- if (file instanceof Blob) {
3
- const slice = file.slice(start, start + length);
4
- return await slice.arrayBuffer();
5
- }
6
-
7
- return await file.read(start, start + length);
8
- }
9
- //# sourceMappingURL=read-array-buffer.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../src/lib/read-array-buffer.ts"],"names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"mappings":"AAEA,OAAO,eAAeA,eAAf,CACLC,IADK,EAELC,KAFK,EAGLC,MAHK,EAIiB;AACtB,MAAIF,IAAI,YAAYG,IAApB,EAA0B;AACxB,UAAMC,KAAK,GAAGJ,IAAI,CAACI,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,CAAd;AACA,WAAO,MAAME,KAAK,CAACC,WAAN,EAAb;AACD;;AACD,SAAO,MAAML,IAAI,CAACM,IAAL,CAAUL,KAAV,EAAiBA,KAAK,GAAGC,MAAzB,CAAb;AACD","sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n file: Blob | ArrayBuffer | any,\n start: number,\n length: number\n): Promise<ArrayBuffer> {\n if (file instanceof Blob) {\n const slice = file.slice(start, start + length);\n return await slice.arrayBuffer();\n }\n return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n return await new Promise((resolve, reject) => {\n const fileReader = new FileReader();\n fileReader.onload = (event: ProgressEvent<FileReader>) =>\n resolve(event?.target?.result as ArrayBuffer);\n // TODO - reject with a proper Error\n fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n fileReader.readAsArrayBuffer(blob);\n });\n}\n*/\n"],"file":"read-array-buffer.js"}
@@ -1,22 +0,0 @@
1
- const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
2
- const DEFAULT_PARQUET_LOADER_OPTIONS = {
3
- parquet: {
4
- type: 'object-row-table',
5
- url: undefined
6
- }
7
- };
8
- export const ParquetLoader = {
9
- name: 'Apache Parquet',
10
- id: 'parquet',
11
- module: 'parquet',
12
- version: VERSION,
13
- worker: true,
14
- category: 'table',
15
- extensions: ['parquet'],
16
- mimeTypes: ['application/octet-stream'],
17
- binary: true,
18
- tests: ['PAR1', 'PARE'],
19
- options: DEFAULT_PARQUET_LOADER_OPTIONS
20
- };
21
- export const _typecheckParquetLoader = ParquetLoader;
22
- //# sourceMappingURL=parquet-loader.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQA,OAAO,MAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;AAcP,OAAO,MAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'object-row-table';\n url?: string;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'object-row-table',\n url: undefined\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}
@@ -1,18 +0,0 @@
1
- const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
2
- const DEFAULT_PARQUET_LOADER_OPTIONS = {};
3
- export const ParquetWriter = {
4
- name: 'Apache Parquet',
5
- id: 'parquet',
6
- module: 'parquet',
7
- version: VERSION,
8
- extensions: ['parquet'],
9
- mimeTypes: ['application/octet-stream'],
10
- encodeSync,
11
- binary: true,
12
- options: DEFAULT_PARQUET_LOADER_OPTIONS
13
- };
14
-
15
- function encodeSync(data, options) {
16
- return new ArrayBuffer(0);
17
- }
18
- //# sourceMappingURL=parquet-writer.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":"AAIA,MAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEA,OAAO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
@@ -1,20 +0,0 @@
1
- The code in the parquetjs is forked from https://github.com/kbajalc/parquets under MIT license
2
-
3
- Copyright (c) 2017 ironSource Ltd.
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy of
6
- this software and associated documentation files (the "Software"), to deal in the
7
- Software without restriction, including without limitation the rights to use,
8
- copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
9
- Software, and to permit persons to whom the Software is furnished to do so,
10
- subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
16
- INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
17
- PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
18
- HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20
- SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=declare.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":[],"names":[],"mappings":"","sourcesContent":[],"file":"declare.js"}
@@ -1,12 +0,0 @@
1
- import { decodeValues as decodeRleValues } from './rle';
2
- export function decodeValues(type, cursor, count, opts) {
3
- opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
4
- cursor.offset += 1;
5
- return decodeRleValues(type, cursor, count, { ...opts,
6
- disableEnvelope: true
7
- });
8
- }
9
- export function encodeValues(type, cursor, count, opts) {
10
- throw new Error('Encode dictionary functionality is not supported');
11
- }
12
- //# sourceMappingURL=dictionary.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../../src/parquetjs/codecs/dictionary.ts"],"names":["decodeValues","decodeRleValues","type","cursor","count","opts","bitWidth","buffer","slice","offset","readInt8","disableEnvelope","encodeValues","Error"],"mappings":"AAAA,SAAQA,YAAY,IAAIC,eAAxB,QAA8C,OAA9C;AAEA,OAAO,SAASD,YAAT,CAAsBE,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtDA,EAAAA,IAAI,CAACC,QAAL,GAAgBH,MAAM,CAACI,MAAP,CAAcC,KAAd,CAAoBL,MAAM,CAACM,MAA3B,EAAmCN,MAAM,CAACM,MAAP,GAAgB,CAAnD,EAAsDC,QAAtD,CAA+D,CAA/D,CAAhB;AACAP,EAAAA,MAAM,CAACM,MAAP,IAAiB,CAAjB;AACA,SAAOR,eAAe,CAACC,IAAD,EAAOC,MAAP,EAAeC,KAAf,EAAsB,EAAC,GAAGC,IAAJ;AAAUM,IAAAA,eAAe,EAAE;AAA3B,GAAtB,CAAtB;AACD;AAED,OAAO,SAASC,YAAT,CAAsBV,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtD,QAAM,IAAIQ,KAAJ,CAAU,kDAAV,CAAN;AACD","sourcesContent":["import {decodeValues as decodeRleValues} from './rle';\n\nexport function decodeValues(type, cursor, count, opts) {\n opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);\n cursor.offset += 1;\n return decodeRleValues(type, cursor, count, {...opts, disableEnvelope: true});\n}\n\nexport function encodeValues(type, cursor, count, opts) {\n throw new Error('Encode dictionary functionality is not supported');\n}\n"],"file":"dictionary.js"}
@@ -1,23 +0,0 @@
1
- import * as PLAIN from './plain';
2
- import * as RLE from './rle';
3
- import * as DICTIONARY from './dictionary';
4
- export * from './declare';
5
- export const PARQUET_CODECS = {
6
- PLAIN: {
7
- encodeValues: PLAIN.encodeValues,
8
- decodeValues: PLAIN.decodeValues
9
- },
10
- RLE: {
11
- encodeValues: RLE.encodeValues,
12
- decodeValues: RLE.decodeValues
13
- },
14
- PLAIN_DICTIONARY: {
15
- encodeValues: DICTIONARY.encodeValues,
16
- decodeValues: DICTIONARY.decodeValues
17
- },
18
- RLE_DICTIONARY: {
19
- encodeValues: DICTIONARY.encodeValues,
20
- decodeValues: DICTIONARY.decodeValues
21
- }
22
- };
23
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PLAIN","RLE","DICTIONARY","PARQUET_CODECS","encodeValues","decodeValues","PLAIN_DICTIONARY","RLE_DICTIONARY"],"mappings":"AAGA,OAAO,KAAKA,KAAZ,MAAuB,SAAvB;AACA,OAAO,KAAKC,GAAZ,MAAqB,OAArB;AACA,OAAO,KAAKC,UAAZ,MAA4B,cAA5B;AAEA,cAAc,WAAd;AAEA,OAAO,MAAMC,cAAqD,GAAG;AACnEH,EAAAA,KAAK,EAAE;AACLI,IAAAA,YAAY,EAAEJ,KAAK,CAACI,YADf;AAELC,IAAAA,YAAY,EAAEL,KAAK,CAACK;AAFf,GAD4D;AAKnEJ,EAAAA,GAAG,EAAE;AACHG,IAAAA,YAAY,EAAEH,GAAG,CAACG,YADf;AAEHC,IAAAA,YAAY,EAAEJ,GAAG,CAACI;AAFf,GAL8D;AAUnEC,EAAAA,gBAAgB,EAAE;AAEhBF,IAAAA,YAAY,EAAEF,UAAU,CAACE,YAFT;AAGhBC,IAAAA,YAAY,EAAEH,UAAU,CAACG;AAHT,GAViD;AAgBnEE,EAAAA,cAAc,EAAE;AAEdH,IAAAA,YAAY,EAAEF,UAAU,CAACE,YAFX;AAGdC,IAAAA,YAAY,EAAEH,UAAU,CAACG;AAHX;AAhBmD,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\nimport * as DICTIONARY from './dictionary';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n PLAIN: {\n encodeValues: PLAIN.encodeValues,\n decodeValues: PLAIN.decodeValues\n },\n RLE: {\n encodeValues: RLE.encodeValues,\n decodeValues: RLE.decodeValues\n },\n // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.\n PLAIN_DICTIONARY: {\n // @ts-ignore\n encodeValues: DICTIONARY.encodeValues,\n decodeValues: DICTIONARY.decodeValues\n },\n // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.\n RLE_DICTIONARY: {\n // @ts-ignore\n encodeValues: DICTIONARY.encodeValues,\n decodeValues: DICTIONARY.decodeValues\n }\n};\n"],"file":"index.js"}