@loaders.gl/parquet 3.3.0-alpha.4 → 3.3.0-alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/dist/dist.min.js +14 -14
  2. package/dist/dist.min.js.map +2 -2
  3. package/dist/es5/bundle.js +0 -1
  4. package/dist/es5/bundle.js.map +1 -1
  5. package/dist/es5/constants.js +3 -1
  6. package/dist/es5/constants.js.map +1 -1
  7. package/dist/es5/index.js +23 -39
  8. package/dist/es5/index.js.map +1 -1
  9. package/dist/es5/lib/convert-schema.js +2 -11
  10. package/dist/es5/lib/convert-schema.js.map +1 -1
  11. package/dist/es5/lib/parse-parquet.js +29 -72
  12. package/dist/es5/lib/parse-parquet.js.map +1 -1
  13. package/dist/es5/lib/read-array-buffer.js +0 -10
  14. package/dist/es5/lib/read-array-buffer.js.map +1 -1
  15. package/dist/es5/lib/wasm/encode-parquet-wasm.js +0 -11
  16. package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -1
  17. package/dist/es5/lib/wasm/load-wasm/index.js +0 -1
  18. package/dist/es5/lib/wasm/load-wasm/index.js.map +1 -1
  19. package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +0 -14
  20. package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
  21. package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +0 -10
  22. package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
  23. package/dist/es5/lib/wasm/parse-parquet-wasm.js +1 -19
  24. package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -1
  25. package/dist/es5/parquet-loader.js +2 -1
  26. package/dist/es5/parquet-loader.js.map +1 -1
  27. package/dist/es5/parquet-wasm-loader.js +2 -1
  28. package/dist/es5/parquet-wasm-loader.js.map +1 -1
  29. package/dist/es5/parquet-wasm-writer.js +1 -3
  30. package/dist/es5/parquet-wasm-writer.js.map +1 -1
  31. package/dist/es5/parquet-writer.js +1 -2
  32. package/dist/es5/parquet-writer.js.map +1 -1
  33. package/dist/es5/parquetjs/codecs/declare.js.map +1 -1
  34. package/dist/es5/parquetjs/codecs/dictionary.js +2 -9
  35. package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
  36. package/dist/es5/parquetjs/codecs/index.js +0 -8
  37. package/dist/es5/parquetjs/codecs/index.js.map +1 -1
  38. package/dist/es5/parquetjs/codecs/plain.js +1 -77
  39. package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
  40. package/dist/es5/parquetjs/codecs/rle.js +1 -39
  41. package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
  42. package/dist/es5/parquetjs/compression.js +5 -30
  43. package/dist/es5/parquetjs/compression.js.map +1 -1
  44. package/dist/es5/parquetjs/encoder/writer.js +31 -149
  45. package/dist/es5/parquetjs/encoder/writer.js.map +1 -1
  46. package/dist/es5/parquetjs/file.js +3 -12
  47. package/dist/es5/parquetjs/file.js.map +1 -1
  48. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +0 -1
  49. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -1
  50. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +0 -15
  51. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  52. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +0 -48
  53. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  54. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +0 -47
  55. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  56. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +0 -82
  57. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  58. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +0 -25
  59. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  60. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +0 -1
  61. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -1
  62. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +0 -1
  63. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +1 -1
  64. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +0 -39
  65. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  66. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -51
  67. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  68. package/dist/es5/parquetjs/parquet-thrift/DateType.js +0 -15
  69. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
  70. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +0 -26
  71. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  72. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -30
  73. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  74. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +0 -1
  75. package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +1 -1
  76. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +0 -15
  77. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  78. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -1
  79. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -1
  80. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +0 -59
  81. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  82. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +0 -15
  83. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  84. package/dist/es5/parquetjs/parquet-thrift/IntType.js +0 -26
  85. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
  86. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +0 -15
  87. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  88. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +0 -26
  89. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  90. package/dist/es5/parquetjs/parquet-thrift/ListType.js +0 -15
  91. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
  92. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +0 -85
  93. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  94. package/dist/es5/parquetjs/parquet-thrift/MapType.js +0 -15
  95. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
  96. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +0 -15
  97. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  98. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +0 -15
  99. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  100. package/dist/es5/parquetjs/parquet-thrift/NullType.js +0 -15
  101. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
  102. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +0 -25
  103. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  104. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +0 -30
  105. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  106. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +0 -54
  107. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  108. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +0 -31
  109. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  110. package/dist/es5/parquetjs/parquet-thrift/PageType.js +0 -1
  111. package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +1 -1
  112. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +0 -41
  113. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  114. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +0 -59
  115. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  116. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +0 -30
  117. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  118. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +0 -42
  119. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  120. package/dist/es5/parquetjs/parquet-thrift/StringType.js +0 -15
  121. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
  122. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +0 -27
  123. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  124. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +0 -30
  125. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  126. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +0 -27
  127. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  128. package/dist/es5/parquetjs/parquet-thrift/Type.js +0 -1
  129. package/dist/es5/parquetjs/parquet-thrift/Type.js.map +1 -1
  130. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -15
  131. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  132. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +0 -15
  133. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  134. package/dist/es5/parquetjs/parquet-thrift/index.js +0 -86
  135. package/dist/es5/parquetjs/parquet-thrift/index.js.map +1 -1
  136. package/dist/es5/parquetjs/parser/decoders.js +3 -82
  137. package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
  138. package/dist/es5/parquetjs/parser/parquet-cursor.js +5 -37
  139. package/dist/es5/parquetjs/parser/parquet-cursor.js.map +1 -1
  140. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +2 -88
  141. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
  142. package/dist/es5/parquetjs/parser/parquet-reader.js +14 -67
  143. package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
  144. package/dist/es5/parquetjs/schema/declare.js +3 -7
  145. package/dist/es5/parquetjs/schema/declare.js.map +1 -1
  146. package/dist/es5/parquetjs/schema/schema.js +6 -34
  147. package/dist/es5/parquetjs/schema/schema.js.map +1 -1
  148. package/dist/es5/parquetjs/schema/shred.js +11 -41
  149. package/dist/es5/parquetjs/schema/shred.js.map +1 -1
  150. package/dist/es5/parquetjs/schema/types.js +3 -84
  151. package/dist/es5/parquetjs/schema/types.js.map +1 -1
  152. package/dist/es5/parquetjs/utils/buffer-utils.js +0 -2
  153. package/dist/es5/parquetjs/utils/buffer-utils.js.map +1 -1
  154. package/dist/es5/parquetjs/utils/file-utils.js +1 -7
  155. package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
  156. package/dist/es5/parquetjs/utils/read-utils.js +6 -38
  157. package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
  158. package/dist/es5/workers/parquet-worker.js +0 -2
  159. package/dist/es5/workers/parquet-worker.js.map +1 -1
  160. package/dist/esm/bundle.js +1 -1
  161. package/dist/esm/bundle.js.map +1 -1
  162. package/dist/esm/constants.js +3 -0
  163. package/dist/esm/constants.js.map +1 -1
  164. package/dist/esm/index.js +10 -2
  165. package/dist/esm/index.js.map +1 -1
  166. package/dist/esm/lib/convert-schema.js +1 -7
  167. package/dist/esm/lib/convert-schema.js.map +1 -1
  168. package/dist/esm/lib/parse-parquet.js +2 -5
  169. package/dist/esm/lib/parse-parquet.js.map +1 -1
  170. package/dist/esm/lib/read-array-buffer.js +2 -1
  171. package/dist/esm/lib/read-array-buffer.js.map +1 -1
  172. package/dist/esm/lib/wasm/encode-parquet-wasm.js +1 -1
  173. package/dist/esm/lib/wasm/encode-parquet-wasm.js.map +1 -1
  174. package/dist/esm/lib/wasm/load-wasm/index.js.map +1 -1
  175. package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
  176. package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
  177. package/dist/esm/lib/wasm/parse-parquet-wasm.js +2 -3
  178. package/dist/esm/lib/wasm/parse-parquet-wasm.js.map +1 -1
  179. package/dist/esm/parquet-loader.js +3 -1
  180. package/dist/esm/parquet-loader.js.map +1 -1
  181. package/dist/esm/parquet-wasm-loader.js +3 -1
  182. package/dist/esm/parquet-wasm-loader.js.map +1 -1
  183. package/dist/esm/parquet-wasm-writer.js +2 -1
  184. package/dist/esm/parquet-wasm-writer.js.map +1 -1
  185. package/dist/esm/parquet-writer.js +2 -2
  186. package/dist/esm/parquet-writer.js.map +1 -1
  187. package/dist/esm/parquetjs/codecs/declare.js.map +1 -1
  188. package/dist/esm/parquetjs/codecs/dictionary.js +2 -1
  189. package/dist/esm/parquetjs/codecs/dictionary.js.map +1 -1
  190. package/dist/esm/parquetjs/codecs/index.js +2 -0
  191. package/dist/esm/parquetjs/codecs/index.js.map +1 -1
  192. package/dist/esm/parquetjs/codecs/plain.js +2 -68
  193. package/dist/esm/parquetjs/codecs/plain.js.map +1 -1
  194. package/dist/esm/parquetjs/codecs/rle.js +3 -29
  195. package/dist/esm/parquetjs/codecs/rle.js.map +1 -1
  196. package/dist/esm/parquetjs/compression.js +9 -5
  197. package/dist/esm/parquetjs/compression.js.map +1 -1
  198. package/dist/esm/parquetjs/encoder/writer.js +21 -51
  199. package/dist/esm/parquetjs/encoder/writer.js.map +1 -1
  200. package/dist/esm/parquetjs/file.js +1 -0
  201. package/dist/esm/parquetjs/file.js.map +1 -1
  202. package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
  203. package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -1
  204. package/dist/esm/parquetjs/parquet-thrift/BsonType.js +1 -8
  205. package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  206. package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +0 -44
  207. package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  208. package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +0 -42
  209. package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  210. package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +0 -82
  211. package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  212. package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +0 -18
  213. package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  214. package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
  215. package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -1
  216. package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +1 -1
  217. package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +1 -1
  218. package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +0 -34
  219. package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  220. package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -49
  221. package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  222. package/dist/esm/parquetjs/parquet-thrift/DateType.js +1 -8
  223. package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +1 -1
  224. package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +0 -19
  225. package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  226. package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -24
  227. package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  228. package/dist/esm/parquetjs/parquet-thrift/Encoding.js +1 -1
  229. package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +1 -1
  230. package/dist/esm/parquetjs/parquet-thrift/EnumType.js +1 -8
  231. package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  232. package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
  233. package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -1
  234. package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +2 -53
  235. package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  236. package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +1 -8
  237. package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  238. package/dist/esm/parquetjs/parquet-thrift/IntType.js +0 -19
  239. package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +1 -1
  240. package/dist/esm/parquetjs/parquet-thrift/JsonType.js +1 -8
  241. package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  242. package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +0 -19
  243. package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  244. package/dist/esm/parquetjs/parquet-thrift/ListType.js +1 -8
  245. package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +1 -1
  246. package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +0 -90
  247. package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  248. package/dist/esm/parquetjs/parquet-thrift/MapType.js +1 -8
  249. package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +1 -1
  250. package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +1 -8
  251. package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  252. package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +1 -8
  253. package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  254. package/dist/esm/parquetjs/parquet-thrift/NullType.js +1 -8
  255. package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +1 -1
  256. package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +0 -16
  257. package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  258. package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +0 -24
  259. package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  260. package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +0 -49
  261. package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  262. package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +0 -24
  263. package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  264. package/dist/esm/parquetjs/parquet-thrift/PageType.js +1 -1
  265. package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +1 -1
  266. package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +0 -33
  267. package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  268. package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +0 -59
  269. package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  270. package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +0 -24
  271. package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  272. package/dist/esm/parquetjs/parquet-thrift/Statistics.js +0 -38
  273. package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  274. package/dist/esm/parquetjs/parquet-thrift/StringType.js +1 -8
  275. package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +1 -1
  276. package/dist/esm/parquetjs/parquet-thrift/TimeType.js +0 -19
  277. package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  278. package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +0 -24
  279. package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  280. package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +0 -19
  281. package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  282. package/dist/esm/parquetjs/parquet-thrift/Type.js +1 -1
  283. package/dist/esm/parquetjs/parquet-thrift/Type.js.map +1 -1
  284. package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -8
  285. package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  286. package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +1 -8
  287. package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  288. package/dist/esm/parquetjs/parquet-thrift/index.js +1 -0
  289. package/dist/esm/parquetjs/parquet-thrift/index.js.map +1 -1
  290. package/dist/esm/parquetjs/parser/decoders.js +9 -39
  291. package/dist/esm/parquetjs/parser/decoders.js.map +1 -1
  292. package/dist/esm/parquetjs/parser/parquet-cursor.js +1 -13
  293. package/dist/esm/parquetjs/parser/parquet-cursor.js.map +1 -1
  294. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +6 -32
  295. package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
  296. package/dist/esm/parquetjs/parser/parquet-reader.js +1 -18
  297. package/dist/esm/parquetjs/parser/parquet-reader.js.map +1 -1
  298. package/dist/esm/parquetjs/schema/declare.js +4 -4
  299. package/dist/esm/parquetjs/schema/declare.js.map +1 -1
  300. package/dist/esm/parquetjs/schema/schema.js +3 -29
  301. package/dist/esm/parquetjs/schema/schema.js.map +1 -1
  302. package/dist/esm/parquetjs/schema/shred.js +7 -22
  303. package/dist/esm/parquetjs/schema/shred.js.map +1 -1
  304. package/dist/esm/parquetjs/schema/types.js +3 -78
  305. package/dist/esm/parquetjs/schema/types.js.map +1 -1
  306. package/dist/esm/parquetjs/utils/buffer-utils.js +2 -1
  307. package/dist/esm/parquetjs/utils/buffer-utils.js.map +1 -1
  308. package/dist/esm/parquetjs/utils/file-utils.js +1 -0
  309. package/dist/esm/parquetjs/utils/file-utils.js.map +1 -1
  310. package/dist/esm/parquetjs/utils/read-utils.js +5 -12
  311. package/dist/esm/parquetjs/utils/read-utils.js.map +1 -1
  312. package/dist/esm/workers/parquet-worker.js.map +1 -1
  313. package/dist/lib/wasm/load-wasm/load-wasm-browser.js +5 -1
  314. package/dist/lib/wasm/load-wasm/load-wasm-node.js +5 -1
  315. package/dist/parquet-worker.js +14 -14
  316. package/dist/parquet-worker.js.map +2 -2
  317. package/dist/parquetjs/codecs/index.js +5 -1
  318. package/dist/parquetjs/encoder/writer.d.ts +1 -0
  319. package/dist/parquetjs/encoder/writer.d.ts.map +1 -1
  320. package/dist/parquetjs/encoder/writer.js +5 -1
  321. package/dist/parquetjs/parquet-thrift/BsonType.js +5 -1
  322. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +5 -1
  323. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +5 -1
  324. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +5 -1
  325. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +5 -1
  326. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +5 -1
  327. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +5 -1
  328. package/dist/parquetjs/parquet-thrift/DateType.js +5 -1
  329. package/dist/parquetjs/parquet-thrift/DecimalType.js +5 -1
  330. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +5 -1
  331. package/dist/parquetjs/parquet-thrift/EnumType.js +5 -1
  332. package/dist/parquetjs/parquet-thrift/FileMetaData.js +5 -1
  333. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +5 -1
  334. package/dist/parquetjs/parquet-thrift/IntType.js +5 -1
  335. package/dist/parquetjs/parquet-thrift/JsonType.js +5 -1
  336. package/dist/parquetjs/parquet-thrift/KeyValue.js +5 -1
  337. package/dist/parquetjs/parquet-thrift/ListType.js +5 -1
  338. package/dist/parquetjs/parquet-thrift/LogicalType.js +5 -1
  339. package/dist/parquetjs/parquet-thrift/MapType.js +5 -1
  340. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +5 -1
  341. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +5 -1
  342. package/dist/parquetjs/parquet-thrift/NullType.js +5 -1
  343. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +5 -1
  344. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +5 -1
  345. package/dist/parquetjs/parquet-thrift/PageHeader.js +5 -1
  346. package/dist/parquetjs/parquet-thrift/PageLocation.js +5 -1
  347. package/dist/parquetjs/parquet-thrift/RowGroup.js +5 -1
  348. package/dist/parquetjs/parquet-thrift/SchemaElement.js +5 -1
  349. package/dist/parquetjs/parquet-thrift/SortingColumn.js +5 -1
  350. package/dist/parquetjs/parquet-thrift/Statistics.js +5 -1
  351. package/dist/parquetjs/parquet-thrift/StringType.js +5 -1
  352. package/dist/parquetjs/parquet-thrift/TimeType.js +5 -1
  353. package/dist/parquetjs/parquet-thrift/TimeUnit.js +5 -1
  354. package/dist/parquetjs/parquet-thrift/TimestampType.js +5 -1
  355. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +5 -1
  356. package/dist/parquetjs/parquet-thrift/UUIDType.js +5 -1
  357. package/dist/parquetjs/parquet-thrift/index.js +5 -1
  358. package/dist/parquetjs/schema/shred.js +5 -1
  359. package/dist/parquetjs/utils/file-utils.d.ts +2 -0
  360. package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
  361. package/package.json +6 -6
@@ -1,69 +1,44 @@
1
1
  "use strict";
2
2
 
3
3
  var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
-
5
4
  var _typeof = require("@babel/runtime/helpers/typeof");
6
-
7
5
  Object.defineProperty(exports, "__esModule", {
8
6
  value: true
9
7
  });
10
- exports.ParquetTransformer = exports.ParquetEnvelopeWriter = exports.ParquetWriter = void 0;
11
-
8
+ exports.ParquetWriter = exports.ParquetTransformer = exports.ParquetEnvelopeWriter = void 0;
12
9
  var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
13
-
14
10
  var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
15
-
16
11
  var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
17
-
18
12
  var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
19
-
20
13
  var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
21
-
22
14
  var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
23
-
24
15
  var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
25
-
26
16
  var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
27
-
28
17
  var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
29
-
30
18
  var _stream = require("stream");
31
-
32
19
  var _codecs = require("../codecs");
33
-
34
20
  var Compression = _interopRequireWildcard(require("../compression"));
35
-
36
21
  var Shred = _interopRequireWildcard(require("../schema/shred"));
37
-
38
22
  var _parquetThrift = require("../parquet-thrift");
39
-
40
23
  var _fileUtils = require("../utils/file-utils");
41
-
42
24
  var _readUtils = require("../utils/read-utils");
43
-
44
25
  var _nodeInt = _interopRequireDefault(require("node-int64"));
45
-
46
26
  function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
47
-
48
27
  function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
49
-
50
28
  function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
51
-
52
29
  function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
53
-
54
30
  function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
55
-
56
31
  function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
57
-
58
32
  function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
59
-
60
33
  var PARQUET_MAGIC = 'PAR1';
34
+
61
35
  var PARQUET_VERSION = 1;
36
+
62
37
  var PARQUET_DEFAULT_PAGE_SIZE = 8192;
63
38
  var PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
39
+
64
40
  var PARQUET_RDLVL_TYPE = 'INT32';
65
41
  var PARQUET_RDLVL_ENCODING = 'RLE';
66
-
67
42
  var ParquetWriter = function () {
68
43
  function ParquetWriter(schema, envelopeWriter, opts) {
69
44
  (0, _classCallCheck2.default)(this, ParquetWriter);
@@ -79,9 +54,9 @@ var ParquetWriter = function () {
79
54
  this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
80
55
  this.closed = false;
81
56
  this.userMetadata = {};
57
+
82
58
  this.writeHeader();
83
59
  }
84
-
85
60
  (0, _createClass2.default)(ParquetWriter, [{
86
61
  key: "writeHeader",
87
62
  value: function () {
@@ -93,20 +68,16 @@ var ParquetWriter = function () {
93
68
  _context.prev = 0;
94
69
  _context.next = 3;
95
70
  return this.envelopeWriter.writeHeader();
96
-
97
71
  case 3:
98
72
  _context.next = 10;
99
73
  break;
100
-
101
74
  case 5:
102
75
  _context.prev = 5;
103
76
  _context.t0 = _context["catch"](0);
104
77
  _context.next = 9;
105
78
  return this.envelopeWriter.close();
106
-
107
79
  case 9:
108
80
  throw _context.t0;
109
-
110
81
  case 10:
111
82
  case "end":
112
83
  return _context.stop();
@@ -114,11 +85,9 @@ var ParquetWriter = function () {
114
85
  }
115
86
  }, _callee, this, [[0, 5]]);
116
87
  }));
117
-
118
88
  function writeHeader() {
119
89
  return _writeHeader.apply(this, arguments);
120
90
  }
121
-
122
91
  return writeHeader;
123
92
  }()
124
93
  }, {
@@ -133,16 +102,12 @@ var ParquetWriter = function () {
133
102
  _context2.next = 2;
134
103
  break;
135
104
  }
136
-
137
105
  throw new Error('writer was closed');
138
-
139
106
  case 2:
140
107
  Shred.shredRecord(this.schema, row, this.rowBuffer);
141
-
142
108
  if (this.rowBuffer.rowCount >= this.rowGroupSize) {
143
109
  this.rowBuffer = {};
144
110
  }
145
-
146
111
  case 4:
147
112
  case "end":
148
113
  return _context2.stop();
@@ -150,11 +115,9 @@ var ParquetWriter = function () {
150
115
  }
151
116
  }, _callee2, this);
152
117
  }));
153
-
154
118
  function appendRow(_x) {
155
119
  return _appendRow.apply(this, arguments);
156
120
  }
157
-
158
121
  return appendRow;
159
122
  }()
160
123
  }, {
@@ -169,28 +132,22 @@ var ParquetWriter = function () {
169
132
  _context3.next = 2;
170
133
  break;
171
134
  }
172
-
173
135
  throw new Error('writer was closed');
174
-
175
136
  case 2:
176
137
  this.closed = true;
177
-
178
138
  if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
179
139
  this.rowBuffer = {};
180
140
  }
181
-
182
141
  _context3.next = 6;
183
142
  return this.envelopeWriter.writeFooter(this.userMetadata);
184
-
185
143
  case 6:
186
144
  _context3.next = 8;
187
145
  return this.envelopeWriter.close();
188
-
189
146
  case 8:
147
+
190
148
  if (callback) {
191
149
  callback();
192
150
  }
193
-
194
151
  case 9:
195
152
  case "end":
196
153
  return _context3.stop();
@@ -198,26 +155,29 @@ var ParquetWriter = function () {
198
155
  }
199
156
  }, _callee3, this);
200
157
  }));
201
-
202
158
  function close(_x2) {
203
159
  return _close.apply(this, arguments);
204
160
  }
205
-
206
161
  return close;
207
162
  }()
208
163
  }, {
209
164
  key: "setMetadata",
210
- value: function setMetadata(key, value) {
165
+ value:
166
+ function setMetadata(key, value) {
211
167
  this.userMetadata[String(key)] = String(value);
212
168
  }
169
+
213
170
  }, {
214
171
  key: "setRowGroupSize",
215
- value: function setRowGroupSize(cnt) {
172
+ value:
173
+ function setRowGroupSize(cnt) {
216
174
  this.rowGroupSize = cnt;
217
175
  }
176
+
218
177
  }, {
219
178
  key: "setPageSize",
220
- value: function setPageSize(cnt) {
179
+ value:
180
+ function setPageSize(cnt) {
221
181
  this.envelopeWriter.setPageSize(cnt);
222
182
  }
223
183
  }], [{
@@ -231,11 +191,9 @@ var ParquetWriter = function () {
231
191
  case 0:
232
192
  _context4.next = 2;
233
193
  return (0, _fileUtils.osopen)(path, opts);
234
-
235
194
  case 2:
236
195
  outputStream = _context4.sent;
237
196
  return _context4.abrupt("return", ParquetWriter.openStream(schema, outputStream, opts));
238
-
239
197
  case 4:
240
198
  case "end":
241
199
  return _context4.stop();
@@ -243,11 +201,9 @@ var ParquetWriter = function () {
243
201
  }
244
202
  }, _callee4);
245
203
  }));
246
-
247
204
  function openFile(_x3, _x4, _x5) {
248
205
  return _openFile.apply(this, arguments);
249
206
  }
250
-
251
207
  return openFile;
252
208
  }()
253
209
  }, {
@@ -262,14 +218,11 @@ var ParquetWriter = function () {
262
218
  if (!opts) {
263
219
  opts = {};
264
220
  }
265
-
266
221
  _context5.next = 3;
267
222
  return ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
268
-
269
223
  case 3:
270
224
  envelopeWriter = _context5.sent;
271
225
  return _context5.abrupt("return", new ParquetWriter(schema, envelopeWriter, opts));
272
-
273
226
  case 5:
274
227
  case "end":
275
228
  return _context5.stop();
@@ -277,19 +230,15 @@ var ParquetWriter = function () {
277
230
  }
278
231
  }, _callee5);
279
232
  }));
280
-
281
233
  function openStream(_x6, _x7, _x8) {
282
234
  return _openStream.apply(this, arguments);
283
235
  }
284
-
285
236
  return openStream;
286
237
  }()
287
238
  }]);
288
239
  return ParquetWriter;
289
240
  }();
290
-
291
241
  exports.ParquetWriter = ParquetWriter;
292
-
293
242
  var ParquetEnvelopeWriter = function () {
294
243
  function ParquetEnvelopeWriter(schema, writeFn, closeFn, fileOffset, opts) {
295
244
  (0, _classCallCheck2.default)(this, ParquetEnvelopeWriter);
@@ -310,18 +259,20 @@ var ParquetEnvelopeWriter = function () {
310
259
  this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
311
260
  this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
312
261
  }
313
-
314
262
  (0, _createClass2.default)(ParquetEnvelopeWriter, [{
315
263
  key: "writeSection",
316
264
  value: function writeSection(buf) {
317
265
  this.offset += buf.length;
318
266
  return this.write(buf);
319
267
  }
268
+
320
269
  }, {
321
270
  key: "writeHeader",
322
- value: function writeHeader() {
271
+ value:
272
+ function writeHeader() {
323
273
  return this.writeSection(Buffer.from(PARQUET_MAGIC));
324
274
  }
275
+
325
276
  }, {
326
277
  key: "writeRowGroup",
327
278
  value: function () {
@@ -337,17 +288,14 @@ var ParquetEnvelopeWriter = function () {
337
288
  pageSize: this.pageSize,
338
289
  useDataPageV2: this.useDataPageV2
339
290
  });
340
-
341
291
  case 2:
342
292
  rgroup = _context6.sent;
343
293
  this.rowCount += records.rowCount;
344
294
  this.rowGroups.push(rgroup.metadata);
345
295
  _context6.next = 7;
346
296
  return this.writeSection(rgroup.body);
347
-
348
297
  case 7:
349
298
  return _context6.abrupt("return", _context6.sent);
350
-
351
299
  case 8:
352
300
  case "end":
353
301
  return _context6.stop();
@@ -355,25 +303,25 @@ var ParquetEnvelopeWriter = function () {
355
303
  }
356
304
  }, _callee6, this);
357
305
  }));
358
-
359
306
  function writeRowGroup(_x9) {
360
307
  return _writeRowGroup.apply(this, arguments);
361
308
  }
362
-
363
309
  return writeRowGroup;
364
310
  }()
365
311
  }, {
366
312
  key: "writeFooter",
367
- value: function writeFooter(userMetadata) {
313
+ value:
314
+ function writeFooter(userMetadata) {
368
315
  if (!userMetadata) {
369
316
  userMetadata = {};
370
317
  }
371
-
372
318
  return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
373
319
  }
320
+
374
321
  }, {
375
322
  key: "setPageSize",
376
- value: function setPageSize(cnt) {
323
+ value:
324
+ function setPageSize(cnt) {
377
325
  this.pageSize = cnt;
378
326
  }
379
327
  }], [{
@@ -388,7 +336,6 @@ var ParquetEnvelopeWriter = function () {
388
336
  writeFn = _fileUtils.oswrite.bind(undefined, outputStream);
389
337
  closeFn = _fileUtils.osclose.bind(undefined, outputStream);
390
338
  return _context7.abrupt("return", new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts));
391
-
392
339
  case 3:
393
340
  case "end":
394
341
  return _context7.stop();
@@ -396,34 +343,26 @@ var ParquetEnvelopeWriter = function () {
396
343
  }
397
344
  }, _callee7);
398
345
  }));
399
-
400
346
  function openStream(_x10, _x11, _x12) {
401
347
  return _openStream2.apply(this, arguments);
402
348
  }
403
-
404
349
  return openStream;
405
350
  }()
406
351
  }]);
407
352
  return ParquetEnvelopeWriter;
408
353
  }();
409
-
410
354
  exports.ParquetEnvelopeWriter = ParquetEnvelopeWriter;
411
-
412
355
  var ParquetTransformer = function (_Transform) {
413
356
  (0, _inherits2.default)(ParquetTransformer, _Transform);
414
-
415
357
  var _super = _createSuper(ParquetTransformer);
416
-
417
358
  function ParquetTransformer(schema) {
418
359
  var _this;
419
-
420
360
  var opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
421
361
  (0, _classCallCheck2.default)(this, ParquetTransformer);
422
362
  _this = _super.call(this, {
423
363
  objectMode: true
424
364
  });
425
365
  (0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "writer", void 0);
426
-
427
366
  var writeProxy = function (t) {
428
367
  return function () {
429
368
  var _ref = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee8(b) {
@@ -432,7 +371,6 @@ var ParquetTransformer = function (_Transform) {
432
371
  switch (_context8.prev = _context8.next) {
433
372
  case 0:
434
373
  t.push(b);
435
-
436
374
  case 1:
437
375
  case "end":
438
376
  return _context8.stop();
@@ -440,13 +378,11 @@ var ParquetTransformer = function (_Transform) {
440
378
  }
441
379
  }, _callee8);
442
380
  }));
443
-
444
381
  return function (_x13) {
445
382
  return _ref.apply(this, arguments);
446
383
  };
447
384
  }();
448
385
  }((0, _assertThisInitialized2.default)(_this));
449
-
450
386
  _this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee9() {
451
387
  return _regenerator.default.wrap(function _callee9$(_context9) {
452
388
  while (1) {
@@ -463,14 +399,15 @@ var ParquetTransformer = function (_Transform) {
463
399
 
464
400
  (0, _createClass2.default)(ParquetTransformer, [{
465
401
  key: "_transform",
466
- value: function _transform(row, encoding, callback) {
402
+ value:
403
+ function _transform(row, encoding, callback) {
467
404
  if (row) {
468
405
  return this.writer.appendRow(row).then(callback);
469
406
  }
470
-
471
407
  callback();
472
408
  return Promise.resolve();
473
409
  }
410
+
474
411
  }, {
475
412
  key: "_flush",
476
413
  value: function () {
@@ -481,7 +418,6 @@ var ParquetTransformer = function (_Transform) {
481
418
  case 0:
482
419
  _context10.next = 2;
483
420
  return this.writer.close(callback);
484
-
485
421
  case 2:
486
422
  case "end":
487
423
  return _context10.stop();
@@ -489,31 +425,25 @@ var ParquetTransformer = function (_Transform) {
489
425
  }
490
426
  }, _callee10, this);
491
427
  }));
492
-
493
428
  function _flush(_x14) {
494
429
  return _flush2.apply(this, arguments);
495
430
  }
496
-
497
431
  return _flush;
498
432
  }()
499
433
  }]);
500
434
  return ParquetTransformer;
501
435
  }(_stream.Transform);
502
-
503
436
  exports.ParquetTransformer = ParquetTransformer;
504
-
505
437
  function encodeValues(type, encoding, values, opts) {
506
438
  if (!(encoding in _codecs.PARQUET_CODECS)) {
507
439
  throw new Error("invalid encoding: ".concat(encoding));
508
440
  }
509
-
510
441
  return _codecs.PARQUET_CODECS[encoding].encodeValues(type, values, opts);
511
442
  }
512
443
 
513
444
  function encodeDataPage(_x15, _x16) {
514
445
  return _encodeDataPage.apply(this, arguments);
515
446
  }
516
-
517
447
  function _encodeDataPage() {
518
448
  _encodeDataPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee11(column, data) {
519
449
  var rLevelsBuf, dLevelsBuf, valuesBuf, dataBuf, compressedBuf, header, headerBuf, page;
@@ -522,15 +452,12 @@ function _encodeDataPage() {
522
452
  switch (_context11.prev = _context11.next) {
523
453
  case 0:
524
454
  rLevelsBuf = Buffer.alloc(0);
525
-
526
455
  if (column.rLevelMax > 0) {
527
456
  rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
528
457
  bitWidth: (0, _readUtils.getBitWidth)(column.rLevelMax)
529
458
  });
530
459
  }
531
-
532
460
  dLevelsBuf = Buffer.alloc(0);
533
-
534
461
  if (column.dLevelMax > 0) {
535
462
  dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
536
463
  bitWidth: (0, _readUtils.getBitWidth)(column.dLevelMax)
@@ -544,7 +471,6 @@ function _encodeDataPage() {
544
471
  dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
545
472
  _context11.next = 8;
546
473
  return Compression.deflate(column.compression, dataBuf);
547
-
548
474
  case 8:
549
475
  compressedBuf = _context11.sent;
550
476
  header = new _parquetThrift.PageHeader({
@@ -555,6 +481,7 @@ function _encodeDataPage() {
555
481
  definition_level_encoding: _parquetThrift.Encoding[PARQUET_RDLVL_ENCODING],
556
482
  repetition_level_encoding: _parquetThrift.Encoding[PARQUET_RDLVL_ENCODING]
557
483
  }),
484
+
558
485
  uncompressed_page_size: dataBuf.length,
559
486
  compressed_page_size: compressedBuf.length
560
487
  });
@@ -565,7 +492,6 @@ function _encodeDataPage() {
565
492
  headerSize: headerBuf.length,
566
493
  page: page
567
494
  });
568
-
569
495
  case 13:
570
496
  case "end":
571
497
  return _context11.stop();
@@ -575,11 +501,9 @@ function _encodeDataPage() {
575
501
  }));
576
502
  return _encodeDataPage.apply(this, arguments);
577
503
  }
578
-
579
504
  function encodeDataPageV2(_x17, _x18, _x19) {
580
505
  return _encodeDataPageV.apply(this, arguments);
581
506
  }
582
-
583
507
  function _encodeDataPageV() {
584
508
  _encodeDataPageV = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee12(column, data, rowCount) {
585
509
  var valuesBuf, compressedBuf, rLevelsBuf, dLevelsBuf, header, headerBuf, page;
@@ -593,20 +517,16 @@ function _encodeDataPageV() {
593
517
  });
594
518
  _context12.next = 3;
595
519
  return Compression.deflate(column.compression, valuesBuf);
596
-
597
520
  case 3:
598
521
  compressedBuf = _context12.sent;
599
522
  rLevelsBuf = Buffer.alloc(0);
600
-
601
523
  if (column.rLevelMax > 0) {
602
524
  rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
603
525
  bitWidth: (0, _readUtils.getBitWidth)(column.rLevelMax),
604
526
  disableEnvelope: true
605
527
  });
606
528
  }
607
-
608
529
  dLevelsBuf = Buffer.alloc(0);
609
-
610
530
  if (column.dLevelMax > 0) {
611
531
  dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
612
532
  bitWidth: (0, _readUtils.getBitWidth)(column.dLevelMax),
@@ -635,7 +555,6 @@ function _encodeDataPageV() {
635
555
  headerSize: headerBuf.length,
636
556
  page: page
637
557
  });
638
-
639
558
  case 12:
640
559
  case "end":
641
560
  return _context12.stop();
@@ -645,11 +564,9 @@ function _encodeDataPageV() {
645
564
  }));
646
565
  return _encodeDataPageV.apply(this, arguments);
647
566
  }
648
-
649
567
  function encodeColumnChunk(_x20, _x21, _x22, _x23) {
650
568
  return _encodeColumnChunk.apply(this, arguments);
651
569
  }
652
-
653
570
  function _encodeColumnChunk() {
654
571
  _encodeColumnChunk = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee13(column, buffer, offset, opts) {
655
572
  var data, baseOffset, pageBuf, total_uncompressed_size, total_compressed_size, result, metadata, metadataOffset, body;
@@ -661,27 +578,21 @@ function _encodeColumnChunk() {
661
578
  baseOffset = (opts.baseOffset || 0) + offset;
662
579
  total_uncompressed_size = 0;
663
580
  total_compressed_size = 0;
664
-
665
581
  if (!opts.useDataPageV2) {
666
582
  _context13.next = 10;
667
583
  break;
668
584
  }
669
-
670
585
  _context13.next = 7;
671
586
  return encodeDataPageV2(column, data, buffer.rowCount);
672
-
673
587
  case 7:
674
588
  _context13.t0 = _context13.sent;
675
589
  _context13.next = 13;
676
590
  break;
677
-
678
591
  case 10:
679
592
  _context13.next = 12;
680
593
  return encodeDataPage(column, data);
681
-
682
594
  case 12:
683
595
  _context13.t0 = _context13.sent;
684
-
685
596
  case 13:
686
597
  result = _context13.t0;
687
598
  pageBuf = result.page;
@@ -699,6 +610,7 @@ function _encodeColumnChunk() {
699
610
  });
700
611
  metadata.encodings.push(_parquetThrift.Encoding[PARQUET_RDLVL_ENCODING]);
701
612
  metadata.encodings.push(_parquetThrift.Encoding[column.encoding]);
613
+
702
614
  metadataOffset = baseOffset + pageBuf.length;
703
615
  body = Buffer.concat([pageBuf, (0, _readUtils.serializeThrift)(metadata)]);
704
616
  return _context13.abrupt("return", {
@@ -706,7 +618,6 @@ function _encodeColumnChunk() {
706
618
  metadata: metadata,
707
619
  metadataOffset: metadataOffset
708
620
  });
709
-
710
621
  case 23:
711
622
  case "end":
712
623
  return _context13.stop();
@@ -716,15 +627,12 @@ function _encodeColumnChunk() {
716
627
  }));
717
628
  return _encodeColumnChunk.apply(this, arguments);
718
629
  }
719
-
720
630
  function encodeRowGroup(_x24, _x25, _x26) {
721
631
  return _encodeRowGroup.apply(this, arguments);
722
632
  }
723
-
724
633
  function _encodeRowGroup() {
725
634
  _encodeRowGroup = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee14(schema, data, opts) {
726
635
  var metadata, body, _iterator2, _step2, field, cchunkData, cchunk;
727
-
728
636
  return _regenerator.default.wrap(function _callee14$(_context14) {
729
637
  while (1) {
730
638
  switch (_context14.prev = _context14.next) {
@@ -737,28 +645,21 @@ function _encodeRowGroup() {
737
645
  body = Buffer.alloc(0);
738
646
  _iterator2 = _createForOfIteratorHelper(schema.fieldList);
739
647
  _context14.prev = 3;
740
-
741
648
  _iterator2.s();
742
-
743
649
  case 5:
744
650
  if ((_step2 = _iterator2.n()).done) {
745
651
  _context14.next = 18;
746
652
  break;
747
653
  }
748
-
749
654
  field = _step2.value;
750
-
751
655
  if (!field.isNested) {
752
656
  _context14.next = 9;
753
657
  break;
754
658
  }
755
-
756
659
  return _context14.abrupt("continue", 16);
757
-
758
660
  case 9:
759
661
  _context14.next = 11;
760
662
  return encodeColumnChunk(field, data, body.length, opts);
761
-
762
663
  case 11:
763
664
  cchunkData = _context14.sent;
764
665
  cchunk = new _parquetThrift.ColumnChunk({
@@ -768,34 +669,25 @@ function _encodeRowGroup() {
768
669
  metadata.columns.push(cchunk);
769
670
  metadata.total_byte_size = new _nodeInt.default(Number(metadata.total_byte_size) + cchunkData.body.length);
770
671
  body = Buffer.concat([body, cchunkData.body]);
771
-
772
672
  case 16:
773
673
  _context14.next = 5;
774
674
  break;
775
-
776
675
  case 18:
777
676
  _context14.next = 23;
778
677
  break;
779
-
780
678
  case 20:
781
679
  _context14.prev = 20;
782
680
  _context14.t0 = _context14["catch"](3);
783
-
784
681
  _iterator2.e(_context14.t0);
785
-
786
682
  case 23:
787
683
  _context14.prev = 23;
788
-
789
684
  _iterator2.f();
790
-
791
685
  return _context14.finish(23);
792
-
793
686
  case 26:
794
687
  return _context14.abrupt("return", {
795
688
  body: body,
796
689
  metadata: metadata
797
690
  });
798
-
799
691
  case 27:
800
692
  case "end":
801
693
  return _context14.stop();
@@ -805,7 +697,6 @@ function _encodeRowGroup() {
805
697
  }));
806
698
  return _encodeRowGroup.apply(this, arguments);
807
699
  }
808
-
809
700
  function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
810
701
  var metadata = new _parquetThrift.FileMetaData({
811
702
  version: PARQUET_VERSION,
@@ -815,17 +706,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
815
706
  schema: [],
816
707
  key_value_metadata: []
817
708
  });
818
-
819
709
  for (var key in userMetadata) {
820
- var _metadata$key_value_m, _metadata$key_value_m2;
821
-
710
+ var _metadata$key_value_m, _metadata$key_value_m2, _metadata$key_value_m3;
822
711
  var kv = new _parquetThrift.KeyValue({
823
712
  key: key,
824
713
  value: userMetadata[key]
825
714
  });
826
- (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = _metadata$key_value_m.push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m, kv);
715
+ (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = (_metadata$key_value_m3 = _metadata$key_value_m).push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m3, kv);
827
716
  }
828
-
829
717
  {
830
718
  var schemaRoot = new _parquetThrift.SchemaElement({
831
719
  name: 'root',
@@ -833,10 +721,8 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
833
721
  });
834
722
  metadata.schema.push(schemaRoot);
835
723
  }
836
-
837
724
  var _iterator = _createForOfIteratorHelper(schema.fieldList),
838
- _step;
839
-
725
+ _step;
840
726
  try {
841
727
  for (_iterator.s(); !(_step = _iterator.n()).done;) {
842
728
  var field = _step.value;
@@ -845,17 +731,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
845
731
  name: field.name,
846
732
  repetition_type: relt
847
733
  });
848
-
849
734
  if (field.isNested) {
850
735
  schemaElem.num_children = field.fieldCount;
851
736
  } else {
852
737
  schemaElem.type = _parquetThrift.Type[field.primitiveType];
853
738
  }
854
-
855
739
  if (field.originalType) {
856
740
  schemaElem.converted_type = _parquetThrift.ConvertedType[field.originalType];
857
741
  }
858
-
859
742
  schemaElem.type_length = field.typeLength;
860
743
  metadata.schema.push(schemaElem);
861
744
  }
@@ -864,7 +747,6 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
864
747
  } finally {
865
748
  _iterator.f();
866
749
  }
867
-
868
750
  var metadataEncoded = (0, _readUtils.serializeThrift)(metadata);
869
751
  var footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
870
752
  metadataEncoded.copy(footerEncoded);