@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (336) hide show
  1. package/LICENSE +1 -1
  2. package/dist/bundled.d.ts +1 -1
  3. package/dist/bundled.d.ts.map +1 -1
  4. package/dist/bundled.js.map +1 -1
  5. package/dist/delta-format.d.ts +9 -0
  6. package/dist/delta-format.d.ts.map +1 -0
  7. package/dist/delta-format.js +12 -0
  8. package/dist/delta-format.js.map +1 -0
  9. package/dist/delta-source-loader-types.d.ts +27 -0
  10. package/dist/delta-source-loader-types.d.ts.map +1 -0
  11. package/dist/delta-source-loader-types.js +28 -0
  12. package/dist/delta-source-loader-types.js.map +1 -0
  13. package/dist/delta-source.d.ts +48 -0
  14. package/dist/delta-source.d.ts.map +1 -0
  15. package/dist/delta-source.js +222 -0
  16. package/dist/delta-source.js.map +1 -0
  17. package/dist/delta-types.d.ts +38 -0
  18. package/dist/delta-types.d.ts.map +1 -0
  19. package/dist/delta-types.js +5 -0
  20. package/dist/delta-types.js.map +1 -0
  21. package/dist/geoparquet-loader.js +1 -1
  22. package/dist/iceberg-rest-catalog.d.ts +49 -0
  23. package/dist/iceberg-rest-catalog.d.ts.map +1 -0
  24. package/dist/iceberg-rest-catalog.js +78 -0
  25. package/dist/iceberg-rest-catalog.js.map +1 -0
  26. package/dist/iceberg-table-source.d.ts +63 -0
  27. package/dist/iceberg-table-source.d.ts.map +1 -0
  28. package/dist/iceberg-table-source.js +797 -0
  29. package/dist/iceberg-table-source.js.map +1 -0
  30. package/dist/iceberg-types.d.ts +150 -0
  31. package/dist/iceberg-types.d.ts.map +1 -0
  32. package/dist/iceberg-types.js +5 -0
  33. package/dist/iceberg-types.js.map +1 -0
  34. package/dist/index.cjs +15841 -3306
  35. package/dist/index.cjs.map +3 -3
  36. package/dist/index.d.ts +8 -3
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +5 -1
  39. package/dist/index.js.map +1 -1
  40. package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
  41. package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
  42. package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
  43. package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  44. package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
  45. package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
  46. package/dist/lib/constants.js +1 -1
  47. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
  48. package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
  49. package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
  50. package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
  51. package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
  52. package/dist/lib/geo/geoparquet-covering.js +137 -0
  53. package/dist/lib/geo/geoparquet-covering.js.map +1 -0
  54. package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
  55. package/dist/lib/geo/geospatial-metadata.js +39 -13
  56. package/dist/lib/geo/geospatial-metadata.js.map +1 -1
  57. package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
  58. package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
  59. package/dist/lib/parquet-bloom-filter-planner.js +57 -0
  60. package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
  61. package/dist/lib/parquet-bloom-filter.d.ts +30 -0
  62. package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
  63. package/dist/lib/parquet-bloom-filter.js +289 -0
  64. package/dist/lib/parquet-bloom-filter.js.map +1 -0
  65. package/dist/lib/parquet-encryption.d.ts +80 -0
  66. package/dist/lib/parquet-encryption.d.ts.map +1 -0
  67. package/dist/lib/parquet-encryption.js +190 -0
  68. package/dist/lib/parquet-encryption.js.map +1 -0
  69. package/dist/lib/parquet-page-index.d.ts +34 -5
  70. package/dist/lib/parquet-page-index.d.ts.map +1 -1
  71. package/dist/lib/parquet-page-index.js +114 -28
  72. package/dist/lib/parquet-page-index.js.map +1 -1
  73. package/dist/lib/parquet-predicate.d.ts +5 -1
  74. package/dist/lib/parquet-predicate.d.ts.map +1 -1
  75. package/dist/lib/parquet-predicate.js +42 -14
  76. package/dist/lib/parquet-predicate.js.map +1 -1
  77. package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
  78. package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
  79. package/dist/lib/parquet-source-worker-decoder.js +83 -17
  80. package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
  81. package/dist/lib/parquet-source-worker-types.d.ts +24 -0
  82. package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
  83. package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
  84. package/dist/lib/parsers/get-parquet-schema.js +3 -1
  85. package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
  86. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
  87. package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
  88. package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
  89. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
  90. package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
  91. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
  92. package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
  93. package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
  94. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  95. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  96. package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
  97. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
  98. package/dist/lib/slice-parquet-batch.d.ts +9 -0
  99. package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
  100. package/dist/lib/slice-parquet-batch.js +28 -0
  101. package/dist/lib/slice-parquet-batch.js.map +1 -0
  102. package/dist/lib/sources/parquet-range-file.d.ts +1 -1
  103. package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
  104. package/dist/lib/sources/parquet-range-file.js +28 -22
  105. package/dist/lib/sources/parquet-range-file.js.map +1 -1
  106. package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
  107. package/dist/lib/utils/normalize-parquet-options.js +15 -1
  108. package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
  109. package/dist/parquet-dataset-source.cjs +16528 -4928
  110. package/dist/parquet-dataset-source.cjs.map +3 -3
  111. package/dist/parquet-dataset-source.d.ts +31 -4
  112. package/dist/parquet-dataset-source.d.ts.map +1 -1
  113. package/dist/parquet-dataset-source.js +296 -74
  114. package/dist/parquet-dataset-source.js.map +1 -1
  115. package/dist/parquet-encoded-page-types.d.ts +108 -0
  116. package/dist/parquet-encoded-page-types.d.ts.map +1 -0
  117. package/dist/parquet-encoded-page-types.js +5 -0
  118. package/dist/parquet-encoded-page-types.js.map +1 -0
  119. package/dist/parquet-js-loader-types.d.ts +2 -0
  120. package/dist/parquet-js-loader-types.d.ts.map +1 -1
  121. package/dist/parquet-js-loader-types.js +2 -1
  122. package/dist/parquet-js-loader-types.js.map +1 -1
  123. package/dist/parquet-js-loader.d.ts +1 -0
  124. package/dist/parquet-js-loader.d.ts.map +1 -1
  125. package/dist/parquet-js-loader.js +2 -1
  126. package/dist/parquet-js-loader.js.map +1 -1
  127. package/dist/parquet-js-writer.d.ts +23 -2
  128. package/dist/parquet-js-writer.d.ts.map +1 -1
  129. package/dist/parquet-js-writer.js +1 -1
  130. package/dist/parquet-js-writer.js.map +1 -1
  131. package/dist/parquet-loader-base.js +1 -1
  132. package/dist/parquet-loader-options.d.ts +11 -0
  133. package/dist/parquet-loader-options.d.ts.map +1 -1
  134. package/dist/parquet-loader-options.js.map +1 -1
  135. package/dist/parquet-source-capabilities.d.ts +8 -1
  136. package/dist/parquet-source-capabilities.d.ts.map +1 -1
  137. package/dist/parquet-source-capabilities.js +10 -0
  138. package/dist/parquet-source-capabilities.js.map +1 -1
  139. package/dist/parquet-source-loader-types.d.ts +10 -0
  140. package/dist/parquet-source-loader-types.d.ts.map +1 -1
  141. package/dist/parquet-source-loader-types.js +11 -1
  142. package/dist/parquet-source-loader-types.js.map +1 -1
  143. package/dist/parquet-source-loader.cjs +14121 -2796
  144. package/dist/parquet-source-loader.cjs.map +3 -3
  145. package/dist/parquet-source-loader.d.ts +51 -4
  146. package/dist/parquet-source-loader.d.ts.map +1 -1
  147. package/dist/parquet-source-loader.js +651 -38
  148. package/dist/parquet-source-loader.js.map +1 -1
  149. package/dist/parquet-source-types.d.ts +180 -30
  150. package/dist/parquet-source-types.d.ts.map +1 -1
  151. package/dist/parquet-source-worker.js +7378 -6989
  152. package/dist/parquet-source-worker.js.map +4 -4
  153. package/dist/parquet-worker-node.cjs +8501 -1045
  154. package/dist/parquet-worker-node.cjs.map +4 -4
  155. package/dist/parquet-worker-url.js +5 -1
  156. package/dist/parquet-worker-url.js.map +1 -1
  157. package/dist/parquet-worker.js +289 -218
  158. package/dist/parquet-worker.js.map +4 -4
  159. package/dist/parquetjs/codecs/declare.d.ts +15 -0
  160. package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
  161. package/dist/parquetjs/codecs/declare.js +10 -0
  162. package/dist/parquetjs/codecs/declare.js.map +1 -1
  163. package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
  164. package/dist/parquetjs/codecs/delta.js +356 -7
  165. package/dist/parquetjs/codecs/delta.js.map +1 -1
  166. package/dist/parquetjs/codecs/index.d.ts.map +1 -1
  167. package/dist/parquetjs/codecs/index.js +4 -0
  168. package/dist/parquetjs/codecs/index.js.map +1 -1
  169. package/dist/parquetjs/codecs/plain.d.ts +1 -1
  170. package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
  171. package/dist/parquetjs/codecs/plain.js +153 -12
  172. package/dist/parquetjs/codecs/plain.js.map +1 -1
  173. package/dist/parquetjs/codecs/rle.d.ts +4 -0
  174. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  175. package/dist/parquetjs/codecs/rle.js +406 -3
  176. package/dist/parquetjs/codecs/rle.js.map +1 -1
  177. package/dist/parquetjs/compression.d.ts +9 -0
  178. package/dist/parquetjs/compression.d.ts.map +1 -1
  179. package/dist/parquetjs/compression.js +86 -67
  180. package/dist/parquetjs/compression.js.map +1 -1
  181. package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
  182. package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
  183. package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
  184. package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
  185. package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
  186. package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
  187. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
  188. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
  189. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
  190. package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
  191. package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
  192. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
  193. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
  194. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
  195. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  196. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
  197. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
  198. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
  199. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
  200. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
  201. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
  202. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
  203. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  204. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
  205. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
  206. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
  207. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
  208. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
  209. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
  210. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
  211. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
  212. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
  213. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
  214. package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
  215. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  216. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
  217. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
  218. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
  219. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
  220. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
  221. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
  222. package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
  223. package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
  224. package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
  225. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
  226. package/dist/parquetjs/parquet-thrift/index.js +6 -0
  227. package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
  228. package/dist/parquetjs/parser/decoders.d.ts +10 -1
  229. package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
  230. package/dist/parquetjs/parser/decoders.js +318 -14
  231. package/dist/parquetjs/parser/decoders.js.map +1 -1
  232. package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
  233. package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
  234. package/dist/parquetjs/parser/encoded-pages.js +206 -0
  235. package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
  236. package/dist/parquetjs/parser/parquet-reader.d.ts +88 -12
  237. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
  238. package/dist/parquetjs/parser/parquet-reader.js +713 -66
  239. package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
  240. package/dist/parquetjs/schema/declare.d.ts +22 -3
  241. package/dist/parquetjs/schema/declare.d.ts.map +1 -1
  242. package/dist/parquetjs/schema/declare.js.map +1 -1
  243. package/dist/parquetjs/schema/schema.d.ts.map +1 -1
  244. package/dist/parquetjs/schema/schema.js +10 -0
  245. package/dist/parquetjs/schema/schema.js.map +1 -1
  246. package/dist/parquetjs/schema/shred.d.ts.map +1 -1
  247. package/dist/parquetjs/schema/shred.js +219 -11
  248. package/dist/parquetjs/schema/shred.js.map +1 -1
  249. package/dist/parquetjs/schema/types.js +19 -21
  250. package/dist/parquetjs/schema/types.js.map +1 -1
  251. package/dist/parquetjs/schema/variant.d.ts +3 -0
  252. package/dist/parquetjs/schema/variant.d.ts.map +1 -0
  253. package/dist/parquetjs/schema/variant.js +248 -0
  254. package/dist/parquetjs/schema/variant.js.map +1 -0
  255. package/dist/parquetjs/utils/crc32.d.ts +3 -0
  256. package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
  257. package/dist/parquetjs/utils/crc32.js +15 -0
  258. package/dist/parquetjs/utils/crc32.js.map +1 -0
  259. package/dist/parquetjs/utils/read-utils.d.ts +11 -1
  260. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
  261. package/dist/parquetjs/utils/read-utils.js +21 -1
  262. package/dist/parquetjs/utils/read-utils.js.map +1 -1
  263. package/dist/unbundled.d.ts +2 -1
  264. package/dist/unbundled.d.ts.map +1 -1
  265. package/dist/unbundled.js +1 -0
  266. package/dist/unbundled.js.map +1 -1
  267. package/package.json +36 -13
  268. package/src/bundled.ts +10 -0
  269. package/src/delta-format.ts +12 -0
  270. package/src/delta-source-loader-types.ts +40 -0
  271. package/src/delta-source.ts +305 -0
  272. package/src/delta-types.ts +40 -0
  273. package/src/iceberg-rest-catalog.ts +122 -0
  274. package/src/iceberg-table-source.ts +1037 -0
  275. package/src/iceberg-types.ts +165 -0
  276. package/src/index.ts +59 -1
  277. package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
  278. package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
  279. package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
  280. package/src/lib/geo/geoparquet-covering.ts +201 -0
  281. package/src/lib/geo/geospatial-metadata.ts +42 -15
  282. package/src/lib/parquet-bloom-filter-planner.ts +86 -0
  283. package/src/lib/parquet-bloom-filter.ts +345 -0
  284. package/src/lib/parquet-encryption.ts +344 -0
  285. package/src/lib/parquet-page-index.ts +180 -34
  286. package/src/lib/parquet-predicate.ts +52 -16
  287. package/src/lib/parquet-source-worker-decoder.ts +116 -20
  288. package/src/lib/parquet-source-worker-types.ts +25 -0
  289. package/src/lib/parsers/get-parquet-schema.ts +3 -1
  290. package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
  291. package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
  292. package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
  293. package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
  294. package/src/lib/slice-parquet-batch.ts +32 -0
  295. package/src/lib/sources/parquet-range-file.ts +35 -29
  296. package/src/lib/utils/normalize-parquet-options.ts +17 -1
  297. package/src/parquet-dataset-source.ts +397 -85
  298. package/src/parquet-encoded-page-types.ts +122 -0
  299. package/src/parquet-js-loader-types.ts +1 -0
  300. package/src/parquet-js-loader.ts +2 -1
  301. package/src/parquet-js-writer.ts +30 -0
  302. package/src/parquet-loader-options.ts +11 -0
  303. package/src/parquet-source-capabilities.ts +18 -1
  304. package/src/parquet-source-loader-types.ts +10 -0
  305. package/src/parquet-source-loader.ts +1024 -52
  306. package/src/parquet-source-types.ts +197 -28
  307. package/src/parquet-worker-url.ts +6 -1
  308. package/src/parquetjs/codecs/declare.ts +27 -0
  309. package/src/parquetjs/codecs/delta.ts +466 -6
  310. package/src/parquetjs/codecs/index.ts +4 -0
  311. package/src/parquetjs/codecs/plain.ts +177 -11
  312. package/src/parquetjs/codecs/rle.ts +573 -3
  313. package/src/parquetjs/compression.ts +102 -76
  314. package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
  315. package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
  316. package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
  317. package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
  318. package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
  319. package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
  320. package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
  321. package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
  322. package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
  323. package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
  324. package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
  325. package/src/parquetjs/parquet-thrift/index.ts +6 -0
  326. package/src/parquetjs/parser/decoders.ts +451 -26
  327. package/src/parquetjs/parser/encoded-pages.ts +278 -0
  328. package/src/parquetjs/parser/parquet-reader.ts +1112 -87
  329. package/src/parquetjs/schema/declare.ts +22 -2
  330. package/src/parquetjs/schema/schema.ts +10 -0
  331. package/src/parquetjs/schema/shred.ts +238 -11
  332. package/src/parquetjs/schema/types.ts +16 -25
  333. package/src/parquetjs/schema/variant.ts +313 -0
  334. package/src/parquetjs/utils/crc32.ts +15 -0
  335. package/src/parquetjs/utils/read-utils.ts +26 -1
  336. package/src/unbundled.ts +14 -0
@@ -6,9 +6,10 @@
6
6
 
7
7
  /* eslint-disable camelcase */
8
8
  import {stream} from '@loaders.gl/loader-utils';
9
+ import {getWKBGeometryStatistics, type WKBGeometryBoundingBox} from '@loaders.gl/gis';
9
10
  import {ParquetCodecOptions, PARQUET_CODECS} from '../codecs/index';
10
11
  import * as Compression from '../compression';
11
- import {
12
+ import type {
12
13
  ParquetRowGroup,
13
14
  ParquetCodec,
14
15
  ParquetColumnChunk,
@@ -21,8 +22,14 @@ import {ParquetSchema} from '../schema/schema';
21
22
  import * as Shred from '../schema/shred';
22
23
  import {
23
24
  BsonType,
25
+ BoundingBox,
24
26
  ColumnChunk,
27
+ ColumnCryptoMetaData,
28
+ ColumnIndex,
25
29
  ColumnMetaData,
30
+ SizeStatistics,
31
+ Statistics,
32
+ SortingColumn,
26
33
  CompressionCodec,
27
34
  ConvertedType,
28
35
  DataPageHeader,
@@ -37,6 +44,7 @@ import {
37
44
  FileMetaData,
38
45
  Float16Type,
39
46
  GeographyType,
47
+ GeospatialStatistics,
40
48
  GeometryType,
41
49
  IntType,
42
50
  JsonType,
@@ -51,6 +59,7 @@ import {
51
59
  PageHeader,
52
60
  PageEncodingStats,
53
61
  PageType,
62
+ PageLocation,
54
63
  RowGroup,
55
64
  SchemaElement,
56
65
  StringType,
@@ -59,20 +68,43 @@ import {
59
68
  TimestampType,
60
69
  Type,
61
70
  UUIDType,
62
- VariantType
71
+ VariantType,
72
+ OffsetIndex
63
73
  } from '../parquet-thrift/index';
64
74
  import {osopen, oswrite, osclose} from '../utils/file-utils';
65
75
  import {getBitWidth, serializeThrift} from '../utils/read-utils';
66
76
  import {concatUint8Arrays, encodeUtf8, writeUInt32LE} from '../utils/binary-utils';
77
+ import {crc32} from '../utils/crc32';
67
78
  import {CompactInt64} from '../utils/uint8-array-compact-protocol';
68
79
  import {planColumnPages} from './page-planner';
69
80
  import {planDictionary, type ParquetDictionaryPolicy} from './dictionary-planner';
81
+ import {
82
+ decodeParquetSplitBlockBloomFilter,
83
+ encodeParquetBloomFilterValue,
84
+ encodeParquetSplitBlockBloomFilter
85
+ } from '../../lib/parquet-bloom-filter';
86
+ import {BoundaryOrder} from '../parquet-thrift/BoundaryOrder';
87
+ import {
88
+ EncryptionWithColumnKey,
89
+ EncryptionWithFooterKey
90
+ } from '../parquet-thrift/ColumnCryptoMetaData';
91
+ import {EncryptionAlgorithm} from '../parquet-thrift/EncryptionAlgorithm';
92
+ import {FileCryptoMetaData} from '../parquet-thrift/FileCryptoMetaData';
93
+ import {PARQUET_MAGIC_ENCRYPTED} from '../../lib/constants';
94
+ import {
95
+ createParquetModuleAad,
96
+ createParquetFooterSignature,
97
+ encryptParquetModule,
98
+ type ParquetWriterEncryptionOptions,
99
+ type ParquetWriterFooterSignatureOptions
100
+ } from '../../lib/parquet-encryption';
70
101
 
71
102
  /**
72
103
  * Parquet File Magic String
73
104
  */
74
105
  const PARQUET_MAGIC = 'PAR1';
75
106
  const PARQUET_MAGIC_BYTES = encodeUtf8(PARQUET_MAGIC);
107
+ const PARQUET_MAGIC_ENCRYPTED_BYTES = encodeUtf8(PARQUET_MAGIC_ENCRYPTED);
76
108
 
77
109
  /**
78
110
  * Parquet File Format Version
@@ -99,6 +131,26 @@ export interface ParquetEncoderOptions {
99
131
  dictionary?: ParquetDictionaryPolicy;
100
132
  columnDictionaries?: Record<string, ParquetDictionaryPolicy>;
101
133
  dictionaryPageSizeLimit?: number;
134
+ bloomFilter?: boolean | Record<string, boolean>;
135
+ pageIndex?: boolean | Record<string, boolean>;
136
+ /** Emit CRC-32 checksums for page bodies. */
137
+ writePageChecksums?: boolean;
138
+ /** Emit optional SizeStatistics metadata for each column chunk. */
139
+ writeSizeStatistics?: boolean;
140
+ /** Emit optional min/max/null-count statistics for each column chunk. */
141
+ writeStatistics?: boolean | Record<string, boolean>;
142
+ /** Declare the row-group sort order using top-level or dotted leaf column names. */
143
+ sortingColumns?: readonly ParquetSortingColumnOption[];
144
+ /** Encode INT96 input values as epoch nanoseconds using the canonical Julian-day layout. */
145
+ int96AsTimestamp?: boolean;
146
+ /** Encrypt the footer using Parquet modular encryption. */
147
+ encryption?: ParquetWriterEncryptionOptions;
148
+ /** Authenticate a plaintext footer with a Parquet modular-encryption signature. */
149
+ footerSignature?: ParquetWriterFooterSignatureOptions;
150
+ /** Stable file identifier shared by encrypted column modules and the footer. */
151
+ encryptionFileUnique?: Uint8Array;
152
+ /** Zero-based row-group ordinal used by encrypted module AAD. */
153
+ rowGroupOrdinal?: number;
102
154
 
103
155
  // Write Stream Options
104
156
  flags?: string;
@@ -109,6 +161,16 @@ export interface ParquetEncoderOptions {
109
161
  start?: number;
110
162
  }
111
163
 
164
+ /** Writer-facing declaration for one row-group sort key. */
165
+ export type ParquetSortingColumnOption = {
166
+ /** Top-level field name or dotted nested leaf path. */
167
+ column: string;
168
+ /** Sort direction; defaults to ascending. */
169
+ descending?: boolean;
170
+ /** Whether null values sort before non-null values; defaults to false. */
171
+ nullsFirst?: boolean;
172
+ };
173
+
112
174
  /**
113
175
  * Write a parquet file to an output stream. The ParquetEncoder will perform
114
176
  * buffering/batching for performance, so close() must be called after all rows
@@ -278,6 +340,18 @@ export class ParquetEnvelopeWriter {
278
340
  public dictionary: ParquetDictionaryPolicy;
279
341
  public columnDictionaries: Record<string, ParquetDictionaryPolicy>;
280
342
  public dictionaryPageSizeLimit: number;
343
+ public bloomFilter?: boolean | Record<string, boolean>;
344
+ public pageIndex?: boolean | Record<string, boolean>;
345
+ public writePageChecksums: boolean;
346
+ public writeSizeStatistics: boolean;
347
+ public writeStatistics?: boolean | Record<string, boolean>;
348
+ public sortingColumns?: readonly ParquetSortingColumnOption[];
349
+ /** Optional modular-encryption configuration for the footer. */
350
+ public encryption?: ParquetWriterEncryptionOptions;
351
+ /** Optional plaintext-footer signature configuration. */
352
+ public footerSignature?: ParquetWriterFooterSignatureOptions;
353
+ /** File identifier used for encrypted column-module AAD. */
354
+ public encryptionFileUnique?: Uint8Array;
281
355
 
282
356
  constructor(
283
357
  schema: ParquetSchema,
@@ -297,6 +371,43 @@ export class ParquetEnvelopeWriter {
297
371
  this.dictionary = opts.dictionary ?? 'auto';
298
372
  this.columnDictionaries = opts.columnDictionaries || {};
299
373
  this.dictionaryPageSizeLimit = opts.dictionaryPageSizeLimit ?? 1024 * 1024;
374
+ this.bloomFilter = opts.bloomFilter;
375
+ this.pageIndex = opts.pageIndex;
376
+ this.writePageChecksums = Boolean(opts.writePageChecksums);
377
+ this.writeSizeStatistics = Boolean(opts.writeSizeStatistics);
378
+ this.writeStatistics = opts.writeStatistics;
379
+ this.sortingColumns = opts.sortingColumns;
380
+ const encryptionFileUnique = opts.encryption?.fileUnique
381
+ ? new Uint8Array(opts.encryption.fileUnique)
382
+ : undefined;
383
+ const footerSignatureFileUnique = opts.footerSignature?.fileUnique
384
+ ? new Uint8Array(opts.footerSignature.fileUnique)
385
+ : undefined;
386
+ if (
387
+ encryptionFileUnique &&
388
+ footerSignatureFileUnique &&
389
+ !encryptionFileUnique.every((value, index) => value === footerSignatureFileUnique[index])
390
+ ) {
391
+ throw new Error('Parquet encryption and footer signature file_unique values must match');
392
+ }
393
+ const sharedFileUnique =
394
+ encryptionFileUnique ??
395
+ footerSignatureFileUnique ??
396
+ (opts.encryption || opts.footerSignature ? createFileUnique() : undefined);
397
+ this.encryption = opts.encryption
398
+ ? {...opts.encryption, fileUnique: new Uint8Array(sharedFileUnique!)}
399
+ : undefined;
400
+ this.footerSignature = opts.footerSignature
401
+ ? {
402
+ ...opts.footerSignature,
403
+ keyMetadata: new Uint8Array(opts.footerSignature.keyMetadata),
404
+ aadPrefix: opts.footerSignature.aadPrefix
405
+ ? new Uint8Array(opts.footerSignature.aadPrefix)
406
+ : undefined,
407
+ fileUnique: new Uint8Array(sharedFileUnique!)
408
+ }
409
+ : undefined;
410
+ this.encryptionFileUnique = sharedFileUnique;
300
411
  }
301
412
 
302
413
  writeSection(buf: Uint8Array): Promise<void> {
@@ -308,7 +419,9 @@ export class ParquetEnvelopeWriter {
308
419
  * Encode the parquet file header
309
420
  */
310
421
  writeHeader(): Promise<void> {
311
- return this.writeSection(PARQUET_MAGIC_BYTES);
422
+ return this.writeSection(
423
+ this.encryption && !this.footerSignature ? PARQUET_MAGIC_ENCRYPTED_BYTES : PARQUET_MAGIC_BYTES
424
+ );
312
425
  }
313
426
 
314
427
  /**
@@ -322,7 +435,16 @@ export class ParquetEnvelopeWriter {
322
435
  useDataPageV2: this.useDataPageV2,
323
436
  dictionary: this.dictionary,
324
437
  columnDictionaries: this.columnDictionaries,
325
- dictionaryPageSizeLimit: this.dictionaryPageSizeLimit
438
+ dictionaryPageSizeLimit: this.dictionaryPageSizeLimit,
439
+ bloomFilter: this.bloomFilter,
440
+ pageIndex: this.pageIndex,
441
+ writePageChecksums: this.writePageChecksums,
442
+ writeSizeStatistics: this.writeSizeStatistics,
443
+ writeStatistics: this.writeStatistics,
444
+ sortingColumns: this.sortingColumns,
445
+ encryption: this.encryption,
446
+ encryptionFileUnique: this.encryptionFileUnique,
447
+ rowGroupOrdinal: this.rowGroups.length
326
448
  });
327
449
 
328
450
  this.rowCount += records.rowCount;
@@ -333,15 +455,30 @@ export class ParquetEnvelopeWriter {
333
455
  /**
334
456
  * Write the parquet file footer
335
457
  */
336
- writeFooter(userMetadata: Record<string, string>): Promise<void> {
458
+ async writeFooter(userMetadata: Record<string, string>): Promise<void> {
337
459
  if (!userMetadata) {
338
460
  // tslint:disable-next-line:no-parameter-reassignment
339
461
  userMetadata = {};
340
462
  }
341
463
 
342
- return this.writeSection(
343
- encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata)
344
- );
464
+ if (this.footerSignature) {
465
+ await this.writeSection(
466
+ await encodeSignedFooter(
467
+ this.schema,
468
+ this.rowCount,
469
+ this.rowGroups,
470
+ userMetadata,
471
+ this.footerSignature
472
+ )
473
+ );
474
+ return;
475
+ }
476
+ const footer = encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata);
477
+ if (!this.encryption) {
478
+ await this.writeSection(footer);
479
+ return;
480
+ }
481
+ await this.writeSection(await encodeEncryptedFooter(footer, this.encryption));
345
482
  }
346
483
 
347
484
  /**
@@ -412,7 +549,10 @@ async function encodeDataPage(
412
549
  column: ParquetField,
413
550
  data: ParquetColumnChunk,
414
551
  valueEncoding: ParquetCodec = column.encoding!,
415
- valueBitWidth?: number
552
+ valueBitWidth?: number,
553
+ writePageChecksums = false,
554
+ statistics?: Statistics,
555
+ int96AsTimestamp = false
416
556
  ): Promise<{
417
557
  header: PageHeader;
418
558
  headerSize: number;
@@ -448,7 +588,8 @@ async function encodeDataPage(
448
588
  /* encode values */
449
589
  const valuesBuf = encodeValues(column.primitiveType!, valueEncoding, data.values as any[], {
450
590
  typeLength: column.typeLength,
451
- bitWidth: valueBitWidth ?? column.typeLength
591
+ bitWidth: valueBitWidth ?? column.typeLength,
592
+ int96AsTimestamp
452
593
  });
453
594
 
454
595
  const dataBuf = concatUint8Arrays([rLevelsBuf, dLevelsBuf, valuesBuf]);
@@ -463,10 +604,12 @@ async function encodeDataPage(
463
604
  num_values: data.count,
464
605
  encoding: Encoding[valueEncoding] as any,
465
606
  definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],
466
- repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]
607
+ repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING]
608
+ statistics
467
609
  }),
468
610
  uncompressed_page_size: dataBuf.length,
469
- compressed_page_size: compressedBuf.length
611
+ compressed_page_size: compressedBuf.length,
612
+ crc: writePageChecksums ? crc32(compressedBuf) : undefined
470
613
  });
471
614
 
472
615
  /* concat page header, repetition and definition levels and values */
@@ -484,7 +627,10 @@ async function encodeDataPageV2(
484
627
  data: ParquetColumnChunk,
485
628
  rowCount: number,
486
629
  valueEncoding: ParquetCodec = column.encoding!,
487
- valueBitWidth?: number
630
+ valueBitWidth?: number,
631
+ writePageChecksums = false,
632
+ statistics?: Statistics,
633
+ int96AsTimestamp = false
488
634
  ): Promise<{
489
635
  header: PageHeader;
490
636
  headerSize: number;
@@ -493,7 +639,8 @@ async function encodeDataPageV2(
493
639
  /* encode values */
494
640
  const valuesBuf = encodeValues(column.primitiveType!, valueEncoding, data.values as any[], {
495
641
  typeLength: column.typeLength,
496
- bitWidth: valueBitWidth ?? column.typeLength
642
+ bitWidth: valueBitWidth ?? column.typeLength,
643
+ int96AsTimestamp
497
644
  });
498
645
 
499
646
  // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
@@ -536,10 +683,14 @@ async function encodeDataPageV2(
536
683
  encoding: Encoding[valueEncoding] as any,
537
684
  definition_levels_byte_length: dLevelsBuf.length,
538
685
  repetition_levels_byte_length: rLevelsBuf.length,
539
- is_compressed: column.compression !== 'UNCOMPRESSED'
686
+ is_compressed: column.compression !== 'UNCOMPRESSED',
687
+ statistics
540
688
  }),
541
689
  uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,
542
- compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length
690
+ compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length,
691
+ crc: writePageChecksums
692
+ ? crc32(concatUint8Arrays([rLevelsBuf, dLevelsBuf, compressedBuf]))
693
+ : undefined
543
694
  });
544
695
 
545
696
  /* concat page header, repetition and definition levels and values */
@@ -551,10 +702,13 @@ async function encodeDataPageV2(
551
702
  /** Encodes one chunk-wide PLAIN dictionary page. */
552
703
  async function encodeDictionaryPage(
553
704
  column: ParquetField,
554
- dictionaryValues: unknown[]
705
+ dictionaryValues: unknown[],
706
+ writePageChecksums = false,
707
+ int96AsTimestamp = false
555
708
  ): Promise<{header: PageHeader; headerSize: number; page: Uint8Array}> {
556
709
  const valuesBuffer = encodeValues(column.primitiveType!, 'PLAIN', dictionaryValues as any[], {
557
- typeLength: column.typeLength
710
+ typeLength: column.typeLength,
711
+ int96AsTimestamp
558
712
  });
559
713
  const compressedBuffer = await Compression.deflate(column.compression!, valuesBuffer);
560
714
  const header = new PageHeader({
@@ -564,7 +718,8 @@ async function encodeDictionaryPage(
564
718
  encoding: Encoding.PLAIN
565
719
  }),
566
720
  uncompressed_page_size: valuesBuffer.length,
567
- compressed_page_size: compressedBuffer.length
721
+ compressed_page_size: compressedBuffer.length,
722
+ crc: writePageChecksums ? crc32(compressedBuffer) : undefined
568
723
  });
569
724
  const headerBuffer = serializeThrift(header);
570
725
  return {
@@ -574,6 +729,65 @@ async function encodeDictionaryPage(
574
729
  };
575
730
  }
576
731
 
732
+ /** Computes native Parquet geospatial statistics for a GEOMETRY or GEOGRAPHY column chunk. */
733
+ function createGeospatialStatistics(
734
+ column: ParquetField,
735
+ values: ParquetColumnChunk['values']
736
+ ): GeospatialStatistics | undefined {
737
+ const logicalType = column.logicalType?.type || column.originalType;
738
+ if (logicalType !== 'GEOMETRY' && logicalType !== 'GEOGRAPHY') return undefined;
739
+ const geometryTypes = new Set<number>();
740
+ let bbox: WKBGeometryBoundingBox | undefined;
741
+ // Vertex bounds are not conservative for GEOGRAPHY: non-linear edge algorithms can reach
742
+ // extrema between vertices. Until edge-aware geodesic bounds are available, omit the bbox while
743
+ // retaining the useful geometry-type statistics.
744
+ const canWriteBoundingBox = logicalType === 'GEOMETRY';
745
+ for (const value of values) {
746
+ if (!(value instanceof ArrayBuffer) && !ArrayBuffer.isView(value)) {
747
+ throw new Error(`${logicalType} columns must contain WKB binary values`);
748
+ }
749
+ const statistics = getWKBGeometryStatistics(value);
750
+ geometryTypes.add(statistics.geometryType);
751
+ if (canWriteBoundingBox) {
752
+ bbox = mergeGeometryBoundingBoxes(bbox, statistics.bbox);
753
+ }
754
+ }
755
+ return new GeospatialStatistics({
756
+ bbox: bbox ? new BoundingBox(bbox) : undefined,
757
+ geospatial_types: [...geometryTypes].sort((left, right) => left - right)
758
+ });
759
+ }
760
+
761
+ /** Merges finite per-dimension bounds without inventing absent Z or M dimensions. */
762
+ function mergeGeometryBoundingBoxes(
763
+ target: WKBGeometryBoundingBox | undefined,
764
+ source: WKBGeometryBoundingBox | undefined
765
+ ): WKBGeometryBoundingBox | undefined {
766
+ if (!source) return target;
767
+ if (!target) return {...source};
768
+ target.xmin = Math.min(target.xmin, source.xmin);
769
+ target.xmax = Math.max(target.xmax, source.xmax);
770
+ target.ymin = Math.min(target.ymin, source.ymin);
771
+ target.ymax = Math.max(target.ymax, source.ymax);
772
+ mergeOptionalBounds(target, source, 'zmin', Math.min);
773
+ mergeOptionalBounds(target, source, 'zmax', Math.max);
774
+ mergeOptionalBounds(target, source, 'mmin', Math.min);
775
+ mergeOptionalBounds(target, source, 'mmax', Math.max);
776
+ return target;
777
+ }
778
+
779
+ /** Merges one optional bound while preserving a dimension absent from every value. */
780
+ function mergeOptionalBounds(
781
+ target: WKBGeometryBoundingBox,
782
+ source: WKBGeometryBoundingBox,
783
+ key: 'zmin' | 'zmax' | 'mmin' | 'mmax',
784
+ merge: (left: number, right: number) => number
785
+ ): void {
786
+ const sourceValue = source[key];
787
+ if (sourceValue === undefined) return;
788
+ target[key] = target[key] === undefined ? sourceValue : merge(target[key]!, sourceValue);
789
+ }
790
+
577
791
  /**
578
792
  * Encode an array of values into a parquet column chunk
579
793
  */
@@ -581,16 +795,29 @@ async function encodeColumnChunk(
581
795
  column: ParquetField,
582
796
  buffer: ParquetRowGroup,
583
797
  offset: number,
584
- opts: ParquetEncoderOptions
798
+ opts: ParquetEncoderOptions,
799
+ rowGroupOrdinal: number,
800
+ columnOrdinal: number
585
801
  ): Promise<{
586
802
  body: Uint8Array;
587
803
  metadata: ColumnMetaData;
804
+ encryptedColumnMetadata?: Uint8Array;
805
+ cryptoMetadata?: ColumnCryptoMetaData;
588
806
  metadataOffset: number;
807
+ offsetIndexOffset?: number;
808
+ offsetIndexLength?: number;
809
+ columnIndexOffset?: number;
810
+ columnIndexLength?: number;
589
811
  }> {
590
812
  const data = buffer.columnData[column.path.join()];
591
813
  const baseOffset = (opts.baseOffset || 0) + offset;
592
814
  /* encode data page(s) */
593
- const pages: Uint8Array[] = [];
815
+ const pageRecords: Array<{
816
+ page: Uint8Array;
817
+ headerSize: number;
818
+ dictionary: boolean;
819
+ }> = [];
820
+ const pageLocations: PageLocation[] = [];
594
821
  // tslint:disable-next-line:variable-name
595
822
  let total_uncompressed_size = 0;
596
823
  // tslint:disable-next-line:variable-name
@@ -610,13 +837,24 @@ async function encodeColumnChunk(
610
837
  opts.dictionaryPageSizeLimit ?? 1024 * 1024
611
838
  );
612
839
  if (dictionaryPlan) {
613
- const result = await encodeDictionaryPage(column, dictionaryPlan.values);
614
- pages.push(result.page);
840
+ const result = await encodeDictionaryPage(
841
+ column,
842
+ dictionaryPlan.values,
843
+ opts.writePageChecksums,
844
+ opts.int96AsTimestamp
845
+ );
846
+ pageRecords.push({page: result.page, headerSize: result.headerSize, dictionary: true});
615
847
  total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
616
848
  total_compressed_size += result.header.compressed_page_size + result.headerSize;
617
849
  }
618
850
 
619
- const valueEncoding: ParquetCodec = dictionaryPlan ? 'RLE_DICTIONARY' : column.encoding!;
851
+ const valueEncoding: ParquetCodec = dictionaryPlan
852
+ ? column.encoding === 'PLAIN_DICTIONARY'
853
+ ? 'PLAIN_DICTIONARY'
854
+ : 'RLE_DICTIONARY'
855
+ : column.encoding === 'PLAIN_DICTIONARY' || column.encoding === 'RLE_DICTIONARY'
856
+ ? 'PLAIN'
857
+ : column.encoding!;
620
858
  let dictionaryIndexOffset = 0;
621
859
  for (const plannedPage of plannedPages) {
622
860
  const pageData = dictionaryPlan
@@ -635,16 +873,90 @@ async function encodeColumnChunk(
635
873
  pageData,
636
874
  plannedPage.rowCount,
637
875
  valueEncoding,
638
- dictionaryPlan?.bitWidth
876
+ dictionaryPlan?.bitWidth,
877
+ opts.writePageChecksums,
878
+ isStatisticsEnabled(opts.writeStatistics, column)
879
+ ? createColumnStatistics(column, plannedPage.data)
880
+ : undefined,
881
+ opts.int96AsTimestamp
639
882
  )
640
- : await encodeDataPage(column, pageData, valueEncoding, dictionaryPlan?.bitWidth);
641
- pages.push(result.page);
883
+ : await encodeDataPage(
884
+ column,
885
+ pageData,
886
+ valueEncoding,
887
+ dictionaryPlan?.bitWidth,
888
+ opts.writePageChecksums,
889
+ isStatisticsEnabled(opts.writeStatistics, column)
890
+ ? createColumnStatistics(column, plannedPage.data)
891
+ : undefined,
892
+ opts.int96AsTimestamp
893
+ );
894
+ pageRecords.push({page: result.page, headerSize: result.headerSize, dictionary: false});
642
895
  total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
643
896
  total_compressed_size += result.header.compressed_page_size + result.headerSize;
644
897
  }
645
898
 
899
+ const encryption = opts.encryption;
900
+ const encryptedColumn = Boolean(encryption && shouldEncryptColumn(encryption, column));
901
+ const columnKeyMetadata = encryption ? getColumnKeyMetadata(encryption, column) : undefined;
902
+ const pages = encryptedColumn
903
+ ? await encryptParquetColumnPages(
904
+ pageRecords,
905
+ encryption!,
906
+ opts.encryptionFileUnique!,
907
+ opts.rowGroupOrdinal ?? 0,
908
+ columnOrdinal,
909
+ columnKeyMetadata
910
+ )
911
+ : pageRecords.map(record => record.page);
912
+ let pageOffset = 0;
913
+ let dataPageIndex = 0;
914
+ let firstRowIndex = 0;
915
+ for (const [pageIndex, page] of pages.entries()) {
916
+ if (!pageRecords[pageIndex].dictionary) {
917
+ pageLocations.push(
918
+ new PageLocation({
919
+ offset: baseOffset + pageOffset,
920
+ compressed_page_size: page.length,
921
+ first_row_index: firstRowIndex
922
+ })
923
+ );
924
+ firstRowIndex += plannedPages[dataPageIndex++].rowCount;
925
+ }
926
+ pageOffset += page.length;
927
+ }
928
+ total_compressed_size = pages.reduce((total, page) => total + page.length, 0);
646
929
  const pagesBuf = concatUint8Arrays(pages);
647
930
  // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;
931
+ const bloomFilterEnabled =
932
+ opts.bloomFilter === true || opts.bloomFilter?.[column.path[0]] === true;
933
+ const bloomFilter = bloomFilterEnabled ? createColumnBloomFilter(column, data.values) : undefined;
934
+ const encodedBloomFilter =
935
+ bloomFilter && encryptedColumn
936
+ ? await encryptParquetBloomFilter(
937
+ bloomFilter,
938
+ encryption!,
939
+ opts.encryptionFileUnique!,
940
+ opts.rowGroupOrdinal ?? 0,
941
+ columnOrdinal,
942
+ columnKeyMetadata
943
+ )
944
+ : bloomFilter;
945
+ const pageIndexEnabled = opts.pageIndex === true || opts.pageIndex?.[column.path[0]] === true;
946
+ const pageIndexes = pageIndexEnabled
947
+ ? createColumnPageIndexes(column, plannedPages, pageLocations, opts.int96AsTimestamp)
948
+ : undefined;
949
+ const encodedPageIndexes =
950
+ pageIndexes && encryptedColumn
951
+ ? await encryptParquetPageIndexes(
952
+ pageIndexes,
953
+ encryption!,
954
+ opts.encryptionFileUnique!,
955
+ opts.rowGroupOrdinal ?? 0,
956
+ columnOrdinal,
957
+ columnKeyMetadata
958
+ )
959
+ : pageIndexes;
648
960
 
649
961
  /* prepare metadata header */
650
962
  const metadata = new ColumnMetaData({
@@ -672,7 +984,14 @@ async function encodeColumnChunk(
672
984
  total_uncompressed_size: int64(total_uncompressed_size), // : pagesBuf.length,
673
985
  total_compressed_size: int64(total_compressed_size),
674
986
  type: Type[column.primitiveType!],
675
- codec: CompressionCodec[column.compression!]
987
+ codec: CompressionCodec[column.compression!],
988
+ bloom_filter_offset: encodedBloomFilter ? int64(baseOffset + pagesBuf.length) : undefined,
989
+ bloom_filter_length: encodedBloomFilter?.length,
990
+ geospatial_statistics: createGeospatialStatistics(column, data.values),
991
+ size_statistics: opts.writeSizeStatistics ? createSizeStatistics(column, data) : undefined,
992
+ statistics: isStatisticsEnabled(opts.writeStatistics, column)
993
+ ? createColumnStatistics(column, data)
994
+ : undefined
676
995
  });
677
996
 
678
997
  /* list encodings */
@@ -681,9 +1000,541 @@ async function encodeColumnChunk(
681
1000
  metadata.encodings.push(Encoding[valueEncoding]);
682
1001
 
683
1002
  /* concat metadata header and data pages */
684
- const metadataOffset = baseOffset + pagesBuf.length;
685
- const body = concatUint8Arrays([pagesBuf, serializeThrift(metadata)]);
686
- return {body, metadata, metadataOffset};
1003
+ const metadataEncoded = serializeThrift(metadata);
1004
+ const metadataSection = encryptedColumn ? undefined : metadataEncoded;
1005
+ const metadataOffset = baseOffset + pagesBuf.length + (encodedBloomFilter?.length || 0);
1006
+ const offsetIndexOffset = encodedPageIndexes
1007
+ ? metadataOffset + (metadataSection?.length || 0)
1008
+ : undefined;
1009
+ const columnIndexOffset = encodedPageIndexes?.columnIndex
1010
+ ? metadataOffset + (metadataSection?.length || 0) + encodedPageIndexes.offsetIndex.length
1011
+ : undefined;
1012
+ const body = concatUint8Arrays([
1013
+ pagesBuf,
1014
+ ...(encodedBloomFilter ? [encodedBloomFilter] : []),
1015
+ ...(metadataSection ? [metadataSection] : []),
1016
+ ...(encodedPageIndexes
1017
+ ? [
1018
+ encodedPageIndexes.offsetIndex,
1019
+ ...(encodedPageIndexes.columnIndex ? [encodedPageIndexes.columnIndex] : [])
1020
+ ]
1021
+ : [])
1022
+ ]);
1023
+ return {
1024
+ body,
1025
+ metadata,
1026
+ metadataOffset,
1027
+ offsetIndexOffset,
1028
+ offsetIndexLength: encodedPageIndexes?.offsetIndex.length,
1029
+ columnIndexOffset,
1030
+ columnIndexLength: encodedPageIndexes?.columnIndex?.length,
1031
+ ...(encryptedColumn
1032
+ ? {
1033
+ encryptedColumnMetadata: await encryptParquetModule(metadataEncoded, {
1034
+ algorithm: encryption!.algorithm ?? 'AES_GCM_V1',
1035
+ aad: createParquetModuleAad(
1036
+ encryption!.aadPrefix,
1037
+ opts.encryptionFileUnique!,
1038
+ 'column-metadata',
1039
+ opts.rowGroupOrdinal ?? 0,
1040
+ columnOrdinal
1041
+ ),
1042
+ keyMetadata: columnKeyMetadata ?? encryption!.keyMetadata,
1043
+ keyRetriever: encryption!.keyRetriever
1044
+ }),
1045
+ cryptoMetadata: new ColumnCryptoMetaData({
1046
+ ...(columnKeyMetadata
1047
+ ? {
1048
+ ENCRYPTION_WITH_COLUMN_KEY: new EncryptionWithColumnKey({
1049
+ path_in_schema: [...column.path],
1050
+ key_metadata: new Uint8Array(columnKeyMetadata)
1051
+ })
1052
+ }
1053
+ : {ENCRYPTION_WITH_FOOTER_KEY: new EncryptionWithFooterKey()})
1054
+ })
1055
+ }
1056
+ : {})
1057
+ };
1058
+ }
1059
+
1060
+ /** Returns whether a writer column should use modular encryption. */
1061
+ function shouldEncryptColumn(
1062
+ options: ParquetWriterEncryptionOptions,
1063
+ column: ParquetField
1064
+ ): boolean {
1065
+ return Boolean(
1066
+ options.columnKeyMetadata?.[column.path[0]] ||
1067
+ options.encryptColumns === true ||
1068
+ options.encryptColumns?.[column.path[0]] === true
1069
+ );
1070
+ }
1071
+
1072
+ /** Returns a defensive copy of the key metadata assigned to one top-level column. */
1073
+ function getColumnKeyMetadata(
1074
+ options: ParquetWriterEncryptionOptions,
1075
+ column: ParquetField
1076
+ ): Uint8Array | undefined {
1077
+ const keyMetadata = options.columnKeyMetadata?.[column.path[0]];
1078
+ return keyMetadata ? new Uint8Array(keyMetadata) : undefined;
1079
+ }
1080
+
1081
+ /** Adds the four-byte module length required by encrypted page and Bloom-filter modules. */
1082
+ function prefixEncryptedModule(module: Uint8Array): Uint8Array {
1083
+ const prefixed = new Uint8Array(module.byteLength + 4);
1084
+ writeUInt32LE(prefixed, module.byteLength, 0);
1085
+ prefixed.set(module, 4);
1086
+ return prefixed;
1087
+ }
1088
+
1089
+ /** Encrypts one column's dictionary and data pages using the Parquet module AAD rules. */
1090
+ async function encryptParquetColumnPages(
1091
+ pageRecords: readonly {page: Uint8Array; headerSize: number; dictionary: boolean}[],
1092
+ options: ParquetWriterEncryptionOptions,
1093
+ fileUnique: Uint8Array,
1094
+ rowGroupOrdinal: number,
1095
+ columnOrdinal: number,
1096
+ keyMetadata?: Uint8Array
1097
+ ): Promise<Uint8Array[]> {
1098
+ const encryptedPages: Uint8Array[] = [];
1099
+ let dataPageOrdinal = 0;
1100
+ const algorithm = options.algorithm ?? 'AES_GCM_V1';
1101
+ for (const pageRecord of pageRecords) {
1102
+ const moduleType = pageRecord.dictionary ? 'dictionary-page' : 'data-page';
1103
+ const headerModuleType = pageRecord.dictionary ? 'dictionary-page-header' : 'data-page-header';
1104
+ const header = await encryptParquetModule(pageRecord.page.subarray(0, pageRecord.headerSize), {
1105
+ algorithm,
1106
+ aad: createParquetModuleAad(
1107
+ options.aadPrefix,
1108
+ fileUnique,
1109
+ headerModuleType,
1110
+ rowGroupOrdinal,
1111
+ columnOrdinal,
1112
+ pageRecord.dictionary ? undefined : dataPageOrdinal
1113
+ ),
1114
+ keyMetadata: keyMetadata ?? options.keyMetadata,
1115
+ keyRetriever: options.keyRetriever
1116
+ });
1117
+ const body = await encryptParquetModule(pageRecord.page.subarray(pageRecord.headerSize), {
1118
+ algorithm,
1119
+ aad: createParquetModuleAad(
1120
+ options.aadPrefix,
1121
+ fileUnique,
1122
+ moduleType,
1123
+ rowGroupOrdinal,
1124
+ columnOrdinal,
1125
+ pageRecord.dictionary ? undefined : dataPageOrdinal
1126
+ ),
1127
+ keyMetadata: keyMetadata ?? options.keyMetadata,
1128
+ keyRetriever: options.keyRetriever,
1129
+ page: true
1130
+ });
1131
+ encryptedPages.push(
1132
+ concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(body)])
1133
+ );
1134
+ if (!pageRecord.dictionary) dataPageOrdinal++;
1135
+ }
1136
+ return encryptedPages;
1137
+ }
1138
+
1139
+ /** Encrypts a split-block Bloom filter header and bitset as independent Parquet modules. */
1140
+ async function encryptParquetBloomFilter(
1141
+ bloomFilter: Uint8Array,
1142
+ options: ParquetWriterEncryptionOptions,
1143
+ fileUnique: Uint8Array,
1144
+ rowGroupOrdinal: number,
1145
+ columnOrdinal: number,
1146
+ keyMetadata?: Uint8Array
1147
+ ): Promise<Uint8Array> {
1148
+ const parsed = decodeParquetSplitBlockBloomFilter(bloomFilter);
1149
+ const algorithm = options.algorithm ?? 'AES_GCM_V1';
1150
+ const keyOptions = {
1151
+ algorithm,
1152
+ keyMetadata: keyMetadata ?? options.keyMetadata,
1153
+ keyRetriever: options.keyRetriever
1154
+ };
1155
+ const header = await encryptParquetModule(bloomFilter.subarray(0, parsed.headerByteLength), {
1156
+ ...keyOptions,
1157
+ aad: createParquetModuleAad(
1158
+ options.aadPrefix,
1159
+ fileUnique,
1160
+ 'bloom-filter-header',
1161
+ rowGroupOrdinal,
1162
+ columnOrdinal
1163
+ )
1164
+ });
1165
+ const bitset = await encryptParquetModule(parsed.bitset, {
1166
+ ...keyOptions,
1167
+ aad: createParquetModuleAad(
1168
+ options.aadPrefix,
1169
+ fileUnique,
1170
+ 'bloom-filter-bitset',
1171
+ rowGroupOrdinal,
1172
+ columnOrdinal
1173
+ )
1174
+ });
1175
+ return concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(bitset)]);
1176
+ }
1177
+
1178
+ /** Encrypts column and offset indexes independently while preserving their footer offsets. */
1179
+ async function encryptParquetPageIndexes(
1180
+ pageIndexes: {offsetIndex: Uint8Array; columnIndex?: Uint8Array},
1181
+ options: ParquetWriterEncryptionOptions,
1182
+ fileUnique: Uint8Array,
1183
+ rowGroupOrdinal: number,
1184
+ columnOrdinal: number,
1185
+ keyMetadata?: Uint8Array
1186
+ ): Promise<{offsetIndex: Uint8Array; columnIndex?: Uint8Array}> {
1187
+ const algorithm = options.algorithm ?? 'AES_GCM_V1';
1188
+ const encryptIndex = (bytes: Uint8Array, module: 'offset-index' | 'column-index') =>
1189
+ encryptParquetModule(bytes, {
1190
+ algorithm,
1191
+ aad: createParquetModuleAad(
1192
+ options.aadPrefix,
1193
+ fileUnique,
1194
+ module,
1195
+ rowGroupOrdinal,
1196
+ columnOrdinal
1197
+ ),
1198
+ keyMetadata: keyMetadata ?? options.keyMetadata,
1199
+ keyRetriever: options.keyRetriever
1200
+ });
1201
+ return {
1202
+ offsetIndex: await encryptIndex(pageIndexes.offsetIndex, 'offset-index'),
1203
+ columnIndex: pageIndexes.columnIndex
1204
+ ? await encryptIndex(pageIndexes.columnIndex, 'column-index')
1205
+ : undefined
1206
+ };
1207
+ }
1208
+
1209
+ /** Builds page indexes for primitive leaves, including nested and repeated leaves. */
1210
+ function createColumnPageIndexes(
1211
+ column: ParquetField,
1212
+ plannedPages: ReturnType<typeof planColumnPages>,
1213
+ pageLocations: PageLocation[],
1214
+ int96AsTimestamp = false
1215
+ ): {offsetIndex: Uint8Array; columnIndex?: Uint8Array} | undefined {
1216
+ if (
1217
+ !column.primitiveType ||
1218
+ !isPageIndexPhysicalType(column.primitiveType) ||
1219
+ plannedPages.length !== pageLocations.length ||
1220
+ plannedPages.length === 0
1221
+ ) {
1222
+ return undefined;
1223
+ }
1224
+
1225
+ const offsetIndex = serializeThrift(new OffsetIndex({page_locations: pageLocations}));
1226
+ if (!supportsStatisticsSortOrder(column)) {
1227
+ return {offsetIndex};
1228
+ }
1229
+
1230
+ const nullPages: boolean[] = [];
1231
+ const minValues: Uint8Array[] = [];
1232
+ const maxValues: Uint8Array[] = [];
1233
+ // A repeated leaf's definition-level count is not a row-level null count.
1234
+ // Omit null_counts for those leaves so readers do not interpret entry counts
1235
+ // as null counts while still benefiting from min/max page pruning.
1236
+ const nullCounts: number[] | undefined = column.rLevelMax === 0 ? [] : undefined;
1237
+ for (const plannedPage of plannedPages) {
1238
+ const values = plannedPage.data.values;
1239
+ const nullCount = plannedPage.data.count - values.length;
1240
+ nullCounts?.push(nullCount);
1241
+ if (values.length === 0) {
1242
+ nullPages.push(true);
1243
+ minValues.push(new Uint8Array(0));
1244
+ maxValues.push(new Uint8Array(0));
1245
+ continue;
1246
+ }
1247
+ const bounds = getPageBounds(
1248
+ values,
1249
+ column.primitiveType!,
1250
+ column.typeLength,
1251
+ int96AsTimestamp
1252
+ );
1253
+ if (!bounds) return undefined;
1254
+ nullPages.push(false);
1255
+ minValues.push(bounds.min);
1256
+ maxValues.push(bounds.max);
1257
+ }
1258
+
1259
+ return {
1260
+ offsetIndex,
1261
+ columnIndex: serializeThrift(
1262
+ new ColumnIndex({
1263
+ null_pages: nullPages,
1264
+ min_values: minValues,
1265
+ max_values: maxValues,
1266
+ boundary_order: BoundaryOrder.UNORDERED,
1267
+ null_counts: nullCounts
1268
+ })
1269
+ )
1270
+ };
1271
+ }
1272
+
1273
+ function isPageIndexPhysicalType(
1274
+ physicalType: ParquetField['primitiveType']
1275
+ ): physicalType is PageIndexPhysicalType {
1276
+ return (
1277
+ physicalType === 'BOOLEAN' ||
1278
+ physicalType === 'INT32' ||
1279
+ physicalType === 'INT64' ||
1280
+ physicalType === 'INT96' ||
1281
+ physicalType === 'FLOAT' ||
1282
+ physicalType === 'DOUBLE' ||
1283
+ physicalType === 'BYTE_ARRAY' ||
1284
+ physicalType === 'FIXED_LEN_BYTE_ARRAY'
1285
+ );
1286
+ }
1287
+
1288
+ type PageIndexPhysicalType =
1289
+ | 'BOOLEAN'
1290
+ | 'INT32'
1291
+ | 'INT64'
1292
+ | 'INT96'
1293
+ | 'FLOAT'
1294
+ | 'DOUBLE'
1295
+ | 'BYTE_ARRAY'
1296
+ | 'FIXED_LEN_BYTE_ARRAY';
1297
+
1298
+ function getPageBounds(
1299
+ values: ParquetColumnChunk['values'],
1300
+ physicalType: PageIndexPhysicalType,
1301
+ typeLength?: number,
1302
+ int96AsTimestamp = false
1303
+ ): {min: Uint8Array; max: Uint8Array} | undefined {
1304
+ if (physicalType === 'INT96' && !int96AsTimestamp) return undefined;
1305
+ const normalizedValues = Array.from(values as Iterable<unknown>, value =>
1306
+ normalizePageIndexValue(value, physicalType)
1307
+ );
1308
+ if (normalizedValues.some(value => value === undefined)) return undefined;
1309
+ const comparableValues = normalizedValues as Array<
1310
+ boolean | number | bigint | string | Uint8Array
1311
+ >;
1312
+ let min = comparableValues[0];
1313
+ let max = comparableValues[0];
1314
+ for (const value of comparableValues.slice(1)) {
1315
+ if (comparePageIndexValues(value, min, physicalType) < 0) min = value;
1316
+ if (comparePageIndexValues(value, max, physicalType) > 0) max = value;
1317
+ }
1318
+ try {
1319
+ return {
1320
+ min: encodePageIndexValue(min, physicalType, typeLength, int96AsTimestamp),
1321
+ max: encodePageIndexValue(max, physicalType, typeLength, int96AsTimestamp)
1322
+ };
1323
+ } catch {
1324
+ return undefined;
1325
+ }
1326
+ }
1327
+
1328
+ function encodePageIndexValue(
1329
+ value: boolean | number | bigint | string | Uint8Array,
1330
+ physicalType: PageIndexPhysicalType,
1331
+ typeLength: number | undefined,
1332
+ int96AsTimestamp: boolean
1333
+ ): Uint8Array {
1334
+ if (physicalType === 'INT96') {
1335
+ if (!int96AsTimestamp) {
1336
+ throw new Error('INT96 page indexes require timestamp mode');
1337
+ }
1338
+ return PARQUET_CODECS.PLAIN.encodeValues('INT96', [value], {int96AsTimestamp});
1339
+ }
1340
+ return encodeParquetBloomFilterValue(value, physicalType, typeLength);
1341
+ }
1342
+
1343
+ function normalizePageIndexValue(
1344
+ value: unknown,
1345
+ physicalType: PageIndexPhysicalType
1346
+ ): boolean | number | bigint | string | Uint8Array | undefined {
1347
+ if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
1348
+ if (typeof value === 'string') return new TextEncoder().encode(value);
1349
+ if (value instanceof ArrayBuffer) return new Uint8Array(value);
1350
+ if (ArrayBuffer.isView(value)) {
1351
+ return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
1352
+ }
1353
+ return undefined;
1354
+ }
1355
+ if (
1356
+ typeof value === 'boolean' ||
1357
+ typeof value === 'number' ||
1358
+ typeof value === 'bigint' ||
1359
+ typeof value === 'string'
1360
+ ) {
1361
+ return value;
1362
+ }
1363
+ return undefined;
1364
+ }
1365
+
1366
+ function comparePageIndexValues(
1367
+ left: boolean | number | bigint | string | Uint8Array,
1368
+ right: boolean | number | bigint | string | Uint8Array,
1369
+ physicalType: PageIndexPhysicalType
1370
+ ): number {
1371
+ if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
1372
+ const leftBytes = typeof left === 'string' ? new TextEncoder().encode(left) : left;
1373
+ const rightBytes = typeof right === 'string' ? new TextEncoder().encode(right) : right;
1374
+ if (!(leftBytes instanceof Uint8Array) || !(rightBytes instanceof Uint8Array)) return 0;
1375
+ const length = Math.min(leftBytes.length, rightBytes.length);
1376
+ for (let index = 0; index < length; index++) {
1377
+ if (leftBytes[index] !== rightBytes[index]) {
1378
+ return leftBytes[index] < rightBytes[index] ? -1 : 1;
1379
+ }
1380
+ }
1381
+ return leftBytes.length - rightBytes.length;
1382
+ }
1383
+ if (typeof left === 'boolean' && typeof right === 'boolean') return Number(left) - Number(right);
1384
+ if (physicalType !== 'INT64' && physicalType !== 'INT96') {
1385
+ const leftNumber = Number(left);
1386
+ const rightNumber = Number(right);
1387
+ return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
1388
+ }
1389
+ try {
1390
+ const leftNumber = BigInt(left as number | bigint | string);
1391
+ const rightNumber = BigInt(right as number | bigint | string);
1392
+ return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
1393
+ } catch {
1394
+ const leftNumber = Number(left);
1395
+ const rightNumber = Number(right);
1396
+ return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
1397
+ }
1398
+ }
1399
+
1400
+ /** Builds a Bloom filter for writer-supported physical scalar columns. */
1401
+ function createColumnBloomFilter(
1402
+ column: ParquetField,
1403
+ values: ParquetColumnChunk['values']
1404
+ ): Uint8Array | undefined {
1405
+ const physicalType = column.primitiveType;
1406
+ if (
1407
+ physicalType !== 'BOOLEAN' &&
1408
+ physicalType !== 'INT32' &&
1409
+ physicalType !== 'INT64' &&
1410
+ physicalType !== 'FLOAT' &&
1411
+ physicalType !== 'DOUBLE' &&
1412
+ physicalType !== 'BYTE_ARRAY' &&
1413
+ physicalType !== 'FIXED_LEN_BYTE_ARRAY'
1414
+ ) {
1415
+ return undefined;
1416
+ }
1417
+ const bloomValues = values.map(value => {
1418
+ if (value instanceof ArrayBuffer) return new Uint8Array(value);
1419
+ if (ArrayBuffer.isView(value)) {
1420
+ return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
1421
+ }
1422
+ return value;
1423
+ });
1424
+ return encodeParquetSplitBlockBloomFilter(
1425
+ bloomValues as Parameters<typeof encodeParquetSplitBlockBloomFilter>[0],
1426
+ physicalType,
1427
+ column.typeLength
1428
+ );
1429
+ }
1430
+
1431
+ /** Builds optional size statistics from one shredded column chunk. */
1432
+ function createSizeStatistics(column: ParquetField, data: ParquetColumnChunk): SizeStatistics {
1433
+ const repetitionLevelHistogram = createLevelHistogram(data.rlevels, column.rLevelMax);
1434
+ const definitionLevelHistogram = createLevelHistogram(data.dlevels, column.dLevelMax);
1435
+ let unencodedByteArrayDataBytes = 0;
1436
+ if (column.primitiveType === 'BYTE_ARRAY') {
1437
+ for (const value of data.values) {
1438
+ if (typeof value === 'string') {
1439
+ unencodedByteArrayDataBytes += encodeUtf8(value).byteLength;
1440
+ } else if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
1441
+ unencodedByteArrayDataBytes += value.byteLength;
1442
+ }
1443
+ }
1444
+ }
1445
+ return new SizeStatistics({
1446
+ unencoded_byte_array_data_bytes:
1447
+ column.primitiveType === 'BYTE_ARRAY' ? unencodedByteArrayDataBytes : undefined,
1448
+ repetition_level_histogram: repetitionLevelHistogram,
1449
+ definition_level_histogram: definitionLevelHistogram
1450
+ });
1451
+ }
1452
+
1453
+ /** Returns whether standard column statistics are enabled for one top-level field. */
1454
+ function isStatisticsEnabled(
1455
+ option: boolean | Record<string, boolean> | undefined,
1456
+ column: ParquetField
1457
+ ): boolean {
1458
+ return option === true || option?.[column.path[0]] === true;
1459
+ }
1460
+
1461
+ /** Builds conservative standard min/max/null-count statistics for one column chunk. */
1462
+ function createColumnStatistics(
1463
+ column: ParquetField,
1464
+ data: ParquetColumnChunk
1465
+ ): Statistics | undefined {
1466
+ if (
1467
+ !column.primitiveType ||
1468
+ !isPageIndexPhysicalType(column.primitiveType) ||
1469
+ !supportsStatisticsSortOrder(column)
1470
+ ) {
1471
+ return undefined;
1472
+ }
1473
+ const statistics = new Statistics({
1474
+ null_count: column.rLevelMax === 0 ? data.count - data.values.length : undefined,
1475
+ is_min_value_exact: true,
1476
+ is_max_value_exact: true
1477
+ });
1478
+ if (data.values.length === 0) return statistics;
1479
+ const bounds = getPageBounds(data.values, column.primitiveType, column.typeLength);
1480
+ if (!bounds) return undefined;
1481
+ statistics.min_value = bounds.min;
1482
+ statistics.max_value = bounds.max;
1483
+ return statistics;
1484
+ }
1485
+
1486
+ /** Returns whether the physical representation preserves the logical sort order. */
1487
+ function supportsStatisticsSortOrder(column: ParquetField): boolean {
1488
+ const logicalType = column.logicalType?.type || column.originalType;
1489
+ if (logicalType === 'FLOAT16') return false;
1490
+ if (logicalType === 'DECIMAL' || logicalType?.startsWith('DECIMAL_')) {
1491
+ return column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY';
1492
+ }
1493
+ if (logicalType === 'INTEGER' || logicalType?.startsWith('UINT_')) {
1494
+ return column.logicalType?.isSigned !== false && !logicalType.startsWith('UINT_');
1495
+ }
1496
+ return true;
1497
+ }
1498
+
1499
+ /** Counts the occurrences of each definition or repetition level. */
1500
+ function createLevelHistogram(
1501
+ levels: ParquetColumnChunk['rlevels'],
1502
+ maximumLevel: number
1503
+ ): number[] {
1504
+ const histogram = new Array<number>(maximumLevel + 1).fill(0);
1505
+ for (const level of levels) {
1506
+ if (level >= 0 && level <= maximumLevel) histogram[level]++;
1507
+ }
1508
+ return histogram;
1509
+ }
1510
+
1511
+ /** Converts writer-facing sort keys into Parquet leaf-column indexes. */
1512
+ function createSortingColumns(
1513
+ schema: ParquetSchema,
1514
+ sortingColumns: readonly ParquetSortingColumnOption[] | undefined
1515
+ ): SortingColumn[] | undefined {
1516
+ if (!sortingColumns?.length) return undefined;
1517
+ const leafFields = schema.fieldList.filter(field => !field.isNested);
1518
+ const usedColumnIndexes = new Set<number>();
1519
+ return sortingColumns.map(sortKey => {
1520
+ const columnIndex = leafFields.findIndex(
1521
+ field =>
1522
+ (field.path.length === 1 && field.name === sortKey.column) ||
1523
+ (field.path.length > 1 && field.path.join('.') === sortKey.column)
1524
+ );
1525
+ if (columnIndex < 0) {
1526
+ throw new Error(`Unknown Parquet sorting column ${sortKey.column}`);
1527
+ }
1528
+ if (usedColumnIndexes.has(columnIndex)) {
1529
+ throw new Error(`Duplicate Parquet sorting column ${sortKey.column}`);
1530
+ }
1531
+ usedColumnIndexes.add(columnIndex);
1532
+ return new SortingColumn({
1533
+ column_idx: columnIndex,
1534
+ descending: Boolean(sortKey.descending),
1535
+ nulls_first: Boolean(sortKey.nullsFirst)
1536
+ });
1537
+ });
687
1538
  }
688
1539
 
689
1540
  /**
@@ -700,7 +1551,8 @@ async function encodeRowGroup(
700
1551
  const metadata = new RowGroup({
701
1552
  num_rows: int64(data.rowCount),
702
1553
  columns: [],
703
- total_byte_size: int64(0)
1554
+ total_byte_size: int64(0),
1555
+ sorting_columns: createSortingColumns(schema, opts.sortingColumns)
704
1556
  });
705
1557
 
706
1558
  let body: Uint8Array = new Uint8Array(0);
@@ -709,11 +1561,30 @@ async function encodeRowGroup(
709
1561
  continue; // eslint-disable-line no-continue
710
1562
  }
711
1563
 
712
- const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
1564
+ const cchunkData = await encodeColumnChunk(
1565
+ field,
1566
+ data,
1567
+ body.length,
1568
+ opts,
1569
+ opts.rowGroupOrdinal ?? 0,
1570
+ metadata.columns.length
1571
+ );
713
1572
 
714
1573
  const cchunk = new ColumnChunk({
715
1574
  file_offset: int64(cchunkData.metadataOffset),
716
- meta_data: cchunkData.metadata
1575
+ meta_data: cchunkData.encryptedColumnMetadata ? undefined : cchunkData.metadata,
1576
+ offset_index_offset:
1577
+ cchunkData.offsetIndexOffset === undefined
1578
+ ? undefined
1579
+ : int64(cchunkData.offsetIndexOffset),
1580
+ offset_index_length: cchunkData.offsetIndexLength,
1581
+ column_index_offset:
1582
+ cchunkData.columnIndexOffset === undefined
1583
+ ? undefined
1584
+ : int64(cchunkData.columnIndexOffset),
1585
+ column_index_length: cchunkData.columnIndexLength,
1586
+ crypto_metadata: cchunkData.cryptoMetadata,
1587
+ encrypted_column_metadata: cchunkData.encryptedColumnMetadata
717
1588
  });
718
1589
 
719
1590
  metadata.columns.push(cchunk);
@@ -728,19 +1599,44 @@ async function encodeRowGroup(
728
1599
  /**
729
1600
  * Encode a parquet file metadata footer
730
1601
  */
731
- function encodeFooter(
1602
+ function encodeFooterMetadata(
732
1603
  schema: ParquetSchema,
733
1604
  rowCount: number,
734
1605
  rowGroups: RowGroup[],
735
- userMetadata: Record<string, string>
1606
+ userMetadata: Record<string, string>,
1607
+ footerSignature?: ParquetWriterFooterSignatureOptions
736
1608
  ): Uint8Array {
1609
+ const signatureAlgorithm = footerSignature?.algorithm ?? 'AES_GCM_V1';
1610
+ const signatureFileUnique = footerSignature?.fileUnique
1611
+ ? new Uint8Array(footerSignature.fileUnique)
1612
+ : undefined;
737
1613
  const metadata = new FileMetaData({
738
1614
  version: PARQUET_VERSION,
739
1615
  created_by: 'parquets',
740
1616
  num_rows: int64(rowCount),
741
1617
  row_groups: rowGroups,
742
1618
  schema: [],
743
- key_value_metadata: []
1619
+ key_value_metadata: [],
1620
+ encryption_algorithm: footerSignature
1621
+ ? new EncryptionAlgorithm({
1622
+ ...(signatureAlgorithm === 'AES_GCM_CTR_V1'
1623
+ ? {
1624
+ AES_GCM_CTR_V1: {
1625
+ aad_prefix: footerSignature.aadPrefix,
1626
+ aad_file_unique: signatureFileUnique
1627
+ }
1628
+ }
1629
+ : {
1630
+ AES_GCM_V1: {
1631
+ aad_prefix: footerSignature.aadPrefix,
1632
+ aad_file_unique: signatureFileUnique
1633
+ }
1634
+ })
1635
+ })
1636
+ : undefined,
1637
+ footer_signing_key_metadata: footerSignature
1638
+ ? new Uint8Array(footerSignature.keyMetadata)
1639
+ : undefined
744
1640
  });
745
1641
 
746
1642
  for (const key in userMetadata) {
@@ -791,7 +1687,17 @@ function encodeFooter(
791
1687
  metadata.schema.push(schemaElem);
792
1688
  }
793
1689
 
794
- const metadataEncoded = serializeThrift(metadata);
1690
+ return serializeThrift(metadata);
1691
+ }
1692
+
1693
+ /** Encodes an unsigned plaintext footer with its standard Parquet trailer. */
1694
+ function encodeFooter(
1695
+ schema: ParquetSchema,
1696
+ rowCount: number,
1697
+ rowGroups: RowGroup[],
1698
+ userMetadata: Record<string, string>
1699
+ ): Uint8Array {
1700
+ const metadataEncoded = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata);
795
1701
  const footerEncoded = new Uint8Array(metadataEncoded.length + 8);
796
1702
 
797
1703
  footerEncoded.set(metadataEncoded);
@@ -800,6 +1706,78 @@ function encodeFooter(
800
1706
  return footerEncoded;
801
1707
  }
802
1708
 
1709
+ /** Encodes a plaintext footer with its trailing 28-byte authenticated signature. */
1710
+ async function encodeSignedFooter(
1711
+ schema: ParquetSchema,
1712
+ rowCount: number,
1713
+ rowGroups: RowGroup[],
1714
+ userMetadata: Record<string, string>,
1715
+ options: ParquetWriterFooterSignatureOptions
1716
+ ): Promise<Uint8Array> {
1717
+ const metadata = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata, options);
1718
+ const signature = await createParquetFooterSignature(metadata, options);
1719
+ const footerEncoded = new Uint8Array(metadata.length + signature.length + 8);
1720
+ footerEncoded.set(metadata);
1721
+ footerEncoded.set(signature, metadata.length);
1722
+ writeUInt32LE(
1723
+ footerEncoded,
1724
+ metadata.length + signature.length,
1725
+ metadata.length + signature.length
1726
+ );
1727
+ footerEncoded.set(PARQUET_MAGIC_BYTES, metadata.length + signature.length + 4);
1728
+ return footerEncoded;
1729
+ }
1730
+
1731
+ /** Wraps a serialized footer in Parquet modular-encryption metadata and AES-GCM. */
1732
+ async function encodeEncryptedFooter(
1733
+ footer: Uint8Array,
1734
+ options: ParquetWriterEncryptionOptions
1735
+ ): Promise<Uint8Array> {
1736
+ const algorithm = options.algorithm ?? 'AES_GCM_V1';
1737
+ const fileUnique = options.fileUnique ? new Uint8Array(options.fileUnique) : createFileUnique();
1738
+ if (fileUnique.byteLength !== 8) {
1739
+ throw new Error(`Parquet encrypted footer file_unique must be 8 bytes`);
1740
+ }
1741
+ const algorithmParameters = {
1742
+ aad_prefix: options.aadPrefix && new Uint8Array(options.aadPrefix),
1743
+ aad_file_unique: fileUnique,
1744
+ supply_aad_prefix: undefined
1745
+ };
1746
+ const encryptionAlgorithm =
1747
+ algorithm === 'AES_GCM_CTR_V1'
1748
+ ? new EncryptionAlgorithm({AES_GCM_CTR_V1: algorithmParameters})
1749
+ : new EncryptionAlgorithm({AES_GCM_V1: algorithmParameters});
1750
+ const cryptoMetadata = serializeThrift(
1751
+ new FileCryptoMetaData({
1752
+ encryption_algorithm: encryptionAlgorithm,
1753
+ key_metadata: options.keyMetadata && new Uint8Array(options.keyMetadata)
1754
+ })
1755
+ );
1756
+ const footerMetadata = footer.subarray(0, footer.byteLength - 8);
1757
+ const encryptedFooter = await encryptParquetModule(footerMetadata, {
1758
+ algorithm,
1759
+ aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'footer'),
1760
+ keyMetadata: options.keyMetadata,
1761
+ keyRetriever: options.keyRetriever
1762
+ });
1763
+ const metadataSize = cryptoMetadata.byteLength + encryptedFooter.byteLength;
1764
+ const trailer = new Uint8Array(8);
1765
+ writeUInt32LE(trailer, metadataSize, 0);
1766
+ trailer.set(encodeUtf8(PARQUET_MAGIC_ENCRYPTED), 4);
1767
+ return concatUint8Arrays([cryptoMetadata, encryptedFooter, trailer]);
1768
+ }
1769
+
1770
+ /** Generates the eight-byte file identifier required by Parquet module AAD. */
1771
+ function createFileUnique(): Uint8Array {
1772
+ const fileUnique = new Uint8Array(8);
1773
+ const cryptoProvider = globalThis.crypto;
1774
+ if (!cryptoProvider?.getRandomValues) {
1775
+ throw new Error('Parquet encrypted footer requires Web Crypto random values');
1776
+ }
1777
+ cryptoProvider.getRandomValues(fileUnique);
1778
+ return fileUnique;
1779
+ }
1780
+
803
1781
  /** Returns an explicit or inferred modern logical annotation for a writer field. */
804
1782
  function getFieldLogicalType(field: ParquetField): ParquetLogicalType | undefined {
805
1783
  if (field.logicalType) return field.logicalType;