@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (326) hide show
  1. package/LICENSE +1 -1
  2. package/dist/bundled.d.ts +1 -1
  3. package/dist/bundled.d.ts.map +1 -1
  4. package/dist/bundled.js.map +1 -1
  5. package/dist/delta-format.d.ts +9 -0
  6. package/dist/delta-format.d.ts.map +1 -0
  7. package/dist/delta-format.js +12 -0
  8. package/dist/delta-format.js.map +1 -0
  9. package/dist/delta-source-loader-types.d.ts +27 -0
  10. package/dist/delta-source-loader-types.d.ts.map +1 -0
  11. package/dist/delta-source-loader-types.js +28 -0
  12. package/dist/delta-source-loader-types.js.map +1 -0
  13. package/dist/delta-source.d.ts +48 -0
  14. package/dist/delta-source.d.ts.map +1 -0
  15. package/dist/delta-source.js +222 -0
  16. package/dist/delta-source.js.map +1 -0
  17. package/dist/delta-types.d.ts +38 -0
  18. package/dist/delta-types.d.ts.map +1 -0
  19. package/dist/delta-types.js +5 -0
  20. package/dist/delta-types.js.map +1 -0
  21. package/dist/geoparquet-loader.js +1 -1
  22. package/dist/iceberg-rest-catalog.d.ts +49 -0
  23. package/dist/iceberg-rest-catalog.d.ts.map +1 -0
  24. package/dist/iceberg-rest-catalog.js +78 -0
  25. package/dist/iceberg-rest-catalog.js.map +1 -0
  26. package/dist/iceberg-table-source.d.ts +63 -0
  27. package/dist/iceberg-table-source.d.ts.map +1 -0
  28. package/dist/iceberg-table-source.js +797 -0
  29. package/dist/iceberg-table-source.js.map +1 -0
  30. package/dist/iceberg-types.d.ts +150 -0
  31. package/dist/iceberg-types.d.ts.map +1 -0
  32. package/dist/iceberg-types.js +5 -0
  33. package/dist/iceberg-types.js.map +1 -0
  34. package/dist/index.cjs +15005 -3143
  35. package/dist/index.cjs.map +3 -3
  36. package/dist/index.d.ts +7 -3
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +5 -1
  39. package/dist/index.js.map +1 -1
  40. package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
  41. package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
  42. package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
  43. package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  44. package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
  45. package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
  46. package/dist/lib/constants.js +1 -1
  47. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
  48. package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
  49. package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
  50. package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
  51. package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
  52. package/dist/lib/geo/geoparquet-covering.js +137 -0
  53. package/dist/lib/geo/geoparquet-covering.js.map +1 -0
  54. package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
  55. package/dist/lib/geo/geospatial-metadata.js +39 -13
  56. package/dist/lib/geo/geospatial-metadata.js.map +1 -1
  57. package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
  58. package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
  59. package/dist/lib/parquet-bloom-filter-planner.js +57 -0
  60. package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
  61. package/dist/lib/parquet-bloom-filter.d.ts +30 -0
  62. package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
  63. package/dist/lib/parquet-bloom-filter.js +289 -0
  64. package/dist/lib/parquet-bloom-filter.js.map +1 -0
  65. package/dist/lib/parquet-encryption.d.ts +80 -0
  66. package/dist/lib/parquet-encryption.d.ts.map +1 -0
  67. package/dist/lib/parquet-encryption.js +190 -0
  68. package/dist/lib/parquet-encryption.js.map +1 -0
  69. package/dist/lib/parquet-page-index.d.ts +34 -5
  70. package/dist/lib/parquet-page-index.d.ts.map +1 -1
  71. package/dist/lib/parquet-page-index.js +114 -28
  72. package/dist/lib/parquet-page-index.js.map +1 -1
  73. package/dist/lib/parquet-predicate.d.ts +5 -1
  74. package/dist/lib/parquet-predicate.d.ts.map +1 -1
  75. package/dist/lib/parquet-predicate.js +42 -14
  76. package/dist/lib/parquet-predicate.js.map +1 -1
  77. package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
  78. package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
  79. package/dist/lib/parquet-source-worker-decoder.js +83 -17
  80. package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
  81. package/dist/lib/parquet-source-worker-types.d.ts +24 -0
  82. package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
  83. package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
  84. package/dist/lib/parsers/get-parquet-schema.js +3 -1
  85. package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
  86. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
  87. package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
  88. package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
  89. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
  90. package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
  91. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
  92. package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
  93. package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
  94. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  95. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  96. package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
  97. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
  98. package/dist/lib/slice-parquet-batch.d.ts +9 -0
  99. package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
  100. package/dist/lib/slice-parquet-batch.js +28 -0
  101. package/dist/lib/slice-parquet-batch.js.map +1 -0
  102. package/dist/lib/sources/parquet-range-file.d.ts +1 -1
  103. package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
  104. package/dist/lib/sources/parquet-range-file.js +28 -22
  105. package/dist/lib/sources/parquet-range-file.js.map +1 -1
  106. package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
  107. package/dist/lib/utils/normalize-parquet-options.js +15 -1
  108. package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
  109. package/dist/parquet-dataset-source.cjs +15766 -4896
  110. package/dist/parquet-dataset-source.cjs.map +3 -3
  111. package/dist/parquet-dataset-source.d.ts +31 -4
  112. package/dist/parquet-dataset-source.d.ts.map +1 -1
  113. package/dist/parquet-dataset-source.js +296 -74
  114. package/dist/parquet-dataset-source.js.map +1 -1
  115. package/dist/parquet-js-loader-types.d.ts +2 -0
  116. package/dist/parquet-js-loader-types.d.ts.map +1 -1
  117. package/dist/parquet-js-loader-types.js +2 -1
  118. package/dist/parquet-js-loader-types.js.map +1 -1
  119. package/dist/parquet-js-loader.d.ts +1 -0
  120. package/dist/parquet-js-loader.d.ts.map +1 -1
  121. package/dist/parquet-js-loader.js +2 -1
  122. package/dist/parquet-js-loader.js.map +1 -1
  123. package/dist/parquet-js-writer.d.ts +23 -2
  124. package/dist/parquet-js-writer.d.ts.map +1 -1
  125. package/dist/parquet-js-writer.js +1 -1
  126. package/dist/parquet-js-writer.js.map +1 -1
  127. package/dist/parquet-loader-base.js +1 -1
  128. package/dist/parquet-loader-options.d.ts +11 -0
  129. package/dist/parquet-loader-options.d.ts.map +1 -1
  130. package/dist/parquet-loader-options.js.map +1 -1
  131. package/dist/parquet-source-capabilities.d.ts +8 -1
  132. package/dist/parquet-source-capabilities.d.ts.map +1 -1
  133. package/dist/parquet-source-capabilities.js +10 -0
  134. package/dist/parquet-source-capabilities.js.map +1 -1
  135. package/dist/parquet-source-loader-types.d.ts +10 -0
  136. package/dist/parquet-source-loader-types.d.ts.map +1 -1
  137. package/dist/parquet-source-loader-types.js +11 -1
  138. package/dist/parquet-source-loader-types.js.map +1 -1
  139. package/dist/parquet-source-loader.cjs +14911 -4316
  140. package/dist/parquet-source-loader.cjs.map +3 -3
  141. package/dist/parquet-source-loader.d.ts +42 -4
  142. package/dist/parquet-source-loader.d.ts.map +1 -1
  143. package/dist/parquet-source-loader.js +585 -38
  144. package/dist/parquet-source-loader.js.map +1 -1
  145. package/dist/parquet-source-types.d.ts +180 -30
  146. package/dist/parquet-source-types.d.ts.map +1 -1
  147. package/dist/parquet-source-worker.js +7055 -6989
  148. package/dist/parquet-source-worker.js.map +4 -4
  149. package/dist/parquet-worker-node.cjs +8505 -1041
  150. package/dist/parquet-worker-node.cjs.map +4 -4
  151. package/dist/parquet-worker-url.js +5 -1
  152. package/dist/parquet-worker-url.js.map +1 -1
  153. package/dist/parquet-worker.js +289 -218
  154. package/dist/parquet-worker.js.map +4 -4
  155. package/dist/parquetjs/codecs/declare.d.ts +15 -0
  156. package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
  157. package/dist/parquetjs/codecs/declare.js +10 -0
  158. package/dist/parquetjs/codecs/declare.js.map +1 -1
  159. package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
  160. package/dist/parquetjs/codecs/delta.js +356 -7
  161. package/dist/parquetjs/codecs/delta.js.map +1 -1
  162. package/dist/parquetjs/codecs/index.d.ts.map +1 -1
  163. package/dist/parquetjs/codecs/index.js +4 -0
  164. package/dist/parquetjs/codecs/index.js.map +1 -1
  165. package/dist/parquetjs/codecs/plain.d.ts +1 -1
  166. package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
  167. package/dist/parquetjs/codecs/plain.js +153 -12
  168. package/dist/parquetjs/codecs/plain.js.map +1 -1
  169. package/dist/parquetjs/codecs/rle.d.ts +4 -0
  170. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  171. package/dist/parquetjs/codecs/rle.js +406 -3
  172. package/dist/parquetjs/codecs/rle.js.map +1 -1
  173. package/dist/parquetjs/compression.d.ts +9 -0
  174. package/dist/parquetjs/compression.d.ts.map +1 -1
  175. package/dist/parquetjs/compression.js +86 -67
  176. package/dist/parquetjs/compression.js.map +1 -1
  177. package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
  178. package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
  179. package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
  180. package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
  181. package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
  182. package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
  183. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
  184. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
  185. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
  186. package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
  187. package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
  188. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
  189. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
  190. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
  191. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  192. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
  193. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
  194. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
  195. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
  196. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
  197. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
  198. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
  199. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  200. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
  201. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
  202. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
  203. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
  204. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
  205. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
  206. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
  207. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
  208. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
  209. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
  210. package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
  211. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  212. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
  213. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
  214. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
  215. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
  216. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
  217. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
  218. package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
  219. package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
  220. package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
  221. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
  222. package/dist/parquetjs/parquet-thrift/index.js +6 -0
  223. package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
  224. package/dist/parquetjs/parser/decoders.d.ts +10 -1
  225. package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
  226. package/dist/parquetjs/parser/decoders.js +318 -14
  227. package/dist/parquetjs/parser/decoders.js.map +1 -1
  228. package/dist/parquetjs/parser/parquet-reader.d.ts +78 -12
  229. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
  230. package/dist/parquetjs/parser/parquet-reader.js +605 -66
  231. package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
  232. package/dist/parquetjs/schema/declare.d.ts +22 -3
  233. package/dist/parquetjs/schema/declare.d.ts.map +1 -1
  234. package/dist/parquetjs/schema/declare.js.map +1 -1
  235. package/dist/parquetjs/schema/schema.d.ts.map +1 -1
  236. package/dist/parquetjs/schema/schema.js +10 -0
  237. package/dist/parquetjs/schema/schema.js.map +1 -1
  238. package/dist/parquetjs/schema/shred.d.ts.map +1 -1
  239. package/dist/parquetjs/schema/shred.js +219 -11
  240. package/dist/parquetjs/schema/shred.js.map +1 -1
  241. package/dist/parquetjs/schema/types.js +19 -21
  242. package/dist/parquetjs/schema/types.js.map +1 -1
  243. package/dist/parquetjs/schema/variant.d.ts +3 -0
  244. package/dist/parquetjs/schema/variant.d.ts.map +1 -0
  245. package/dist/parquetjs/schema/variant.js +248 -0
  246. package/dist/parquetjs/schema/variant.js.map +1 -0
  247. package/dist/parquetjs/utils/crc32.d.ts +3 -0
  248. package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
  249. package/dist/parquetjs/utils/crc32.js +15 -0
  250. package/dist/parquetjs/utils/crc32.js.map +1 -0
  251. package/dist/parquetjs/utils/read-utils.d.ts +11 -1
  252. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
  253. package/dist/parquetjs/utils/read-utils.js +21 -1
  254. package/dist/parquetjs/utils/read-utils.js.map +1 -1
  255. package/dist/unbundled.d.ts +2 -1
  256. package/dist/unbundled.d.ts.map +1 -1
  257. package/dist/unbundled.js +1 -0
  258. package/dist/unbundled.js.map +1 -1
  259. package/package.json +36 -11
  260. package/src/bundled.ts +3 -0
  261. package/src/delta-format.ts +12 -0
  262. package/src/delta-source-loader-types.ts +40 -0
  263. package/src/delta-source.ts +305 -0
  264. package/src/delta-types.ts +40 -0
  265. package/src/iceberg-rest-catalog.ts +122 -0
  266. package/src/iceberg-table-source.ts +1037 -0
  267. package/src/iceberg-types.ts +165 -0
  268. package/src/index.ts +50 -1
  269. package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
  270. package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
  271. package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
  272. package/src/lib/geo/geoparquet-covering.ts +201 -0
  273. package/src/lib/geo/geospatial-metadata.ts +42 -15
  274. package/src/lib/parquet-bloom-filter-planner.ts +86 -0
  275. package/src/lib/parquet-bloom-filter.ts +345 -0
  276. package/src/lib/parquet-encryption.ts +344 -0
  277. package/src/lib/parquet-page-index.ts +180 -34
  278. package/src/lib/parquet-predicate.ts +52 -16
  279. package/src/lib/parquet-source-worker-decoder.ts +116 -20
  280. package/src/lib/parquet-source-worker-types.ts +25 -0
  281. package/src/lib/parsers/get-parquet-schema.ts +3 -1
  282. package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
  283. package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
  284. package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
  285. package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
  286. package/src/lib/slice-parquet-batch.ts +32 -0
  287. package/src/lib/sources/parquet-range-file.ts +35 -29
  288. package/src/lib/utils/normalize-parquet-options.ts +17 -1
  289. package/src/parquet-dataset-source.ts +397 -85
  290. package/src/parquet-js-loader-types.ts +1 -0
  291. package/src/parquet-js-loader.ts +2 -1
  292. package/src/parquet-js-writer.ts +30 -0
  293. package/src/parquet-loader-options.ts +11 -0
  294. package/src/parquet-source-capabilities.ts +18 -1
  295. package/src/parquet-source-loader-types.ts +10 -0
  296. package/src/parquet-source-loader.ts +929 -52
  297. package/src/parquet-source-types.ts +197 -28
  298. package/src/parquet-worker-url.ts +6 -1
  299. package/src/parquetjs/codecs/declare.ts +27 -0
  300. package/src/parquetjs/codecs/delta.ts +466 -6
  301. package/src/parquetjs/codecs/index.ts +4 -0
  302. package/src/parquetjs/codecs/plain.ts +177 -11
  303. package/src/parquetjs/codecs/rle.ts +573 -3
  304. package/src/parquetjs/compression.ts +102 -76
  305. package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
  306. package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
  307. package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
  308. package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
  309. package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
  310. package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
  311. package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
  312. package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
  313. package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
  314. package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
  315. package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
  316. package/src/parquetjs/parquet-thrift/index.ts +6 -0
  317. package/src/parquetjs/parser/decoders.ts +451 -26
  318. package/src/parquetjs/parser/parquet-reader.ts +935 -87
  319. package/src/parquetjs/schema/declare.ts +22 -2
  320. package/src/parquetjs/schema/schema.ts +10 -0
  321. package/src/parquetjs/schema/shred.ts +238 -11
  322. package/src/parquetjs/schema/types.ts +16 -25
  323. package/src/parquetjs/schema/variant.ts +313 -0
  324. package/src/parquetjs/utils/crc32.ts +15 -0
  325. package/src/parquetjs/utils/read-utils.ts +26 -1
  326. package/src/unbundled.ts +14 -0
@@ -17,7 +17,7 @@ import {
17
17
  SchemaDefinition
18
18
  } from '../schema/declare';
19
19
  import {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs/index';
20
- import type {ParquetValueBuffer} from '../codecs/declare';
20
+ import type {ParquetByteArrayOutput, ParquetValueBuffer} from '../codecs/declare';
21
21
  import {
22
22
  ConvertedType,
23
23
  EdgeInterpolationAlgorithm,
@@ -32,11 +32,16 @@ import {decompress} from '../compression';
32
32
  import type {TimeUnit as ParquetThriftTimeUnit} from '../parquet-thrift/TimeUnit';
33
33
  import {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../lib/constants';
34
34
  import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';
35
+ import {crc32} from '../utils/crc32';
36
+
37
+ /** Shared empty level storage for required flat columns, whose levels are implicit. */
38
+ const EMPTY_PARQUET_LEVEL_BUFFER = new Uint8Array(0);
35
39
 
36
40
  /** Preallocated column destination used to bypass page-local value and level arrays. */
37
41
  type ParquetPageDecodeTarget = {
38
42
  values: ParquetValueBuffer;
39
43
  valueOffset: number;
44
+ byteArrayOutput?: ParquetByteArrayOutput;
40
45
  dictionary: readonly unknown[];
41
46
  rlevels: ParquetLevelBuffer;
42
47
  dlevels: ParquetLevelBuffer;
@@ -70,12 +75,14 @@ export async function decodeDataPages(
70
75
  }
71
76
 
72
77
  const outputCapacity = expectedLevelCount ?? 0;
78
+ const byteArrayOutput = createArrowByteArrayOutput(buffer, context, outputCapacity);
73
79
  const data: ParquetColumnChunk = {
74
80
  rlevels: createParquetLevelBuffer(context, outputCapacity, context.rLevelMax),
75
81
  dlevels: createParquetLevelBuffer(context, outputCapacity, context.dLevelMax),
76
- values: createParquetColumnValueBuffer(context, outputCapacity),
82
+ values: byteArrayOutput ? [] : createParquetColumnValueBuffer(context, outputCapacity),
77
83
  pageHeaders: [],
78
- count: 0
84
+ count: 0,
85
+ nullCount: 0
79
86
  };
80
87
 
81
88
  let dictionary = context.dictionary || [];
@@ -88,14 +95,19 @@ export async function decodeDataPages(
88
95
  (expectedLevelCount === undefined || levelOffset < expectedLevelCount)
89
96
  ) {
90
97
  // Looks like we have to decode these in sequence due to cursor updates?
91
- const page = await decodePage(cursor, context, {
98
+ const target = {
92
99
  values: data.values,
93
100
  valueOffset,
101
+ byteArrayOutput,
94
102
  dictionary,
95
103
  rlevels: data.rlevels,
96
104
  dlevels: data.dlevels,
97
105
  levelOffset
98
- });
106
+ };
107
+ const page =
108
+ context.compression === 'UNCOMPRESSED'
109
+ ? decodeUncompressedPage(cursor, context, target)
110
+ : await decodePage(cursor, context, target);
99
111
 
100
112
  if (page.dictionary) {
101
113
  dictionary = page.dictionary;
@@ -107,6 +119,7 @@ export async function decodeDataPages(
107
119
 
108
120
  if (page.directValuesWritten !== undefined) {
109
121
  valueOffset += page.directValuesWritten;
122
+ data.nullCount! += page.count - page.directValuesWritten;
110
123
  } else {
111
124
  const valueEncoding = getThriftEnum(
112
125
  Encoding,
@@ -129,11 +142,114 @@ export async function decodeDataPages(
129
142
 
130
143
  data.rlevels = trimParquetLevelBuffer(data.rlevels, levelOffset);
131
144
  data.dlevels = trimParquetLevelBuffer(data.dlevels, levelOffset);
132
- data.values = trimParquetValueBuffer(data.values, valueOffset);
145
+ if (byteArrayOutput) {
146
+ data.values = [];
147
+ data.byteArrayData = {
148
+ data: byteArrayOutput.data.subarray(0, byteArrayOutput.byteLength),
149
+ valueOffsets: byteArrayOutput.valueOffsets.subarray(0, valueOffset + 1)
150
+ };
151
+ } else {
152
+ data.values = trimParquetValueBuffer(data.values, valueOffset);
153
+ }
133
154
 
134
155
  return data;
135
156
  }
136
157
 
158
+ /**
159
+ * Decodes an uncompressed column chunk synchronously.
160
+ *
161
+ * In-memory Parquet reads use this entry point to avoid one resolved Promise per selected column.
162
+ */
163
+ export function decodeUncompressedDataPages(
164
+ buffer: Uint8Array,
165
+ context: ParquetReaderContext
166
+ ): ParquetColumnChunk {
167
+ const cursor: CursorBuffer = {buffer, offset: 0, size: buffer.length};
168
+ const expectedLevelCount =
169
+ context.numValues === undefined ? undefined : Number(context.numValues);
170
+ if (
171
+ expectedLevelCount !== undefined &&
172
+ (!Number.isSafeInteger(expectedLevelCount) || expectedLevelCount < 0)
173
+ ) {
174
+ throw new Error(`Invalid Parquet column value count ${expectedLevelCount}`);
175
+ }
176
+
177
+ const outputCapacity = expectedLevelCount ?? 0;
178
+ const byteArrayOutput = createArrowByteArrayOutput(buffer, context, outputCapacity);
179
+ const data: ParquetColumnChunk = {
180
+ rlevels: createParquetLevelBuffer(context, outputCapacity, context.rLevelMax),
181
+ dlevels: createParquetLevelBuffer(context, outputCapacity, context.dLevelMax),
182
+ values: byteArrayOutput ? [] : createParquetColumnValueBuffer(context, outputCapacity),
183
+ pageHeaders: [],
184
+ count: 0,
185
+ nullCount: 0
186
+ };
187
+ let dictionary = context.dictionary || [];
188
+ let levelOffset = 0;
189
+ let valueOffset = 0;
190
+
191
+ while (
192
+ cursor.offset < (cursor.size ?? cursor.buffer.length) &&
193
+ (expectedLevelCount === undefined || levelOffset < expectedLevelCount)
194
+ ) {
195
+ const page = decodeUncompressedPage(cursor, context, {
196
+ values: data.values,
197
+ valueOffset,
198
+ byteArrayOutput,
199
+ dictionary,
200
+ rlevels: data.rlevels,
201
+ dlevels: data.dlevels,
202
+ levelOffset
203
+ });
204
+ if (page.dictionary) {
205
+ dictionary = page.dictionary;
206
+ continue;
207
+ }
208
+
209
+ levelOffset += page.count;
210
+ if (page.directValuesWritten !== undefined) {
211
+ valueOffset += page.directValuesWritten;
212
+ data.nullCount! += page.count - page.directValuesWritten;
213
+ } else {
214
+ const valueEncoding = getThriftEnum(
215
+ Encoding,
216
+ page.pageHeader.data_page_header?.encoding ?? page.pageHeader.data_page_header_v2?.encoding!
217
+ ) as ParquetCodec;
218
+ const usesDictionary =
219
+ dictionary.length &&
220
+ (valueEncoding === 'PLAIN_DICTIONARY' || valueEncoding === 'RLE_DICTIONARY');
221
+ for (let index = 0; index < page.values.length; index++) {
222
+ const value = usesDictionary ? dictionary[Number(page.values[index])] : page.values[index];
223
+ if (value !== undefined) data.values[valueOffset++] = value;
224
+ }
225
+ }
226
+ data.count += page.count;
227
+ data.pageHeaders.push(page.pageHeader);
228
+ }
229
+
230
+ data.rlevels = trimParquetLevelBuffer(data.rlevels, levelOffset);
231
+ data.dlevels = trimParquetLevelBuffer(data.dlevels, levelOffset);
232
+ if (byteArrayOutput) {
233
+ data.values = [];
234
+ data.byteArrayData = {
235
+ data: byteArrayOutput.data.subarray(0, byteArrayOutput.byteLength),
236
+ valueOffsets: byteArrayOutput.valueOffsets.subarray(0, valueOffset + 1)
237
+ };
238
+ } else {
239
+ data.values = trimParquetValueBuffer(data.values, valueOffset);
240
+ }
241
+ return data;
242
+ }
243
+
244
+ /** Decodes a dictionary page from an uncompressed in-memory column chunk. */
245
+ export function decodeUncompressedDictionaryBuffer(
246
+ buffer: Uint8Array,
247
+ context: ParquetReaderContext
248
+ ): unknown[] {
249
+ const cursor: CursorBuffer = {buffer, offset: 0, size: buffer.length};
250
+ return decodeUncompressedPage(cursor, context).dictionary!;
251
+ }
252
+
137
253
  /**
138
254
  * Decode parquet page based on page type
139
255
  * @param cursor
@@ -150,6 +266,8 @@ export async function decodePage(
150
266
  cursor.offset += length;
151
267
 
152
268
  const pageType = getThriftEnum(PageType, pageHeader.type);
269
+ const pageEnd = cursor.offset + pageHeader.compressed_page_size;
270
+ verifyPageChecksum(cursor, pageHeader, pageEnd, context);
153
271
 
154
272
  switch (pageType) {
155
273
  case 'DATA_PAGE':
@@ -171,6 +289,56 @@ export async function decodePage(
171
289
  return page;
172
290
  }
173
291
 
292
+ /** Decodes one uncompressed page without introducing a Promise/microtask boundary. */
293
+ function decodeUncompressedPage(
294
+ cursor: CursorBuffer,
295
+ context: ParquetReaderContext,
296
+ target?: ParquetPageDecodeTarget
297
+ ): ParquetPageData {
298
+ const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);
299
+ cursor.offset += length;
300
+ const pageEnd = cursor.offset + pageHeader.compressed_page_size;
301
+ verifyPageChecksum(cursor, pageHeader, pageEnd, context);
302
+
303
+ const pageType = getThriftEnum(PageType, pageHeader.type);
304
+ switch (pageType) {
305
+ case 'DATA_PAGE':
306
+ return decodeDataPageValues(cursor, pageHeader, context, target);
307
+ case 'DATA_PAGE_V2':
308
+ return decodeUncompressedDataPageV2(cursor, pageHeader, context, target);
309
+ case 'DICTIONARY_PAGE':
310
+ return {
311
+ dictionary: decodeUncompressedDictionaryPage(cursor, pageHeader, context),
312
+ dlevels: EMPTY_PARQUET_LEVEL_BUFFER,
313
+ rlevels: EMPTY_PARQUET_LEVEL_BUFFER,
314
+ values: [],
315
+ count: 0,
316
+ pageHeader
317
+ };
318
+ default:
319
+ throw new Error(`invalid page type: ${pageType}`);
320
+ }
321
+ }
322
+
323
+ /** Verifies a page checksum before either the synchronous or asynchronous decode path consumes it. */
324
+ function verifyPageChecksum(
325
+ cursor: CursorBuffer,
326
+ pageHeader: PageHeader,
327
+ pageEnd: number,
328
+ context: ParquetReaderContext
329
+ ): void {
330
+ if (!context.verifyPageChecksums) return;
331
+ if (pageEnd > (cursor.size ?? cursor.buffer.length)) {
332
+ throw new Error('Parquet page extends beyond the available buffer');
333
+ }
334
+ if (pageHeader.crc === undefined) return;
335
+ const expected = pageHeader.crc >>> 0;
336
+ const actual = crc32(cursor.buffer.subarray(cursor.offset, pageEnd));
337
+ if (actual !== expected) {
338
+ throw new Error(`Parquet page checksum mismatch: expected ${expected}, calculated ${actual}`);
339
+ }
340
+ }
341
+
174
342
  /**
175
343
  * Decode parquet schema
176
344
  * @param schemaElements input schema elements data
@@ -437,17 +605,15 @@ async function decodeDataPage(
437
605
  target?: ParquetPageDecodeTarget
438
606
  ): Promise<ParquetPageData> {
439
607
  const cursorEnd = cursor.offset + header.compressed_page_size;
440
- const valueCount = header.data_page_header?.num_values;
441
608
 
442
609
  /* uncompress page */
443
610
  let dataCursor = cursor;
444
611
 
445
612
  if (context.compression !== 'UNCOMPRESSED') {
446
- const valuesBuf = await decompress(
447
- context.compression,
448
- cursor.buffer.slice(cursor.offset, cursorEnd),
449
- header.uncompressed_page_size
450
- );
613
+ const compressedValues = cursor.buffer.slice(cursor.offset, cursorEnd);
614
+ const valuesBuf = context.decompressPage
615
+ ? await context.decompressPage(compressedValues, header.uncompressed_page_size)
616
+ : await decompress(context.compression, compressedValues, header.uncompressed_page_size);
451
617
  dataCursor = {
452
618
  buffer: valuesBuf,
453
619
  offset: 0,
@@ -456,6 +622,18 @@ async function decodeDataPage(
456
622
  cursor.offset = cursorEnd;
457
623
  }
458
624
 
625
+ return decodeDataPageValues(dataCursor, header, context, target);
626
+ }
627
+
628
+ /** Decodes one uncompressed Data Page V1 body without introducing an asynchronous boundary. */
629
+ function decodeDataPageValues(
630
+ dataCursor: CursorBuffer,
631
+ header: PageHeader,
632
+ context: ParquetReaderContext,
633
+ target?: ParquetPageDecodeTarget
634
+ ): ParquetPageData {
635
+ const valueCount = header.data_page_header?.num_values;
636
+
459
637
  /* read repetition levels */
460
638
  const rLevelEncoding = getThriftEnum(
461
639
  Encoding,
@@ -506,7 +684,9 @@ async function decodeDataPage(
506
684
  output: target?.values,
507
685
  outputOffset: target?.valueOffset,
508
686
  dictionary: isDictionaryEncoding(valueEncoding) ? target?.dictionary : undefined,
509
- int64AsBigInt: shouldDecodeInt64AsBigInt(context)
687
+ byteArrayOutput: target?.byteArrayOutput,
688
+ int64AsBigInt: shouldDecodeInt64AsBigInt(context),
689
+ int96AsTimestamp: context.int96AsTimestamp
510
690
  };
511
691
 
512
692
  const values = decodeValues(
@@ -615,14 +795,125 @@ async function decodeDataPageV2(
615
795
  let valuesBuffer = cursor.buffer.subarray(valuesOffset, cursorEnd);
616
796
 
617
797
  if (dataPageHeader.is_compressed !== false && context.compression !== 'UNCOMPRESSED') {
618
- valuesBuffer = await decompress(
619
- context.compression,
620
- valuesBuffer,
621
- valuesUncompressedByteLength
798
+ valuesBuffer = context.decompressPage
799
+ ? await context.decompressPage(valuesBuffer, valuesUncompressedByteLength)
800
+ : await decompress(context.compression, valuesBuffer, valuesUncompressedByteLength);
801
+ }
802
+ cursor.offset = cursorEnd;
803
+ return decodeDataPageV2Values(
804
+ valuesBuffer,
805
+ header,
806
+ context,
807
+ target,
808
+ rLevels,
809
+ dLevels,
810
+ valueCount,
811
+ valueCountNonNull,
812
+ valueEncoding
813
+ );
814
+ }
815
+
816
+ /** Decodes one uncompressed Data Page V2 without awaiting an unused decompressor. */
817
+ function decodeUncompressedDataPageV2(
818
+ cursor: CursorBuffer,
819
+ header: PageHeader,
820
+ context: ParquetReaderContext,
821
+ target?: ParquetPageDecodeTarget
822
+ ): ParquetPageData {
823
+ const dataPageHeader = header.data_page_header_v2;
824
+ if (!dataPageHeader) {
825
+ throw new Error('Missing Parquet data page v2 header');
826
+ }
827
+ const cursorEnd = cursor.offset + header.compressed_page_size;
828
+ const levelsOffset = cursor.offset;
829
+ const valueCount = dataPageHeader.num_values;
830
+ const valueCountNonNull = valueCount - dataPageHeader.num_nulls;
831
+ const valueEncoding = getThriftEnum(Encoding, dataPageHeader.encoding) as ParquetCodec;
832
+ if (
833
+ header.compressed_page_size < 0 ||
834
+ cursorEnd > (cursor.size ?? cursor.buffer.length) ||
835
+ valueCountNonNull < 0
836
+ ) {
837
+ throw new Error('Invalid Parquet data page v2 header');
838
+ }
839
+
840
+ let rLevels: number[] = [];
841
+ if (context.column.rLevelMax > 0) {
842
+ const repetitionLevelCursor = createPageSliceCursor(
843
+ cursor,
844
+ levelsOffset,
845
+ dataPageHeader.repetition_levels_byte_length
846
+ );
847
+ rLevels = decodeLevels(
848
+ repetitionLevelCursor,
849
+ valueCount,
850
+ context.column.rLevelMax,
851
+ PARQUET_RDLVL_ENCODING,
852
+ true,
853
+ target?.rlevels,
854
+ target?.levelOffset
622
855
  );
856
+ } else if (!target) {
857
+ rLevels = new Array(valueCount).fill(0);
623
858
  }
624
- const valuesBufCursor = {buffer: valuesBuffer, offset: 0, size: valuesBuffer.length};
859
+
860
+ const definitionLevelsOffset = levelsOffset + dataPageHeader.repetition_levels_byte_length;
861
+ let dLevels: number[] = [];
862
+ if (context.column.dLevelMax > 0) {
863
+ const definitionLevelCursor = createPageSliceCursor(
864
+ cursor,
865
+ definitionLevelsOffset,
866
+ dataPageHeader.definition_levels_byte_length
867
+ );
868
+ dLevels = decodeLevels(
869
+ definitionLevelCursor,
870
+ valueCount,
871
+ context.column.dLevelMax,
872
+ PARQUET_RDLVL_ENCODING,
873
+ true,
874
+ target?.dlevels,
875
+ target?.levelOffset
876
+ );
877
+ } else if (!target) {
878
+ dLevels = new Array(valueCount).fill(0);
879
+ }
880
+
881
+ const valuesOffset = definitionLevelsOffset + dataPageHeader.definition_levels_byte_length;
882
+ const valuesUncompressedByteLength =
883
+ header.uncompressed_page_size -
884
+ dataPageHeader.repetition_levels_byte_length -
885
+ dataPageHeader.definition_levels_byte_length;
886
+ if (valuesOffset > cursorEnd || valuesUncompressedByteLength < 0) {
887
+ throw new Error('Invalid Parquet data page v2 level lengths');
888
+ }
889
+ const valuesBuffer = cursor.buffer.subarray(valuesOffset, cursorEnd);
625
890
  cursor.offset = cursorEnd;
891
+ return decodeDataPageV2Values(
892
+ valuesBuffer,
893
+ header,
894
+ context,
895
+ target,
896
+ rLevels,
897
+ dLevels,
898
+ valueCount,
899
+ valueCountNonNull,
900
+ valueEncoding
901
+ );
902
+ }
903
+
904
+ /** Decodes a prepared Data Page V2 value section without an asynchronous boundary. */
905
+ function decodeDataPageV2Values(
906
+ valuesBuffer: Uint8Array,
907
+ header: PageHeader,
908
+ context: ParquetReaderContext,
909
+ target: ParquetPageDecodeTarget | undefined,
910
+ rLevels: number[],
911
+ dLevels: number[],
912
+ valueCount: number,
913
+ valueCountNonNull: number,
914
+ valueEncoding: ParquetCodec
915
+ ): ParquetPageData {
916
+ const valuesBufCursor = {buffer: valuesBuffer, offset: 0, size: valuesBuffer.length};
626
917
 
627
918
  const decodeOptions = {
628
919
  typeLength: context.column.typeLength,
@@ -631,7 +922,9 @@ async function decodeDataPageV2(
631
922
  output: target?.values,
632
923
  outputOffset: target?.valueOffset,
633
924
  dictionary: isDictionaryEncoding(valueEncoding) ? target?.dictionary : undefined,
634
- int64AsBigInt: shouldDecodeInt64AsBigInt(context)
925
+ byteArrayOutput: target?.byteArrayOutput,
926
+ int64AsBigInt: shouldDecodeInt64AsBigInt(context),
927
+ int96AsTimestamp: context.int96AsTimestamp
635
928
  };
636
929
 
637
930
  const values = decodeValues(
@@ -674,12 +967,114 @@ function decodeLevels(
674
967
  return output ? [] : levels;
675
968
  }
676
969
 
970
+ /**
971
+ * Allocates a compact Arrow-compatible destination for supported uncompressed byte pages.
972
+ *
973
+ * Scanning only the small Thrift page headers avoids committing to the fast path when a column
974
+ * mixes in an encoding that still requires materialized values. The page buffer is a useful
975
+ * initial capacity; the destination grows for prefix-reconstructed delta values when necessary.
976
+ */
977
+ function createArrowByteArrayOutput(
978
+ buffer: Uint8Array,
979
+ context: ParquetReaderContext,
980
+ valueCapacity: number
981
+ ): ParquetByteArrayOutput | undefined {
982
+ if (
983
+ !context.useArrowByteArrayBuffers ||
984
+ context.compression !== 'UNCOMPRESSED' ||
985
+ !supportsArrowByteArrayOutput(context.column) ||
986
+ valueCapacity === 0
987
+ ) {
988
+ return undefined;
989
+ }
990
+ if (
991
+ !context.hasOnlyArrowByteArrayDataPages &&
992
+ !hasOnlyArrowByteArrayDataPages(buffer, valueCapacity)
993
+ ) {
994
+ return undefined;
995
+ }
996
+ return {
997
+ data:
998
+ context.column.primitiveType === 'FIXED_LEN_BYTE_ARRAY'
999
+ ? new Uint8Array(0)
1000
+ : new Uint8Array(Math.max(1, buffer.byteLength)),
1001
+ valueOffsets: new Int32Array(valueCapacity + 1),
1002
+ byteLength: 0
1003
+ };
1004
+ }
1005
+
1006
+ /** Returns whether a BYTE_ARRAY field maps directly to Arrow Utf8 or Binary without conversion. */
1007
+ function supportsArrowByteArrayOutput(column: ParquetReaderContext['column']): boolean {
1008
+ if (column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY') {
1009
+ return false;
1010
+ }
1011
+ if (column.logicalType) {
1012
+ return (
1013
+ column.logicalType.type === 'STRING' ||
1014
+ column.logicalType.type === 'ENUM' ||
1015
+ column.logicalType.type === 'JSON' ||
1016
+ column.logicalType.type === 'BSON' ||
1017
+ column.logicalType.type === 'VARIANT' ||
1018
+ column.logicalType.type === 'GEOMETRY' ||
1019
+ column.logicalType.type === 'GEOGRAPHY'
1020
+ );
1021
+ }
1022
+ return (
1023
+ !column.originalType ||
1024
+ column.originalType === 'UTF8' ||
1025
+ column.originalType === 'ENUM' ||
1026
+ column.originalType === 'JSON' ||
1027
+ column.originalType === 'BSON' ||
1028
+ column.originalType === 'INTERVAL'
1029
+ );
1030
+ }
1031
+
1032
+ /** Returns whether a complete column buffer contains only direct Arrow byte-array data pages. */
1033
+ function hasOnlyArrowByteArrayDataPages(buffer: Uint8Array, expectedLevelCount: number): boolean {
1034
+ let offset = 0;
1035
+ let levelCount = 0;
1036
+ while (offset < buffer.byteLength && levelCount < expectedLevelCount) {
1037
+ const {pageHeader, length} = decodePageHeader(buffer, offset);
1038
+ const pageType = getThriftEnum(PageType, pageHeader.type);
1039
+ if (pageType !== 'DATA_PAGE' && pageType !== 'DATA_PAGE_V2') {
1040
+ return false;
1041
+ }
1042
+ const encoding = getThriftEnum(
1043
+ Encoding,
1044
+ pageHeader.data_page_header?.encoding ?? pageHeader.data_page_header_v2?.encoding!
1045
+ );
1046
+ if (
1047
+ encoding !== 'PLAIN' &&
1048
+ encoding !== 'DELTA_LENGTH_BYTE_ARRAY' &&
1049
+ encoding !== 'DELTA_BYTE_ARRAY'
1050
+ ) {
1051
+ return false;
1052
+ }
1053
+ const valueCount =
1054
+ pageHeader.data_page_header?.num_values ?? pageHeader.data_page_header_v2?.num_values;
1055
+ if (valueCount === undefined || valueCount < 0) {
1056
+ return false;
1057
+ }
1058
+ levelCount += valueCount;
1059
+ offset += length + pageHeader.compressed_page_size;
1060
+ if (offset > buffer.byteLength) {
1061
+ return false;
1062
+ }
1063
+ }
1064
+ return levelCount === expectedLevelCount;
1065
+ }
1066
+
677
1067
  /** Allocates compact unsigned storage for repetition and definition levels. */
678
1068
  function createParquetLevelBuffer(
679
1069
  context: ParquetReaderContext,
680
1070
  capacity: number,
681
1071
  levelMax: number
682
1072
  ): ParquetLevelBuffer {
1073
+ if (context.useTypedLevelBuffers && levelMax === 0) {
1074
+ // Required flat columns have no encoded levels. Avoid allocating two row-sized zero buffers
1075
+ // that neither direct Arrow construction nor level decoding will inspect.
1076
+ return EMPTY_PARQUET_LEVEL_BUFFER;
1077
+ }
683
1078
  if (!context.useTypedLevelBuffers || capacity === 0) {
684
1079
  return levelMax > 0 ? new Array<number>(capacity) : new Array<number>(capacity).fill(0);
685
1080
  }
@@ -698,6 +1093,7 @@ function trimParquetLevelBuffer(levels: ParquetLevelBuffer, length: number): Par
698
1093
  levels.length = length;
699
1094
  return levels;
700
1095
  }
1096
+ if (levels.length === 0 || levels.length === length) return levels;
701
1097
  return levels.subarray(0, length) as ParquetLevelBuffer;
702
1098
  }
703
1099
 
@@ -727,6 +1123,7 @@ function createParquetColumnValueBuffer(
727
1123
  case 'INT64':
728
1124
  return new BigInt64Array(capacity);
729
1125
  case 'INT96':
1126
+ return context.int96AsTimestamp ? new BigInt64Array(capacity) : new Float64Array(capacity);
730
1127
  case 'DOUBLE':
731
1128
  return new Float64Array(capacity);
732
1129
  case 'FLOAT':
@@ -792,11 +1189,10 @@ async function decodeDictionaryPage(
792
1189
  cursor.offset = cursorEnd;
793
1190
 
794
1191
  if (context.compression !== 'UNCOMPRESSED') {
795
- const valuesBuf = await decompress(
796
- context.compression,
797
- dictCursor.buffer.subarray(dictCursor.offset),
798
- pageHeader.uncompressed_page_size
799
- );
1192
+ const compressedValues = dictCursor.buffer.subarray(dictCursor.offset);
1193
+ const valuesBuf = context.decompressPage
1194
+ ? await context.decompressPage(compressedValues, pageHeader.uncompressed_page_size)
1195
+ : await decompress(context.compression, compressedValues, pageHeader.uncompressed_page_size);
800
1196
 
801
1197
  dictCursor = {
802
1198
  buffer: valuesBuf,
@@ -807,6 +1203,34 @@ async function decodeDictionaryPage(
807
1203
  cursor.offset = cursorEnd;
808
1204
  }
809
1205
 
1206
+ return decodeDictionaryValues(dictCursor, pageHeader, context);
1207
+ }
1208
+
1209
+ /** Decodes an uncompressed dictionary page directly from a view of the column chunk. */
1210
+ function decodeUncompressedDictionaryPage(
1211
+ cursor: CursorBuffer,
1212
+ pageHeader: PageHeader,
1213
+ context: ParquetReaderContext
1214
+ ): (string | ArrayBuffer)[] {
1215
+ const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
1216
+ if (cursorEnd > (cursor.size ?? cursor.buffer.length)) {
1217
+ throw new Error('Parquet dictionary page exceeds the available buffer');
1218
+ }
1219
+ const dictionaryCursor: CursorBuffer = {
1220
+ buffer: cursor.buffer.subarray(cursor.offset, cursorEnd),
1221
+ offset: 0,
1222
+ size: pageHeader.compressed_page_size
1223
+ };
1224
+ cursor.offset = cursorEnd;
1225
+ return decodeDictionaryValues(dictionaryCursor, pageHeader, context);
1226
+ }
1227
+
1228
+ /** Decodes one dictionary body from its current cursor without copying uncompressed page bytes. */
1229
+ function decodeDictionaryValues(
1230
+ dictionaryCursor: CursorBuffer,
1231
+ pageHeader: PageHeader,
1232
+ context: ParquetReaderContext
1233
+ ): (string | ArrayBuffer)[] {
810
1234
  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
811
1235
  const declaredEncoding = getThriftEnum(
812
1236
  Encoding,
@@ -822,12 +1246,13 @@ async function decodeDictionaryPage(
822
1246
  const decodedDictionaryValues = decodeValues(
823
1247
  context.column.primitiveType!,
824
1248
  dictionaryValueEncoding,
825
- dictCursor,
1249
+ dictionaryCursor,
826
1250
  numValues,
827
1251
  {
828
1252
  ...context,
829
1253
  typeLength: context.column.typeLength,
830
- int64AsBigInt: shouldDecodeInt64AsBigInt(context)
1254
+ int64AsBigInt: shouldDecodeInt64AsBigInt(context),
1255
+ int96AsTimestamp: context.int96AsTimestamp
831
1256
  } as ParquetCodecOptions
832
1257
  );
833
1258