@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (336) hide show
  1. package/LICENSE +1 -1
  2. package/dist/bundled.d.ts +1 -1
  3. package/dist/bundled.d.ts.map +1 -1
  4. package/dist/bundled.js.map +1 -1
  5. package/dist/delta-format.d.ts +9 -0
  6. package/dist/delta-format.d.ts.map +1 -0
  7. package/dist/delta-format.js +12 -0
  8. package/dist/delta-format.js.map +1 -0
  9. package/dist/delta-source-loader-types.d.ts +27 -0
  10. package/dist/delta-source-loader-types.d.ts.map +1 -0
  11. package/dist/delta-source-loader-types.js +28 -0
  12. package/dist/delta-source-loader-types.js.map +1 -0
  13. package/dist/delta-source.d.ts +48 -0
  14. package/dist/delta-source.d.ts.map +1 -0
  15. package/dist/delta-source.js +222 -0
  16. package/dist/delta-source.js.map +1 -0
  17. package/dist/delta-types.d.ts +38 -0
  18. package/dist/delta-types.d.ts.map +1 -0
  19. package/dist/delta-types.js +5 -0
  20. package/dist/delta-types.js.map +1 -0
  21. package/dist/geoparquet-loader.js +1 -1
  22. package/dist/iceberg-rest-catalog.d.ts +49 -0
  23. package/dist/iceberg-rest-catalog.d.ts.map +1 -0
  24. package/dist/iceberg-rest-catalog.js +78 -0
  25. package/dist/iceberg-rest-catalog.js.map +1 -0
  26. package/dist/iceberg-table-source.d.ts +63 -0
  27. package/dist/iceberg-table-source.d.ts.map +1 -0
  28. package/dist/iceberg-table-source.js +797 -0
  29. package/dist/iceberg-table-source.js.map +1 -0
  30. package/dist/iceberg-types.d.ts +150 -0
  31. package/dist/iceberg-types.d.ts.map +1 -0
  32. package/dist/iceberg-types.js +5 -0
  33. package/dist/iceberg-types.js.map +1 -0
  34. package/dist/index.cjs +15841 -3306
  35. package/dist/index.cjs.map +3 -3
  36. package/dist/index.d.ts +8 -3
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +5 -1
  39. package/dist/index.js.map +1 -1
  40. package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
  41. package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
  42. package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
  43. package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  44. package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
  45. package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
  46. package/dist/lib/constants.js +1 -1
  47. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
  48. package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
  49. package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
  50. package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
  51. package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
  52. package/dist/lib/geo/geoparquet-covering.js +137 -0
  53. package/dist/lib/geo/geoparquet-covering.js.map +1 -0
  54. package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
  55. package/dist/lib/geo/geospatial-metadata.js +39 -13
  56. package/dist/lib/geo/geospatial-metadata.js.map +1 -1
  57. package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
  58. package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
  59. package/dist/lib/parquet-bloom-filter-planner.js +57 -0
  60. package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
  61. package/dist/lib/parquet-bloom-filter.d.ts +30 -0
  62. package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
  63. package/dist/lib/parquet-bloom-filter.js +289 -0
  64. package/dist/lib/parquet-bloom-filter.js.map +1 -0
  65. package/dist/lib/parquet-encryption.d.ts +80 -0
  66. package/dist/lib/parquet-encryption.d.ts.map +1 -0
  67. package/dist/lib/parquet-encryption.js +190 -0
  68. package/dist/lib/parquet-encryption.js.map +1 -0
  69. package/dist/lib/parquet-page-index.d.ts +34 -5
  70. package/dist/lib/parquet-page-index.d.ts.map +1 -1
  71. package/dist/lib/parquet-page-index.js +114 -28
  72. package/dist/lib/parquet-page-index.js.map +1 -1
  73. package/dist/lib/parquet-predicate.d.ts +5 -1
  74. package/dist/lib/parquet-predicate.d.ts.map +1 -1
  75. package/dist/lib/parquet-predicate.js +42 -14
  76. package/dist/lib/parquet-predicate.js.map +1 -1
  77. package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
  78. package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
  79. package/dist/lib/parquet-source-worker-decoder.js +83 -17
  80. package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
  81. package/dist/lib/parquet-source-worker-types.d.ts +24 -0
  82. package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
  83. package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
  84. package/dist/lib/parsers/get-parquet-schema.js +3 -1
  85. package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
  86. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
  87. package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
  88. package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
  89. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
  90. package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
  91. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
  92. package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
  93. package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
  94. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  95. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  96. package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
  97. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
  98. package/dist/lib/slice-parquet-batch.d.ts +9 -0
  99. package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
  100. package/dist/lib/slice-parquet-batch.js +28 -0
  101. package/dist/lib/slice-parquet-batch.js.map +1 -0
  102. package/dist/lib/sources/parquet-range-file.d.ts +1 -1
  103. package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
  104. package/dist/lib/sources/parquet-range-file.js +28 -22
  105. package/dist/lib/sources/parquet-range-file.js.map +1 -1
  106. package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
  107. package/dist/lib/utils/normalize-parquet-options.js +15 -1
  108. package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
  109. package/dist/parquet-dataset-source.cjs +16528 -4928
  110. package/dist/parquet-dataset-source.cjs.map +3 -3
  111. package/dist/parquet-dataset-source.d.ts +31 -4
  112. package/dist/parquet-dataset-source.d.ts.map +1 -1
  113. package/dist/parquet-dataset-source.js +296 -74
  114. package/dist/parquet-dataset-source.js.map +1 -1
  115. package/dist/parquet-encoded-page-types.d.ts +108 -0
  116. package/dist/parquet-encoded-page-types.d.ts.map +1 -0
  117. package/dist/parquet-encoded-page-types.js +5 -0
  118. package/dist/parquet-encoded-page-types.js.map +1 -0
  119. package/dist/parquet-js-loader-types.d.ts +2 -0
  120. package/dist/parquet-js-loader-types.d.ts.map +1 -1
  121. package/dist/parquet-js-loader-types.js +2 -1
  122. package/dist/parquet-js-loader-types.js.map +1 -1
  123. package/dist/parquet-js-loader.d.ts +1 -0
  124. package/dist/parquet-js-loader.d.ts.map +1 -1
  125. package/dist/parquet-js-loader.js +2 -1
  126. package/dist/parquet-js-loader.js.map +1 -1
  127. package/dist/parquet-js-writer.d.ts +23 -2
  128. package/dist/parquet-js-writer.d.ts.map +1 -1
  129. package/dist/parquet-js-writer.js +1 -1
  130. package/dist/parquet-js-writer.js.map +1 -1
  131. package/dist/parquet-loader-base.js +1 -1
  132. package/dist/parquet-loader-options.d.ts +11 -0
  133. package/dist/parquet-loader-options.d.ts.map +1 -1
  134. package/dist/parquet-loader-options.js.map +1 -1
  135. package/dist/parquet-source-capabilities.d.ts +8 -1
  136. package/dist/parquet-source-capabilities.d.ts.map +1 -1
  137. package/dist/parquet-source-capabilities.js +10 -0
  138. package/dist/parquet-source-capabilities.js.map +1 -1
  139. package/dist/parquet-source-loader-types.d.ts +10 -0
  140. package/dist/parquet-source-loader-types.d.ts.map +1 -1
  141. package/dist/parquet-source-loader-types.js +11 -1
  142. package/dist/parquet-source-loader-types.js.map +1 -1
  143. package/dist/parquet-source-loader.cjs +14121 -2796
  144. package/dist/parquet-source-loader.cjs.map +3 -3
  145. package/dist/parquet-source-loader.d.ts +51 -4
  146. package/dist/parquet-source-loader.d.ts.map +1 -1
  147. package/dist/parquet-source-loader.js +651 -38
  148. package/dist/parquet-source-loader.js.map +1 -1
  149. package/dist/parquet-source-types.d.ts +180 -30
  150. package/dist/parquet-source-types.d.ts.map +1 -1
  151. package/dist/parquet-source-worker.js +7378 -6989
  152. package/dist/parquet-source-worker.js.map +4 -4
  153. package/dist/parquet-worker-node.cjs +8501 -1045
  154. package/dist/parquet-worker-node.cjs.map +4 -4
  155. package/dist/parquet-worker-url.js +5 -1
  156. package/dist/parquet-worker-url.js.map +1 -1
  157. package/dist/parquet-worker.js +289 -218
  158. package/dist/parquet-worker.js.map +4 -4
  159. package/dist/parquetjs/codecs/declare.d.ts +15 -0
  160. package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
  161. package/dist/parquetjs/codecs/declare.js +10 -0
  162. package/dist/parquetjs/codecs/declare.js.map +1 -1
  163. package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
  164. package/dist/parquetjs/codecs/delta.js +356 -7
  165. package/dist/parquetjs/codecs/delta.js.map +1 -1
  166. package/dist/parquetjs/codecs/index.d.ts.map +1 -1
  167. package/dist/parquetjs/codecs/index.js +4 -0
  168. package/dist/parquetjs/codecs/index.js.map +1 -1
  169. package/dist/parquetjs/codecs/plain.d.ts +1 -1
  170. package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
  171. package/dist/parquetjs/codecs/plain.js +153 -12
  172. package/dist/parquetjs/codecs/plain.js.map +1 -1
  173. package/dist/parquetjs/codecs/rle.d.ts +4 -0
  174. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  175. package/dist/parquetjs/codecs/rle.js +406 -3
  176. package/dist/parquetjs/codecs/rle.js.map +1 -1
  177. package/dist/parquetjs/compression.d.ts +9 -0
  178. package/dist/parquetjs/compression.d.ts.map +1 -1
  179. package/dist/parquetjs/compression.js +86 -67
  180. package/dist/parquetjs/compression.js.map +1 -1
  181. package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
  182. package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
  183. package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
  184. package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
  185. package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
  186. package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
  187. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
  188. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
  189. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
  190. package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
  191. package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
  192. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
  193. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
  194. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
  195. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  196. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
  197. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
  198. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
  199. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
  200. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
  201. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
  202. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
  203. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  204. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
  205. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
  206. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
  207. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
  208. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
  209. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
  210. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
  211. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
  212. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
  213. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
  214. package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
  215. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  216. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
  217. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
  218. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
  219. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
  220. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
  221. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
  222. package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
  223. package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
  224. package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
  225. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
  226. package/dist/parquetjs/parquet-thrift/index.js +6 -0
  227. package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
  228. package/dist/parquetjs/parser/decoders.d.ts +10 -1
  229. package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
  230. package/dist/parquetjs/parser/decoders.js +318 -14
  231. package/dist/parquetjs/parser/decoders.js.map +1 -1
  232. package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
  233. package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
  234. package/dist/parquetjs/parser/encoded-pages.js +206 -0
  235. package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
  236. package/dist/parquetjs/parser/parquet-reader.d.ts +88 -12
  237. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
  238. package/dist/parquetjs/parser/parquet-reader.js +713 -66
  239. package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
  240. package/dist/parquetjs/schema/declare.d.ts +22 -3
  241. package/dist/parquetjs/schema/declare.d.ts.map +1 -1
  242. package/dist/parquetjs/schema/declare.js.map +1 -1
  243. package/dist/parquetjs/schema/schema.d.ts.map +1 -1
  244. package/dist/parquetjs/schema/schema.js +10 -0
  245. package/dist/parquetjs/schema/schema.js.map +1 -1
  246. package/dist/parquetjs/schema/shred.d.ts.map +1 -1
  247. package/dist/parquetjs/schema/shred.js +219 -11
  248. package/dist/parquetjs/schema/shred.js.map +1 -1
  249. package/dist/parquetjs/schema/types.js +19 -21
  250. package/dist/parquetjs/schema/types.js.map +1 -1
  251. package/dist/parquetjs/schema/variant.d.ts +3 -0
  252. package/dist/parquetjs/schema/variant.d.ts.map +1 -0
  253. package/dist/parquetjs/schema/variant.js +248 -0
  254. package/dist/parquetjs/schema/variant.js.map +1 -0
  255. package/dist/parquetjs/utils/crc32.d.ts +3 -0
  256. package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
  257. package/dist/parquetjs/utils/crc32.js +15 -0
  258. package/dist/parquetjs/utils/crc32.js.map +1 -0
  259. package/dist/parquetjs/utils/read-utils.d.ts +11 -1
  260. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
  261. package/dist/parquetjs/utils/read-utils.js +21 -1
  262. package/dist/parquetjs/utils/read-utils.js.map +1 -1
  263. package/dist/unbundled.d.ts +2 -1
  264. package/dist/unbundled.d.ts.map +1 -1
  265. package/dist/unbundled.js +1 -0
  266. package/dist/unbundled.js.map +1 -1
  267. package/package.json +36 -13
  268. package/src/bundled.ts +10 -0
  269. package/src/delta-format.ts +12 -0
  270. package/src/delta-source-loader-types.ts +40 -0
  271. package/src/delta-source.ts +305 -0
  272. package/src/delta-types.ts +40 -0
  273. package/src/iceberg-rest-catalog.ts +122 -0
  274. package/src/iceberg-table-source.ts +1037 -0
  275. package/src/iceberg-types.ts +165 -0
  276. package/src/index.ts +59 -1
  277. package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
  278. package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
  279. package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
  280. package/src/lib/geo/geoparquet-covering.ts +201 -0
  281. package/src/lib/geo/geospatial-metadata.ts +42 -15
  282. package/src/lib/parquet-bloom-filter-planner.ts +86 -0
  283. package/src/lib/parquet-bloom-filter.ts +345 -0
  284. package/src/lib/parquet-encryption.ts +344 -0
  285. package/src/lib/parquet-page-index.ts +180 -34
  286. package/src/lib/parquet-predicate.ts +52 -16
  287. package/src/lib/parquet-source-worker-decoder.ts +116 -20
  288. package/src/lib/parquet-source-worker-types.ts +25 -0
  289. package/src/lib/parsers/get-parquet-schema.ts +3 -1
  290. package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
  291. package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
  292. package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
  293. package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
  294. package/src/lib/slice-parquet-batch.ts +32 -0
  295. package/src/lib/sources/parquet-range-file.ts +35 -29
  296. package/src/lib/utils/normalize-parquet-options.ts +17 -1
  297. package/src/parquet-dataset-source.ts +397 -85
  298. package/src/parquet-encoded-page-types.ts +122 -0
  299. package/src/parquet-js-loader-types.ts +1 -0
  300. package/src/parquet-js-loader.ts +2 -1
  301. package/src/parquet-js-writer.ts +30 -0
  302. package/src/parquet-loader-options.ts +11 -0
  303. package/src/parquet-source-capabilities.ts +18 -1
  304. package/src/parquet-source-loader-types.ts +10 -0
  305. package/src/parquet-source-loader.ts +1024 -52
  306. package/src/parquet-source-types.ts +197 -28
  307. package/src/parquet-worker-url.ts +6 -1
  308. package/src/parquetjs/codecs/declare.ts +27 -0
  309. package/src/parquetjs/codecs/delta.ts +466 -6
  310. package/src/parquetjs/codecs/index.ts +4 -0
  311. package/src/parquetjs/codecs/plain.ts +177 -11
  312. package/src/parquetjs/codecs/rle.ts +573 -3
  313. package/src/parquetjs/compression.ts +102 -76
  314. package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
  315. package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
  316. package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
  317. package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
  318. package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
  319. package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
  320. package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
  321. package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
  322. package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
  323. package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
  324. package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
  325. package/src/parquetjs/parquet-thrift/index.ts +6 -0
  326. package/src/parquetjs/parser/decoders.ts +451 -26
  327. package/src/parquetjs/parser/encoded-pages.ts +278 -0
  328. package/src/parquetjs/parser/parquet-reader.ts +1112 -87
  329. package/src/parquetjs/schema/declare.ts +22 -2
  330. package/src/parquetjs/schema/schema.ts +10 -0
  331. package/src/parquetjs/schema/shred.ts +238 -11
  332. package/src/parquetjs/schema/types.ts +16 -25
  333. package/src/parquetjs/schema/variant.ts +313 -0
  334. package/src/parquetjs/utils/crc32.ts +15 -0
  335. package/src/parquetjs/utils/read-utils.ts +26 -1
  336. package/src/unbundled.ts +14 -0
@@ -143,6 +143,85 @@ export function decodeValues(
143
143
  return output;
144
144
  }
145
145
 
146
+ /** Encodes legacy Parquet BIT_PACKED values, including the optional page length envelope. */
147
+ export function encodeBitPackedValues(
148
+ type: PrimitiveType,
149
+ values: any[],
150
+ options: ParquetCodecOptions
151
+ ): Uint8Array {
152
+ if (!('bitWidth' in options)) {
153
+ throw new Error('bitWidth is required');
154
+ }
155
+ const bitWidth = options.bitWidth!;
156
+ if (!Number.isInteger(bitWidth) || bitWidth < 0 || bitWidth > 32) {
157
+ throw new Error(`invalid bit width: ${bitWidth}`);
158
+ }
159
+ if (type !== 'BOOLEAN' && type !== 'INT32' && type !== 'INT64') {
160
+ throw new Error(`unsupported type: ${type}`);
161
+ }
162
+ const paddedValueCount = Math.ceil(values.length / 8) * 8;
163
+ const encoded = new Uint8Array(Math.ceil((paddedValueCount * bitWidth) / 8));
164
+ for (let valueIndex = 0; valueIndex < values.length; valueIndex++) {
165
+ let value = Number(values[valueIndex]);
166
+ for (let bitIndex = 0; bitIndex < bitWidth; bitIndex++) {
167
+ if (value & (1 << bitIndex)) {
168
+ const packedBit = valueIndex * bitWidth + bitIndex;
169
+ encoded[Math.floor(packedBit / 8)] |= 1 << (packedBit % 8);
170
+ }
171
+ }
172
+ }
173
+ if (options.disableEnvelope) {
174
+ return encoded;
175
+ }
176
+ const envelope = new Uint8Array(encoded.length + 4);
177
+ writeUInt32LE(envelope, encoded.length, 0);
178
+ envelope.set(encoded, 4);
179
+ return envelope;
180
+ }
181
+
182
+ /** Decodes legacy Parquet BIT_PACKED values, including the optional page length envelope. */
183
+ export function decodeBitPackedValues(
184
+ type: PrimitiveType,
185
+ cursor: CursorBuffer,
186
+ count: number,
187
+ options: ParquetCodecOptions
188
+ ): ParquetValueBuffer {
189
+ if (!('bitWidth' in options)) {
190
+ throw new Error('bitWidth is required');
191
+ }
192
+ const bitWidth = options.bitWidth!;
193
+ if (!Number.isInteger(bitWidth) || bitWidth < 0 || bitWidth > 64) {
194
+ throw new Error(`invalid bit width: ${bitWidth}`);
195
+ }
196
+ if (!Number.isInteger(count) || count < 0) {
197
+ throw new Error(`invalid value count: ${count}`);
198
+ }
199
+ const envelopeEnd = options.disableEnvelope
200
+ ? undefined
201
+ : (() => {
202
+ assertReadable(cursor, 4);
203
+ const length =
204
+ cursor.buffer[cursor.offset] |
205
+ (cursor.buffer[cursor.offset + 1] << 8) |
206
+ (cursor.buffer[cursor.offset + 2] << 16) |
207
+ (cursor.buffer[cursor.offset + 3] << 24);
208
+ cursor.offset += 4;
209
+ if (length < 0) throw new Error('invalid BIT_PACKED length');
210
+ assertReadable(cursor, length);
211
+ return cursor.offset + length;
212
+ })();
213
+ const paddedValueCount = Math.ceil(count / 8) * 8;
214
+ const packedByteLength = Math.ceil((paddedValueCount * bitWidth) / 8);
215
+ assertReadable(cursor, packedByteLength);
216
+ const {output, outputOffset} = getParquetValueOutput(options, count);
217
+ decodeBitPackedRun(cursor, paddedValueCount, count, bitWidth, output, outputOffset);
218
+ if (envelopeEnd !== undefined) {
219
+ if (cursor.offset > envelopeEnd) throw new Error('invalid BIT_PACKED length');
220
+ cursor.offset = envelopeEnd;
221
+ }
222
+ return output;
223
+ }
224
+
146
225
  /** Decodes one complete bit-packed run directly into the caller's output array. */
147
226
  function decodeBitPackedRun(
148
227
  cursor: CursorBuffer,
@@ -237,10 +316,43 @@ function decodeUint32BitPackedRun(
237
316
  outputOffset: number,
238
317
  dictionary?: readonly unknown[]
239
318
  ): void {
319
+ if (!dictionary && bitWidth <= 4) {
320
+ decodeNarrowUnsignedBitPackedRun(buffer, packedOffset, count, bitWidth, output, outputOffset);
321
+ return;
322
+ }
323
+ if (dictionary && bitWidth <= 4) {
324
+ decodeNarrowDictionaryBitPackedRun(
325
+ buffer,
326
+ packedOffset,
327
+ count,
328
+ bitWidth,
329
+ output,
330
+ outputOffset,
331
+ dictionary
332
+ );
333
+ return;
334
+ }
335
+ if (dictionary && bitWidth === 10) {
336
+ decode10BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
337
+ return;
338
+ }
339
+ if (dictionary && bitWidth === 12) {
340
+ decode12BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
341
+ return;
342
+ }
240
343
  if (bitWidth === 8) {
344
+ if (dictionary) {
345
+ for (let valueIndex = 0; valueIndex < count; valueIndex++) {
346
+ const value = buffer[packedOffset + valueIndex] ?? 0;
347
+ if (value >= dictionary.length) {
348
+ throw new Error(`Invalid Parquet dictionary index ${value}`);
349
+ }
350
+ output[outputOffset + valueIndex] = dictionary[value];
351
+ }
352
+ return;
353
+ }
241
354
  for (let valueIndex = 0; valueIndex < count; valueIndex++) {
242
- const value = buffer[packedOffset + valueIndex] ?? 0;
243
- output[outputOffset + valueIndex] = resolveDictionaryValue(value, dictionary);
355
+ output[outputOffset + valueIndex] = buffer[packedOffset + valueIndex] ?? 0;
244
356
  }
245
357
  return;
246
358
  }
@@ -249,18 +361,476 @@ function decodeUint32BitPackedRun(
249
361
  let packedBits = 0;
250
362
  let packedBitCount = 0;
251
363
  let byteOffset = packedOffset;
364
+ if (dictionary) {
365
+ for (let valueIndex = 0; valueIndex < count; valueIndex++) {
366
+ while (packedBitCount < bitWidth) {
367
+ packedBits |= (buffer[byteOffset++] ?? 0) << packedBitCount;
368
+ packedBitCount += 8;
369
+ }
370
+ const value = packedBits & mask;
371
+ if (value >= dictionary.length) {
372
+ throw new Error(`Invalid Parquet dictionary index ${value}`);
373
+ }
374
+ output[outputOffset + valueIndex] = dictionary[value];
375
+ packedBits >>>= bitWidth;
376
+ packedBitCount -= bitWidth;
377
+ }
378
+ return;
379
+ }
252
380
  for (let valueIndex = 0; valueIndex < count; valueIndex++) {
253
381
  while (packedBitCount < bitWidth) {
254
382
  packedBits |= (buffer[byteOffset++] ?? 0) << packedBitCount;
255
383
  packedBitCount += 8;
256
384
  }
257
385
  const value = packedBits & mask;
258
- output[outputOffset + valueIndex] = resolveDictionaryValue(value, dictionary);
386
+ output[outputOffset + valueIndex] = value;
387
+ packedBits >>>= bitWidth;
388
+ packedBitCount -= bitWidth;
389
+ }
390
+ }
391
+
392
+ /** Decodes complete groups of eight narrow unsigned values without a per-value reservoir. */
393
+ function decodeNarrowUnsignedBitPackedRun(
394
+ buffer: Uint8Array,
395
+ packedOffset: number,
396
+ count: number,
397
+ bitWidth: number,
398
+ output: ParquetValueBuffer,
399
+ outputOffset: number
400
+ ): void {
401
+ if (bitWidth === 1) {
402
+ decode1BitUnsignedRun(buffer, packedOffset, count, output, outputOffset);
403
+ return;
404
+ }
405
+ if (bitWidth === 2) {
406
+ decode2BitUnsignedRun(buffer, packedOffset, count, output, outputOffset);
407
+ return;
408
+ }
409
+ const mask = 2 ** bitWidth - 1;
410
+ let byteOffset = packedOffset;
411
+ let valueIndex = 0;
412
+ while (valueIndex < count) {
413
+ const packedBits =
414
+ (buffer[byteOffset] ?? 0) |
415
+ ((buffer[byteOffset + 1] ?? 0) << 8) |
416
+ ((buffer[byteOffset + 2] ?? 0) << 16) |
417
+ ((buffer[byteOffset + 3] ?? 0) << 24);
418
+ const groupValueCount = Math.min(8, count - valueIndex);
419
+ for (let groupIndex = 0; groupIndex < groupValueCount; groupIndex++) {
420
+ output[outputOffset + valueIndex + groupIndex] =
421
+ (packedBits >>> (groupIndex * bitWidth)) & mask;
422
+ }
423
+ byteOffset += bitWidth;
424
+ valueIndex += groupValueCount;
425
+ }
426
+ }
427
+
428
+ /** Expands byte-aligned groups of eight one-bit level values. */
429
+ function decode1BitUnsignedRun(
430
+ buffer: Uint8Array,
431
+ packedOffset: number,
432
+ count: number,
433
+ output: ParquetValueBuffer,
434
+ outputOffset: number
435
+ ): void {
436
+ let byteOffset = packedOffset;
437
+ let valueIndex = 0;
438
+ while (valueIndex + 8 <= count) {
439
+ const packedBits = buffer[byteOffset++] ?? 0;
440
+ output[outputOffset + valueIndex] = packedBits & 1;
441
+ output[outputOffset + valueIndex + 1] = (packedBits >>> 1) & 1;
442
+ output[outputOffset + valueIndex + 2] = (packedBits >>> 2) & 1;
443
+ output[outputOffset + valueIndex + 3] = (packedBits >>> 3) & 1;
444
+ output[outputOffset + valueIndex + 4] = (packedBits >>> 4) & 1;
445
+ output[outputOffset + valueIndex + 5] = (packedBits >>> 5) & 1;
446
+ output[outputOffset + valueIndex + 6] = (packedBits >>> 6) & 1;
447
+ output[outputOffset + valueIndex + 7] = packedBits >>> 7;
448
+ valueIndex += 8;
449
+ }
450
+ const packedBits = buffer[byteOffset] ?? 0;
451
+ for (let tailIndex = 0; valueIndex + tailIndex < count; tailIndex++) {
452
+ output[outputOffset + valueIndex + tailIndex] = (packedBits >>> tailIndex) & 1;
453
+ }
454
+ }
455
+
456
+ /** Expands two-byte groups of eight two-bit level values. */
457
+ function decode2BitUnsignedRun(
458
+ buffer: Uint8Array,
459
+ packedOffset: number,
460
+ count: number,
461
+ output: ParquetValueBuffer,
462
+ outputOffset: number
463
+ ): void {
464
+ let byteOffset = packedOffset;
465
+ let valueIndex = 0;
466
+ while (valueIndex + 8 <= count) {
467
+ const packedBits = (buffer[byteOffset] ?? 0) | ((buffer[byteOffset + 1] ?? 0) << 8);
468
+ output[outputOffset + valueIndex] = packedBits & 3;
469
+ output[outputOffset + valueIndex + 1] = (packedBits >>> 2) & 3;
470
+ output[outputOffset + valueIndex + 2] = (packedBits >>> 4) & 3;
471
+ output[outputOffset + valueIndex + 3] = (packedBits >>> 6) & 3;
472
+ output[outputOffset + valueIndex + 4] = (packedBits >>> 8) & 3;
473
+ output[outputOffset + valueIndex + 5] = (packedBits >>> 10) & 3;
474
+ output[outputOffset + valueIndex + 6] = (packedBits >>> 12) & 3;
475
+ output[outputOffset + valueIndex + 7] = (packedBits >>> 14) & 3;
476
+ byteOffset += 2;
477
+ valueIndex += 8;
478
+ }
479
+ const packedBits = (buffer[byteOffset] ?? 0) | ((buffer[byteOffset + 1] ?? 0) << 8);
480
+ for (let tailIndex = 0; valueIndex + tailIndex < count; tailIndex++) {
481
+ output[outputOffset + valueIndex + tailIndex] = (packedBits >>> (tailIndex * 2)) & 3;
482
+ }
483
+ }
484
+
485
+ /** Resolves aligned groups of four 10-bit dictionary indices from five input bytes. */
486
+ function decode10BitDictionaryRun(
487
+ buffer: Uint8Array,
488
+ packedOffset: number,
489
+ count: number,
490
+ output: ParquetValueBuffer,
491
+ outputOffset: number,
492
+ dictionary: readonly unknown[]
493
+ ): void {
494
+ let byteOffset = packedOffset;
495
+ let valueIndex = 0;
496
+ while (valueIndex + 4 <= count) {
497
+ const byte0 = buffer[byteOffset] ?? 0;
498
+ const byte1 = buffer[byteOffset + 1] ?? 0;
499
+ const byte2 = buffer[byteOffset + 2] ?? 0;
500
+ const byte3 = buffer[byteOffset + 3] ?? 0;
501
+ const byte4 = buffer[byteOffset + 4] ?? 0;
502
+ const index0 = byte0 | ((byte1 & 0x03) << 8);
503
+ const index1 = (byte1 >>> 2) | ((byte2 & 0x0f) << 6);
504
+ const index2 = (byte2 >>> 4) | ((byte3 & 0x3f) << 4);
505
+ const index3 = (byte3 >>> 6) | (byte4 << 2);
506
+ if (
507
+ index0 >= dictionary.length ||
508
+ index1 >= dictionary.length ||
509
+ index2 >= dictionary.length ||
510
+ index3 >= dictionary.length
511
+ ) {
512
+ const invalidIndex = [index0, index1, index2, index3].find(
513
+ dictionaryIndex => dictionaryIndex >= dictionary.length
514
+ );
515
+ throw new Error(`Invalid Parquet dictionary index ${invalidIndex}`);
516
+ }
517
+ output[outputOffset + valueIndex] = dictionary[index0];
518
+ output[outputOffset + valueIndex + 1] = dictionary[index1];
519
+ output[outputOffset + valueIndex + 2] = dictionary[index2];
520
+ output[outputOffset + valueIndex + 3] = dictionary[index3];
521
+ byteOffset += 5;
522
+ valueIndex += 4;
523
+ }
524
+ decodeDictionaryBitPackedTail(
525
+ buffer,
526
+ byteOffset,
527
+ count - valueIndex,
528
+ 10,
529
+ output,
530
+ outputOffset + valueIndex,
531
+ dictionary
532
+ );
533
+ }
534
+
535
+ /** Resolves aligned pairs of 12-bit dictionary indices from three input bytes. */
536
+ function decode12BitDictionaryRun(
537
+ buffer: Uint8Array,
538
+ packedOffset: number,
539
+ count: number,
540
+ output: ParquetValueBuffer,
541
+ outputOffset: number,
542
+ dictionary: readonly unknown[]
543
+ ): void {
544
+ let byteOffset = packedOffset;
545
+ let valueIndex = 0;
546
+ while (valueIndex + 2 <= count) {
547
+ const byte0 = buffer[byteOffset] ?? 0;
548
+ const byte1 = buffer[byteOffset + 1] ?? 0;
549
+ const byte2 = buffer[byteOffset + 2] ?? 0;
550
+ const index0 = byte0 | ((byte1 & 0x0f) << 8);
551
+ const index1 = (byte1 >>> 4) | (byte2 << 4);
552
+ if (index0 >= dictionary.length || index1 >= dictionary.length) {
553
+ const invalidIndex = index0 >= dictionary.length ? index0 : index1;
554
+ throw new Error(`Invalid Parquet dictionary index ${invalidIndex}`);
555
+ }
556
+ output[outputOffset + valueIndex] = dictionary[index0];
557
+ output[outputOffset + valueIndex + 1] = dictionary[index1];
558
+ byteOffset += 3;
559
+ valueIndex += 2;
560
+ }
561
+ decodeDictionaryBitPackedTail(
562
+ buffer,
563
+ byteOffset,
564
+ count - valueIndex,
565
+ 12,
566
+ output,
567
+ outputOffset + valueIndex,
568
+ dictionary
569
+ );
570
+ }
571
+
572
+ /** Resolves the byte-aligned remainder after a fixed-width dictionary group. */
573
+ function decodeDictionaryBitPackedTail(
574
+ buffer: Uint8Array,
575
+ packedOffset: number,
576
+ count: number,
577
+ bitWidth: number,
578
+ output: ParquetValueBuffer,
579
+ outputOffset: number,
580
+ dictionary: readonly unknown[]
581
+ ): void {
582
+ const mask = 2 ** bitWidth - 1;
583
+ let packedBits = 0;
584
+ let packedBitCount = 0;
585
+ let byteOffset = packedOffset;
586
+ for (let valueIndex = 0; valueIndex < count; valueIndex++) {
587
+ while (packedBitCount < bitWidth) {
588
+ packedBits |= (buffer[byteOffset++] ?? 0) << packedBitCount;
589
+ packedBitCount += 8;
590
+ }
591
+ const dictionaryIndex = packedBits & mask;
592
+ if (dictionaryIndex >= dictionary.length) {
593
+ throw new Error(`Invalid Parquet dictionary index ${dictionaryIndex}`);
594
+ }
595
+ output[outputOffset + valueIndex] = dictionary[dictionaryIndex];
259
596
  packedBits >>>= bitWidth;
260
597
  packedBitCount -= bitWidth;
261
598
  }
262
599
  }
263
600
 
601
+ /** Resolves complete groups of eight narrow dictionary indices without a per-value reservoir. */
602
+ function decodeNarrowDictionaryBitPackedRun(
603
+ buffer: Uint8Array,
604
+ packedOffset: number,
605
+ count: number,
606
+ bitWidth: number,
607
+ output: ParquetValueBuffer,
608
+ outputOffset: number,
609
+ dictionary: readonly unknown[]
610
+ ): void {
611
+ if (bitWidth === 1) {
612
+ decode1BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
613
+ return;
614
+ }
615
+ if (bitWidth === 2) {
616
+ decode2BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
617
+ return;
618
+ }
619
+ if (bitWidth === 3) {
620
+ decode3BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
621
+ return;
622
+ }
623
+ if (bitWidth === 4) {
624
+ decode4BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
625
+ return;
626
+ }
627
+ const mask = 2 ** bitWidth - 1;
628
+ let byteOffset = packedOffset;
629
+ let valueIndex = 0;
630
+ while (valueIndex < count) {
631
+ const packedBits =
632
+ (buffer[byteOffset] ?? 0) |
633
+ ((buffer[byteOffset + 1] ?? 0) << 8) |
634
+ ((buffer[byteOffset + 2] ?? 0) << 16) |
635
+ ((buffer[byteOffset + 3] ?? 0) << 24);
636
+ const groupValueCount = Math.min(8, count - valueIndex);
637
+ for (let groupIndex = 0; groupIndex < groupValueCount; groupIndex++) {
638
+ const dictionaryIndex = (packedBits >>> (groupIndex * bitWidth)) & mask;
639
+ if (dictionaryIndex >= dictionary.length) {
640
+ throw new Error(`Invalid Parquet dictionary index ${dictionaryIndex}`);
641
+ }
642
+ output[outputOffset + valueIndex + groupIndex] = dictionary[dictionaryIndex];
643
+ }
644
+ byteOffset += bitWidth;
645
+ valueIndex += groupValueCount;
646
+ }
647
+ }
648
+
649
+ /** Resolves byte-aligned groups of eight one-bit dictionary indices. */
650
+ function decode1BitDictionaryRun(
651
+ buffer: Uint8Array,
652
+ packedOffset: number,
653
+ count: number,
654
+ output: ParquetValueBuffer,
655
+ outputOffset: number,
656
+ dictionary: readonly unknown[]
657
+ ): void {
658
+ let byteOffset = packedOffset;
659
+ let valueIndex = 0;
660
+ while (valueIndex + 8 <= count) {
661
+ const packedBits = buffer[byteOffset++] ?? 0;
662
+ if (dictionary.length < 2 && packedBits !== 0) {
663
+ throw new Error('Invalid Parquet dictionary index 1');
664
+ }
665
+ output[outputOffset + valueIndex] = dictionary[packedBits & 1];
666
+ output[outputOffset + valueIndex + 1] = dictionary[(packedBits >>> 1) & 1];
667
+ output[outputOffset + valueIndex + 2] = dictionary[(packedBits >>> 2) & 1];
668
+ output[outputOffset + valueIndex + 3] = dictionary[(packedBits >>> 3) & 1];
669
+ output[outputOffset + valueIndex + 4] = dictionary[(packedBits >>> 4) & 1];
670
+ output[outputOffset + valueIndex + 5] = dictionary[(packedBits >>> 5) & 1];
671
+ output[outputOffset + valueIndex + 6] = dictionary[(packedBits >>> 6) & 1];
672
+ output[outputOffset + valueIndex + 7] = dictionary[packedBits >>> 7];
673
+ valueIndex += 8;
674
+ }
675
+ decodeDictionaryBitPackedTail(
676
+ buffer,
677
+ byteOffset,
678
+ count - valueIndex,
679
+ 1,
680
+ output,
681
+ outputOffset + valueIndex,
682
+ dictionary
683
+ );
684
+ }
685
+
686
+ /** Resolves two-byte groups of eight two-bit dictionary indices. */
687
+ function decode2BitDictionaryRun(
688
+ buffer: Uint8Array,
689
+ packedOffset: number,
690
+ count: number,
691
+ output: ParquetValueBuffer,
692
+ outputOffset: number,
693
+ dictionary: readonly unknown[]
694
+ ): void {
695
+ let byteOffset = packedOffset;
696
+ let valueIndex = 0;
697
+ while (valueIndex + 8 <= count) {
698
+ const packedBits = (buffer[byteOffset] ?? 0) | ((buffer[byteOffset + 1] ?? 0) << 8);
699
+ const index0 = packedBits & 3;
700
+ const index1 = (packedBits >>> 2) & 3;
701
+ const index2 = (packedBits >>> 4) & 3;
702
+ const index3 = (packedBits >>> 6) & 3;
703
+ const index4 = (packedBits >>> 8) & 3;
704
+ const index5 = (packedBits >>> 10) & 3;
705
+ const index6 = (packedBits >>> 12) & 3;
706
+ const index7 = packedBits >>> 14;
707
+ if (
708
+ index0 >= dictionary.length ||
709
+ index1 >= dictionary.length ||
710
+ index2 >= dictionary.length ||
711
+ index3 >= dictionary.length ||
712
+ index4 >= dictionary.length ||
713
+ index5 >= dictionary.length ||
714
+ index6 >= dictionary.length ||
715
+ index7 >= dictionary.length
716
+ ) {
717
+ throw new Error('Invalid Parquet dictionary index');
718
+ }
719
+ output[outputOffset + valueIndex] = dictionary[index0];
720
+ output[outputOffset + valueIndex + 1] = dictionary[index1];
721
+ output[outputOffset + valueIndex + 2] = dictionary[index2];
722
+ output[outputOffset + valueIndex + 3] = dictionary[index3];
723
+ output[outputOffset + valueIndex + 4] = dictionary[index4];
724
+ output[outputOffset + valueIndex + 5] = dictionary[index5];
725
+ output[outputOffset + valueIndex + 6] = dictionary[index6];
726
+ output[outputOffset + valueIndex + 7] = dictionary[index7];
727
+ byteOffset += 2;
728
+ valueIndex += 8;
729
+ }
730
+ decodeDictionaryBitPackedTail(
731
+ buffer,
732
+ byteOffset,
733
+ count - valueIndex,
734
+ 2,
735
+ output,
736
+ outputOffset + valueIndex,
737
+ dictionary
738
+ );
739
+ }
740
+
741
+ /** Resolves byte-aligned groups of eight three-bit dictionary indices. */
742
+ function decode3BitDictionaryRun(
743
+ buffer: Uint8Array,
744
+ packedOffset: number,
745
+ count: number,
746
+ output: ParquetValueBuffer,
747
+ outputOffset: number,
748
+ dictionary: readonly unknown[]
749
+ ): void {
750
+ let byteOffset = packedOffset;
751
+ let valueIndex = 0;
752
+ while (valueIndex + 8 <= count) {
753
+ const packedBits =
754
+ (buffer[byteOffset] ?? 0) |
755
+ ((buffer[byteOffset + 1] ?? 0) << 8) |
756
+ ((buffer[byteOffset + 2] ?? 0) << 16);
757
+ const index0 = packedBits & 7;
758
+ const index1 = (packedBits >>> 3) & 7;
759
+ const index2 = (packedBits >>> 6) & 7;
760
+ const index3 = (packedBits >>> 9) & 7;
761
+ const index4 = (packedBits >>> 12) & 7;
762
+ const index5 = (packedBits >>> 15) & 7;
763
+ const index6 = (packedBits >>> 18) & 7;
764
+ const index7 = (packedBits >>> 21) & 7;
765
+ if (
766
+ index0 >= dictionary.length ||
767
+ index1 >= dictionary.length ||
768
+ index2 >= dictionary.length ||
769
+ index3 >= dictionary.length ||
770
+ index4 >= dictionary.length ||
771
+ index5 >= dictionary.length ||
772
+ index6 >= dictionary.length ||
773
+ index7 >= dictionary.length
774
+ ) {
775
+ throw new Error('Invalid Parquet dictionary index');
776
+ }
777
+ output[outputOffset + valueIndex] = dictionary[index0];
778
+ output[outputOffset + valueIndex + 1] = dictionary[index1];
779
+ output[outputOffset + valueIndex + 2] = dictionary[index2];
780
+ output[outputOffset + valueIndex + 3] = dictionary[index3];
781
+ output[outputOffset + valueIndex + 4] = dictionary[index4];
782
+ output[outputOffset + valueIndex + 5] = dictionary[index5];
783
+ output[outputOffset + valueIndex + 6] = dictionary[index6];
784
+ output[outputOffset + valueIndex + 7] = dictionary[index7];
785
+ byteOffset += 3;
786
+ valueIndex += 8;
787
+ }
788
+ decodeDictionaryBitPackedTail(
789
+ buffer,
790
+ byteOffset,
791
+ count - valueIndex,
792
+ 3,
793
+ output,
794
+ outputOffset + valueIndex,
795
+ dictionary
796
+ );
797
+ }
798
+
799
+ /** Resolves pairs of four-bit dictionary indices from each input byte. */
800
+ function decode4BitDictionaryRun(
801
+ buffer: Uint8Array,
802
+ packedOffset: number,
803
+ count: number,
804
+ output: ParquetValueBuffer,
805
+ outputOffset: number,
806
+ dictionary: readonly unknown[]
807
+ ): void {
808
+ let byteOffset = packedOffset;
809
+ let valueIndex = 0;
810
+ while (valueIndex + 2 <= count) {
811
+ const packedByte = buffer[byteOffset++] ?? 0;
812
+ const index0 = packedByte & 15;
813
+ const index1 = packedByte >>> 4;
814
+ if (index0 >= dictionary.length || index1 >= dictionary.length) {
815
+ throw new Error(
816
+ `Invalid Parquet dictionary index ${index0 >= dictionary.length ? index0 : index1}`
817
+ );
818
+ }
819
+ output[outputOffset + valueIndex] = dictionary[index0];
820
+ output[outputOffset + valueIndex + 1] = dictionary[index1];
821
+ valueIndex += 2;
822
+ }
823
+ decodeDictionaryBitPackedTail(
824
+ buffer,
825
+ byteOffset,
826
+ count - valueIndex,
827
+ 4,
828
+ output,
829
+ outputOffset + valueIndex,
830
+ dictionary
831
+ );
832
+ }
833
+
264
834
  /** Decodes a bit-packed run with an exact number-based bit reservoir. */
265
835
  function decodeNumberBitPackedRun(
266
836
  buffer: Uint8Array,