@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (336) hide show
  1. package/LICENSE +1 -1
  2. package/dist/bundled.d.ts +1 -1
  3. package/dist/bundled.d.ts.map +1 -1
  4. package/dist/bundled.js.map +1 -1
  5. package/dist/delta-format.d.ts +9 -0
  6. package/dist/delta-format.d.ts.map +1 -0
  7. package/dist/delta-format.js +12 -0
  8. package/dist/delta-format.js.map +1 -0
  9. package/dist/delta-source-loader-types.d.ts +27 -0
  10. package/dist/delta-source-loader-types.d.ts.map +1 -0
  11. package/dist/delta-source-loader-types.js +28 -0
  12. package/dist/delta-source-loader-types.js.map +1 -0
  13. package/dist/delta-source.d.ts +48 -0
  14. package/dist/delta-source.d.ts.map +1 -0
  15. package/dist/delta-source.js +222 -0
  16. package/dist/delta-source.js.map +1 -0
  17. package/dist/delta-types.d.ts +38 -0
  18. package/dist/delta-types.d.ts.map +1 -0
  19. package/dist/delta-types.js +5 -0
  20. package/dist/delta-types.js.map +1 -0
  21. package/dist/geoparquet-loader.js +1 -1
  22. package/dist/iceberg-rest-catalog.d.ts +49 -0
  23. package/dist/iceberg-rest-catalog.d.ts.map +1 -0
  24. package/dist/iceberg-rest-catalog.js +78 -0
  25. package/dist/iceberg-rest-catalog.js.map +1 -0
  26. package/dist/iceberg-table-source.d.ts +63 -0
  27. package/dist/iceberg-table-source.d.ts.map +1 -0
  28. package/dist/iceberg-table-source.js +797 -0
  29. package/dist/iceberg-table-source.js.map +1 -0
  30. package/dist/iceberg-types.d.ts +150 -0
  31. package/dist/iceberg-types.d.ts.map +1 -0
  32. package/dist/iceberg-types.js +5 -0
  33. package/dist/iceberg-types.js.map +1 -0
  34. package/dist/index.cjs +15841 -3306
  35. package/dist/index.cjs.map +3 -3
  36. package/dist/index.d.ts +8 -3
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +5 -1
  39. package/dist/index.js.map +1 -1
  40. package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
  41. package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
  42. package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
  43. package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  44. package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
  45. package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
  46. package/dist/lib/constants.js +1 -1
  47. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
  48. package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
  49. package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
  50. package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
  51. package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
  52. package/dist/lib/geo/geoparquet-covering.js +137 -0
  53. package/dist/lib/geo/geoparquet-covering.js.map +1 -0
  54. package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
  55. package/dist/lib/geo/geospatial-metadata.js +39 -13
  56. package/dist/lib/geo/geospatial-metadata.js.map +1 -1
  57. package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
  58. package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
  59. package/dist/lib/parquet-bloom-filter-planner.js +57 -0
  60. package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
  61. package/dist/lib/parquet-bloom-filter.d.ts +30 -0
  62. package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
  63. package/dist/lib/parquet-bloom-filter.js +289 -0
  64. package/dist/lib/parquet-bloom-filter.js.map +1 -0
  65. package/dist/lib/parquet-encryption.d.ts +80 -0
  66. package/dist/lib/parquet-encryption.d.ts.map +1 -0
  67. package/dist/lib/parquet-encryption.js +190 -0
  68. package/dist/lib/parquet-encryption.js.map +1 -0
  69. package/dist/lib/parquet-page-index.d.ts +34 -5
  70. package/dist/lib/parquet-page-index.d.ts.map +1 -1
  71. package/dist/lib/parquet-page-index.js +114 -28
  72. package/dist/lib/parquet-page-index.js.map +1 -1
  73. package/dist/lib/parquet-predicate.d.ts +5 -1
  74. package/dist/lib/parquet-predicate.d.ts.map +1 -1
  75. package/dist/lib/parquet-predicate.js +42 -14
  76. package/dist/lib/parquet-predicate.js.map +1 -1
  77. package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
  78. package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
  79. package/dist/lib/parquet-source-worker-decoder.js +83 -17
  80. package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
  81. package/dist/lib/parquet-source-worker-types.d.ts +24 -0
  82. package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
  83. package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
  84. package/dist/lib/parsers/get-parquet-schema.js +3 -1
  85. package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
  86. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
  87. package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
  88. package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
  89. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
  90. package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
  91. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
  92. package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
  93. package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
  94. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  95. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  96. package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
  97. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
  98. package/dist/lib/slice-parquet-batch.d.ts +9 -0
  99. package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
  100. package/dist/lib/slice-parquet-batch.js +28 -0
  101. package/dist/lib/slice-parquet-batch.js.map +1 -0
  102. package/dist/lib/sources/parquet-range-file.d.ts +1 -1
  103. package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
  104. package/dist/lib/sources/parquet-range-file.js +28 -22
  105. package/dist/lib/sources/parquet-range-file.js.map +1 -1
  106. package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
  107. package/dist/lib/utils/normalize-parquet-options.js +15 -1
  108. package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
  109. package/dist/parquet-dataset-source.cjs +16528 -4928
  110. package/dist/parquet-dataset-source.cjs.map +3 -3
  111. package/dist/parquet-dataset-source.d.ts +31 -4
  112. package/dist/parquet-dataset-source.d.ts.map +1 -1
  113. package/dist/parquet-dataset-source.js +296 -74
  114. package/dist/parquet-dataset-source.js.map +1 -1
  115. package/dist/parquet-encoded-page-types.d.ts +108 -0
  116. package/dist/parquet-encoded-page-types.d.ts.map +1 -0
  117. package/dist/parquet-encoded-page-types.js +5 -0
  118. package/dist/parquet-encoded-page-types.js.map +1 -0
  119. package/dist/parquet-js-loader-types.d.ts +2 -0
  120. package/dist/parquet-js-loader-types.d.ts.map +1 -1
  121. package/dist/parquet-js-loader-types.js +2 -1
  122. package/dist/parquet-js-loader-types.js.map +1 -1
  123. package/dist/parquet-js-loader.d.ts +1 -0
  124. package/dist/parquet-js-loader.d.ts.map +1 -1
  125. package/dist/parquet-js-loader.js +2 -1
  126. package/dist/parquet-js-loader.js.map +1 -1
  127. package/dist/parquet-js-writer.d.ts +23 -2
  128. package/dist/parquet-js-writer.d.ts.map +1 -1
  129. package/dist/parquet-js-writer.js +1 -1
  130. package/dist/parquet-js-writer.js.map +1 -1
  131. package/dist/parquet-loader-base.js +1 -1
  132. package/dist/parquet-loader-options.d.ts +11 -0
  133. package/dist/parquet-loader-options.d.ts.map +1 -1
  134. package/dist/parquet-loader-options.js.map +1 -1
  135. package/dist/parquet-source-capabilities.d.ts +8 -1
  136. package/dist/parquet-source-capabilities.d.ts.map +1 -1
  137. package/dist/parquet-source-capabilities.js +10 -0
  138. package/dist/parquet-source-capabilities.js.map +1 -1
  139. package/dist/parquet-source-loader-types.d.ts +10 -0
  140. package/dist/parquet-source-loader-types.d.ts.map +1 -1
  141. package/dist/parquet-source-loader-types.js +11 -1
  142. package/dist/parquet-source-loader-types.js.map +1 -1
  143. package/dist/parquet-source-loader.cjs +14121 -2796
  144. package/dist/parquet-source-loader.cjs.map +3 -3
  145. package/dist/parquet-source-loader.d.ts +51 -4
  146. package/dist/parquet-source-loader.d.ts.map +1 -1
  147. package/dist/parquet-source-loader.js +651 -38
  148. package/dist/parquet-source-loader.js.map +1 -1
  149. package/dist/parquet-source-types.d.ts +180 -30
  150. package/dist/parquet-source-types.d.ts.map +1 -1
  151. package/dist/parquet-source-worker.js +7378 -6989
  152. package/dist/parquet-source-worker.js.map +4 -4
  153. package/dist/parquet-worker-node.cjs +8501 -1045
  154. package/dist/parquet-worker-node.cjs.map +4 -4
  155. package/dist/parquet-worker-url.js +5 -1
  156. package/dist/parquet-worker-url.js.map +1 -1
  157. package/dist/parquet-worker.js +289 -218
  158. package/dist/parquet-worker.js.map +4 -4
  159. package/dist/parquetjs/codecs/declare.d.ts +15 -0
  160. package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
  161. package/dist/parquetjs/codecs/declare.js +10 -0
  162. package/dist/parquetjs/codecs/declare.js.map +1 -1
  163. package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
  164. package/dist/parquetjs/codecs/delta.js +356 -7
  165. package/dist/parquetjs/codecs/delta.js.map +1 -1
  166. package/dist/parquetjs/codecs/index.d.ts.map +1 -1
  167. package/dist/parquetjs/codecs/index.js +4 -0
  168. package/dist/parquetjs/codecs/index.js.map +1 -1
  169. package/dist/parquetjs/codecs/plain.d.ts +1 -1
  170. package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
  171. package/dist/parquetjs/codecs/plain.js +153 -12
  172. package/dist/parquetjs/codecs/plain.js.map +1 -1
  173. package/dist/parquetjs/codecs/rle.d.ts +4 -0
  174. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  175. package/dist/parquetjs/codecs/rle.js +406 -3
  176. package/dist/parquetjs/codecs/rle.js.map +1 -1
  177. package/dist/parquetjs/compression.d.ts +9 -0
  178. package/dist/parquetjs/compression.d.ts.map +1 -1
  179. package/dist/parquetjs/compression.js +86 -67
  180. package/dist/parquetjs/compression.js.map +1 -1
  181. package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
  182. package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
  183. package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
  184. package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
  185. package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
  186. package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
  187. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
  188. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
  189. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
  190. package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
  191. package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
  192. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
  193. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
  194. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
  195. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  196. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
  197. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
  198. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
  199. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
  200. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
  201. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
  202. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
  203. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  204. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
  205. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
  206. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
  207. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
  208. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
  209. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
  210. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
  211. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
  212. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
  213. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
  214. package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
  215. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  216. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
  217. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
  218. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
  219. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
  220. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
  221. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
  222. package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
  223. package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
  224. package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
  225. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
  226. package/dist/parquetjs/parquet-thrift/index.js +6 -0
  227. package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
  228. package/dist/parquetjs/parser/decoders.d.ts +10 -1
  229. package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
  230. package/dist/parquetjs/parser/decoders.js +318 -14
  231. package/dist/parquetjs/parser/decoders.js.map +1 -1
  232. package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
  233. package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
  234. package/dist/parquetjs/parser/encoded-pages.js +206 -0
  235. package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
  236. package/dist/parquetjs/parser/parquet-reader.d.ts +88 -12
  237. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
  238. package/dist/parquetjs/parser/parquet-reader.js +713 -66
  239. package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
  240. package/dist/parquetjs/schema/declare.d.ts +22 -3
  241. package/dist/parquetjs/schema/declare.d.ts.map +1 -1
  242. package/dist/parquetjs/schema/declare.js.map +1 -1
  243. package/dist/parquetjs/schema/schema.d.ts.map +1 -1
  244. package/dist/parquetjs/schema/schema.js +10 -0
  245. package/dist/parquetjs/schema/schema.js.map +1 -1
  246. package/dist/parquetjs/schema/shred.d.ts.map +1 -1
  247. package/dist/parquetjs/schema/shred.js +219 -11
  248. package/dist/parquetjs/schema/shred.js.map +1 -1
  249. package/dist/parquetjs/schema/types.js +19 -21
  250. package/dist/parquetjs/schema/types.js.map +1 -1
  251. package/dist/parquetjs/schema/variant.d.ts +3 -0
  252. package/dist/parquetjs/schema/variant.d.ts.map +1 -0
  253. package/dist/parquetjs/schema/variant.js +248 -0
  254. package/dist/parquetjs/schema/variant.js.map +1 -0
  255. package/dist/parquetjs/utils/crc32.d.ts +3 -0
  256. package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
  257. package/dist/parquetjs/utils/crc32.js +15 -0
  258. package/dist/parquetjs/utils/crc32.js.map +1 -0
  259. package/dist/parquetjs/utils/read-utils.d.ts +11 -1
  260. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
  261. package/dist/parquetjs/utils/read-utils.js +21 -1
  262. package/dist/parquetjs/utils/read-utils.js.map +1 -1
  263. package/dist/unbundled.d.ts +2 -1
  264. package/dist/unbundled.d.ts.map +1 -1
  265. package/dist/unbundled.js +1 -0
  266. package/dist/unbundled.js.map +1 -1
  267. package/package.json +36 -13
  268. package/src/bundled.ts +10 -0
  269. package/src/delta-format.ts +12 -0
  270. package/src/delta-source-loader-types.ts +40 -0
  271. package/src/delta-source.ts +305 -0
  272. package/src/delta-types.ts +40 -0
  273. package/src/iceberg-rest-catalog.ts +122 -0
  274. package/src/iceberg-table-source.ts +1037 -0
  275. package/src/iceberg-types.ts +165 -0
  276. package/src/index.ts +59 -1
  277. package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
  278. package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
  279. package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
  280. package/src/lib/geo/geoparquet-covering.ts +201 -0
  281. package/src/lib/geo/geospatial-metadata.ts +42 -15
  282. package/src/lib/parquet-bloom-filter-planner.ts +86 -0
  283. package/src/lib/parquet-bloom-filter.ts +345 -0
  284. package/src/lib/parquet-encryption.ts +344 -0
  285. package/src/lib/parquet-page-index.ts +180 -34
  286. package/src/lib/parquet-predicate.ts +52 -16
  287. package/src/lib/parquet-source-worker-decoder.ts +116 -20
  288. package/src/lib/parquet-source-worker-types.ts +25 -0
  289. package/src/lib/parsers/get-parquet-schema.ts +3 -1
  290. package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
  291. package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
  292. package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
  293. package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
  294. package/src/lib/slice-parquet-batch.ts +32 -0
  295. package/src/lib/sources/parquet-range-file.ts +35 -29
  296. package/src/lib/utils/normalize-parquet-options.ts +17 -1
  297. package/src/parquet-dataset-source.ts +397 -85
  298. package/src/parquet-encoded-page-types.ts +122 -0
  299. package/src/parquet-js-loader-types.ts +1 -0
  300. package/src/parquet-js-loader.ts +2 -1
  301. package/src/parquet-js-writer.ts +30 -0
  302. package/src/parquet-loader-options.ts +11 -0
  303. package/src/parquet-source-capabilities.ts +18 -1
  304. package/src/parquet-source-loader-types.ts +10 -0
  305. package/src/parquet-source-loader.ts +1024 -52
  306. package/src/parquet-source-types.ts +197 -28
  307. package/src/parquet-worker-url.ts +6 -1
  308. package/src/parquetjs/codecs/declare.ts +27 -0
  309. package/src/parquetjs/codecs/delta.ts +466 -6
  310. package/src/parquetjs/codecs/index.ts +4 -0
  311. package/src/parquetjs/codecs/plain.ts +177 -11
  312. package/src/parquetjs/codecs/rle.ts +573 -3
  313. package/src/parquetjs/compression.ts +102 -76
  314. package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
  315. package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
  316. package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
  317. package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
  318. package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
  319. package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
  320. package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
  321. package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
  322. package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
  323. package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
  324. package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
  325. package/src/parquetjs/parquet-thrift/index.ts +6 -0
  326. package/src/parquetjs/parser/decoders.ts +451 -26
  327. package/src/parquetjs/parser/encoded-pages.ts +278 -0
  328. package/src/parquetjs/parser/parquet-reader.ts +1112 -87
  329. package/src/parquetjs/schema/declare.ts +22 -2
  330. package/src/parquetjs/schema/schema.ts +10 -0
  331. package/src/parquetjs/schema/shred.ts +238 -11
  332. package/src/parquetjs/schema/types.ts +16 -25
  333. package/src/parquetjs/schema/variant.ts +313 -0
  334. package/src/parquetjs/utils/crc32.ts +15 -0
  335. package/src/parquetjs/utils/read-utils.ts +26 -1
  336. package/src/unbundled.ts +14 -0
@@ -0,0 +1,108 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ /* eslint-disable camelcase */
6
+ import * as thrift from '../utils/thrift-runtime';
7
+
8
+ /** Constructor arguments for the Parquet geospatial bounding-box Thrift structure. */
9
+ export interface IBoundingBoxArgs {
10
+ xmin: number;
11
+ xmax: number;
12
+ ymin: number;
13
+ ymax: number;
14
+ zmin?: number;
15
+ zmax?: number;
16
+ mmin?: number;
17
+ mmax?: number;
18
+ }
19
+
20
+ /** Parquet geospatial bounding-box Thrift structure. */
21
+ export class BoundingBox {
22
+ public xmin: number;
23
+ public xmax: number;
24
+ public ymin: number;
25
+ public ymax: number;
26
+ public zmin?: number;
27
+ public zmax?: number;
28
+ public mmin?: number;
29
+ public mmax?: number;
30
+
31
+ constructor(args: IBoundingBoxArgs) {
32
+ this.xmin = getRequiredCoordinate(args.xmin, 'xmin');
33
+ this.xmax = getRequiredCoordinate(args.xmax, 'xmax');
34
+ this.ymin = getRequiredCoordinate(args.ymin, 'ymin');
35
+ this.ymax = getRequiredCoordinate(args.ymax, 'ymax');
36
+ this.zmin = args.zmin;
37
+ this.zmax = args.zmax;
38
+ this.mmin = args.mmin;
39
+ this.mmax = args.mmax;
40
+ }
41
+
42
+ /** Writes this bounding box to a Thrift protocol. */
43
+ public write(output: thrift.TProtocol): void {
44
+ output.writeStructBegin('BoundingBox');
45
+ writeCoordinate(output, 'xmin', 1, this.xmin);
46
+ writeCoordinate(output, 'xmax', 2, this.xmax);
47
+ writeCoordinate(output, 'ymin', 3, this.ymin);
48
+ writeCoordinate(output, 'ymax', 4, this.ymax);
49
+ writeCoordinate(output, 'zmin', 5, this.zmin);
50
+ writeCoordinate(output, 'zmax', 6, this.zmax);
51
+ writeCoordinate(output, 'mmin', 7, this.mmin);
52
+ writeCoordinate(output, 'mmax', 8, this.mmax);
53
+ output.writeFieldStop();
54
+ output.writeStructEnd();
55
+ }
56
+
57
+ /** Reads a bounding box from a Thrift protocol. */
58
+ public static read(input: thrift.TProtocol): BoundingBox {
59
+ input.readStructBegin();
60
+ const args: Partial<IBoundingBoxArgs> = {};
61
+ while (true) {
62
+ const field = input.readFieldBegin();
63
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
64
+ if (field.fid >= 1 && field.fid <= 8 && field.ftype === thrift.Thrift.Type.DOUBLE) {
65
+ const coordinateNames: Array<keyof IBoundingBoxArgs> = [
66
+ 'xmin',
67
+ 'xmax',
68
+ 'ymin',
69
+ 'ymax',
70
+ 'zmin',
71
+ 'zmax',
72
+ 'mmin',
73
+ 'mmax'
74
+ ];
75
+ args[coordinateNames[field.fid - 1]] = input.readDouble();
76
+ } else {
77
+ input.skip(field.ftype);
78
+ }
79
+ input.readFieldEnd();
80
+ }
81
+ input.readStructEnd();
82
+ return new BoundingBox(args as IBoundingBoxArgs);
83
+ }
84
+ }
85
+
86
+ /** Writes one optional coordinate to a Thrift protocol. */
87
+ function writeCoordinate(
88
+ output: thrift.TProtocol,
89
+ name: string,
90
+ fieldId: number,
91
+ value: number | undefined
92
+ ): void {
93
+ if (value === undefined) return;
94
+ output.writeFieldBegin(name, thrift.Thrift.Type.DOUBLE, fieldId);
95
+ output.writeDouble(value);
96
+ output.writeFieldEnd();
97
+ }
98
+
99
+ /** Validates one required coordinate reported by the Thrift structure. */
100
+ function getRequiredCoordinate(value: number | undefined, name: string): number {
101
+ if (value === undefined) {
102
+ throw new thrift.Thrift.TProtocolException(
103
+ thrift.Thrift.TProtocolExceptionType.UNKNOWN,
104
+ `Required field[${name}] is unset!`
105
+ );
106
+ }
107
+ return value;
108
+ }
@@ -13,6 +13,7 @@
13
13
  import {CompactInt64 as Int64} from '../utils/uint8-array-compact-protocol';
14
14
  import * as thrift from '../utils/thrift-runtime';
15
15
  import * as ColumnMetaData from './ColumnMetaData';
16
+ import * as ColumnCryptoMetaData from './ColumnCryptoMetaData';
16
17
  export interface IColumnChunkArgs {
17
18
  file_path?: string;
18
19
  file_offset: number | Int64;
@@ -21,6 +22,8 @@ export interface IColumnChunkArgs {
21
22
  offset_index_length?: number;
22
23
  column_index_offset?: number | Int64;
23
24
  column_index_length?: number;
25
+ crypto_metadata?: ColumnCryptoMetaData.ColumnCryptoMetaData;
26
+ encrypted_column_metadata?: Uint8Array;
24
27
  }
25
28
  export class ColumnChunk {
26
29
  public file_path?: string;
@@ -30,6 +33,8 @@ export class ColumnChunk {
30
33
  public offset_index_length?: number;
31
34
  public column_index_offset?: Int64;
32
35
  public column_index_length?: number;
36
+ public crypto_metadata?: ColumnCryptoMetaData.ColumnCryptoMetaData;
37
+ public encrypted_column_metadata?: Uint8Array;
33
38
  constructor(args: IColumnChunkArgs) {
34
39
  if (args != null && args.file_path != null) {
35
40
  this.file_path = args.file_path;
@@ -69,6 +74,12 @@ export class ColumnChunk {
69
74
  if (args != null && args.column_index_length != null) {
70
75
  this.column_index_length = args.column_index_length;
71
76
  }
77
+ if (args != null && args.crypto_metadata != null) {
78
+ this.crypto_metadata = args.crypto_metadata;
79
+ }
80
+ if (args != null && args.encrypted_column_metadata != null) {
81
+ this.encrypted_column_metadata = args.encrypted_column_metadata;
82
+ }
72
83
  }
73
84
  public write(output: thrift.TProtocol): void {
74
85
  output.writeStructBegin('ColumnChunk');
@@ -107,6 +118,16 @@ export class ColumnChunk {
107
118
  output.writeI32(this.column_index_length);
108
119
  output.writeFieldEnd();
109
120
  }
121
+ if (this.crypto_metadata != null) {
122
+ output.writeFieldBegin('crypto_metadata', thrift.Thrift.Type.STRUCT, 8);
123
+ this.crypto_metadata.write(output);
124
+ output.writeFieldEnd();
125
+ }
126
+ if (this.encrypted_column_metadata != null) {
127
+ output.writeFieldBegin('encrypted_column_metadata', thrift.Thrift.Type.STRING, 9);
128
+ output.writeBinary(this.encrypted_column_metadata);
129
+ output.writeFieldEnd();
130
+ }
110
131
  output.writeFieldStop();
111
132
  output.writeStructEnd();
112
133
  return;
@@ -179,6 +200,20 @@ export class ColumnChunk {
179
200
  input.skip(fieldType);
180
201
  }
181
202
  break;
203
+ case 8:
204
+ if (fieldType === thrift.Thrift.Type.STRUCT) {
205
+ _args.crypto_metadata = ColumnCryptoMetaData.ColumnCryptoMetaData.read(input);
206
+ } else {
207
+ input.skip(fieldType);
208
+ }
209
+ break;
210
+ case 9:
211
+ if (fieldType === thrift.Thrift.Type.STRING) {
212
+ _args.encrypted_column_metadata = input.readBinary();
213
+ } else {
214
+ input.skip(fieldType);
215
+ }
216
+ break;
182
217
  default: {
183
218
  input.skip(fieldType);
184
219
  }
@@ -0,0 +1,139 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as thrift from '../utils/thrift-runtime';
6
+
7
+ /** Marker for a column encrypted with the file footer key. */
8
+ export class EncryptionWithFooterKey {
9
+ /** Writes the empty footer-key marker. */
10
+ public write(output: thrift.TProtocol): void {
11
+ output.writeStructBegin('EncryptionWithFooterKey');
12
+ output.writeFieldStop();
13
+ output.writeStructEnd();
14
+ }
15
+ /** Reads the empty footer-key marker. */
16
+ public static read(input: thrift.TProtocol): EncryptionWithFooterKey {
17
+ input.readStructBegin();
18
+ while (true) {
19
+ const field = input.readFieldBegin();
20
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
21
+ input.skip(field.ftype);
22
+ input.readFieldEnd();
23
+ }
24
+ input.readStructEnd();
25
+ return new EncryptionWithFooterKey();
26
+ }
27
+ }
28
+
29
+ /** Metadata needed to retrieve a column-specific encryption key. */
30
+ export class EncryptionWithColumnKey {
31
+ public path_in_schema: string[];
32
+ public key_metadata?: Uint8Array;
33
+ constructor(args: {path_in_schema: string[]; key_metadata?: Uint8Array}) {
34
+ this.path_in_schema = args.path_in_schema;
35
+ this.key_metadata = args.key_metadata;
36
+ }
37
+
38
+ /** Writes column-key retrieval metadata. */
39
+ public write(output: thrift.TProtocol): void {
40
+ output.writeStructBegin('EncryptionWithColumnKey');
41
+ output.writeFieldBegin('path_in_schema', thrift.Thrift.Type.LIST, 1);
42
+ output.writeListBegin(thrift.Thrift.Type.STRING, this.path_in_schema.length);
43
+ for (const pathPart of this.path_in_schema) output.writeString(pathPart);
44
+ output.writeListEnd();
45
+ output.writeFieldEnd();
46
+ if (this.key_metadata) {
47
+ output.writeFieldBegin('key_metadata', thrift.Thrift.Type.STRING, 2);
48
+ output.writeBinary(this.key_metadata);
49
+ output.writeFieldEnd();
50
+ }
51
+ output.writeFieldStop();
52
+ output.writeStructEnd();
53
+ }
54
+
55
+ /** Reads column-key retrieval metadata. */
56
+ public static read(input: thrift.TProtocol): EncryptionWithColumnKey {
57
+ input.readStructBegin();
58
+ let pathInSchema: string[] | undefined;
59
+ let keyMetadata: Uint8Array | undefined;
60
+ while (true) {
61
+ const field = input.readFieldBegin();
62
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
63
+ if (field.fid === 1 && field.ftype === thrift.Thrift.Type.LIST) {
64
+ const list = input.readListBegin();
65
+ pathInSchema = [];
66
+ for (let index = 0; index < list.size; index++) pathInSchema.push(input.readString());
67
+ input.readListEnd();
68
+ } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.STRING) {
69
+ keyMetadata = input.readBinary();
70
+ } else {
71
+ input.skip(field.ftype);
72
+ }
73
+ input.readFieldEnd();
74
+ }
75
+ input.readStructEnd();
76
+ if (!pathInSchema) throw new Error('EncryptionWithColumnKey path_in_schema is required');
77
+ return new EncryptionWithColumnKey({path_in_schema: pathInSchema, key_metadata: keyMetadata});
78
+ }
79
+ }
80
+
81
+ /** Column encryption metadata union. */
82
+ export class ColumnCryptoMetaData {
83
+ public ENCRYPTION_WITH_FOOTER_KEY?: EncryptionWithFooterKey;
84
+ public ENCRYPTION_WITH_COLUMN_KEY?: EncryptionWithColumnKey;
85
+ constructor(args: {
86
+ ENCRYPTION_WITH_FOOTER_KEY?: EncryptionWithFooterKey;
87
+ ENCRYPTION_WITH_COLUMN_KEY?: EncryptionWithColumnKey;
88
+ }) {
89
+ if (
90
+ (args.ENCRYPTION_WITH_FOOTER_KEY ? 1 : 0) + (args.ENCRYPTION_WITH_COLUMN_KEY ? 1 : 0) !==
91
+ 1
92
+ ) {
93
+ throw new thrift.Thrift.TProtocolException(
94
+ thrift.Thrift.TProtocolExceptionType.INVALID_DATA,
95
+ 'ColumnCryptoMetaData must contain exactly one key reference'
96
+ );
97
+ }
98
+ Object.assign(this, args);
99
+ }
100
+
101
+ /** Writes the selected column encryption key reference. */
102
+ public write(output: thrift.TProtocol): void {
103
+ output.writeStructBegin('ColumnCryptoMetaData');
104
+ if (this.ENCRYPTION_WITH_FOOTER_KEY) {
105
+ output.writeFieldBegin('ENCRYPTION_WITH_FOOTER_KEY', thrift.Thrift.Type.STRUCT, 1);
106
+ this.ENCRYPTION_WITH_FOOTER_KEY.write(output);
107
+ output.writeFieldEnd();
108
+ } else if (this.ENCRYPTION_WITH_COLUMN_KEY) {
109
+ output.writeFieldBegin('ENCRYPTION_WITH_COLUMN_KEY', thrift.Thrift.Type.STRUCT, 2);
110
+ this.ENCRYPTION_WITH_COLUMN_KEY.write(output);
111
+ output.writeFieldEnd();
112
+ }
113
+ output.writeFieldStop();
114
+ output.writeStructEnd();
115
+ }
116
+
117
+ /** Reads column encryption metadata. */
118
+ public static read(input: thrift.TProtocol): ColumnCryptoMetaData {
119
+ input.readStructBegin();
120
+ let footerKey: EncryptionWithFooterKey | undefined;
121
+ let columnKey: EncryptionWithColumnKey | undefined;
122
+ while (true) {
123
+ const field = input.readFieldBegin();
124
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
125
+ if (field.fid === 1 && field.ftype === thrift.Thrift.Type.STRUCT) {
126
+ footerKey = EncryptionWithFooterKey.read(input);
127
+ } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.STRUCT) {
128
+ columnKey = EncryptionWithColumnKey.read(input);
129
+ } else {
130
+ input.skip(field.ftype);
131
+ }
132
+ input.readFieldEnd();
133
+ }
134
+ input.readStructEnd();
135
+ return footerKey
136
+ ? new ColumnCryptoMetaData({ENCRYPTION_WITH_FOOTER_KEY: footerKey})
137
+ : new ColumnCryptoMetaData({ENCRYPTION_WITH_COLUMN_KEY: columnKey!});
138
+ }
139
+ }
@@ -14,10 +14,12 @@ import {CompactInt64 as Int64} from '../utils/uint8-array-compact-protocol';
14
14
  import * as thrift from '../utils/thrift-runtime';
15
15
  import * as CompressionCodec from './CompressionCodec';
16
16
  import * as Encoding from './Encoding';
17
+ import * as GeospatialStatistics from './GeospatialStatistics';
17
18
  import * as KeyValue from './KeyValue';
18
19
  import * as PageEncodingStats from './PageEncodingStats';
19
20
  import * as Statistics from './Statistics';
20
21
  import * as Type from './Type';
22
+ import * as SizeStatistics from './SizeStatistics';
21
23
  export interface IColumnMetaDataArgs {
22
24
  type: Type.Type;
23
25
  encodings: Array<Encoding.Encoding>;
@@ -32,6 +34,10 @@ export interface IColumnMetaDataArgs {
32
34
  dictionary_page_offset?: number | Int64;
33
35
  statistics?: Statistics.Statistics;
34
36
  encoding_stats?: Array<PageEncodingStats.PageEncodingStats>;
37
+ bloom_filter_offset?: number | Int64;
38
+ bloom_filter_length?: number;
39
+ geospatial_statistics?: GeospatialStatistics.GeospatialStatistics;
40
+ size_statistics?: SizeStatistics.SizeStatistics;
35
41
  }
36
42
  export class ColumnMetaData {
37
43
  public type: Type.Type;
@@ -47,6 +53,10 @@ export class ColumnMetaData {
47
53
  public dictionary_page_offset?: Int64;
48
54
  public statistics?: Statistics.Statistics;
49
55
  public encoding_stats?: Array<PageEncodingStats.PageEncodingStats>;
56
+ public bloom_filter_offset?: Int64;
57
+ public bloom_filter_length?: number;
58
+ public geospatial_statistics?: GeospatialStatistics.GeospatialStatistics;
59
+ public size_statistics?: SizeStatistics.SizeStatistics;
50
60
  constructor(args: IColumnMetaDataArgs) {
51
61
  if (args != null && args.type != null) {
52
62
  this.type = args.type;
@@ -151,6 +161,21 @@ export class ColumnMetaData {
151
161
  if (args != null && args.encoding_stats != null) {
152
162
  this.encoding_stats = args.encoding_stats;
153
163
  }
164
+ if (args != null && args.bloom_filter_offset != null) {
165
+ this.bloom_filter_offset =
166
+ typeof args.bloom_filter_offset === 'number'
167
+ ? new Int64(args.bloom_filter_offset)
168
+ : args.bloom_filter_offset;
169
+ }
170
+ if (args != null && args.bloom_filter_length != null) {
171
+ this.bloom_filter_length = args.bloom_filter_length;
172
+ }
173
+ if (args != null && args.geospatial_statistics != null) {
174
+ this.geospatial_statistics = args.geospatial_statistics;
175
+ }
176
+ if (args != null && args.size_statistics != null) {
177
+ this.size_statistics = args.size_statistics;
178
+ }
154
179
  }
155
180
  public write(output: thrift.TProtocol): void {
156
181
  output.writeStructBegin('ColumnMetaData');
@@ -235,6 +260,26 @@ export class ColumnMetaData {
235
260
  output.writeListEnd();
236
261
  output.writeFieldEnd();
237
262
  }
263
+ if (this.bloom_filter_offset != null) {
264
+ output.writeFieldBegin('bloom_filter_offset', thrift.Thrift.Type.I64, 14);
265
+ output.writeI64(this.bloom_filter_offset);
266
+ output.writeFieldEnd();
267
+ }
268
+ if (this.bloom_filter_length != null) {
269
+ output.writeFieldBegin('bloom_filter_length', thrift.Thrift.Type.I32, 15);
270
+ output.writeI32(this.bloom_filter_length);
271
+ output.writeFieldEnd();
272
+ }
273
+ if (this.geospatial_statistics != null) {
274
+ output.writeFieldBegin('geospatial_statistics', thrift.Thrift.Type.STRUCT, 17);
275
+ this.geospatial_statistics.write(output);
276
+ output.writeFieldEnd();
277
+ }
278
+ if (this.size_statistics != null) {
279
+ output.writeFieldBegin('size_statistics', thrift.Thrift.Type.STRUCT, 16);
280
+ this.size_statistics.write(output);
281
+ output.writeFieldEnd();
282
+ }
238
283
  output.writeFieldStop();
239
284
  output.writeStructEnd();
240
285
  return;
@@ -384,6 +429,34 @@ export class ColumnMetaData {
384
429
  input.skip(fieldType);
385
430
  }
386
431
  break;
432
+ case 14:
433
+ if (fieldType === thrift.Thrift.Type.I64) {
434
+ _args.bloom_filter_offset = input.readI64();
435
+ } else {
436
+ input.skip(fieldType);
437
+ }
438
+ break;
439
+ case 15:
440
+ if (fieldType === thrift.Thrift.Type.I32) {
441
+ _args.bloom_filter_length = input.readI32();
442
+ } else {
443
+ input.skip(fieldType);
444
+ }
445
+ break;
446
+ case 17:
447
+ if (fieldType === thrift.Thrift.Type.STRUCT) {
448
+ _args.geospatial_statistics = GeospatialStatistics.GeospatialStatistics.read(input);
449
+ } else {
450
+ input.skip(fieldType);
451
+ }
452
+ break;
453
+ case 16:
454
+ if (fieldType === thrift.Thrift.Type.STRUCT) {
455
+ _args.size_statistics = SizeStatistics.SizeStatistics.read(input);
456
+ } else {
457
+ input.skip(fieldType);
458
+ }
459
+ break;
387
460
  default: {
388
461
  input.skip(fieldType);
389
462
  }
@@ -0,0 +1,127 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as thrift from '../utils/thrift-runtime';
6
+
7
+ /** AES-GCM algorithm parameters from the Parquet modular-encryption spec. */
8
+ export class AesGcmV1 {
9
+ public aad_prefix?: Uint8Array;
10
+ public aad_file_unique?: Uint8Array;
11
+ public supply_aad_prefix?: boolean;
12
+
13
+ constructor(args: Partial<AesGcmV1> = {}) {
14
+ Object.assign(this, args);
15
+ }
16
+ }
17
+
18
+ /** AES-GCM metadata algorithm with AES-CTR page encryption. */
19
+ export class AesGcmCtrV1 extends AesGcmV1 {}
20
+
21
+ /** Parquet modular-encryption algorithm union. */
22
+ export class EncryptionAlgorithm {
23
+ public AES_GCM_V1?: AesGcmV1;
24
+ public AES_GCM_CTR_V1?: AesGcmCtrV1;
25
+
26
+ constructor(args: Partial<EncryptionAlgorithm> = {}) {
27
+ const fields = [args.AES_GCM_V1, args.AES_GCM_CTR_V1].filter(Boolean);
28
+ if (fields.length !== 1) {
29
+ throw new thrift.Thrift.TProtocolException(
30
+ thrift.Thrift.TProtocolExceptionType.INVALID_DATA,
31
+ 'EncryptionAlgorithm must contain exactly one algorithm'
32
+ );
33
+ }
34
+ Object.assign(this, args);
35
+ }
36
+
37
+ /** Writes the selected encryption algorithm union. */
38
+ public write(output: thrift.TProtocol): void {
39
+ output.writeStructBegin('EncryptionAlgorithm');
40
+ if (this.AES_GCM_V1) {
41
+ writeAlgorithm(output, 'AES_GCM_V1', 1, this.AES_GCM_V1);
42
+ } else if (this.AES_GCM_CTR_V1) {
43
+ writeAlgorithm(output, 'AES_GCM_CTR_V1', 2, this.AES_GCM_CTR_V1);
44
+ }
45
+ output.writeFieldStop();
46
+ output.writeStructEnd();
47
+ }
48
+
49
+ /** Reads an encryption algorithm union. */
50
+ public static read(input: thrift.TProtocol): EncryptionAlgorithm {
51
+ input.readStructBegin();
52
+ let algorithm: EncryptionAlgorithm | undefined;
53
+ while (true) {
54
+ const field = input.readFieldBegin();
55
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
56
+ if (field.fid === 1 && field.ftype === thrift.Thrift.Type.STRUCT) {
57
+ algorithm = new EncryptionAlgorithm({AES_GCM_V1: readAlgorithm(input)});
58
+ } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.STRUCT) {
59
+ algorithm = new EncryptionAlgorithm({AES_GCM_CTR_V1: readAlgorithm(input)});
60
+ } else {
61
+ input.skip(field.ftype);
62
+ }
63
+ input.readFieldEnd();
64
+ }
65
+ input.readStructEnd();
66
+ if (!algorithm) {
67
+ throw new thrift.Thrift.TProtocolException(
68
+ thrift.Thrift.TProtocolExceptionType.INVALID_DATA,
69
+ 'EncryptionAlgorithm union has no selected algorithm'
70
+ );
71
+ }
72
+ return algorithm;
73
+ }
74
+ }
75
+
76
+ function writeAlgorithm(
77
+ output: thrift.TProtocol,
78
+ name: string,
79
+ fieldId: number,
80
+ algorithm: AesGcmV1
81
+ ): void {
82
+ output.writeFieldBegin(name, thrift.Thrift.Type.STRUCT, fieldId);
83
+ output.writeStructBegin(name);
84
+ if (algorithm.aad_prefix) writeBinary(output, 'aad_prefix', 1, algorithm.aad_prefix);
85
+ if (algorithm.aad_file_unique)
86
+ writeBinary(output, 'aad_file_unique', 2, algorithm.aad_file_unique);
87
+ if (algorithm.supply_aad_prefix !== undefined) {
88
+ output.writeFieldBegin('supply_aad_prefix', thrift.Thrift.Type.BOOL, 3);
89
+ output.writeBool(algorithm.supply_aad_prefix);
90
+ output.writeFieldEnd();
91
+ }
92
+ output.writeFieldStop();
93
+ output.writeStructEnd();
94
+ output.writeFieldEnd();
95
+ }
96
+
97
+ function readAlgorithm(input: thrift.TProtocol): AesGcmV1 {
98
+ input.readStructBegin();
99
+ const result = new AesGcmV1();
100
+ while (true) {
101
+ const field = input.readFieldBegin();
102
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
103
+ if (field.fid === 1 && field.ftype === thrift.Thrift.Type.STRING) {
104
+ result.aad_prefix = input.readBinary();
105
+ } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.STRING) {
106
+ result.aad_file_unique = input.readBinary();
107
+ } else if (field.fid === 3 && field.ftype === thrift.Thrift.Type.BOOL) {
108
+ result.supply_aad_prefix = input.readBool();
109
+ } else {
110
+ input.skip(field.ftype);
111
+ }
112
+ input.readFieldEnd();
113
+ }
114
+ input.readStructEnd();
115
+ return result;
116
+ }
117
+
118
+ function writeBinary(
119
+ output: thrift.TProtocol,
120
+ name: string,
121
+ fieldId: number,
122
+ value: Uint8Array
123
+ ): void {
124
+ output.writeFieldBegin(name, thrift.Thrift.Type.STRING, fieldId);
125
+ output.writeBinary(value);
126
+ output.writeFieldEnd();
127
+ }
@@ -0,0 +1,60 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as thrift from '../utils/thrift-runtime';
6
+ import * as EncryptionAlgorithm from './EncryptionAlgorithm';
7
+
8
+ /** File-level crypto metadata stored before an encrypted Parquet footer. */
9
+ export class FileCryptoMetaData {
10
+ public encryption_algorithm: EncryptionAlgorithm.EncryptionAlgorithm;
11
+ public key_metadata?: Uint8Array;
12
+ constructor(args: {
13
+ encryption_algorithm: EncryptionAlgorithm.EncryptionAlgorithm;
14
+ key_metadata?: Uint8Array;
15
+ }) {
16
+ this.encryption_algorithm = args.encryption_algorithm;
17
+ this.key_metadata = args.key_metadata;
18
+ }
19
+
20
+ /** Writes file-level encryption metadata. */
21
+ public write(output: thrift.TProtocol): void {
22
+ output.writeStructBegin('FileCryptoMetaData');
23
+ output.writeFieldBegin('encryption_algorithm', thrift.Thrift.Type.STRUCT, 1);
24
+ this.encryption_algorithm.write(output);
25
+ output.writeFieldEnd();
26
+ if (this.key_metadata) {
27
+ output.writeFieldBegin('key_metadata', thrift.Thrift.Type.STRING, 2);
28
+ output.writeBinary(this.key_metadata);
29
+ output.writeFieldEnd();
30
+ }
31
+ output.writeFieldStop();
32
+ output.writeStructEnd();
33
+ }
34
+
35
+ /** Reads file-level encryption metadata. */
36
+ public static read(input: thrift.TProtocol): FileCryptoMetaData {
37
+ input.readStructBegin();
38
+ let encryptionAlgorithm: EncryptionAlgorithm.EncryptionAlgorithm | undefined;
39
+ let keyMetadata: Uint8Array | undefined;
40
+ while (true) {
41
+ const field = input.readFieldBegin();
42
+ if (field.ftype === thrift.Thrift.Type.STOP) break;
43
+ if (field.fid === 1 && field.ftype === thrift.Thrift.Type.STRUCT) {
44
+ encryptionAlgorithm = EncryptionAlgorithm.EncryptionAlgorithm.read(input);
45
+ } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.STRING) {
46
+ keyMetadata = input.readBinary();
47
+ } else {
48
+ input.skip(field.ftype);
49
+ }
50
+ input.readFieldEnd();
51
+ }
52
+ input.readStructEnd();
53
+ if (!encryptionAlgorithm)
54
+ throw new Error('FileCryptoMetaData encryption_algorithm is required');
55
+ return new FileCryptoMetaData({
56
+ encryption_algorithm: encryptionAlgorithm,
57
+ key_metadata: keyMetadata
58
+ });
59
+ }
60
+ }