@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (790) hide show
  1. package/LICENSE +14 -3
  2. package/dist/bundled.d.ts +9 -0
  3. package/dist/bundled.d.ts.map +1 -0
  4. package/dist/bundled.js +10 -0
  5. package/dist/bundled.js.map +1 -0
  6. package/dist/delta-format.d.ts +9 -0
  7. package/dist/delta-format.d.ts.map +1 -0
  8. package/dist/delta-format.js +12 -0
  9. package/dist/delta-format.js.map +1 -0
  10. package/dist/delta-source-loader-types.d.ts +27 -0
  11. package/dist/delta-source-loader-types.d.ts.map +1 -0
  12. package/dist/delta-source-loader-types.js +28 -0
  13. package/dist/delta-source-loader-types.js.map +1 -0
  14. package/dist/delta-source.d.ts +48 -0
  15. package/dist/delta-source.d.ts.map +1 -0
  16. package/dist/delta-source.js +222 -0
  17. package/dist/delta-source.js.map +1 -0
  18. package/dist/delta-types.d.ts +38 -0
  19. package/dist/delta-types.d.ts.map +1 -0
  20. package/dist/delta-types.js +5 -0
  21. package/dist/delta-types.js.map +1 -0
  22. package/dist/geoparquet-loader-with-parser.d.ts +33 -0
  23. package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
  24. package/dist/geoparquet-loader-with-parser.js +52 -0
  25. package/dist/geoparquet-loader-with-parser.js.map +1 -0
  26. package/dist/geoparquet-loader.d.ts +59 -0
  27. package/dist/geoparquet-loader.d.ts.map +1 -0
  28. package/dist/geoparquet-loader.js +30 -0
  29. package/dist/geoparquet-loader.js.map +1 -0
  30. package/dist/iceberg-rest-catalog.d.ts +49 -0
  31. package/dist/iceberg-rest-catalog.d.ts.map +1 -0
  32. package/dist/iceberg-rest-catalog.js +78 -0
  33. package/dist/iceberg-rest-catalog.js.map +1 -0
  34. package/dist/iceberg-table-source.d.ts +63 -0
  35. package/dist/iceberg-table-source.d.ts.map +1 -0
  36. package/dist/iceberg-table-source.js +797 -0
  37. package/dist/iceberg-table-source.js.map +1 -0
  38. package/dist/iceberg-types.d.ts +150 -0
  39. package/dist/iceberg-types.d.ts.map +1 -0
  40. package/dist/iceberg-types.js +5 -0
  41. package/dist/iceberg-types.js.map +1 -0
  42. package/dist/index.cjs +41568 -6781
  43. package/dist/index.cjs.map +3 -4
  44. package/dist/index.d.ts +15 -9
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +11 -12
  47. package/dist/index.js.map +1 -1
  48. package/dist/lib/arrow/convert-columns-to-row-group.js +3 -0
  49. package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -1
  50. package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
  51. package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
  52. package/dist/lib/arrow/convert-schema-from-parquet.js +164 -29
  53. package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  54. package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -1
  55. package/dist/lib/arrow/convert-schema-to-parquet.js +16 -7
  56. package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -1
  57. package/dist/lib/arrow/create-nested-arrow-vector.d.ts +8 -0
  58. package/dist/lib/arrow/create-nested-arrow-vector.d.ts.map +1 -0
  59. package/dist/lib/arrow/create-nested-arrow-vector.js +485 -0
  60. package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -0
  61. package/dist/lib/constants.d.ts +0 -1
  62. package/dist/lib/constants.d.ts.map +1 -1
  63. package/dist/lib/constants.js +1 -2
  64. package/dist/lib/constants.js.map +1 -1
  65. package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
  66. package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
  67. package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
  68. package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
  69. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
  70. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
  71. package/dist/lib/encoders/encode-table-to-parquet-js.js +394 -0
  72. package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
  73. package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
  74. package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
  75. package/dist/lib/geo/geoparquet-covering.js +137 -0
  76. package/dist/lib/geo/geoparquet-covering.js.map +1 -0
  77. package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
  78. package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
  79. package/dist/lib/geo/geospatial-metadata.js +412 -0
  80. package/dist/lib/geo/geospatial-metadata.js.map +1 -0
  81. package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
  82. package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
  83. package/dist/lib/parquet-bloom-filter-planner.js +57 -0
  84. package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
  85. package/dist/lib/parquet-bloom-filter.d.ts +30 -0
  86. package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
  87. package/dist/lib/parquet-bloom-filter.js +289 -0
  88. package/dist/lib/parquet-bloom-filter.js.map +1 -0
  89. package/dist/lib/parquet-encryption.d.ts +80 -0
  90. package/dist/lib/parquet-encryption.d.ts.map +1 -0
  91. package/dist/lib/parquet-encryption.js +190 -0
  92. package/dist/lib/parquet-encryption.js.map +1 -0
  93. package/dist/lib/parquet-page-index.d.ts +91 -0
  94. package/dist/lib/parquet-page-index.d.ts.map +1 -0
  95. package/dist/lib/parquet-page-index.js +463 -0
  96. package/dist/lib/parquet-page-index.js.map +1 -0
  97. package/dist/lib/parquet-predicate.d.ts +21 -0
  98. package/dist/lib/parquet-predicate.d.ts.map +1 -0
  99. package/dist/lib/parquet-predicate.js +328 -0
  100. package/dist/lib/parquet-predicate.js.map +1 -0
  101. package/dist/lib/parquet-source-worker-client.d.ts +6 -0
  102. package/dist/lib/parquet-source-worker-client.d.ts.map +1 -0
  103. package/dist/lib/parquet-source-worker-client.js +24 -0
  104. package/dist/lib/parquet-source-worker-client.js.map +1 -0
  105. package/dist/lib/parquet-source-worker-decoder.d.ts +7 -0
  106. package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -0
  107. package/dist/lib/parquet-source-worker-decoder.js +181 -0
  108. package/dist/lib/parquet-source-worker-decoder.js.map +1 -0
  109. package/dist/lib/parquet-source-worker-file.d.ts +24 -0
  110. package/dist/lib/parquet-source-worker-file.d.ts.map +1 -0
  111. package/dist/lib/parquet-source-worker-file.js +40 -0
  112. package/dist/lib/parquet-source-worker-file.js.map +1 -0
  113. package/dist/lib/parquet-source-worker-types.d.ts +121 -0
  114. package/dist/lib/parquet-source-worker-types.d.ts.map +1 -0
  115. package/dist/lib/parquet-source-worker-types.js +5 -0
  116. package/dist/lib/parquet-source-worker-types.js.map +1 -0
  117. package/dist/lib/parquet-worker-transport.d.ts +8 -0
  118. package/dist/lib/parquet-worker-transport.d.ts.map +1 -0
  119. package/dist/lib/parquet-worker-transport.js +40 -0
  120. package/dist/lib/parquet-worker-transport.js.map +1 -0
  121. package/dist/lib/parsers/convert-parquet-tables.d.ts +16 -0
  122. package/dist/lib/parsers/convert-parquet-tables.d.ts.map +1 -0
  123. package/dist/lib/parsers/convert-parquet-tables.js +30 -0
  124. package/dist/lib/parsers/convert-parquet-tables.js.map +1 -0
  125. package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
  126. package/dist/lib/parsers/get-parquet-schema.js +6 -2
  127. package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
  128. package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
  129. package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
  130. package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
  131. package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
  132. package/dist/lib/parsers/parse-parquet-tables.d.ts +37 -0
  133. package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
  134. package/dist/lib/parsers/parse-parquet-tables.js +50 -0
  135. package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
  136. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +19 -0
  137. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
  138. package/dist/lib/parsers/parse-parquet-to-arrow-js.js +921 -0
  139. package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
  140. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
  141. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
  142. package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
  143. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
  144. package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
  145. package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
  146. package/dist/lib/parsers/parse-parquet-to-columns.js +4 -5
  147. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  148. package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
  149. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  150. package/dist/lib/parsers/parse-parquet-to-json.js +95 -28
  151. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
  152. package/dist/lib/slice-parquet-batch.d.ts +9 -0
  153. package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
  154. package/dist/lib/slice-parquet-batch.js +28 -0
  155. package/dist/lib/slice-parquet-batch.js.map +1 -0
  156. package/dist/lib/sources/parquet-range-file.d.ts +80 -0
  157. package/dist/lib/sources/parquet-range-file.d.ts.map +1 -0
  158. package/dist/lib/sources/parquet-range-file.js +267 -0
  159. package/dist/lib/sources/parquet-range-file.js.map +1 -0
  160. package/dist/lib/sources/single-batch-queue.d.ts +24 -0
  161. package/dist/lib/sources/single-batch-queue.d.ts.map +1 -0
  162. package/dist/lib/sources/single-batch-queue.js +91 -0
  163. package/dist/lib/sources/single-batch-queue.js.map +1 -0
  164. package/dist/lib/utils/load-wasm-node.d.ts +3 -0
  165. package/dist/lib/utils/load-wasm-node.d.ts.map +1 -0
  166. package/dist/lib/utils/load-wasm-node.js +23 -0
  167. package/dist/lib/utils/load-wasm-node.js.map +1 -0
  168. package/dist/lib/utils/load-wasm.d.ts +2 -2
  169. package/dist/lib/utils/load-wasm.d.ts.map +1 -1
  170. package/dist/lib/utils/load-wasm.js +28 -12
  171. package/dist/lib/utils/load-wasm.js.map +1 -1
  172. package/dist/lib/utils/make-stream-iterator.d.ts +2 -0
  173. package/dist/lib/utils/make-stream-iterator.d.ts.map +1 -1
  174. package/dist/lib/utils/make-stream-iterator.js +123 -14
  175. package/dist/lib/utils/make-stream-iterator.js.map +1 -1
  176. package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
  177. package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
  178. package/dist/lib/utils/normalize-parquet-options.js +25 -0
  179. package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
  180. package/dist/parquet-dataset-source.cjs +38974 -0
  181. package/dist/parquet-dataset-source.cjs.map +6 -0
  182. package/dist/parquet-dataset-source.d.ts +81 -0
  183. package/dist/parquet-dataset-source.d.ts.map +1 -0
  184. package/dist/parquet-dataset-source.js +602 -0
  185. package/dist/parquet-dataset-source.js.map +1 -0
  186. package/dist/parquet-encoded-page-types.d.ts +108 -0
  187. package/dist/parquet-encoded-page-types.d.ts.map +1 -0
  188. package/dist/parquet-encoded-page-types.js +5 -0
  189. package/dist/parquet-encoded-page-types.js.map +1 -0
  190. package/dist/parquet-format.d.ts +2 -0
  191. package/dist/parquet-format.d.ts.map +1 -1
  192. package/dist/parquet-format.js +2 -0
  193. package/dist/parquet-format.js.map +1 -1
  194. package/dist/parquet-js-loader-types.d.ts +59 -0
  195. package/dist/parquet-js-loader-types.d.ts.map +1 -0
  196. package/dist/parquet-js-loader-types.js +34 -0
  197. package/dist/parquet-js-loader-types.js.map +1 -0
  198. package/dist/parquet-js-loader.d.ts +18 -22
  199. package/dist/parquet-js-loader.d.ts.map +1 -1
  200. package/dist/parquet-js-loader.js +53 -14
  201. package/dist/parquet-js-loader.js.map +1 -1
  202. package/dist/parquet-js-writer.d.ts +65 -0
  203. package/dist/parquet-js-writer.d.ts.map +1 -0
  204. package/dist/parquet-js-writer.js +28 -0
  205. package/dist/parquet-js-writer.js.map +1 -0
  206. package/dist/parquet-loader-base.d.ts +25 -0
  207. package/dist/parquet-loader-base.d.ts.map +1 -0
  208. package/dist/parquet-loader-base.js +23 -0
  209. package/dist/parquet-loader-base.js.map +1 -0
  210. package/dist/parquet-loader-options.d.ts +55 -0
  211. package/dist/parquet-loader-options.d.ts.map +1 -0
  212. package/dist/parquet-loader-options.js +10 -0
  213. package/dist/parquet-loader-options.js.map +1 -0
  214. package/dist/parquet-loader-types.d.ts +61 -0
  215. package/dist/parquet-loader-types.d.ts.map +1 -0
  216. package/dist/parquet-loader-types.js +17 -0
  217. package/dist/parquet-loader-types.js.map +1 -0
  218. package/dist/parquet-loader.d.ts +42 -0
  219. package/dist/parquet-loader.d.ts.map +1 -0
  220. package/dist/parquet-loader.js +68 -0
  221. package/dist/parquet-loader.js.map +1 -0
  222. package/dist/parquet-source-capabilities.d.ts +43 -0
  223. package/dist/parquet-source-capabilities.d.ts.map +1 -0
  224. package/dist/parquet-source-capabilities.js +32 -0
  225. package/dist/parquet-source-capabilities.js.map +1 -0
  226. package/dist/parquet-source-loader-types.d.ts +82 -0
  227. package/dist/parquet-source-loader-types.d.ts.map +1 -0
  228. package/dist/parquet-source-loader-types.js +84 -0
  229. package/dist/parquet-source-loader-types.js.map +1 -0
  230. package/dist/parquet-source-loader.cjs +38261 -0
  231. package/dist/parquet-source-loader.cjs.map +6 -0
  232. package/dist/parquet-source-loader.d.ts +167 -0
  233. package/dist/parquet-source-loader.d.ts.map +1 -0
  234. package/dist/parquet-source-loader.js +1438 -0
  235. package/dist/parquet-source-loader.js.map +1 -0
  236. package/dist/parquet-source-types.d.ts +534 -0
  237. package/dist/parquet-source-types.d.ts.map +1 -0
  238. package/dist/parquet-source-types.js +5 -0
  239. package/dist/parquet-source-types.js.map +1 -0
  240. package/dist/parquet-source-worker-factory.d.ts +4 -0
  241. package/dist/parquet-source-worker-factory.d.ts.map +1 -0
  242. package/dist/parquet-source-worker-factory.js +6 -0
  243. package/dist/parquet-source-worker-factory.js.map +1 -0
  244. package/dist/parquet-source-worker-url.d.ts +3 -0
  245. package/dist/parquet-source-worker-url.d.ts.map +1 -0
  246. package/dist/parquet-source-worker-url.js +22 -0
  247. package/dist/parquet-source-worker-url.js.map +1 -0
  248. package/dist/parquet-source-worker.js +38871 -0
  249. package/dist/parquet-source-worker.js.map +7 -0
  250. package/dist/parquet-worker-node.cjs +78507 -0
  251. package/dist/parquet-worker-node.cjs.map +7 -0
  252. package/dist/parquet-worker-url.d.ts +3 -0
  253. package/dist/parquet-worker-url.d.ts.map +1 -0
  254. package/dist/parquet-worker-url.js +19 -0
  255. package/dist/parquet-worker-url.js.map +1 -0
  256. package/dist/parquet-worker.js +20094 -0
  257. package/dist/parquet-worker.js.map +7 -0
  258. package/dist/parquet-writer.d.ts +10 -0
  259. package/dist/parquet-writer.d.ts.map +1 -0
  260. package/dist/parquet-writer.js +27 -0
  261. package/dist/parquet-writer.js.map +1 -0
  262. package/dist/parquet_wasm_bg.wasm +0 -0
  263. package/dist/parquetjs/codecs/byte-stream-split.d.ts +9 -0
  264. package/dist/parquetjs/codecs/byte-stream-split.d.ts.map +1 -0
  265. package/dist/parquetjs/codecs/byte-stream-split.js +159 -0
  266. package/dist/parquetjs/codecs/byte-stream-split.js.map +1 -0
  267. package/dist/parquetjs/codecs/declare.d.ts +35 -3
  268. package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
  269. package/dist/parquetjs/codecs/declare.js +17 -1
  270. package/dist/parquetjs/codecs/declare.js.map +1 -1
  271. package/dist/parquetjs/codecs/delta.d.ts +17 -0
  272. package/dist/parquetjs/codecs/delta.d.ts.map +1 -0
  273. package/dist/parquetjs/codecs/delta.js +788 -0
  274. package/dist/parquetjs/codecs/delta.js.map +1 -0
  275. package/dist/parquetjs/codecs/dictionary.d.ts +5 -2
  276. package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -1
  277. package/dist/parquetjs/codecs/dictionary.js +16 -4
  278. package/dist/parquetjs/codecs/dictionary.js.map +1 -1
  279. package/dist/parquetjs/codecs/index.d.ts.map +1 -1
  280. package/dist/parquetjs/codecs/index.js +22 -0
  281. package/dist/parquetjs/codecs/index.js.map +1 -1
  282. package/dist/parquetjs/codecs/plain.d.ts +3 -3
  283. package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
  284. package/dist/parquetjs/codecs/plain.js +231 -78
  285. package/dist/parquetjs/codecs/plain.js.map +1 -1
  286. package/dist/parquetjs/codecs/rle.d.ts +7 -3
  287. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  288. package/dist/parquetjs/codecs/rle.js +613 -69
  289. package/dist/parquetjs/codecs/rle.js.map +1 -1
  290. package/dist/parquetjs/compression.d.ts +16 -8
  291. package/dist/parquetjs/compression.d.ts.map +1 -1
  292. package/dist/parquetjs/compression.js +114 -141
  293. package/dist/parquetjs/compression.js.map +1 -1
  294. package/dist/parquetjs/encoder/dictionary-planner.d.ts +17 -0
  295. package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -0
  296. package/dist/parquetjs/encoder/dictionary-planner.js +67 -0
  297. package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -0
  298. package/dist/parquetjs/encoder/page-planner.d.ts +11 -0
  299. package/dist/parquetjs/encoder/page-planner.d.ts.map +1 -0
  300. package/dist/parquetjs/encoder/page-planner.js +60 -0
  301. package/dist/parquetjs/encoder/page-planner.js.map +1 -0
  302. package/dist/parquetjs/encoder/parquet-encoder.d.ts +54 -5
  303. package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
  304. package/dist/parquetjs/encoder/parquet-encoder.js +930 -66
  305. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
  306. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
  307. package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
  308. package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
  309. package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
  310. package/dist/parquetjs/parquet-thrift/BsonType.d.ts +1 -1
  311. package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -1
  312. package/dist/parquetjs/parquet-thrift/BsonType.js +1 -1
  313. package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  314. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +7 -2
  315. package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
  316. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +37 -2
  317. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  318. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
  319. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
  320. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
  321. package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
  322. package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +6 -6
  323. package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
  324. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +3 -3
  325. package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  326. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +12 -2
  327. package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
  328. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +75 -2
  329. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  330. package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +1 -1
  331. package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
  332. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -1
  333. package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  334. package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +1 -1
  335. package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
  336. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -1
  337. package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  338. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +1 -1
  339. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
  340. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -1
  341. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  342. package/dist/parquetjs/parquet-thrift/DateType.d.ts +1 -1
  343. package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -1
  344. package/dist/parquetjs/parquet-thrift/DateType.js +1 -1
  345. package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -1
  346. package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +1 -1
  347. package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -1
  348. package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -1
  349. package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  350. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +1 -1
  351. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
  352. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -1
  353. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  354. package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts +8 -0
  355. package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts.map +1 -0
  356. package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js +14 -0
  357. package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js.map +1 -0
  358. package/dist/parquetjs/parquet-thrift/Encoding.d.ts +2 -1
  359. package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -1
  360. package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
  361. package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -1
  362. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
  363. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
  364. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
  365. package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
  366. package/dist/parquetjs/parquet-thrift/EnumType.d.ts +1 -1
  367. package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -1
  368. package/dist/parquetjs/parquet-thrift/EnumType.js +1 -1
  369. package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  370. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
  371. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
  372. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
  373. package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
  374. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +7 -2
  375. package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
  376. package/dist/parquetjs/parquet-thrift/FileMetaData.js +37 -2
  377. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  378. package/dist/parquetjs/parquet-thrift/Float16Type.d.ts +9 -0
  379. package/dist/parquetjs/parquet-thrift/Float16Type.d.ts.map +1 -0
  380. package/dist/parquetjs/parquet-thrift/Float16Type.js +27 -0
  381. package/dist/parquetjs/parquet-thrift/Float16Type.js.map +1 -0
  382. package/dist/parquetjs/parquet-thrift/GeographyType.d.ts +14 -0
  383. package/dist/parquetjs/parquet-thrift/GeographyType.d.ts.map +1 -0
  384. package/dist/parquetjs/parquet-thrift/GeographyType.js +51 -0
  385. package/dist/parquetjs/parquet-thrift/GeographyType.js.map +1 -0
  386. package/dist/parquetjs/parquet-thrift/GeometryType.d.ts +11 -0
  387. package/dist/parquetjs/parquet-thrift/GeometryType.d.ts.map +1 -0
  388. package/dist/parquetjs/parquet-thrift/GeometryType.js +41 -0
  389. package/dist/parquetjs/parquet-thrift/GeometryType.js.map +1 -0
  390. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
  391. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
  392. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
  393. package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
  394. package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +1 -1
  395. package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -1
  396. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -1
  397. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  398. package/dist/parquetjs/parquet-thrift/IntType.d.ts +1 -1
  399. package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -1
  400. package/dist/parquetjs/parquet-thrift/IntType.js +1 -1
  401. package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -1
  402. package/dist/parquetjs/parquet-thrift/JsonType.d.ts +1 -1
  403. package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -1
  404. package/dist/parquetjs/parquet-thrift/JsonType.js +1 -1
  405. package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  406. package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +1 -1
  407. package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -1
  408. package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -1
  409. package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  410. package/dist/parquetjs/parquet-thrift/ListType.d.ts +1 -1
  411. package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -1
  412. package/dist/parquetjs/parquet-thrift/ListType.js +1 -1
  413. package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -1
  414. package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +17 -1
  415. package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
  416. package/dist/parquetjs/parquet-thrift/LogicalType.js +97 -1
  417. package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  418. package/dist/parquetjs/parquet-thrift/MapType.d.ts +1 -1
  419. package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -1
  420. package/dist/parquetjs/parquet-thrift/MapType.js +1 -1
  421. package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -1
  422. package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +1 -1
  423. package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -1
  424. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -1
  425. package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  426. package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +1 -1
  427. package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -1
  428. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -1
  429. package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  430. package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts +9 -0
  431. package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts.map +1 -0
  432. package/dist/parquetjs/parquet-thrift/NanoSeconds.js +27 -0
  433. package/dist/parquetjs/parquet-thrift/NanoSeconds.js.map +1 -0
  434. package/dist/parquetjs/parquet-thrift/NullType.d.ts +1 -1
  435. package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -1
  436. package/dist/parquetjs/parquet-thrift/NullType.js +1 -1
  437. package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -1
  438. package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +1 -1
  439. package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
  440. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -1
  441. package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  442. package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +1 -1
  443. package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
  444. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -1
  445. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  446. package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +1 -1
  447. package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
  448. package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -1
  449. package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  450. package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +2 -2
  451. package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -1
  452. package/dist/parquetjs/parquet-thrift/PageLocation.js +2 -2
  453. package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  454. package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +2 -2
  455. package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
  456. package/dist/parquetjs/parquet-thrift/RowGroup.js +2 -2
  457. package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  458. package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +1 -1
  459. package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
  460. package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -1
  461. package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  462. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
  463. package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
  464. package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
  465. package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
  466. package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +1 -1
  467. package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -1
  468. package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -1
  469. package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  470. package/dist/parquetjs/parquet-thrift/Statistics.d.ts +14 -10
  471. package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -1
  472. package/dist/parquetjs/parquet-thrift/Statistics.js +38 -2
  473. package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  474. package/dist/parquetjs/parquet-thrift/StringType.d.ts +1 -1
  475. package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -1
  476. package/dist/parquetjs/parquet-thrift/StringType.js +1 -1
  477. package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -1
  478. package/dist/parquetjs/parquet-thrift/TimeType.d.ts +1 -1
  479. package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
  480. package/dist/parquetjs/parquet-thrift/TimeType.js +1 -1
  481. package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  482. package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +5 -1
  483. package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
  484. package/dist/parquetjs/parquet-thrift/TimeUnit.js +25 -1
  485. package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  486. package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +1 -1
  487. package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
  488. package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -1
  489. package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  490. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +1 -1
  491. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -1
  492. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -1
  493. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  494. package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +1 -1
  495. package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -1
  496. package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -1
  497. package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  498. package/dist/parquetjs/parquet-thrift/VariantType.d.ts +11 -0
  499. package/dist/parquetjs/parquet-thrift/VariantType.d.ts.map +1 -0
  500. package/dist/parquetjs/parquet-thrift/VariantType.js +41 -0
  501. package/dist/parquetjs/parquet-thrift/VariantType.js.map +1 -0
  502. package/dist/parquetjs/parquet-thrift/index.d.ts +12 -2
  503. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
  504. package/dist/parquetjs/parquet-thrift/index.js +12 -2
  505. package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
  506. package/dist/parquetjs/parser/decoders.d.ts +23 -3
  507. package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
  508. package/dist/parquetjs/parser/decoders.js +687 -119
  509. package/dist/parquetjs/parser/decoders.js.map +1 -1
  510. package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
  511. package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
  512. package/dist/parquetjs/parser/encoded-pages.js +206 -0
  513. package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
  514. package/dist/parquetjs/parser/parquet-reader.d.ts +111 -10
  515. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
  516. package/dist/parquetjs/parser/parquet-reader.js +836 -69
  517. package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
  518. package/dist/parquetjs/schema/declare.d.ts +83 -10
  519. package/dist/parquetjs/schema/declare.d.ts.map +1 -1
  520. package/dist/parquetjs/schema/declare.js.map +1 -1
  521. package/dist/parquetjs/schema/schema.d.ts +3 -0
  522. package/dist/parquetjs/schema/schema.d.ts.map +1 -1
  523. package/dist/parquetjs/schema/schema.js +31 -3
  524. package/dist/parquetjs/schema/schema.js.map +1 -1
  525. package/dist/parquetjs/schema/shred.d.ts +2 -0
  526. package/dist/parquetjs/schema/shred.d.ts.map +1 -1
  527. package/dist/parquetjs/schema/shred.js +285 -29
  528. package/dist/parquetjs/schema/shred.js.map +1 -1
  529. package/dist/parquetjs/schema/types.d.ts.map +1 -1
  530. package/dist/parquetjs/schema/types.js +290 -71
  531. package/dist/parquetjs/schema/types.js.map +1 -1
  532. package/dist/parquetjs/schema/variant.d.ts +3 -0
  533. package/dist/parquetjs/schema/variant.d.ts.map +1 -0
  534. package/dist/parquetjs/schema/variant.js +248 -0
  535. package/dist/parquetjs/schema/variant.js.map +1 -0
  536. package/dist/parquetjs/utils/binary-utils.d.ts +41 -0
  537. package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
  538. package/dist/parquetjs/utils/binary-utils.js +106 -0
  539. package/dist/parquetjs/utils/binary-utils.js.map +1 -0
  540. package/dist/parquetjs/utils/crc32.d.ts +3 -0
  541. package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
  542. package/dist/parquetjs/utils/crc32.js +15 -0
  543. package/dist/parquetjs/utils/crc32.js.map +1 -0
  544. package/dist/parquetjs/utils/file-utils.d.ts +1 -1
  545. package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
  546. package/dist/parquetjs/utils/file-utils.js +3 -3
  547. package/dist/parquetjs/utils/file-utils.js.map +1 -1
  548. package/dist/parquetjs/utils/read-utils.d.ts +25 -7
  549. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
  550. package/dist/parquetjs/utils/read-utils.js +66 -39
  551. package/dist/parquetjs/utils/read-utils.js.map +1 -1
  552. package/dist/parquetjs/utils/thrift-runtime.d.ts +106 -0
  553. package/dist/parquetjs/utils/thrift-runtime.d.ts.map +1 -0
  554. package/dist/parquetjs/utils/thrift-runtime.js +50 -0
  555. package/dist/parquetjs/utils/thrift-runtime.js.map +1 -0
  556. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
  557. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
  558. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
  559. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
  560. package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +80 -0
  561. package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
  562. package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
  563. package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
  564. package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
  565. package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
  566. package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
  567. package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
  568. package/dist/polyfills/process.d.ts +12 -0
  569. package/dist/polyfills/process.d.ts.map +1 -0
  570. package/dist/polyfills/process.js +17 -0
  571. package/dist/polyfills/process.js.map +1 -0
  572. package/dist/unbundled.d.ts +10 -0
  573. package/dist/unbundled.d.ts.map +1 -0
  574. package/dist/unbundled.js +11 -0
  575. package/dist/unbundled.js.map +1 -0
  576. package/dist/wasm.d.ts +3 -0
  577. package/dist/wasm.d.ts.map +1 -0
  578. package/dist/wasm.js +6 -0
  579. package/dist/wasm.js.map +1 -0
  580. package/dist/workers/parquet-source-worker.d.ts +2 -0
  581. package/dist/workers/parquet-source-worker.d.ts.map +1 -0
  582. package/dist/workers/parquet-source-worker.js +17 -0
  583. package/dist/workers/parquet-source-worker.js.map +1 -0
  584. package/dist/workers/parquet-worker-node.d.ts +2 -0
  585. package/dist/workers/parquet-worker-node.d.ts.map +1 -0
  586. package/dist/workers/parquet-worker-node.js +5 -0
  587. package/dist/workers/parquet-worker-node.js.map +1 -0
  588. package/dist/workers/parquet-worker.d.ts +2 -0
  589. package/dist/workers/parquet-worker.d.ts.map +1 -0
  590. package/dist/workers/parquet-worker.js +27 -0
  591. package/dist/workers/parquet-worker.js.map +1 -0
  592. package/package.json +84 -40
  593. package/src/bundled.ts +43 -0
  594. package/src/delta-format.ts +12 -0
  595. package/src/delta-source-loader-types.ts +40 -0
  596. package/src/delta-source.ts +305 -0
  597. package/src/delta-types.ts +40 -0
  598. package/src/geoparquet-loader-with-parser.ts +95 -0
  599. package/src/geoparquet-loader.ts +54 -0
  600. package/src/iceberg-rest-catalog.ts +122 -0
  601. package/src/iceberg-table-source.ts +1037 -0
  602. package/src/iceberg-types.ts +165 -0
  603. package/src/index.ts +105 -26
  604. package/src/lib/arrow/convert-columns-to-row-group.ts +3 -0
  605. package/src/lib/arrow/convert-schema-from-parquet.ts +193 -32
  606. package/src/lib/arrow/convert-schema-to-parquet.ts +16 -7
  607. package/src/lib/arrow/create-nested-arrow-vector.ts +705 -0
  608. package/src/lib/constants.ts +0 -2
  609. package/src/lib/encoders/encode-arrow-to-parquet.ts +10 -16
  610. package/src/lib/encoders/encode-table-to-parquet-js.ts +526 -0
  611. package/src/lib/geo/geoparquet-covering.ts +201 -0
  612. package/src/lib/geo/geospatial-metadata.ts +547 -0
  613. package/src/lib/parquet-bloom-filter-planner.ts +86 -0
  614. package/src/lib/parquet-bloom-filter.ts +345 -0
  615. package/src/lib/parquet-encryption.ts +344 -0
  616. package/src/lib/parquet-page-index.ts +727 -0
  617. package/src/lib/parquet-predicate.ts +410 -0
  618. package/src/lib/parquet-source-worker-client.ts +41 -0
  619. package/src/lib/parquet-source-worker-decoder.ts +252 -0
  620. package/src/lib/parquet-source-worker-file.ts +56 -0
  621. package/src/lib/parquet-source-worker-types.ts +132 -0
  622. package/src/lib/parquet-worker-transport.ts +57 -0
  623. package/src/lib/parsers/convert-parquet-tables.ts +39 -0
  624. package/src/lib/parsers/get-parquet-schema.ts +6 -2
  625. package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
  626. package/src/lib/parsers/parse-parquet-tables.ts +82 -0
  627. package/src/lib/parsers/parse-parquet-to-arrow-js.ts +1213 -0
  628. package/src/lib/parsers/parse-parquet-to-arrow.ts +62 -36
  629. package/src/lib/parsers/parse-parquet-to-columns.ts +9 -8
  630. package/src/lib/parsers/parse-parquet-to-json.ts +119 -35
  631. package/src/lib/slice-parquet-batch.ts +32 -0
  632. package/src/lib/sources/parquet-range-file.ts +365 -0
  633. package/src/lib/sources/single-batch-queue.ts +95 -0
  634. package/src/lib/utils/load-wasm-node.ts +25 -0
  635. package/src/lib/utils/load-wasm.ts +36 -13
  636. package/src/lib/utils/make-stream-iterator.ts +126 -14
  637. package/src/lib/utils/normalize-parquet-options.ts +30 -0
  638. package/src/parquet-dataset-source.ts +800 -0
  639. package/src/parquet-encoded-page-types.ts +122 -0
  640. package/src/parquet-format.ts +2 -0
  641. package/src/parquet-js-loader-types.ts +53 -0
  642. package/src/parquet-js-loader.ts +76 -19
  643. package/src/parquet-js-writer.ts +98 -0
  644. package/src/parquet-loader-base.ts +34 -0
  645. package/src/parquet-loader-options.ts +62 -0
  646. package/src/parquet-loader-types.ts +35 -0
  647. package/src/parquet-loader.ts +115 -0
  648. package/src/parquet-source-capabilities.ts +73 -0
  649. package/src/parquet-source-loader-types.ts +97 -0
  650. package/src/parquet-source-loader.ts +2254 -0
  651. package/src/parquet-source-types.ts +627 -0
  652. package/src/parquet-source-worker-factory.ts +8 -0
  653. package/src/parquet-source-worker-url.ts +23 -0
  654. package/src/parquet-worker-url.ts +23 -0
  655. package/src/parquet-writer.ts +42 -0
  656. package/src/parquetjs/LICENSE +23 -1
  657. package/src/parquetjs/codecs/byte-stream-split.ts +221 -0
  658. package/src/parquetjs/codecs/declare.ts +62 -3
  659. package/src/parquetjs/codecs/delta.ts +1047 -0
  660. package/src/parquetjs/codecs/dictionary.ts +22 -4
  661. package/src/parquetjs/codecs/index.ts +22 -0
  662. package/src/parquetjs/codecs/plain.ts +317 -89
  663. package/src/parquetjs/codecs/rle.ts +852 -82
  664. package/src/parquetjs/compression.ts +137 -156
  665. package/src/parquetjs/encoder/dictionary-planner.ts +104 -0
  666. package/src/parquetjs/encoder/page-planner.ts +83 -0
  667. package/src/parquetjs/encoder/parquet-encoder.ts +1374 -95
  668. package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
  669. package/src/parquetjs/parquet-thrift/BsonType.ts +1 -1
  670. package/src/parquetjs/parquet-thrift/ColumnChunk.ts +37 -2
  671. package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
  672. package/src/parquetjs/parquet-thrift/ColumnIndex.ts +13 -13
  673. package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +75 -2
  674. package/src/parquetjs/parquet-thrift/ColumnOrder.ts +1 -1
  675. package/src/parquetjs/parquet-thrift/DataPageHeader.ts +1 -1
  676. package/src/parquetjs/parquet-thrift/DataPageHeaderV2.ts +1 -1
  677. package/src/parquetjs/parquet-thrift/DateType.ts +1 -1
  678. package/src/parquetjs/parquet-thrift/DecimalType.ts +1 -1
  679. package/src/parquetjs/parquet-thrift/DictionaryPageHeader.ts +1 -1
  680. package/src/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.ts +13 -0
  681. package/src/parquetjs/parquet-thrift/Encoding.ts +2 -1
  682. package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
  683. package/src/parquetjs/parquet-thrift/EnumType.ts +1 -1
  684. package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
  685. package/src/parquetjs/parquet-thrift/FileMetaData.ts +37 -2
  686. package/src/parquetjs/parquet-thrift/Float16Type.ts +28 -0
  687. package/src/parquetjs/parquet-thrift/GeographyType.ts +54 -0
  688. package/src/parquetjs/parquet-thrift/GeometryType.ts +43 -0
  689. package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
  690. package/src/parquetjs/parquet-thrift/IndexPageHeader.ts +1 -1
  691. package/src/parquetjs/parquet-thrift/IntType.ts +1 -1
  692. package/src/parquetjs/parquet-thrift/JsonType.ts +1 -1
  693. package/src/parquetjs/parquet-thrift/KeyValue.ts +1 -1
  694. package/src/parquetjs/parquet-thrift/ListType.ts +1 -1
  695. package/src/parquetjs/parquet-thrift/LogicalType.ts +97 -1
  696. package/src/parquetjs/parquet-thrift/MapType.ts +1 -1
  697. package/src/parquetjs/parquet-thrift/MicroSeconds.ts +1 -1
  698. package/src/parquetjs/parquet-thrift/MilliSeconds.ts +1 -1
  699. package/src/parquetjs/parquet-thrift/NanoSeconds.ts +28 -0
  700. package/src/parquetjs/parquet-thrift/NullType.ts +1 -1
  701. package/src/parquetjs/parquet-thrift/OffsetIndex.ts +1 -1
  702. package/src/parquetjs/parquet-thrift/PageEncodingStats.ts +1 -1
  703. package/src/parquetjs/parquet-thrift/PageHeader.ts +1 -1
  704. package/src/parquetjs/parquet-thrift/PageLocation.ts +2 -2
  705. package/src/parquetjs/parquet-thrift/RowGroup.ts +2 -2
  706. package/src/parquetjs/parquet-thrift/SchemaElement.ts +1 -1
  707. package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
  708. package/src/parquetjs/parquet-thrift/SortingColumn.ts +1 -1
  709. package/src/parquetjs/parquet-thrift/Statistics.ts +50 -14
  710. package/src/parquetjs/parquet-thrift/StringType.ts +1 -1
  711. package/src/parquetjs/parquet-thrift/TimeType.ts +1 -1
  712. package/src/parquetjs/parquet-thrift/TimeUnit.ts +25 -1
  713. package/src/parquetjs/parquet-thrift/TimestampType.ts +1 -1
  714. package/src/parquetjs/parquet-thrift/TypeDefinedOrder.ts +1 -1
  715. package/src/parquetjs/parquet-thrift/UUIDType.ts +1 -1
  716. package/src/parquetjs/parquet-thrift/VariantType.ts +43 -0
  717. package/src/parquetjs/parquet-thrift/index.ts +12 -4
  718. package/src/parquetjs/parser/decoders.ts +922 -143
  719. package/src/parquetjs/parser/encoded-pages.ts +278 -0
  720. package/src/parquetjs/parser/parquet-reader.ts +1334 -83
  721. package/src/parquetjs/schema/declare.ts +121 -9
  722. package/src/parquetjs/schema/schema.ts +33 -3
  723. package/src/parquetjs/schema/shred.ts +345 -41
  724. package/src/parquetjs/schema/types.ts +308 -84
  725. package/src/parquetjs/schema/variant.ts +313 -0
  726. package/src/parquetjs/utils/binary-utils.ts +132 -0
  727. package/src/parquetjs/utils/crc32.ts +15 -0
  728. package/src/parquetjs/utils/file-utils.ts +4 -4
  729. package/src/parquetjs/utils/read-utils.ts +77 -51
  730. package/src/parquetjs/utils/thrift-runtime.ts +109 -0
  731. package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
  732. package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
  733. package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
  734. package/src/polyfills/process.ts +17 -0
  735. package/src/unbundled.ts +47 -0
  736. package/src/wasm.ts +6 -0
  737. package/src/workers/parquet-source-worker.ts +21 -0
  738. package/src/workers/parquet-worker-node.ts +5 -0
  739. package/src/workers/parquet-worker.ts +31 -0
  740. package/dist/parquet-arrow-loader.d.ts +0 -43
  741. package/dist/parquet-arrow-loader.d.ts.map +0 -1
  742. package/dist/parquet-arrow-loader.js +0 -49
  743. package/dist/parquet-arrow-loader.js.map +0 -1
  744. package/dist/parquet-arrow-writer.d.ts +0 -9
  745. package/dist/parquet-arrow-writer.d.ts.map +0 -1
  746. package/dist/parquet-arrow-writer.js +0 -23
  747. package/dist/parquet-arrow-writer.js.map +0 -1
  748. package/dist/parquet-json-loader.d.ts +0 -148
  749. package/dist/parquet-json-loader.d.ts.map +0 -1
  750. package/dist/parquet-json-loader.js +0 -106
  751. package/dist/parquet-json-loader.js.map +0 -1
  752. package/dist/parquet-json-writer.d.ts +0 -19
  753. package/dist/parquet-json-writer.d.ts.map +0 -1
  754. package/dist/parquet-json-writer.js +0 -20
  755. package/dist/parquet-json-writer.js.map +0 -1
  756. package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
  757. package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
  758. package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
  759. package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
  760. package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
  761. package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
  762. package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
  763. package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
  764. package/dist/polyfills/buffer/buffer.d.ts +0 -222
  765. package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
  766. package/dist/polyfills/buffer/buffer.js +0 -1868
  767. package/dist/polyfills/buffer/buffer.js.map +0 -1
  768. package/dist/polyfills/buffer/index.d.ts +0 -4
  769. package/dist/polyfills/buffer/index.d.ts.map +0 -1
  770. package/dist/polyfills/buffer/index.js +0 -9
  771. package/dist/polyfills/buffer/index.js.map +0 -1
  772. package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
  773. package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
  774. package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
  775. package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
  776. package/dist/workers/parquet-json-worker.d.ts +0 -2
  777. package/dist/workers/parquet-json-worker.d.ts.map +0 -1
  778. package/dist/workers/parquet-json-worker.js +0 -7
  779. package/dist/workers/parquet-json-worker.js.map +0 -1
  780. package/src/parquet-arrow-loader.ts +0 -74
  781. package/src/parquet-arrow-writer.ts +0 -33
  782. package/src/parquet-json-loader.ts +0 -160
  783. package/src/parquet-json-writer.ts +0 -27
  784. package/src/parquetjs/modules.d.ts +0 -25
  785. package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
  786. package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
  787. package/src/polyfills/buffer/buffer.ts +0 -2205
  788. package/src/polyfills/buffer/index.ts +0 -11
  789. package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
  790. package/src/workers/parquet-json-worker.ts +0 -8
@@ -0,0 +1,109 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ /** Minimal Thrift runtime types needed by the generated Parquet metadata classes. */
6
+ export namespace Thrift {
7
+ /** Thrift wire types used by the Parquet compact protocol. */
8
+ export enum Type {
9
+ STOP = 0,
10
+ VOID = 1,
11
+ BOOL = 2,
12
+ BYTE = 3,
13
+ DOUBLE = 4,
14
+ I16 = 6,
15
+ I32 = 8,
16
+ I64 = 10,
17
+ STRING = 11,
18
+ STRUCT = 12,
19
+ MAP = 13,
20
+ SET = 14,
21
+ LIST = 15,
22
+ UTF8 = 16,
23
+ UTF16 = 17
24
+ }
25
+
26
+ /** Protocol failure categories retained for generated-code compatibility. */
27
+ export enum TProtocolExceptionType {
28
+ UNKNOWN = 0,
29
+ INVALID_DATA = 1,
30
+ NEGATIVE_SIZE = 2,
31
+ SIZE_LIMIT = 3,
32
+ BAD_VERSION = 4,
33
+ NOT_IMPLEMENTED = 5,
34
+ DEPTH_LIMIT = 6
35
+ }
36
+
37
+ /** Error raised when generated metadata cannot be encoded or decoded. */
38
+ export class TProtocolException extends Error {
39
+ /** Thrift protocol error category. */
40
+ readonly type: TProtocolExceptionType;
41
+
42
+ /** Creates a protocol exception without importing the Node-oriented thrift package. */
43
+ constructor(type: TProtocolExceptionType, message = 'Thrift protocol error') {
44
+ super(message);
45
+ this.name = 'TProtocolException';
46
+ this.type = type;
47
+ }
48
+ }
49
+ }
50
+
51
+ /** Thrift struct descriptor returned by compact-protocol readers. */
52
+ export type TStruct = {fname: string};
53
+
54
+ /** Thrift field descriptor returned by compact-protocol readers. */
55
+ export type TField = {fname: string; ftype: Thrift.Type; fid: number};
56
+
57
+ /** Thrift map descriptor returned by compact-protocol readers. */
58
+ export type TMap = {ktype: Thrift.Type; vtype: Thrift.Type; size: number};
59
+
60
+ /** Thrift list descriptor returned by compact-protocol readers. */
61
+ export type TList = {etype: Thrift.Type; size: number};
62
+
63
+ /** Thrift set descriptor returned by compact-protocol readers. */
64
+ export type TSet = TList;
65
+
66
+ /**
67
+ * Compact-protocol surface consumed by the generated Parquet metadata classes.
68
+ * Binary values use typed-array views and remain browser-native.
69
+ */
70
+ export interface TProtocol {
71
+ writeStructBegin(name: string): void;
72
+ writeStructEnd(): void;
73
+ writeFieldBegin(name: string, fieldType: Thrift.Type, fieldId: number): void;
74
+ writeFieldEnd(): void;
75
+ writeFieldStop(): void;
76
+ writeMapBegin(keyType: Thrift.Type, valueType: Thrift.Type, size: number): void;
77
+ writeMapEnd(): void;
78
+ writeListBegin(elementType: Thrift.Type, size: number): void;
79
+ writeListEnd(): void;
80
+ writeSetBegin(elementType: Thrift.Type, size: number): void;
81
+ writeSetEnd(): void;
82
+ writeBool(value: boolean): void;
83
+ writeByte(value: number): void;
84
+ writeI16(value: number): void;
85
+ writeI32(value: number): void;
86
+ writeI64(value: unknown): void;
87
+ writeDouble(value: number): void;
88
+ writeString(value: string): void;
89
+ writeBinary(value: ArrayBuffer | ArrayBufferView): void;
90
+ readStructBegin(): TStruct;
91
+ readStructEnd(): void;
92
+ readFieldBegin(): TField;
93
+ readFieldEnd(): void;
94
+ readMapBegin(): TMap;
95
+ readMapEnd(): void;
96
+ readListBegin(): TList;
97
+ readListEnd(): void;
98
+ readSetBegin(): TSet;
99
+ readSetEnd(): void;
100
+ readBool(): boolean;
101
+ readByte(): number;
102
+ readI16(): number;
103
+ readI32(): number;
104
+ readI64(): any;
105
+ readDouble(): number;
106
+ readString(): string;
107
+ readBinary(): Uint8Array;
108
+ skip(type: Thrift.Type): void;
109
+ }
@@ -0,0 +1,254 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as thrift from './thrift-runtime';
6
+ import {concatUint8Arrays, encodeUtf8, toUint8Array, writeDoubleLE} from './binary-utils';
7
+
8
+ const COMPACT_TYPE_STOP = 0x00;
9
+ const COMPACT_TYPE_BOOLEAN_TRUE = 0x01;
10
+ const COMPACT_TYPE_BOOLEAN_FALSE = 0x02;
11
+ const COMPACT_TYPE_BYTE = 0x03;
12
+ const COMPACT_TYPE_I16 = 0x04;
13
+ const COMPACT_TYPE_I32 = 0x05;
14
+ const COMPACT_TYPE_I64 = 0x06;
15
+ const COMPACT_TYPE_DOUBLE = 0x07;
16
+ const COMPACT_TYPE_BINARY = 0x08;
17
+ const COMPACT_TYPE_LIST = 0x09;
18
+ const COMPACT_TYPE_SET = 0x0a;
19
+ const COMPACT_TYPE_MAP = 0x0b;
20
+ const COMPACT_TYPE_STRUCT = 0x0c;
21
+
22
+ type Int64Like = number | bigint | {toString(radix?: number): string; valueOf?: () => number};
23
+
24
+ /** Write-only Thrift compact protocol that assembles exact Uint8Array output. */
25
+ export class Uint8ArrayCompactProtocolWriter {
26
+ /** Serialized byte chunks accumulated by thrift write calls. */
27
+ private readonly chunks: Uint8Array[] = [];
28
+
29
+ /** Field id at the current nesting depth, used to emit compact field-id deltas. */
30
+ private lastFieldId = 0;
31
+
32
+ /** Parent struct field ids restored when nested structs end. */
33
+ private readonly lastFieldIds: number[] = [];
34
+
35
+ /** Boolean field header waiting for the following boolean payload. */
36
+ private pendingBooleanField: {fid: number} | null = null;
37
+
38
+ /** Enter a struct and reset the delta-encoded field id state. */
39
+ writeStructBegin(_name: string): void {
40
+ this.lastFieldIds.push(this.lastFieldId);
41
+ this.lastFieldId = 0;
42
+ }
43
+
44
+ /** Exit a struct and restore the parent field id state. */
45
+ writeStructEnd(): void {
46
+ this.lastFieldId = this.lastFieldIds.pop() || 0;
47
+ }
48
+
49
+ /** Write a field header, deferring boolean fields until their value is known. */
50
+ writeFieldBegin(_name: string, fieldType: thrift.Thrift.Type, fid: number): void {
51
+ if (fieldType === thrift.Thrift.Type.BOOL) {
52
+ this.pendingBooleanField = {fid};
53
+ return;
54
+ }
55
+
56
+ this.writeFieldHeader(this.getCompactType(fieldType), fid);
57
+ }
58
+
59
+ /** Complete the current field; compact protocol has no field-end bytes. */
60
+ writeFieldEnd(): void {}
61
+
62
+ /** Write the stop marker that terminates a thrift struct. */
63
+ writeFieldStop(): void {
64
+ this.writeByteDirect(COMPACT_TYPE_STOP);
65
+ }
66
+
67
+ /** Write a map header with compact key/value element types. */
68
+ writeMapBegin(keyType: thrift.Thrift.Type, valueType: thrift.Thrift.Type, size: number): void {
69
+ if (size === 0) {
70
+ this.writeByteDirect(0);
71
+ return;
72
+ }
73
+
74
+ this.writeVarint32(size);
75
+ this.writeByteDirect((this.getCompactType(keyType) << 4) | this.getCompactType(valueType));
76
+ }
77
+
78
+ /** Complete the current map; compact protocol has no map-end bytes. */
79
+ writeMapEnd(): void {}
80
+
81
+ /** Write a list header with compact element type and element count. */
82
+ writeListBegin(elementType: thrift.Thrift.Type, size: number): void {
83
+ this.writeCollectionBegin(elementType, size);
84
+ }
85
+
86
+ /** Complete the current list; compact protocol has no list-end bytes. */
87
+ writeListEnd(): void {}
88
+
89
+ /** Write a set header with compact element type and element count. */
90
+ writeSetBegin(elementType: thrift.Thrift.Type, size: number): void {
91
+ this.writeCollectionBegin(elementType, size);
92
+ }
93
+
94
+ /** Complete the current set; compact protocol has no set-end bytes. */
95
+ writeSetEnd(): void {}
96
+
97
+ /** Write a boolean, either inline in its field header or as a value byte. */
98
+ writeBool(value: boolean): void {
99
+ const compactType = value ? COMPACT_TYPE_BOOLEAN_TRUE : COMPACT_TYPE_BOOLEAN_FALSE;
100
+ if (this.pendingBooleanField) {
101
+ this.writeFieldHeader(compactType, this.pendingBooleanField.fid);
102
+ this.pendingBooleanField = null;
103
+ return;
104
+ }
105
+
106
+ this.writeByteDirect(compactType);
107
+ }
108
+
109
+ /** Write a signed byte. */
110
+ writeByte(value: number): void {
111
+ this.writeByteDirect(value);
112
+ }
113
+
114
+ /** Write a signed 16-bit integer as zig-zag varint. */
115
+ writeI16(value: number): void {
116
+ this.writeI32(value);
117
+ }
118
+
119
+ /** Write a signed 32-bit integer as zig-zag varint. */
120
+ writeI32(value: number): void {
121
+ this.writeVarint32(this.encodeZigZag32(value));
122
+ }
123
+
124
+ /** Write a signed 64-bit-like value as zig-zag varint. */
125
+ writeI64(value: Int64Like): void {
126
+ this.writeVarint64(this.encodeZigZag64(this.toBigInt(value)));
127
+ }
128
+
129
+ /** Write a little-endian 64-bit float. */
130
+ writeDouble(value: number): void {
131
+ const bytes = new Uint8Array(8);
132
+ writeDoubleLE(bytes, value, 0);
133
+ this.writeBytes(bytes);
134
+ }
135
+
136
+ /** Write a length-prefixed UTF-8 string. */
137
+ writeString(value: string): void {
138
+ this.writeBinary(encodeUtf8(value));
139
+ }
140
+
141
+ /** Write a length-prefixed binary payload from any typed-array view. */
142
+ writeBinary(value: ArrayBuffer | ArrayBufferView): void {
143
+ const bytes = toUint8Array(value);
144
+ this.writeVarint32(bytes.length);
145
+ this.writeBytes(bytes);
146
+ }
147
+
148
+ /** Return all serialized chunks as one contiguous byte array. */
149
+ getBytes(): Uint8Array {
150
+ return concatUint8Arrays(this.chunks);
151
+ }
152
+
153
+ /** Write a compact collection header for list or set values. */
154
+ private writeCollectionBegin(elementType: thrift.Thrift.Type, size: number): void {
155
+ const compactType = this.getCompactType(elementType);
156
+ if (size <= 14) {
157
+ this.writeByteDirect((size << 4) | compactType);
158
+ return;
159
+ }
160
+
161
+ this.writeByteDirect(0xf0 | compactType);
162
+ this.writeVarint32(size);
163
+ }
164
+
165
+ /** Write a compact field header using a field-id delta when possible. */
166
+ private writeFieldHeader(compactType: number, fid: number): void {
167
+ const fieldIdDelta = fid - this.lastFieldId;
168
+ if (fieldIdDelta > 0 && fieldIdDelta <= 15) {
169
+ this.writeByteDirect((fieldIdDelta << 4) | compactType);
170
+ } else {
171
+ this.writeByteDirect(compactType);
172
+ this.writeI16(fid);
173
+ }
174
+ this.lastFieldId = fid;
175
+ }
176
+
177
+ /** Append an already serialized byte range without copying. */
178
+ private writeBytes(bytes: Uint8Array): void {
179
+ this.chunks.push(bytes);
180
+ }
181
+
182
+ /** Append one byte to the serialized output. */
183
+ private writeByteDirect(value: number): void {
184
+ this.chunks.push(new Uint8Array([value & 0xff]));
185
+ }
186
+
187
+ /** Write an unsigned 32-bit value as a compact varint. */
188
+ private writeVarint32(value: number): void {
189
+ this.writeVarint64(BigInt(value >>> 0));
190
+ }
191
+
192
+ /** Write an unsigned 64-bit value as a compact varint. */
193
+ private writeVarint64(value: bigint): void {
194
+ let nextValue = value;
195
+ while (true) {
196
+ if ((nextValue & ~0x7fn) === 0n) {
197
+ this.writeByteDirect(Number(nextValue));
198
+ return;
199
+ }
200
+
201
+ this.writeByteDirect(Number((nextValue & 0x7fn) | 0x80n));
202
+ nextValue >>= 7n;
203
+ }
204
+ }
205
+
206
+ /** Encode a signed 32-bit integer as an unsigned zig-zag value. */
207
+ private encodeZigZag32(value: number): number {
208
+ return ((value << 1) ^ (value >> 31)) >>> 0;
209
+ }
210
+
211
+ /** Encode a signed 64-bit integer as an unsigned zig-zag value. */
212
+ private encodeZigZag64(value: bigint): bigint {
213
+ return (value << 1n) ^ (value >> 63n);
214
+ }
215
+
216
+ /** Normalize thrift int64-like values to bigint before zig-zag encoding. */
217
+ private toBigInt(value: Int64Like): bigint {
218
+ return typeof value === 'bigint' || typeof value === 'number'
219
+ ? BigInt(value)
220
+ : BigInt(value.toString());
221
+ }
222
+
223
+ /** Convert a generated-thrift type to a compact protocol wire type. */
224
+ private getCompactType(thriftType: thrift.Thrift.Type): number {
225
+ switch (thriftType) {
226
+ case thrift.Thrift.Type.STOP:
227
+ return COMPACT_TYPE_STOP;
228
+ case thrift.Thrift.Type.BOOL:
229
+ return COMPACT_TYPE_BOOLEAN_TRUE;
230
+ case thrift.Thrift.Type.BYTE:
231
+ return COMPACT_TYPE_BYTE;
232
+ case thrift.Thrift.Type.I16:
233
+ return COMPACT_TYPE_I16;
234
+ case thrift.Thrift.Type.I32:
235
+ return COMPACT_TYPE_I32;
236
+ case thrift.Thrift.Type.I64:
237
+ return COMPACT_TYPE_I64;
238
+ case thrift.Thrift.Type.DOUBLE:
239
+ return COMPACT_TYPE_DOUBLE;
240
+ case thrift.Thrift.Type.STRING:
241
+ return COMPACT_TYPE_BINARY;
242
+ case thrift.Thrift.Type.LIST:
243
+ return COMPACT_TYPE_LIST;
244
+ case thrift.Thrift.Type.SET:
245
+ return COMPACT_TYPE_SET;
246
+ case thrift.Thrift.Type.MAP:
247
+ return COMPACT_TYPE_MAP;
248
+ case thrift.Thrift.Type.STRUCT:
249
+ return COMPACT_TYPE_STRUCT;
250
+ default:
251
+ throw new Error(`Unsupported thrift type: ${thriftType}`);
252
+ }
253
+ }
254
+ }
@@ -0,0 +1,322 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as thrift from './thrift-runtime';
6
+ import {readDoubleLE} from './binary-utils';
7
+ import {Uint8ArrayTransport} from './uint8-array-transport';
8
+
9
+ const COMPACT_TYPE_STOP = 0x00;
10
+ const COMPACT_TYPE_BOOLEAN_TRUE = 0x01;
11
+ const COMPACT_TYPE_BOOLEAN_FALSE = 0x02;
12
+ const COMPACT_TYPE_BYTE = 0x03;
13
+ const COMPACT_TYPE_I16 = 0x04;
14
+ const COMPACT_TYPE_I32 = 0x05;
15
+ const COMPACT_TYPE_I64 = 0x06;
16
+ const COMPACT_TYPE_DOUBLE = 0x07;
17
+ const COMPACT_TYPE_BINARY = 0x08;
18
+ const COMPACT_TYPE_LIST = 0x09;
19
+ const COMPACT_TYPE_SET = 0x0a;
20
+ const COMPACT_TYPE_MAP = 0x0b;
21
+ const COMPACT_TYPE_STRUCT = 0x0c;
22
+
23
+ /** Number-like replacement for node-int64 used by the compact protocol. */
24
+ export class CompactInt64 {
25
+ /** Signed integer payload represented without precision loss. */
26
+ private readonly value: bigint;
27
+
28
+ /** Create a compact-protocol int64 wrapper from a bigint. */
29
+ constructor(value: bigint | number | string | {toString(): string}) {
30
+ this.value = typeof value === 'bigint' ? value : BigInt(value.toString());
31
+ }
32
+
33
+ /** Return the payload as a JavaScript number. */
34
+ toNumber(): number {
35
+ return Number(this.value);
36
+ }
37
+
38
+ /** Return the payload as a JavaScript number for generated thrift arithmetic. */
39
+ valueOf(): number {
40
+ return this.toNumber();
41
+ }
42
+
43
+ /** Format the payload as a string. */
44
+ toString(radix?: number): string {
45
+ return this.value.toString(radix);
46
+ }
47
+ }
48
+
49
+ /** Read-only Thrift compact protocol backed by Uint8Array. */
50
+ export class Uint8ArrayCompactProtocol {
51
+ /** Byte transport that provides thrift-compatible read, borrow, and consume methods. */
52
+ private readonly transport: Uint8ArrayTransport;
53
+
54
+ /** Field id at the current nesting depth, used for compact field-id deltas. */
55
+ private lastFieldId = 0;
56
+
57
+ /** Parent struct field ids restored when nested structs end. */
58
+ private readonly lastFieldIds: number[] = [];
59
+
60
+ /** Boolean payload encoded in a compact field header. */
61
+ private boolValue: boolean | null = null;
62
+
63
+ /** Create a compact protocol reader over a Uint8Array-backed transport. */
64
+ constructor(transport: Uint8ArrayTransport) {
65
+ this.transport = transport;
66
+ }
67
+
68
+ /** Enter a struct and reset the delta-encoded field id state. */
69
+ readStructBegin(): thrift.TStruct {
70
+ this.lastFieldIds.push(this.lastFieldId);
71
+ this.lastFieldId = 0;
72
+ return {fname: ''};
73
+ }
74
+
75
+ readStructEnd(): void {
76
+ this.lastFieldId = this.lastFieldIds.pop() || 0;
77
+ }
78
+
79
+ /** Read a compact field header and expand it to a thrift field descriptor. */
80
+ readFieldBegin(): thrift.TField {
81
+ const fieldHeader = this.transport.readByte();
82
+ const compactType = fieldHeader & 0x0f;
83
+
84
+ if (compactType === COMPACT_TYPE_STOP) {
85
+ return {fname: null!, ftype: thrift.Thrift.Type.STOP, fid: 0};
86
+ }
87
+
88
+ const fieldIdDelta = (fieldHeader & 0xf0) >>> 4;
89
+ const fieldId = fieldIdDelta === 0 ? this.readI16() : this.lastFieldId + fieldIdDelta;
90
+ const fieldType = this.getThriftType(compactType);
91
+
92
+ if (compactType === COMPACT_TYPE_BOOLEAN_TRUE || compactType === COMPACT_TYPE_BOOLEAN_FALSE) {
93
+ this.boolValue = compactType === COMPACT_TYPE_BOOLEAN_TRUE;
94
+ }
95
+
96
+ this.lastFieldId = fieldId;
97
+ return {fname: null!, ftype: fieldType, fid: fieldId};
98
+ }
99
+
100
+ readFieldEnd(): void {}
101
+
102
+ /** Read a map header, including element compact types when the map is non-empty. */
103
+ readMapBegin(): thrift.TMap {
104
+ const size = this.readVarint32();
105
+ const keyAndValueType = size === 0 ? 0 : this.transport.readByte();
106
+ return {
107
+ ktype: this.getThriftType((keyAndValueType & 0xf0) >>> 4),
108
+ vtype: this.getThriftType(keyAndValueType & 0x0f),
109
+ size
110
+ };
111
+ }
112
+
113
+ readMapEnd(): void {}
114
+
115
+ /** Read a list header and expand the inline or varint element count. */
116
+ readListBegin(): thrift.TList {
117
+ const sizeAndType = this.transport.readByte();
118
+ let size = (sizeAndType >>> 4) & 0x0f;
119
+ if (size === 15) {
120
+ size = this.readVarint32();
121
+ }
122
+
123
+ return {
124
+ etype: this.getThriftType(sizeAndType & 0x0f),
125
+ size
126
+ };
127
+ }
128
+
129
+ readListEnd(): void {}
130
+
131
+ /** Read a set header; compact protocol encodes sets and lists the same way. */
132
+ readSetBegin(): thrift.TSet {
133
+ return this.readListBegin();
134
+ }
135
+
136
+ readSetEnd(): void {}
137
+
138
+ /** Read a boolean from the field header or from the next compact boolean byte. */
139
+ readBool(): boolean {
140
+ if (this.boolValue !== null) {
141
+ const value = this.boolValue;
142
+ this.boolValue = null;
143
+ return value;
144
+ }
145
+
146
+ return this.transport.readByte() === COMPACT_TYPE_BOOLEAN_TRUE;
147
+ }
148
+
149
+ /** Read a signed byte from the transport. */
150
+ readByte(): number {
151
+ return this.transport.readByte();
152
+ }
153
+
154
+ /** Read a signed 16-bit integer encoded as zig-zag varint. */
155
+ readI16(): number {
156
+ return this.readI32();
157
+ }
158
+
159
+ /** Read a signed 32-bit integer encoded as zig-zag varint. */
160
+ readI32(): number {
161
+ return this.decodeZigZag32(this.readVarint32());
162
+ }
163
+
164
+ /** Read a signed 64-bit integer encoded as zig-zag varint. */
165
+ readI64(): CompactInt64 {
166
+ return new CompactInt64(this.decodeZigZag64(this.readVarint64()));
167
+ }
168
+
169
+ /** Read a little-endian 64-bit float. */
170
+ readDouble(): number {
171
+ return readDoubleLE(this.transport.read(8), 0);
172
+ }
173
+
174
+ /** Read a length-prefixed binary payload as an exact Uint8Array. */
175
+ readBinary(): Uint8Array {
176
+ const size = this.readVarint32();
177
+ return size === 0 ? new Uint8Array(0) : this.transport.read(size);
178
+ }
179
+
180
+ /** Read a length-prefixed UTF-8 string. */
181
+ readString(): string {
182
+ const size = this.readVarint32();
183
+ return size === 0 ? '' : this.transport.readString(size);
184
+ }
185
+
186
+ /** Consume and discard a value with the supplied thrift type. */
187
+ skip(type: thrift.Thrift.Type): void {
188
+ switch (type) {
189
+ case thrift.Thrift.Type.STOP:
190
+ return;
191
+ case thrift.Thrift.Type.BOOL:
192
+ this.readBool();
193
+ return;
194
+ case thrift.Thrift.Type.BYTE:
195
+ this.readByte();
196
+ return;
197
+ case thrift.Thrift.Type.I16:
198
+ this.readI16();
199
+ return;
200
+ case thrift.Thrift.Type.I32:
201
+ this.readI32();
202
+ return;
203
+ case thrift.Thrift.Type.I64:
204
+ this.readI64();
205
+ return;
206
+ case thrift.Thrift.Type.DOUBLE:
207
+ this.readDouble();
208
+ return;
209
+ case thrift.Thrift.Type.STRING:
210
+ this.readBinary();
211
+ return;
212
+ case thrift.Thrift.Type.STRUCT:
213
+ this.skipStruct();
214
+ return;
215
+ case thrift.Thrift.Type.MAP:
216
+ this.skipMap();
217
+ return;
218
+ case thrift.Thrift.Type.SET:
219
+ case thrift.Thrift.Type.LIST:
220
+ this.skipList();
221
+ return;
222
+ default:
223
+ throw new Error(`Unsupported thrift type: ${type}`);
224
+ }
225
+ }
226
+
227
+ /** Read an unsigned 32-bit compact varint. */
228
+ private readVarint32(): number {
229
+ return Number(this.readVarint64());
230
+ }
231
+
232
+ /** Read an unsigned 64-bit compact varint. */
233
+ private readVarint64(): bigint {
234
+ let value = 0n;
235
+ let shift = 0n;
236
+
237
+ while (true) {
238
+ const byte = BigInt(this.transport.readByte() & 0xff);
239
+ value |= (byte & 0x7fn) << shift;
240
+ if ((byte & 0x80n) === 0n) {
241
+ return value;
242
+ }
243
+ shift += 7n;
244
+ }
245
+ }
246
+
247
+ /** Decode a zig-zag 32-bit integer to a signed integer. */
248
+ private decodeZigZag32(value: number): number {
249
+ return (value >>> 1) ^ -(value & 1);
250
+ }
251
+
252
+ /** Decode a zig-zag 64-bit integer to a signed integer. */
253
+ private decodeZigZag64(value: bigint): bigint {
254
+ return (value >> 1n) ^ -(value & 1n);
255
+ }
256
+
257
+ /** Convert a compact wire type to a generated-thrift type. */
258
+ private getThriftType(compactType: number): thrift.Thrift.Type {
259
+ switch (compactType) {
260
+ case COMPACT_TYPE_STOP:
261
+ return thrift.Thrift.Type.STOP;
262
+ case COMPACT_TYPE_BOOLEAN_TRUE:
263
+ case COMPACT_TYPE_BOOLEAN_FALSE:
264
+ return thrift.Thrift.Type.BOOL;
265
+ case COMPACT_TYPE_BYTE:
266
+ return thrift.Thrift.Type.BYTE;
267
+ case COMPACT_TYPE_I16:
268
+ return thrift.Thrift.Type.I16;
269
+ case COMPACT_TYPE_I32:
270
+ return thrift.Thrift.Type.I32;
271
+ case COMPACT_TYPE_I64:
272
+ return thrift.Thrift.Type.I64;
273
+ case COMPACT_TYPE_DOUBLE:
274
+ return thrift.Thrift.Type.DOUBLE;
275
+ case COMPACT_TYPE_BINARY:
276
+ return thrift.Thrift.Type.STRING;
277
+ case COMPACT_TYPE_LIST:
278
+ return thrift.Thrift.Type.LIST;
279
+ case COMPACT_TYPE_SET:
280
+ return thrift.Thrift.Type.SET;
281
+ case COMPACT_TYPE_MAP:
282
+ return thrift.Thrift.Type.MAP;
283
+ case COMPACT_TYPE_STRUCT:
284
+ return thrift.Thrift.Type.STRUCT;
285
+ default:
286
+ throw new Error(`Unknown compact thrift type: ${compactType}`);
287
+ }
288
+ }
289
+
290
+ /** Skip every field in the current struct. */
291
+ private skipStruct(): void {
292
+ this.readStructBegin();
293
+ while (true) {
294
+ const field = this.readFieldBegin();
295
+ if (field.ftype === thrift.Thrift.Type.STOP) {
296
+ break;
297
+ }
298
+ this.skip(field.ftype);
299
+ this.readFieldEnd();
300
+ }
301
+ this.readStructEnd();
302
+ }
303
+
304
+ /** Skip every key and value in the current map. */
305
+ private skipMap(): void {
306
+ const map = this.readMapBegin();
307
+ for (let index = 0; index < map.size; index++) {
308
+ this.skip(map.ktype);
309
+ this.skip(map.vtype);
310
+ }
311
+ this.readMapEnd();
312
+ }
313
+
314
+ /** Skip every element in the current list or set. */
315
+ private skipList(): void {
316
+ const list = this.readListBegin();
317
+ for (let index = 0; index < list.size; index++) {
318
+ this.skip(list.etype);
319
+ }
320
+ this.readListEnd();
321
+ }
322
+ }