@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/bundled.d.ts +9 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +10 -0
- package/dist/bundled.js.map +1 -0
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader-with-parser.d.ts +33 -0
- package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
- package/dist/geoparquet-loader-with-parser.js +52 -0
- package/dist/geoparquet-loader-with-parser.js.map +1 -0
- package/dist/geoparquet-loader.d.ts +59 -0
- package/dist/geoparquet-loader.d.ts.map +1 -0
- package/dist/geoparquet-loader.js +30 -0
- package/dist/geoparquet-loader.js.map +1 -0
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +41568 -6781
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +15 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -12
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-columns-to-row-group.js +3 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +164 -29
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.js +16 -7
- package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts +8 -0
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts.map +1 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js +485 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -0
- package/dist/lib/constants.d.ts +0 -1
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +1 -2
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js +394 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
- package/dist/lib/geo/geospatial-metadata.js +412 -0
- package/dist/lib/geo/geospatial-metadata.js.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +91 -0
- package/dist/lib/parquet-page-index.d.ts.map +1 -0
- package/dist/lib/parquet-page-index.js +463 -0
- package/dist/lib/parquet-page-index.js.map +1 -0
- package/dist/lib/parquet-predicate.d.ts +21 -0
- package/dist/lib/parquet-predicate.d.ts.map +1 -0
- package/dist/lib/parquet-predicate.js +328 -0
- package/dist/lib/parquet-predicate.js.map +1 -0
- package/dist/lib/parquet-source-worker-client.d.ts +6 -0
- package/dist/lib/parquet-source-worker-client.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-client.js +24 -0
- package/dist/lib/parquet-source-worker-client.js.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts +7 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.js +181 -0
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -0
- package/dist/lib/parquet-source-worker-file.d.ts +24 -0
- package/dist/lib/parquet-source-worker-file.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-file.js +40 -0
- package/dist/lib/parquet-source-worker-file.js.map +1 -0
- package/dist/lib/parquet-source-worker-types.d.ts +121 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-types.js +5 -0
- package/dist/lib/parquet-source-worker-types.js.map +1 -0
- package/dist/lib/parquet-worker-transport.d.ts +8 -0
- package/dist/lib/parquet-worker-transport.d.ts.map +1 -0
- package/dist/lib/parquet-worker-transport.js +40 -0
- package/dist/lib/parquet-worker-transport.js.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts +16 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.js +30 -0
- package/dist/lib/parsers/convert-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +6 -2
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-tables.d.ts +37 -0
- package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-tables.js +50 -0
- package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +19 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +921 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +4 -5
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +95 -28
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +80 -0
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -0
- package/dist/lib/sources/parquet-range-file.js +267 -0
- package/dist/lib/sources/parquet-range-file.js.map +1 -0
- package/dist/lib/sources/single-batch-queue.d.ts +24 -0
- package/dist/lib/sources/single-batch-queue.d.ts.map +1 -0
- package/dist/lib/sources/single-batch-queue.js +91 -0
- package/dist/lib/sources/single-batch-queue.js.map +1 -0
- package/dist/lib/utils/load-wasm-node.d.ts +3 -0
- package/dist/lib/utils/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/utils/load-wasm-node.js +23 -0
- package/dist/lib/utils/load-wasm-node.js.map +1 -0
- package/dist/lib/utils/load-wasm.d.ts +2 -2
- package/dist/lib/utils/load-wasm.d.ts.map +1 -1
- package/dist/lib/utils/load-wasm.js +28 -12
- package/dist/lib/utils/load-wasm.js.map +1 -1
- package/dist/lib/utils/make-stream-iterator.d.ts +2 -0
- package/dist/lib/utils/make-stream-iterator.d.ts.map +1 -1
- package/dist/lib/utils/make-stream-iterator.js +123 -14
- package/dist/lib/utils/make-stream-iterator.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
- package/dist/lib/utils/normalize-parquet-options.js +25 -0
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
- package/dist/parquet-dataset-source.cjs +38974 -0
- package/dist/parquet-dataset-source.cjs.map +6 -0
- package/dist/parquet-dataset-source.d.ts +81 -0
- package/dist/parquet-dataset-source.d.ts.map +1 -0
- package/dist/parquet-dataset-source.js +602 -0
- package/dist/parquet-dataset-source.js.map +1 -0
- package/dist/parquet-encoded-page-types.d.ts +108 -0
- package/dist/parquet-encoded-page-types.d.ts.map +1 -0
- package/dist/parquet-encoded-page-types.js +5 -0
- package/dist/parquet-encoded-page-types.js.map +1 -0
- package/dist/parquet-format.d.ts +2 -0
- package/dist/parquet-format.d.ts.map +1 -1
- package/dist/parquet-format.js +2 -0
- package/dist/parquet-format.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +59 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -0
- package/dist/parquet-js-loader-types.js +34 -0
- package/dist/parquet-js-loader-types.js.map +1 -0
- package/dist/parquet-js-loader.d.ts +18 -22
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +53 -14
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +65 -0
- package/dist/parquet-js-writer.d.ts.map +1 -0
- package/dist/parquet-js-writer.js +28 -0
- package/dist/parquet-js-writer.js.map +1 -0
- package/dist/parquet-loader-base.d.ts +25 -0
- package/dist/parquet-loader-base.d.ts.map +1 -0
- package/dist/parquet-loader-base.js +23 -0
- package/dist/parquet-loader-base.js.map +1 -0
- package/dist/parquet-loader-options.d.ts +55 -0
- package/dist/parquet-loader-options.d.ts.map +1 -0
- package/dist/parquet-loader-options.js +10 -0
- package/dist/parquet-loader-options.js.map +1 -0
- package/dist/parquet-loader-types.d.ts +61 -0
- package/dist/parquet-loader-types.d.ts.map +1 -0
- package/dist/parquet-loader-types.js +17 -0
- package/dist/parquet-loader-types.js.map +1 -0
- package/dist/parquet-loader.d.ts +42 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +68 -0
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-source-capabilities.d.ts +43 -0
- package/dist/parquet-source-capabilities.d.ts.map +1 -0
- package/dist/parquet-source-capabilities.js +32 -0
- package/dist/parquet-source-capabilities.js.map +1 -0
- package/dist/parquet-source-loader-types.d.ts +82 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -0
- package/dist/parquet-source-loader-types.js +84 -0
- package/dist/parquet-source-loader-types.js.map +1 -0
- package/dist/parquet-source-loader.cjs +38261 -0
- package/dist/parquet-source-loader.cjs.map +6 -0
- package/dist/parquet-source-loader.d.ts +167 -0
- package/dist/parquet-source-loader.d.ts.map +1 -0
- package/dist/parquet-source-loader.js +1438 -0
- package/dist/parquet-source-loader.js.map +1 -0
- package/dist/parquet-source-types.d.ts +534 -0
- package/dist/parquet-source-types.d.ts.map +1 -0
- package/dist/parquet-source-types.js +5 -0
- package/dist/parquet-source-types.js.map +1 -0
- package/dist/parquet-source-worker-factory.d.ts +4 -0
- package/dist/parquet-source-worker-factory.d.ts.map +1 -0
- package/dist/parquet-source-worker-factory.js +6 -0
- package/dist/parquet-source-worker-factory.js.map +1 -0
- package/dist/parquet-source-worker-url.d.ts +3 -0
- package/dist/parquet-source-worker-url.d.ts.map +1 -0
- package/dist/parquet-source-worker-url.js +22 -0
- package/dist/parquet-source-worker-url.js.map +1 -0
- package/dist/parquet-source-worker.js +38871 -0
- package/dist/parquet-source-worker.js.map +7 -0
- package/dist/parquet-worker-node.cjs +78507 -0
- package/dist/parquet-worker-node.cjs.map +7 -0
- package/dist/parquet-worker-url.d.ts +3 -0
- package/dist/parquet-worker-url.d.ts.map +1 -0
- package/dist/parquet-worker-url.js +19 -0
- package/dist/parquet-worker-url.js.map +1 -0
- package/dist/parquet-worker.js +20094 -0
- package/dist/parquet-worker.js.map +7 -0
- package/dist/parquet-writer.d.ts +10 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +27 -0
- package/dist/parquet-writer.js.map +1 -0
- package/dist/parquet_wasm_bg.wasm +0 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts +9 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts.map +1 -0
- package/dist/parquetjs/codecs/byte-stream-split.js +159 -0
- package/dist/parquetjs/codecs/byte-stream-split.js.map +1 -0
- package/dist/parquetjs/codecs/declare.d.ts +35 -3
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +17 -1
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts +17 -0
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -0
- package/dist/parquetjs/codecs/delta.js +788 -0
- package/dist/parquetjs/codecs/delta.js.map +1 -0
- package/dist/parquetjs/codecs/dictionary.d.ts +5 -2
- package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +16 -4
- package/dist/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +22 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +3 -3
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +231 -78
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +7 -3
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +613 -69
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +16 -8
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +114 -141
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts +17 -0
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/dictionary-planner.js +67 -0
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -0
- package/dist/parquetjs/encoder/page-planner.d.ts +11 -0
- package/dist/parquetjs/encoder/page-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/page-planner.js +60 -0
- package/dist/parquetjs/encoder/page-planner.js.map +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +54 -5
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +930 -66
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +7 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +37 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +6 -6
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +3 -3
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +12 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +75 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts +8 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js +14 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts +2 -1
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +7 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +37 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js +27 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts +14 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js +51 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js +41 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +17 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +97 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js +27 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts +14 -10
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +38 -2
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +5 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +25 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js +41 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +12 -2
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +12 -2
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +23 -3
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +687 -119
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
- package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
- package/dist/parquetjs/parser/encoded-pages.js +206 -0
- package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts +111 -10
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +836 -69
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +83 -10
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts +3 -0
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +31 -3
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts +2 -0
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +285 -29
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +290 -71
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/binary-utils.d.ts +41 -0
- package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/binary-utils.js +106 -0
- package/dist/parquetjs/utils/binary-utils.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.d.ts +1 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +3 -3
- package/dist/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts +25 -7
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +66 -39
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/parquetjs/utils/thrift-runtime.d.ts +106 -0
- package/dist/parquetjs/utils/thrift-runtime.d.ts.map +1 -0
- package/dist/parquetjs/utils/thrift-runtime.js +50 -0
- package/dist/parquetjs/utils/thrift-runtime.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +80 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
- package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
- package/dist/polyfills/process.d.ts +12 -0
- package/dist/polyfills/process.d.ts.map +1 -0
- package/dist/polyfills/process.js +17 -0
- package/dist/polyfills/process.js.map +1 -0
- package/dist/unbundled.d.ts +10 -0
- package/dist/unbundled.d.ts.map +1 -0
- package/dist/unbundled.js +11 -0
- package/dist/unbundled.js.map +1 -0
- package/dist/wasm.d.ts +3 -0
- package/dist/wasm.d.ts.map +1 -0
- package/dist/wasm.js +6 -0
- package/dist/wasm.js.map +1 -0
- package/dist/workers/parquet-source-worker.d.ts +2 -0
- package/dist/workers/parquet-source-worker.d.ts.map +1 -0
- package/dist/workers/parquet-source-worker.js +17 -0
- package/dist/workers/parquet-source-worker.js.map +1 -0
- package/dist/workers/parquet-worker-node.d.ts +2 -0
- package/dist/workers/parquet-worker-node.d.ts.map +1 -0
- package/dist/workers/parquet-worker-node.js +5 -0
- package/dist/workers/parquet-worker-node.js.map +1 -0
- package/dist/workers/parquet-worker.d.ts +2 -0
- package/dist/workers/parquet-worker.d.ts.map +1 -0
- package/dist/workers/parquet-worker.js +27 -0
- package/dist/workers/parquet-worker.js.map +1 -0
- package/package.json +84 -40
- package/src/bundled.ts +43 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/geoparquet-loader-with-parser.ts +95 -0
- package/src/geoparquet-loader.ts +54 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +105 -26
- package/src/lib/arrow/convert-columns-to-row-group.ts +3 -0
- package/src/lib/arrow/convert-schema-from-parquet.ts +193 -32
- package/src/lib/arrow/convert-schema-to-parquet.ts +16 -7
- package/src/lib/arrow/create-nested-arrow-vector.ts +705 -0
- package/src/lib/constants.ts +0 -2
- package/src/lib/encoders/encode-arrow-to-parquet.ts +10 -16
- package/src/lib/encoders/encode-table-to-parquet-js.ts +526 -0
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +547 -0
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +727 -0
- package/src/lib/parquet-predicate.ts +410 -0
- package/src/lib/parquet-source-worker-client.ts +41 -0
- package/src/lib/parquet-source-worker-decoder.ts +252 -0
- package/src/lib/parquet-source-worker-file.ts +56 -0
- package/src/lib/parquet-source-worker-types.ts +132 -0
- package/src/lib/parquet-worker-transport.ts +57 -0
- package/src/lib/parsers/convert-parquet-tables.ts +39 -0
- package/src/lib/parsers/get-parquet-schema.ts +6 -2
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
- package/src/lib/parsers/parse-parquet-tables.ts +82 -0
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +1213 -0
- package/src/lib/parsers/parse-parquet-to-arrow.ts +62 -36
- package/src/lib/parsers/parse-parquet-to-columns.ts +9 -8
- package/src/lib/parsers/parse-parquet-to-json.ts +119 -35
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +365 -0
- package/src/lib/sources/single-batch-queue.ts +95 -0
- package/src/lib/utils/load-wasm-node.ts +25 -0
- package/src/lib/utils/load-wasm.ts +36 -13
- package/src/lib/utils/make-stream-iterator.ts +126 -14
- package/src/lib/utils/normalize-parquet-options.ts +30 -0
- package/src/parquet-dataset-source.ts +800 -0
- package/src/parquet-encoded-page-types.ts +122 -0
- package/src/parquet-format.ts +2 -0
- package/src/parquet-js-loader-types.ts +53 -0
- package/src/parquet-js-loader.ts +76 -19
- package/src/parquet-js-writer.ts +98 -0
- package/src/parquet-loader-base.ts +34 -0
- package/src/parquet-loader-options.ts +62 -0
- package/src/parquet-loader-types.ts +35 -0
- package/src/parquet-loader.ts +115 -0
- package/src/parquet-source-capabilities.ts +73 -0
- package/src/parquet-source-loader-types.ts +97 -0
- package/src/parquet-source-loader.ts +2254 -0
- package/src/parquet-source-types.ts +627 -0
- package/src/parquet-source-worker-factory.ts +8 -0
- package/src/parquet-source-worker-url.ts +23 -0
- package/src/parquet-worker-url.ts +23 -0
- package/src/parquet-writer.ts +42 -0
- package/src/parquetjs/LICENSE +23 -1
- package/src/parquetjs/codecs/byte-stream-split.ts +221 -0
- package/src/parquetjs/codecs/declare.ts +62 -3
- package/src/parquetjs/codecs/delta.ts +1047 -0
- package/src/parquetjs/codecs/dictionary.ts +22 -4
- package/src/parquetjs/codecs/index.ts +22 -0
- package/src/parquetjs/codecs/plain.ts +317 -89
- package/src/parquetjs/codecs/rle.ts +852 -82
- package/src/parquetjs/compression.ts +137 -156
- package/src/parquetjs/encoder/dictionary-planner.ts +104 -0
- package/src/parquetjs/encoder/page-planner.ts +83 -0
- package/src/parquetjs/encoder/parquet-encoder.ts +1374 -95
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/BsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +37 -2
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +13 -13
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +75 -2
- package/src/parquetjs/parquet-thrift/ColumnOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeaderV2.ts +1 -1
- package/src/parquetjs/parquet-thrift/DateType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DecimalType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DictionaryPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.ts +13 -0
- package/src/parquetjs/parquet-thrift/Encoding.ts +2 -1
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/EnumType.ts +1 -1
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +37 -2
- package/src/parquetjs/parquet-thrift/Float16Type.ts +28 -0
- package/src/parquetjs/parquet-thrift/GeographyType.ts +54 -0
- package/src/parquetjs/parquet-thrift/GeometryType.ts +43 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/IndexPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/IntType.ts +1 -1
- package/src/parquetjs/parquet-thrift/JsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/KeyValue.ts +1 -1
- package/src/parquetjs/parquet-thrift/ListType.ts +1 -1
- package/src/parquetjs/parquet-thrift/LogicalType.ts +97 -1
- package/src/parquetjs/parquet-thrift/MapType.ts +1 -1
- package/src/parquetjs/parquet-thrift/MicroSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/MilliSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/NanoSeconds.ts +28 -0
- package/src/parquetjs/parquet-thrift/NullType.ts +1 -1
- package/src/parquetjs/parquet-thrift/OffsetIndex.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageEncodingStats.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageLocation.ts +2 -2
- package/src/parquetjs/parquet-thrift/RowGroup.ts +2 -2
- package/src/parquetjs/parquet-thrift/SchemaElement.ts +1 -1
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/SortingColumn.ts +1 -1
- package/src/parquetjs/parquet-thrift/Statistics.ts +50 -14
- package/src/parquetjs/parquet-thrift/StringType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeUnit.ts +25 -1
- package/src/parquetjs/parquet-thrift/TimestampType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TypeDefinedOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/UUIDType.ts +1 -1
- package/src/parquetjs/parquet-thrift/VariantType.ts +43 -0
- package/src/parquetjs/parquet-thrift/index.ts +12 -4
- package/src/parquetjs/parser/decoders.ts +922 -143
- package/src/parquetjs/parser/encoded-pages.ts +278 -0
- package/src/parquetjs/parser/parquet-reader.ts +1334 -83
- package/src/parquetjs/schema/declare.ts +121 -9
- package/src/parquetjs/schema/schema.ts +33 -3
- package/src/parquetjs/schema/shred.ts +345 -41
- package/src/parquetjs/schema/types.ts +308 -84
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/binary-utils.ts +132 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/file-utils.ts +4 -4
- package/src/parquetjs/utils/read-utils.ts +77 -51
- package/src/parquetjs/utils/thrift-runtime.ts +109 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
- package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
- package/src/polyfills/process.ts +17 -0
- package/src/unbundled.ts +47 -0
- package/src/wasm.ts +6 -0
- package/src/workers/parquet-source-worker.ts +21 -0
- package/src/workers/parquet-worker-node.ts +5 -0
- package/src/workers/parquet-worker.ts +31 -0
- package/dist/parquet-arrow-loader.d.ts +0 -43
- package/dist/parquet-arrow-loader.d.ts.map +0 -1
- package/dist/parquet-arrow-loader.js +0 -49
- package/dist/parquet-arrow-loader.js.map +0 -1
- package/dist/parquet-arrow-writer.d.ts +0 -9
- package/dist/parquet-arrow-writer.d.ts.map +0 -1
- package/dist/parquet-arrow-writer.js +0 -23
- package/dist/parquet-arrow-writer.js.map +0 -1
- package/dist/parquet-json-loader.d.ts +0 -148
- package/dist/parquet-json-loader.d.ts.map +0 -1
- package/dist/parquet-json-loader.js +0 -106
- package/dist/parquet-json-loader.js.map +0 -1
- package/dist/parquet-json-writer.d.ts +0 -19
- package/dist/parquet-json-writer.d.ts.map +0 -1
- package/dist/parquet-json-writer.js +0 -20
- package/dist/parquet-json-writer.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.d.ts +0 -222
- package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer.js +0 -1868
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.d.ts +0 -4
- package/dist/polyfills/buffer/index.d.ts.map +0 -1
- package/dist/polyfills/buffer/index.js +0 -9
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/workers/parquet-json-worker.d.ts +0 -2
- package/dist/workers/parquet-json-worker.d.ts.map +0 -1
- package/dist/workers/parquet-json-worker.js +0 -7
- package/dist/workers/parquet-json-worker.js.map +0 -1
- package/src/parquet-arrow-loader.ts +0 -74
- package/src/parquet-arrow-writer.ts +0 -33
- package/src/parquet-json-loader.ts +0 -160
- package/src/parquet-json-writer.ts +0 -27
- package/src/parquetjs/modules.d.ts +0 -25
- package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
- package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
- package/src/polyfills/buffer/buffer.ts +0 -2205
- package/src/polyfills/buffer/index.ts +0 -11
- package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
- package/src/workers/parquet-json-worker.ts +0 -8
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { ArrowTable, ArrowTableBatch, GeoJSONTable, GeoJSONTableBatch } from '@loaders.gl/schema';
|
|
2
|
+
import { type ParquetLoaderOptions as SharedParquetLoaderOptions } from "./parquet-loader-options.js";
|
|
3
|
+
/** Options for the GeoParquet loader. */
|
|
4
|
+
export type GeoParquetLoaderOptions = SharedParquetLoaderOptions;
|
|
5
|
+
/** Preloads the parser-bearing GeoParquet loader implementation. */
|
|
6
|
+
declare function preloadGeoParquetLoader(): Promise<{
|
|
7
|
+
readonly parse: (arrayBuffer: ArrayBuffer, options?: GeoParquetLoaderOptions) => Promise<ArrowTable | GeoJSONTable>;
|
|
8
|
+
readonly parseFile: (file: import("@loaders.gl/loader-utils").ReadableFile, options?: GeoParquetLoaderOptions) => Promise<ArrowTable | GeoJSONTable>;
|
|
9
|
+
readonly parseFileInBatches: (file: import("@loaders.gl/loader-utils").ReadableFile, options?: GeoParquetLoaderOptions) => AsyncIterable<ArrowTableBatch | GeoJSONTableBatch>;
|
|
10
|
+
readonly parseInBatches: (asyncIterator: AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>, options?: GeoParquetLoaderOptions, _context?: unknown) => AsyncGenerator<ArrowTableBatch | GeoJSONTableBatch, void, any>;
|
|
11
|
+
readonly dataType: GeoJSONTable | ArrowTable;
|
|
12
|
+
readonly batchType: GeoJSONTableBatch | ArrowTableBatch;
|
|
13
|
+
readonly id: "parquet";
|
|
14
|
+
readonly module: "parquet";
|
|
15
|
+
readonly version: any;
|
|
16
|
+
readonly worker: false;
|
|
17
|
+
readonly options: {
|
|
18
|
+
readonly parquet: {
|
|
19
|
+
readonly columns: undefined;
|
|
20
|
+
readonly preserveBinary: false;
|
|
21
|
+
readonly shape: "arrow-table";
|
|
22
|
+
};
|
|
23
|
+
};
|
|
24
|
+
readonly name: "Apache Parquet";
|
|
25
|
+
readonly category: "table";
|
|
26
|
+
readonly encoding: "parquet";
|
|
27
|
+
readonly format: "parquet";
|
|
28
|
+
readonly extensions: ["parquet"];
|
|
29
|
+
readonly mimeTypes: ["application/octet-stream"];
|
|
30
|
+
readonly binary: true;
|
|
31
|
+
readonly tests: ["PAR1", "PARE"];
|
|
32
|
+
}>;
|
|
33
|
+
/** Metadata-only GeoParquet table loader. */
|
|
34
|
+
export declare const GeoParquetLoader: {
|
|
35
|
+
readonly dataType: GeoJSONTable | ArrowTable;
|
|
36
|
+
readonly batchType: GeoJSONTableBatch | ArrowTableBatch;
|
|
37
|
+
readonly id: "parquet";
|
|
38
|
+
readonly module: "parquet";
|
|
39
|
+
readonly version: any;
|
|
40
|
+
readonly worker: false;
|
|
41
|
+
readonly options: {
|
|
42
|
+
readonly parquet: {
|
|
43
|
+
readonly columns: undefined;
|
|
44
|
+
readonly preserveBinary: false;
|
|
45
|
+
readonly shape: "arrow-table";
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
readonly preload: typeof preloadGeoParquetLoader;
|
|
49
|
+
readonly name: "Apache Parquet";
|
|
50
|
+
readonly category: "table";
|
|
51
|
+
readonly encoding: "parquet";
|
|
52
|
+
readonly format: "parquet";
|
|
53
|
+
readonly extensions: ["parquet"];
|
|
54
|
+
readonly mimeTypes: ["application/octet-stream"];
|
|
55
|
+
readonly binary: true;
|
|
56
|
+
readonly tests: ["PAR1", "PARE"];
|
|
57
|
+
};
|
|
58
|
+
export {};
|
|
59
|
+
//# sourceMappingURL=geoparquet-loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geoparquet-loader.d.ts","sourceRoot":"","sources":["../src/geoparquet-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,YAAY,EACZ,iBAAiB,EAClB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAEL,KAAK,oBAAoB,IAAI,0BAA0B,EACxD,oCAAiC;AAMlC,yCAAyC;AACzC,MAAM,MAAM,uBAAuB,GAAG,0BAA0B,CAAC;AAEjE,oEAAoE;AACpE,iBAAe,uBAAuB;;;;;uBASP,YAAY,GAAG,UAAU;wBACxB,iBAAiB,GAAG,eAAe;;;;;;;;;;;;;;;;;;;;GAPlE;AAED,6CAA6C;AAC7C,eAAO,MAAM,gBAAgB;uBAGE,YAAY,GAAG,UAAU;wBACxB,iBAAiB,GAAG,eAAe;;;;;;;;;;;;;;;;;;;;;CAiBlE,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
import { ParquetFormat } from "./parquet-format.js";
|
|
5
|
+
import { PARQUET_LOADER_DEFAULT_OPTIONS } from "./parquet-loader-options.js";
|
|
6
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
7
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
8
|
+
const VERSION = typeof "5.0.0-alpha.10" !== 'undefined' ? "5.0.0-alpha.10" : 'latest';
|
|
9
|
+
/** Preloads the parser-bearing GeoParquet loader implementation. */
|
|
10
|
+
async function preloadGeoParquetLoader() {
|
|
11
|
+
const { GeoParquetLoaderWithParser } = await import("./geoparquet-loader-with-parser.js");
|
|
12
|
+
return GeoParquetLoaderWithParser;
|
|
13
|
+
}
|
|
14
|
+
/** Metadata-only GeoParquet table loader. */
|
|
15
|
+
export const GeoParquetLoader = {
|
|
16
|
+
...ParquetFormat,
|
|
17
|
+
dataType: null,
|
|
18
|
+
batchType: null,
|
|
19
|
+
id: 'parquet',
|
|
20
|
+
module: 'parquet',
|
|
21
|
+
version: VERSION,
|
|
22
|
+
worker: false,
|
|
23
|
+
options: {
|
|
24
|
+
parquet: {
|
|
25
|
+
...PARQUET_LOADER_DEFAULT_OPTIONS
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
preload: preloadGeoParquetLoader
|
|
29
|
+
};
|
|
30
|
+
//# sourceMappingURL=geoparquet-loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geoparquet-loader.js","sourceRoot":"","sources":["../src/geoparquet-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAUpC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EACL,8BAA8B,EAE/B,oCAAiC;AAElC,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,uBAAkB,KAAK,WAAW,CAAC,CAAC,kBAAa,CAAC,CAAC,QAAQ,CAAC;AAK5E,oEAAoE;AACpE,KAAK,UAAU,uBAAuB;IACpC,MAAM,EAAC,0BAA0B,EAAC,GAAG,MAAM,MAAM,sCAAmC,CAAC;IACrF,OAAO,0BAA0B,CAAC;AACpC,CAAC;AAED,6CAA6C;AAC7C,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,GAAG,aAAa;IAEhB,QAAQ,EAAE,IAA4C;IACtD,SAAS,EAAE,IAAsD;IAEjE,EAAE,EAAE,SAAS;IACb,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,MAAM,EAAE,KAAK;IAEb,OAAO,EAAE;QACP,OAAO,EAAE;YACP,GAAG,8BAA8B;SAClC;KACF;IACD,OAAO,EAAE,uBAAuB;CAKjC,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { IcebergTableSource } from "./iceberg-table-source.js";
|
|
2
|
+
import type { IcebergTableMetadata } from "./iceberg-types.js";
|
|
3
|
+
import type { RequestScheduler } from '@loaders.gl/loader-utils';
|
|
4
|
+
/** Configuration for the minimal Iceberg REST Catalog adapter. */
|
|
5
|
+
export type IcebergRestCatalogOptions = {
|
|
6
|
+
/** Catalog base URL, for example `https://catalog.example.com`. */
|
|
7
|
+
readonly endpoint: string;
|
|
8
|
+
/** Headers sent to catalog and metadata requests. */
|
|
9
|
+
readonly headers?: HeadersInit;
|
|
10
|
+
/** Fetch implementation, useful for browser auth and deterministic tests. */
|
|
11
|
+
readonly fetch?: (url: string, options?: RequestInit) => Promise<Response>;
|
|
12
|
+
/** Optional shared request gate for catalog metadata calls. */
|
|
13
|
+
readonly requestScheduler?: RequestScheduler;
|
|
14
|
+
};
|
|
15
|
+
/** Fully qualified Iceberg REST table identifier. */
|
|
16
|
+
export type IcebergRestTableIdentifier = {
|
|
17
|
+
/** Namespace components in catalog order. */
|
|
18
|
+
readonly namespace: readonly string[];
|
|
19
|
+
/** Table name within the namespace. */
|
|
20
|
+
readonly table: string;
|
|
21
|
+
};
|
|
22
|
+
/** A table source returned by the REST Catalog adapter. */
|
|
23
|
+
export type IcebergRestTable = {
|
|
24
|
+
/** Inline metadata returned by the catalog, when available. */
|
|
25
|
+
readonly metadata?: IcebergTableMetadata;
|
|
26
|
+
/** Metadata location returned by the catalog, when available. */
|
|
27
|
+
readonly metadataLocation?: string;
|
|
28
|
+
/** Read-only source that plans and scans the table. */
|
|
29
|
+
readonly source: IcebergTableSource;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Minimal read-only Iceberg REST Catalog client.
|
|
33
|
+
*
|
|
34
|
+
* This deliberately stops at table loading. Scan planning and file access remain in
|
|
35
|
+
* `IcebergTableSource`, so catalog authentication and table-format decoding stay separate.
|
|
36
|
+
*/
|
|
37
|
+
export declare class IcebergRestCatalog {
|
|
38
|
+
private readonly options;
|
|
39
|
+
constructor(options: IcebergRestCatalogOptions);
|
|
40
|
+
/** Loads one table's metadata through the Iceberg REST Catalog `/v1` endpoint. */
|
|
41
|
+
loadTable(identifier: IcebergRestTableIdentifier, signal?: AbortSignal): Promise<IcebergRestTable>;
|
|
42
|
+
/** Fetches one catalog response through the optional shared request scheduler. */
|
|
43
|
+
private fetchJson;
|
|
44
|
+
/** Builds the standard REST Catalog table endpoint. */
|
|
45
|
+
private getTableUrl;
|
|
46
|
+
/** Carries catalog headers and fetch through to the metadata/data source. */
|
|
47
|
+
private getSourceOptions;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=iceberg-rest-catalog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iceberg-rest-catalog.d.ts","sourceRoot":"","sources":["../src/iceberg-rest-catalog.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,kBAAkB,EAAC,kCAA+B;AAE1D,OAAO,KAAK,EAAC,oBAAoB,EAAC,2BAAwB;AAC1D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAE/D,kEAAkE;AAClE,MAAM,MAAM,yBAAyB,GAAG;IACtC,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC;IAC/B,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3E,+DAA+D;IAC/D,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CAC9C,CAAC;AAEF,qDAAqD;AACrD,MAAM,MAAM,0BAA0B,GAAG;IACvC,6CAA6C;IAC7C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,uCAAuC;IACvC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,2DAA2D;AAC3D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IACzC,iEAAiE;IACjE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,uDAAuD;IACvD,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAC;CACrC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;gBAExC,OAAO,EAAE,yBAAyB;IAI9C,kFAAkF;IAC5E,SAAS,CACb,UAAU,EAAE,0BAA0B,EACtC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,gBAAgB,CAAC;IAoB5B,kFAAkF;YACpE,SAAS;IAgBvB,uDAAuD;IACvD,OAAO,CAAC,WAAW;IAKnB,6EAA6E;IAC7E,OAAO,CAAC,gBAAgB;CAOzB"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
import { IcebergTableSource } from "./iceberg-table-source.js";
|
|
5
|
+
/**
|
|
6
|
+
* Minimal read-only Iceberg REST Catalog client.
|
|
7
|
+
*
|
|
8
|
+
* This deliberately stops at table loading. Scan planning and file access remain in
|
|
9
|
+
* `IcebergTableSource`, so catalog authentication and table-format decoding stay separate.
|
|
10
|
+
*/
|
|
11
|
+
export class IcebergRestCatalog {
|
|
12
|
+
options;
|
|
13
|
+
constructor(options) {
|
|
14
|
+
this.options = options;
|
|
15
|
+
}
|
|
16
|
+
/** Loads one table's metadata through the Iceberg REST Catalog `/v1` endpoint. */
|
|
17
|
+
async loadTable(identifier, signal) {
|
|
18
|
+
const tableUrl = this.getTableUrl(identifier);
|
|
19
|
+
const response = await this.fetchJson(tableUrl, signal);
|
|
20
|
+
if (!response.ok) {
|
|
21
|
+
throw new Error(`Iceberg REST Catalog request failed: ${response.status} ${response.statusText}`);
|
|
22
|
+
}
|
|
23
|
+
const responseJson = (await response.json());
|
|
24
|
+
const metadataLocation = getString(responseJson['metadata-location']);
|
|
25
|
+
const metadata = isIcebergMetadata(responseJson.metadata) ? responseJson.metadata : undefined;
|
|
26
|
+
if (!metadataLocation && !metadata) {
|
|
27
|
+
throw new Error('Iceberg REST Catalog response did not include table metadata');
|
|
28
|
+
}
|
|
29
|
+
const source = metadata
|
|
30
|
+
? new IcebergTableSource(createMetadataUrl(metadata), this.getSourceOptions())
|
|
31
|
+
: new IcebergTableSource(metadataLocation, this.getSourceOptions());
|
|
32
|
+
return { metadata, metadataLocation, source };
|
|
33
|
+
}
|
|
34
|
+
/** Fetches one catalog response through the optional shared request scheduler. */
|
|
35
|
+
async fetchJson(url, signal) {
|
|
36
|
+
const scheduler = this.options.requestScheduler;
|
|
37
|
+
const request = scheduler
|
|
38
|
+
? await scheduler.scheduleRequest(url, () => (signal?.aborted ? -1 : 0))
|
|
39
|
+
: { done: () => { } };
|
|
40
|
+
if (!request)
|
|
41
|
+
throw new DOMException('The operation was aborted', 'AbortError');
|
|
42
|
+
try {
|
|
43
|
+
return await (this.options.fetch ?? fetch)(url, {
|
|
44
|
+
headers: this.options.headers,
|
|
45
|
+
signal
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
request.done();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Builds the standard REST Catalog table endpoint. */
|
|
53
|
+
getTableUrl(identifier) {
|
|
54
|
+
const namespace = identifier.namespace.map(encodeURIComponent).join('%1F');
|
|
55
|
+
return `${this.options.endpoint.replace(/\/$/, '')}/v1/namespaces/${namespace}/tables/${encodeURIComponent(identifier.table)}`;
|
|
56
|
+
}
|
|
57
|
+
/** Carries catalog headers and fetch through to the metadata/data source. */
|
|
58
|
+
getSourceOptions() {
|
|
59
|
+
const options = { iceberg: { headers: this.options.headers } };
|
|
60
|
+
if (this.options.fetch) {
|
|
61
|
+
options.core = { fetch: this.options.fetch };
|
|
62
|
+
}
|
|
63
|
+
return options;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function getString(value) {
|
|
67
|
+
return typeof value === 'string' && value.length > 0 ? value : undefined;
|
|
68
|
+
}
|
|
69
|
+
function isIcebergMetadata(value) {
|
|
70
|
+
if (!value || typeof value !== 'object')
|
|
71
|
+
return false;
|
|
72
|
+
const metadata = value;
|
|
73
|
+
return typeof metadata['format-version'] === 'number' && typeof metadata.location === 'string';
|
|
74
|
+
}
|
|
75
|
+
function createMetadataUrl(metadata) {
|
|
76
|
+
return `data:application/json,${encodeURIComponent(JSON.stringify(metadata))}`;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=iceberg-rest-catalog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iceberg-rest-catalog.js","sourceRoot":"","sources":["../src/iceberg-rest-catalog.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,kBAAkB,EAAC,kCAA+B;AAmC1D;;;;;GAKG;AACH,MAAM,OAAO,kBAAkB;IACZ,OAAO,CAA4B;IAEpD,YAAY,OAAkC;QAC5C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,SAAS,CACb,UAAsC,EACtC,MAAoB;QAEpB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACjF,CAAC;QACJ,CAAC;QACD,MAAM,YAAY,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;QACxE,MAAM,gBAAgB,GAAG,SAAS,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9F,IAAI,CAAC,gBAAgB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ;YACrB,CAAC,CAAC,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9E,CAAC,CAAC,IAAI,kBAAkB,CAAC,gBAA0B,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAChF,OAAO,EAAC,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAC,CAAC;IAC9C,CAAC;IAED,kFAAkF;IAC1E,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,MAAoB;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;QAChD,MAAM,OAAO,GAAG,SAAS;YACvB,CAAC,CAAC,MAAM,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,CAAC,CAAC,EAAC,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAC,CAAC;QACrB,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,YAAY,CAAC,2BAA2B,EAAE,YAAY,CAAC,CAAC;QAChF,IAAI,CAAC;YACH,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,GAAG,EAAE;gBAC9C,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,MAAM;aACP,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,uDAAuD;IAC/C,WAAW,CAAC,UAAsC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3E,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,SAAS,WAAW,kBAAkB,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;IACjI,CAAC;IAED,6EAA6E;IACrE,gBAAgB;QACtB,MAAM,OAAO,GAAyB,EAAC,OAAO,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAC,EAAC,CAAC;QACjF,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,GAAG,EAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAC,CAAC;QAC7C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3E,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,QAAQ,GAAG,KAAgC,CAAC;IAClD,OAAO,OAAO,QAAQ,CAAC,gBAAgB,CAAC,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC;AACjG,CAAC;AAED,SAAS,iBAAiB,CAAC,QAA8B;IACvD,OAAO,yBAAyB,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;AACjF,CAAC"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { CoreAPI, ScanQueryMetadata, ScanQueryMetadataOptions, TableScanSource } from '@loaders.gl/loader-utils';
|
|
2
|
+
import { DataSource as BaseDataSource } from '@loaders.gl/loader-utils';
|
|
3
|
+
import type { IcebergSpatialFilter, IcebergDeleteFile, IcebergParquetFile, IcebergScanPlan, IcebergSnapshot, IcebergTableMetadata, IcebergTableSourceOptions } from "./iceberg-types.js";
|
|
4
|
+
import type { ParquetDatasetBatch, ParquetPredicate, ParquetDatasetReadOptions, ParquetDatasetExplain, ParquetDatasetSourceOptions } from "./parquet-source-types.js";
|
|
5
|
+
/** Options accepted by `IcebergTableSource`. */
|
|
6
|
+
export type IcebergSourceOptions = ParquetDatasetSourceOptions & {
|
|
7
|
+
iceberg?: IcebergTableSourceOptions;
|
|
8
|
+
};
|
|
9
|
+
/** Options for scanning the current Iceberg snapshot as Parquet batches. */
|
|
10
|
+
export type IcebergScanOptions = ParquetDatasetReadOptions & {
|
|
11
|
+
/** Reads a specific snapshot instead of the table's current snapshot. */
|
|
12
|
+
snapshotId?: number;
|
|
13
|
+
/** Reads the snapshot selected by a named Iceberg branch or tag. */
|
|
14
|
+
snapshotRef?: string;
|
|
15
|
+
/** Applies Iceberg position deletes after Parquet batches are decoded. */
|
|
16
|
+
applyDeletes?: boolean;
|
|
17
|
+
/** Conservatively prunes files whose declared geometry bounds miss this envelope. */
|
|
18
|
+
spatialFilter?: IcebergSpatialFilter;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Read-only Iceberg table metadata source.
|
|
22
|
+
*
|
|
23
|
+
* The source owns Iceberg metadata and manifest discovery, then delegates selected data files to
|
|
24
|
+
* `ParquetDatasetSource` for projection, filtering, range access, workers, and Arrow materialization.
|
|
25
|
+
*/
|
|
26
|
+
export declare class IcebergTableSource extends BaseDataSource<string, IcebergSourceOptions> implements TableScanSource<ParquetDatasetBatch, ParquetPredicate> {
|
|
27
|
+
private metadataPromise;
|
|
28
|
+
private readonly closeController;
|
|
29
|
+
private closed;
|
|
30
|
+
constructor(data: string, options?: IcebergSourceOptions, coreApi?: CoreAPI);
|
|
31
|
+
/** Loads and validates the table metadata JSON. Results are cached for the source lifetime. */
|
|
32
|
+
getMetadata(signal?: AbortSignal): Promise<IcebergTableMetadata>;
|
|
33
|
+
/** Returns the snapshot selected by the table metadata, if one exists. */
|
|
34
|
+
getCurrentSnapshot(signal?: AbortSignal): Promise<IcebergSnapshot | undefined>;
|
|
35
|
+
/** Returns a selected snapshot, defaulting to the table's current snapshot. */
|
|
36
|
+
private getSnapshot;
|
|
37
|
+
/** Builds a data and delete-file plan from one snapshot without opening data files. */
|
|
38
|
+
getScanPlan(signal?: AbortSignal, snapshotId?: number, snapshotRef?: string): Promise<IcebergScanPlan>;
|
|
39
|
+
/** Reads the selected snapshot's active Parquet data files. */
|
|
40
|
+
getParquetFiles(signal?: AbortSignal, snapshotId?: number, snapshotRef?: string): Promise<readonly IcebergParquetFile[]>;
|
|
41
|
+
/** Reads delete files from the selected snapshot without applying them to Parquet rows. */
|
|
42
|
+
getDeleteFiles(signal?: AbortSignal, snapshotId?: number, snapshotRef?: string): Promise<readonly IcebergDeleteFile[]>;
|
|
43
|
+
/** Discovers the current snapshot schema and aggregate file statistics without decoding rows. */
|
|
44
|
+
getQueryMetadata(options?: ScanQueryMetadataOptions): Promise<ScanQueryMetadata>;
|
|
45
|
+
/** Explains Iceberg manifest pruning and the delegated Parquet dataset plan. */
|
|
46
|
+
explain(options?: IcebergScanOptions): Promise<ParquetDatasetExplain>;
|
|
47
|
+
/** Common table-scan entry point for the selected Iceberg snapshot. */
|
|
48
|
+
read(options?: IcebergScanOptions): AsyncIterable<ParquetDatasetBatch>;
|
|
49
|
+
/** Reads the current snapshot as Arrow batches through the existing Parquet dataset source. */
|
|
50
|
+
scan(options?: IcebergScanOptions): AsyncIterable<ParquetDatasetBatch>;
|
|
51
|
+
/** Creates the delegated Parquet dataset after Iceberg metadata-level pruning. */
|
|
52
|
+
private createParquetDataset;
|
|
53
|
+
/** Permanently closes the source and aborts an in-flight metadata request. */
|
|
54
|
+
close(): Promise<void>;
|
|
55
|
+
private loadMetadata;
|
|
56
|
+
private readAvroRecords;
|
|
57
|
+
/** Reads position-delete files into a source-file to row-position index. */
|
|
58
|
+
private loadPositionDeletes;
|
|
59
|
+
/** Reads equality-delete files and resolves their Iceberg field IDs to current schema names. */
|
|
60
|
+
private loadEqualityDeletes;
|
|
61
|
+
private assertOpen;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=iceberg-table-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iceberg-table-source.d.ts","sourceRoot":"","sources":["../src/iceberg-table-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,OAAO,EACP,iBAAiB,EACjB,wBAAwB,EACxB,eAAe,EAChB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,UAAU,IAAI,cAAc,EAE7B,MAAM,0BAA0B,CAAC;AAGlC,OAAO,KAAK,EAGV,oBAAoB,EACpB,iBAAiB,EAEjB,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,oBAAoB,EACpB,yBAAyB,EAC1B,2BAAwB;AAIzB,OAAO,KAAK,EACV,mBAAmB,EAGnB,gBAAgB,EAEhB,yBAAyB,EACzB,qBAAqB,EACrB,2BAA2B,EAC5B,kCAA+B;AAmBhC,gDAAgD;AAChD,MAAM,MAAM,oBAAoB,GAAG,2BAA2B,GAAG;IAC/D,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACrC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,kBAAkB,GAAG,yBAAyB,GAAG;IAC3D,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qFAAqF;IACrF,aAAa,CAAC,EAAE,oBAAoB,CAAC;CACtC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,kBACX,SAAQ,cAAc,CAAC,MAAM,EAAE,oBAAoB,CACnD,YAAW,eAAe,CAAC,mBAAmB,EAAE,gBAAgB,CAAC;IAEjE,OAAO,CAAC,eAAe,CAA8C;IACrE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IACzD,OAAO,CAAC,MAAM,CAAS;gBAEX,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,EAAE,OAAO,CAAC,EAAE,OAAO;IAI/E,+FAA+F;IACzF,WAAW,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAWtE,0EAA0E;IACpE,kBAAkB,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAIpF,+EAA+E;YACjE,WAAW;IAkBzB,uFAAuF;IACjF,WAAW,CACf,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,eAAe,CAAC;IAiD3B,+DAA+D;IACzD,eAAe,CACnB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC;IAIzC,2FAA2F;IACrF,cAAc,CAClB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIxC,iGAAiG;IAC3F,gBAAgB,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA2B1F,gFAAgF;IAC1E,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAY/E,uEAAuE;IACvE,IAAI,CAAC,OAAO,GAAE,kBAAuB,GAAG,aAAa,CAAC,mBAAmB,CAAC;IAI1E,+FAA+F;IACxF,IAAI,CAAC,OAAO,GAAE,kBAAuB,GAAG,aAAa,CAAC,mBAAmB,CAAC;IAyDjF,kFAAkF;IAClF,OAAO,CAAC,oBAAoB;IAqC5B,8EAA8E;IACxE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAOd,YAAY;YAmBZ,eAAe;IAiB7B,4EAA4E;YAC9D,mBAAmB;IA0BjC,gGAAgG;YAClF,mBAAmB;IA2BjC,OAAO,CAAC,UAAU;CAGnB"}
|