@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15841 -3306
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +16528 -4928
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-encoded-page-types.d.ts +108 -0
- package/dist/parquet-encoded-page-types.d.ts.map +1 -0
- package/dist/parquet-encoded-page-types.js +5 -0
- package/dist/parquet-encoded-page-types.js.map +1 -0
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14121 -2796
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +51 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +651 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7378 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8501 -1045
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
- package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
- package/dist/parquetjs/parser/encoded-pages.js +206 -0
- package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts +88 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +713 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -13
- package/src/bundled.ts +10 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +59 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-encoded-page-types.ts +122 -0
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +1024 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/encoded-pages.ts +278 -0
- package/src/parquetjs/parser/parquet-reader.ts +1112 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type { ParquetBoundingBox, ParquetPredicate, ParquetRowGroupMetadata } from "./parquet-source-types.js";
|
|
2
|
+
/** Byte range within an encoded Parquet page body. */
|
|
3
|
+
export type ParquetEncodedPageSection = Readonly<{
|
|
4
|
+
/** Byte offset relative to {@link ParquetEncodedPage.data}. */
|
|
5
|
+
byteOffset: number;
|
|
6
|
+
/** Number of bytes in this section. */
|
|
7
|
+
byteLength: number;
|
|
8
|
+
}>;
|
|
9
|
+
/** Whether the page value payload still uses the column chunk's compression codec. */
|
|
10
|
+
export type ParquetPageCompressionState = 'compressed' | 'decompressed';
|
|
11
|
+
/** Encoded Parquet page body suitable for deferred CPU or GPU decoding. */
|
|
12
|
+
export type ParquetEncodedPage = Readonly<{
|
|
13
|
+
/** Page role and data-page version. */
|
|
14
|
+
type: 'dictionary' | 'data-v1' | 'data-v2';
|
|
15
|
+
/** Zero-based data-page ordinal within the column chunk; dictionaries use `-1`. */
|
|
16
|
+
pageOrdinal: number;
|
|
17
|
+
/** Value encoding declared by the page header. */
|
|
18
|
+
encoding: string;
|
|
19
|
+
/** Repetition-level encoding for V1 pages. V2 levels always use RLE. */
|
|
20
|
+
repetitionLevelEncoding?: string;
|
|
21
|
+
/** Definition-level encoding for V1 pages. V2 levels always use RLE. */
|
|
22
|
+
definitionLevelEncoding?: string;
|
|
23
|
+
/** Compression codec declared by the containing column chunk. */
|
|
24
|
+
compression: string;
|
|
25
|
+
/** Whether the page value payload remains compressed. */
|
|
26
|
+
compressionState: ParquetPageCompressionState;
|
|
27
|
+
/** Number of encoded values declared by the page header. */
|
|
28
|
+
valueCount: number;
|
|
29
|
+
/** Number of non-null values when the page header reports it directly. */
|
|
30
|
+
nonNullValueCount?: number;
|
|
31
|
+
/** Encoded page body without its Thrift page header. */
|
|
32
|
+
data: Uint8Array;
|
|
33
|
+
/** Repetition-level bytes, when their range can be known without decoding values. */
|
|
34
|
+
repetitionLevels?: ParquetEncodedPageSection;
|
|
35
|
+
/** Definition-level bytes, when their range can be known without decoding values. */
|
|
36
|
+
definitionLevels?: ParquetEncodedPageSection;
|
|
37
|
+
/** Value or dictionary bytes, when their range can be known before deferred decoding. */
|
|
38
|
+
values?: ParquetEncodedPageSection;
|
|
39
|
+
/** Compressed body byte length reported by the page header. */
|
|
40
|
+
compressedByteLength: number;
|
|
41
|
+
/** Uncompressed body byte length reported by the page header. */
|
|
42
|
+
uncompressedByteLength: number;
|
|
43
|
+
}>;
|
|
44
|
+
/** One projected Parquet leaf column whose encoded pages have not been value-decoded. */
|
|
45
|
+
export type ParquetEncodedColumnChunk = Readonly<{
|
|
46
|
+
/** Nested Parquet schema path. */
|
|
47
|
+
path: readonly string[];
|
|
48
|
+
/** Physical Parquet scalar type. */
|
|
49
|
+
physicalType: string;
|
|
50
|
+
/** Fixed byte width for FIXED_LEN_BYTE_ARRAY columns. */
|
|
51
|
+
typeLength?: number;
|
|
52
|
+
/** Maximum repetition level for this leaf column. */
|
|
53
|
+
maxRepetitionLevel: number;
|
|
54
|
+
/** Maximum definition level for this leaf column. */
|
|
55
|
+
maxDefinitionLevel: number;
|
|
56
|
+
/** Compression codec declared by the column chunk. */
|
|
57
|
+
compression: string;
|
|
58
|
+
/** Number of encoded values declared by the column chunk. */
|
|
59
|
+
valueCount: number;
|
|
60
|
+
/** Optional dictionary page associated only with this column chunk. */
|
|
61
|
+
dictionary?: ParquetEncodedPage;
|
|
62
|
+
/** Ordered data pages in this column chunk. */
|
|
63
|
+
pages: readonly ParquetEncodedPage[];
|
|
64
|
+
}>;
|
|
65
|
+
/** Exact filters a deferred decoder must apply after conservative metadata pruning. */
|
|
66
|
+
export type ParquetDeferredPageFilter = Readonly<{
|
|
67
|
+
/** Exact scalar predicate requested by the caller. */
|
|
68
|
+
predicate?: ParquetPredicate;
|
|
69
|
+
/** Exact spatial extent requested by the caller. */
|
|
70
|
+
bbox?: ParquetBoundingBox;
|
|
71
|
+
/** Geometry column associated with `bbox`. */
|
|
72
|
+
geometryColumn?: string;
|
|
73
|
+
}>;
|
|
74
|
+
/** Candidate encoded pages returned for one Parquet row group. */
|
|
75
|
+
export type ParquetEncodedPageBatch = Readonly<{
|
|
76
|
+
/** Stable discriminator for transport-neutral deferred Parquet pages. */
|
|
77
|
+
shape: 'parquet-encoded-pages';
|
|
78
|
+
/** Normalized row-group metadata. */
|
|
79
|
+
rowGroup: ParquetRowGroupMetadata;
|
|
80
|
+
/** Columns explicitly requested for final output. Empty means all columns. */
|
|
81
|
+
projectedColumns: readonly string[];
|
|
82
|
+
/** Hidden columns included so a deferred decoder can evaluate residual filters. */
|
|
83
|
+
filterColumns: readonly string[];
|
|
84
|
+
/** Ordered selected column chunks. Each chunk retains its complete nested schema path. */
|
|
85
|
+
columns: readonly ParquetEncodedColumnChunk[];
|
|
86
|
+
/** Exact work left after loaders.gl's conservative statistics and Bloom-filter pruning. */
|
|
87
|
+
residualFilter?: ParquetDeferredPageFilter;
|
|
88
|
+
}>;
|
|
89
|
+
/** Options for reading encoded Parquet pages without materializing values or Arrow arrays. */
|
|
90
|
+
export type ParquetEncodedPageReadOptions = Readonly<{
|
|
91
|
+
/** Zero-based row-group indexes to inspect, in output order. Defaults to all row groups. */
|
|
92
|
+
rowGroups?: readonly number[];
|
|
93
|
+
/** Top-level output columns. Defaults to all columns. */
|
|
94
|
+
columns?: readonly string[];
|
|
95
|
+
/** Retains candidate row groups for which the callback returns true. */
|
|
96
|
+
rowGroupFilter?: (rowGroup: ParquetRowGroupMetadata) => boolean;
|
|
97
|
+
/** Exact predicate used for conservative pruning and returned as residual work. */
|
|
98
|
+
predicate?: ParquetPredicate;
|
|
99
|
+
/** Spatial extent used for conservative pruning and returned as residual work. */
|
|
100
|
+
bbox?: ParquetBoundingBox;
|
|
101
|
+
/** Geometry column associated with `bbox`; defaults to the GeoParquet primary column. */
|
|
102
|
+
geometryColumn?: string;
|
|
103
|
+
/** Compression codecs to preserve for a downstream decoder instead of inflating on the CPU. */
|
|
104
|
+
preserveCompression?: readonly string[];
|
|
105
|
+
/** Abort this page read and its outstanding range requests. */
|
|
106
|
+
signal?: AbortSignal;
|
|
107
|
+
}>;
|
|
108
|
+
//# sourceMappingURL=parquet-encoded-page-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet-encoded-page-types.d.ts","sourceRoot":"","sources":["../src/parquet-encoded-page-types.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACxB,kCAA+B;AAEhC,sDAAsD;AACtD,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,sFAAsF;AACtF,MAAM,MAAM,2BAA2B,GAAG,YAAY,GAAG,cAAc,CAAC;AAExE,2EAA2E;AAC3E,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC;IACxC,uCAAuC;IACvC,IAAI,EAAE,YAAY,GAAG,SAAS,GAAG,SAAS,CAAC;IAC3C,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,wEAAwE;IACxE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,iEAAiE;IACjE,WAAW,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,gBAAgB,EAAE,2BAA2B,CAAC;IAC9C,4DAA4D;IAC5D,UAAU,EAAE,MAAM,CAAC;IACnB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,IAAI,EAAE,UAAU,CAAC;IACjB,qFAAqF;IACrF,gBAAgB,CAAC,EAAE,yBAAyB,CAAC;IAC7C,qFAAqF;IACrF,gBAAgB,CAAC,EAAE,yBAAyB,CAAC;IAC7C,yFAAyF;IACzF,MAAM,CAAC,EAAE,yBAAyB,CAAC;IACnC,+DAA+D;IAC/D,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC,CAAC;AAEH,yFAAyF;AACzF,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,kCAAkC;IAClC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qDAAqD;IACrD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,+CAA+C;IAC/C,KAAK,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACtC,CAAC,CAAC;AAEH,uFAAuF;AACvF,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,sDAAsD;IACtD,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,oDAAoD;IACpD,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAC1B,8CAA8C;IAC9C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAAC;AAEH,kEAAkE;AAClE,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAC7C,yEAAyE;IACzE,KAAK,EAAE,uBAAuB,CAAC;IAC/B,qCAAqC;IACrC,QAAQ,EAAE,uBAAuB,CAAC;IAClC,8EAA8E;IAC9E,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,mFAAmF;IACnF,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,0FAA0F;IAC1F,OAAO,EAAE,SAAS,yBAAyB,EAAE,CAAC;IAC9C,2FAA2F;IAC3F,cAAc,CAAC,EAAE,yBAAyB,CAAC;CAC5C,CAAC,CAAC;AAEH,8FAA8F;AAC9F,MAAM,MAAM,6BAA6B,GAAG,QAAQ,CAAC;IACnD,4FAA4F;IAC5F,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,yDAAyD;IACzD,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,wEAAwE;IACxE,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,uBAAuB,KAAK,OAAO,CAAC;IAChE,mFAAmF;IACnF,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,kFAAkF;IAClF,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAC1B,yFAAyF;IACzF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+FAA+F;IAC/F,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,+DAA+D;IAC/D,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet-encoded-page-types.js","sourceRoot":"","sources":["../src/parquet-encoded-page-types.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC"}
|
|
@@ -16,6 +16,7 @@ declare function preloadParquetJSLoader(): Promise<{
|
|
|
16
16
|
readonly parquet: {
|
|
17
17
|
columns: undefined;
|
|
18
18
|
preserveBinary: boolean;
|
|
19
|
+
verifyFooterSignature: boolean;
|
|
19
20
|
shape: "object-row-table";
|
|
20
21
|
};
|
|
21
22
|
};
|
|
@@ -40,6 +41,7 @@ export declare const ParquetJSLoader: {
|
|
|
40
41
|
readonly parquet: {
|
|
41
42
|
columns: undefined;
|
|
42
43
|
preserveBinary: boolean;
|
|
44
|
+
verifyFooterSignature: boolean;
|
|
43
45
|
shape: "object-row-table";
|
|
44
46
|
};
|
|
45
47
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-js-loader-types.d.ts","sourceRoot":"","sources":["../src/parquet-js-loader-types.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,KAAK,EAAC,sBAAsB,EAAC,oCAAiC;
|
|
1
|
+
{"version":3,"file":"parquet-js-loader-types.d.ts","sourceRoot":"","sources":["../src/parquet-js-loader-types.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,KAAK,EAAC,sBAAsB,EAAC,oCAAiC;AAcrE,4EAA4E;AAC5E,iBAAe,sBAAsB;;;;;;uBASN,cAAc,GAAG,UAAU;wBAC1B,mBAAmB,GAAG,eAAe;;;;;;;;;;;;;;;;;;;;GAPpE;AAED,yFAAyF;AACzF,eAAO,MAAM,eAAe;uBAGG,cAAc,GAAG,UAAU;wBAC1B,mBAAmB,GAAG,eAAe;;;;;;;;;;;;;;;;;;;;;;CAcpE,CAAC"}
|
|
@@ -4,11 +4,12 @@
|
|
|
4
4
|
import { ParquetFormat } from "./parquet-format.js";
|
|
5
5
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
6
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
7
|
+
const VERSION = typeof "5.0.0-alpha.4" !== 'undefined' ? "5.0.0-alpha.4" : 'latest';
|
|
8
8
|
/** Default option bag for the experimental parquetjs plain-row loader. */
|
|
9
9
|
const DEFAULT_PARQUET_JS_OPTIONS = {
|
|
10
10
|
columns: undefined,
|
|
11
11
|
preserveBinary: false,
|
|
12
|
+
verifyFooterSignature: true,
|
|
12
13
|
shape: 'object-row-table'
|
|
13
14
|
};
|
|
14
15
|
/** Preloads the parser-bearing TypeScript Parquet loader implementation. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-js-loader-types.js","sourceRoot":"","sources":["../src/parquet-js-loader-types.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAUpC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAG/C,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAE5E,0EAA0E;AAC1E,MAAM,0BAA0B,GAAG;IACjC,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,KAAK;IACrB,KAAK,EAAE,kBAA2B;CACnC,CAAC;AAEF,4EAA4E;AAC5E,KAAK,UAAU,sBAAsB;IACnC,MAAM,EAAC,yBAAyB,EAAC,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAC;IAC1F,OAAO,yBAAyB,CAAC;AACnC,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,GAAG,aAAa;IAEhB,QAAQ,EAAE,IAA8C;IACxD,SAAS,EAAE,IAAwD;IAEnE,EAAE,EAAE,YAAY;IAChB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,MAAM,EAAE,KAAK;IACb,OAAO,EAAE;QACP,OAAO,EAAE,0BAA0B;KACpC;IACD,OAAO,EAAE,sBAAsB;CAKhC,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet-js-loader-types.js","sourceRoot":"","sources":["../src/parquet-js-loader-types.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAUpC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAG/C,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAE5E,0EAA0E;AAC1E,MAAM,0BAA0B,GAAG;IACjC,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,KAAK;IACrB,qBAAqB,EAAE,IAAI;IAC3B,KAAK,EAAE,kBAA2B;CACnC,CAAC;AAEF,4EAA4E;AAC5E,KAAK,UAAU,sBAAsB;IACnC,MAAM,EAAC,yBAAyB,EAAC,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAC;IAC1F,OAAO,yBAAyB,CAAC;AACnC,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,GAAG,aAAa;IAEhB,QAAQ,EAAE,IAA8C;IACxD,SAAS,EAAE,IAAwD;IAEnE,EAAE,EAAE,YAAY;IAChB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,MAAM,EAAE,KAAK;IACb,OAAO,EAAE;QACP,OAAO,EAAE,0BAA0B;KACpC;IACD,OAAO,EAAE,sBAAsB;CAKhC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-js-loader.d.ts","sourceRoot":"","sources":["../src/parquet-js-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE7E,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,KAAK,EAAC,sBAAsB,EAAC,oCAAiC;
|
|
1
|
+
{"version":3,"file":"parquet-js-loader.d.ts","sourceRoot":"","sources":["../src/parquet-js-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE7E,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,KAAK,EAAC,sBAAsB,EAAC,oCAAiC;AAYrE,oEAAoE;AACpE,eAAO,MAAM,yBAAyB;;kCAGjB,WAAW,YAAY,sBAAsB;+BAOhD,YAAY,YAAY,sBAAsB;wCAMrC,YAAY,YAAY,sBAAsB;6CAQjE,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,YACrC,sBAAsB,aACrB,OAAO;;;;;;;;;;;;;;;;;;;;;;CAerB,CAAC"}
|
|
@@ -10,7 +10,8 @@ const { preload: _ParquetJSLoaderPreload, ...ParquetJSLoaderMetadataWithoutPrelo
|
|
|
10
10
|
/** Default option bag for the experimental parquetjs plain-row loader. */
|
|
11
11
|
const DEFAULT_PARQUET_JS_OPTIONS = {
|
|
12
12
|
columns: undefined,
|
|
13
|
-
preserveBinary: false
|
|
13
|
+
preserveBinary: false,
|
|
14
|
+
verifyFooterSignature: true
|
|
14
15
|
};
|
|
15
16
|
/** Parser-bearing TypeScript-only Parquet loader implementation. */
|
|
16
17
|
export const ParquetJSLoaderWithParser = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-js-loader.js","sourceRoot":"","sources":["../src/parquet-js-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,eAAe,EAAE,4BAA4B,EAAC,MAAM,0BAA0B,CAAC;AAOvF,OAAO,EAAC,gBAAgB,EAAE,yBAAyB,EAAC,+CAA4C;AAChG,OAAO,EACL,sCAAsC,EACtC,6BAA6B,EAC9B,mDAAgD;AACjD,OAAO,EAAC,uBAAuB,EAAC,iDAA8C;AAC9E,OAAO,EAAC,eAAe,IAAI,uBAAuB,EAAC,qCAAkC;AAGrF,MAAM,EAAC,OAAO,EAAE,uBAAuB,EAAE,GAAG,qCAAqC,EAAC,GAChF,uBAAuB,CAAC;AAE1B,0EAA0E;AAC1E,MAAM,0BAA0B,GAAG;IACjC,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,KAAK;
|
|
1
|
+
{"version":3,"file":"parquet-js-loader.js","sourceRoot":"","sources":["../src/parquet-js-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,eAAe,EAAE,4BAA4B,EAAC,MAAM,0BAA0B,CAAC;AAOvF,OAAO,EAAC,gBAAgB,EAAE,yBAAyB,EAAC,+CAA4C;AAChG,OAAO,EACL,sCAAsC,EACtC,6BAA6B,EAC9B,mDAAgD;AACjD,OAAO,EAAC,uBAAuB,EAAC,iDAA8C;AAC9E,OAAO,EAAC,eAAe,IAAI,uBAAuB,EAAC,qCAAkC;AAGrF,MAAM,EAAC,OAAO,EAAE,uBAAuB,EAAE,GAAG,qCAAqC,EAAC,GAChF,uBAAuB,CAAC;AAE1B,0EAA0E;AAC1E,MAAM,0BAA0B,GAAG;IACjC,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,KAAK;IACrB,qBAAqB,EAAE,IAAI;CAC5B,CAAC;AAEF,oEAAoE;AACpE,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,GAAG,qCAAqC;IACxC,MAAM,EAAE,KAAK;IACb,KAAK,CAAC,WAAwB,EAAE,OAAgC;QAC9D,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,WAAW,CAAC,CAAC;QAC9C,OAAO,cAAc,CAAC,OAAO,EAAE,KAAK,KAAK,aAAa;YACpD,CAAC,CAAC,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC;YACrD,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAC7C,CAAC;IACD,SAAS,CAAC,IAAkB,EAAE,OAAgC;QAC5D,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,OAAO,EAAE,KAAK,KAAK,aAAa;YACpD,CAAC,CAAC,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC;YACrD,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAC7C,CAAC;IACD,kBAAkB,CAAC,IAAkB,EAAE,OAAgC;QACrE,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,OAAO,EAAE,KAAK,KAAK,aAAa;YACpD,CAAC,CAAC,sCAAsC,CAAC,IAAI,EAAE,cAAc,CAAC;YAC9D,CAAC,CAAC,yBAAyB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACtD,CAAC;IACD,KAAK,CAAC,CAAC,cAAc,CACnB,aAE+C,EAC/C,OAAgC,EAChC,QAAkB;QAElB,MAAM,WAAW,GAAG,MAAM,4BAA4B,CAAC,aAAa,CAAC,CAAC;QACtE,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,KAAK,aAAa,EAAE,CAAC;YACpD,KAAK,CAAC,CAAC,sCAAsC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,CAAC,yBAAyB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;CAKF,CAAC;AAEF;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,OAAgC;IACzD,OAAO,uBAAuB,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;AACtE,CAAC"}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import type { WriterOptions } from '@loaders.gl/loader-utils';
|
|
2
2
|
import type { Table } from '@loaders.gl/schema';
|
|
3
|
+
import type { ParquetSortingColumnOption } from "./parquetjs/encoder/parquet-encoder.js";
|
|
4
|
+
import type { ParquetWriterEncryptionOptions, ParquetWriterFooterSignatureOptions } from "./lib/parquet-encryption.js";
|
|
5
|
+
export type { ParquetSortingColumnOption } from "./parquetjs/encoder/parquet-encoder.js";
|
|
6
|
+
export type { ParquetWriterEncryptionOptions, ParquetWriterFooterSignatureOptions } from "./lib/parquet-encryption.js";
|
|
3
7
|
/** Stable value encodings selectable by the TypeScript Parquet writer. */
|
|
4
|
-
export type ParquetJSWriterEncoding = 'PLAIN' | 'BYTE_STREAM_SPLIT' | 'DELTA_BINARY_PACKED' | 'DELTA_LENGTH_BYTE_ARRAY' | 'DELTA_BYTE_ARRAY';
|
|
8
|
+
export type ParquetJSWriterEncoding = 'PLAIN' | 'PLAIN_DICTIONARY' | 'BYTE_STREAM_SPLIT' | 'DELTA_BINARY_PACKED' | 'DELTA_LENGTH_BYTE_ARRAY' | 'DELTA_BYTE_ARRAY';
|
|
5
9
|
/** Encoder-specific options for the experimental parquetjs writer. */
|
|
6
10
|
type ParquetJSWriterEncoderOptions = {
|
|
7
11
|
/** Value encoding overrides keyed by top-level column name. */
|
|
@@ -12,6 +16,24 @@ type ParquetJSWriterEncoderOptions = {
|
|
|
12
16
|
columnDictionaries?: Record<string, boolean | 'auto'>;
|
|
13
17
|
/** Maximum uncompressed PLAIN dictionary payload per column chunk. */
|
|
14
18
|
dictionaryPageSizeLimit?: number;
|
|
19
|
+
/** Emits Parquet split-block Bloom filters for selected columns. */
|
|
20
|
+
bloomFilter?: boolean | Record<string, boolean>;
|
|
21
|
+
/** Emits Parquet column and offset indexes for selected non-repeated columns. */
|
|
22
|
+
pageIndex?: boolean | Record<string, boolean>;
|
|
23
|
+
/** Emits CRC-32 checksums for every data and dictionary page. */
|
|
24
|
+
writePageChecksums?: boolean;
|
|
25
|
+
/** Emits optional SizeStatistics metadata for every column chunk. */
|
|
26
|
+
writeSizeStatistics?: boolean;
|
|
27
|
+
/** Emits min/max/null-count statistics for every column chunk. */
|
|
28
|
+
writeStatistics?: boolean | Record<string, boolean>;
|
|
29
|
+
/** Declares row-group sort keys using top-level or dotted nested leaf names. */
|
|
30
|
+
sortingColumns?: readonly ParquetSortingColumnOption[];
|
|
31
|
+
/** Encodes INT96 input values as epoch nanoseconds using the canonical Julian-day layout. */
|
|
32
|
+
int96AsTimestamp?: boolean;
|
|
33
|
+
/** Encrypt the footer using Parquet modular encryption. */
|
|
34
|
+
encryption?: ParquetWriterEncryptionOptions;
|
|
35
|
+
/** Authenticate a plaintext footer with a Parquet modular-encryption signature. */
|
|
36
|
+
footerSignature?: ParquetWriterFooterSignatureOptions;
|
|
15
37
|
rowGroupSize?: number;
|
|
16
38
|
pageSize?: number;
|
|
17
39
|
useDataPageV2?: boolean;
|
|
@@ -40,5 +62,4 @@ export declare const ParquetJSWriter: {
|
|
|
40
62
|
readonly binary: true;
|
|
41
63
|
readonly tests: ["PAR1", "PARE"];
|
|
42
64
|
};
|
|
43
|
-
export {};
|
|
44
65
|
//# sourceMappingURL=parquet-js-writer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-js-writer.d.ts","sourceRoot":"","sources":["../src/parquet-js-writer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAoB,MAAM,0BAA0B,CAAC;AAC/E,OAAO,KAAK,EAAiB,KAAK,EAAa,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"parquet-js-writer.d.ts","sourceRoot":"","sources":["../src/parquet-js-writer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAoB,MAAM,0BAA0B,CAAC;AAC/E,OAAO,KAAK,EAAiB,KAAK,EAAa,MAAM,oBAAoB,CAAC;AAM1E,OAAO,KAAK,EAAC,0BAA0B,EAAC,+CAA4C;AACpF,OAAO,KAAK,EACV,8BAA8B,EAC9B,mCAAmC,EACpC,oCAAiC;AAElC,YAAY,EAAC,0BAA0B,EAAC,+CAA4C;AACpF,YAAY,EACV,8BAA8B,EAC9B,mCAAmC,EACpC,oCAAiC;AAMlC,0EAA0E;AAC1E,MAAM,MAAM,uBAAuB,GAC/B,OAAO,GACP,kBAAkB,GAClB,mBAAmB,GACnB,qBAAqB,GACrB,yBAAyB,GACzB,kBAAkB,CAAC;AAEvB,sEAAsE;AACtE,KAAK,6BAA6B,GAAG;IACnC,+DAA+D;IAC/D,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IAC1D,wEAAwE;IACxE,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9B,kEAAkE;IAClE,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC;IACtD,sEAAsE;IACtE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,oEAAoE;IACpE,WAAW,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChD,iFAAiF;IACjF,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,iEAAiE;IACjE,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,kEAAkE;IAClE,eAAe,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpD,gFAAgF;IAChF,cAAc,CAAC,EAAE,SAAS,0BAA0B,EAAE,CAAC;IACvD,6FAA6F;IAC7F,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,2DAA2D;IAC3D,UAAU,CAAC,EAAE,8BAA8B,CAAC;IAC5C,mFAAmF;IACnF,eAAe,CAAC,EAAE,mCAAmC,CAAC;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAEF,kEAAkE;AAClE,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG;IACnD,OAAO,CAAC,EAAE;SACP,GAAG,IAAI,MAAM,6BAA6B,CAAC,CAAC,EAAE,6BAA6B,CAAC,GAAG,CAAC;KAClF,CAAC;CACH,CAAC;AAKF,oFAAoF;AACpF,eAAO,MAAM,eAAe;;;;;;;6BAQN,KAAK,YAAY,sBAAsB;;;;;;;;;CASoB,CAAC"}
|
|
@@ -7,7 +7,7 @@ import { encodeTableToParquetJs } from "./lib/encoders/encode-table-to-parquet-j
|
|
|
7
7
|
import { ParquetFormat } from "./parquet-format.js";
|
|
8
8
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
9
9
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
10
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
10
|
+
const VERSION = typeof "5.0.0-alpha.4" !== 'undefined' ? "5.0.0-alpha.4" : 'latest';
|
|
11
11
|
/** Default option bag for the experimental parquetjs writer. */
|
|
12
12
|
const DEFAULT_PARQUET_JS_OPTIONS = {};
|
|
13
13
|
/** Plain-row Parquet writer backed by the experimental parquetjs implementation. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-js-writer.js","sourceRoot":"","sources":["../src/parquet-js-writer.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,YAAY,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEzE,OAAO,EAAC,uBAAuB,EAAC,iDAA8C;AAC9E,OAAO,EAAC,sBAAsB,EAAC,qDAAkD;AACjF,OAAO,EAAC,aAAa,EAAC,4BAAyB;
|
|
1
|
+
{"version":3,"file":"parquet-js-writer.js","sourceRoot":"","sources":["../src/parquet-js-writer.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,YAAY,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEzE,OAAO,EAAC,uBAAuB,EAAC,iDAA8C;AAC9E,OAAO,EAAC,sBAAsB,EAAC,qDAAkD;AACjF,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAa/C,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAmD5E,gEAAgE;AAChE,MAAM,0BAA0B,GAAG,EAAE,CAAC;AAEtC,oFAAoF;AACpF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,GAAG,aAAa;IAChB,EAAE,EAAE,YAAY;IAChB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE;QACP,OAAO,EAAE,0BAA0B;KACpC;IACD,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,OAAgC;QACzD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACxD,MAAM,cAAc,GAAG,YAAY,CAAC,EAAC,GAAG,KAAK,EAAE,MAAM,EAAC,EAAE,kBAAkB,CAAmB,CAAC;QAC9F,OAAO,MAAM,sBAAsB,CACjC,EAAC,GAAG,KAAK,EAAE,MAAM,EAAC,EAClB,cAAc,EACd,uBAAuB,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAC7D,CAAC;IACJ,CAAC;CAC8E,CAAC"}
|
|
@@ -5,7 +5,7 @@ import { ParquetFormat } from "./parquet-format.js";
|
|
|
5
5
|
import { PARQUET_LOADER_DEFAULT_OPTIONS } from "./parquet-loader-options.js";
|
|
6
6
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
7
7
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
8
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
8
|
+
const VERSION = typeof "5.0.0-alpha.4" !== 'undefined' ? "5.0.0-alpha.4" : 'latest';
|
|
9
9
|
/** Shared Parquet loader metadata without parser preloading or worker delivery policy. */
|
|
10
10
|
export const PARQUET_LOADER_BASE = {
|
|
11
11
|
...ParquetFormat,
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import type { LoaderOptions } from '@loaders.gl/loader-utils';
|
|
2
|
+
import type { ParquetKeyRetriever } from "./lib/parquet-encryption.js";
|
|
2
3
|
/** Shared row-level options supported by both Parquet backends. */
|
|
3
4
|
type ParquetCommonLoaderOptions = {
|
|
5
|
+
/** @deprecated Use `_scan.limit` instead. */
|
|
4
6
|
limit?: number;
|
|
5
7
|
offset?: number;
|
|
6
8
|
batchSize?: number;
|
|
9
|
+
/** @deprecated Use `_scan.columns` instead for logical projection. */
|
|
7
10
|
columns?: string[];
|
|
8
11
|
preserveBinary?: boolean;
|
|
12
|
+
/** Decode legacy INT96 values as Arrow timestamp-nanosecond values. */
|
|
13
|
+
int96AsTimestamp?: boolean;
|
|
9
14
|
};
|
|
10
15
|
/** Additional options supported by the wasm-backed Parquet reader. */
|
|
11
16
|
type ParquetWasmLoaderOptions = ParquetCommonLoaderOptions & {
|
|
@@ -30,6 +35,12 @@ export type ParquetLoaderOptions = LoaderOptions & {
|
|
|
30
35
|
export type ParquetJSLoaderOptions = LoaderOptions & {
|
|
31
36
|
parquet?: {
|
|
32
37
|
shape?: 'object-row-table' | 'arrow-table';
|
|
38
|
+
/** Resolves modular-encryption keys from file and column key metadata. */
|
|
39
|
+
keyRetriever?: ParquetKeyRetriever;
|
|
40
|
+
/** AAD prefix for encrypted files that intentionally omit it from metadata. */
|
|
41
|
+
aadPrefix?: Uint8Array;
|
|
42
|
+
/** Verify plaintext-footer signatures when present. Enabled by default. */
|
|
43
|
+
verifyFooterSignature?: boolean;
|
|
33
44
|
} & {
|
|
34
45
|
[Key in keyof ParquetCommonLoaderOptions]?: ParquetCommonLoaderOptions[Key];
|
|
35
46
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-loader-options.d.ts","sourceRoot":"","sources":["../src/parquet-loader-options.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"parquet-loader-options.d.ts","sourceRoot":"","sources":["../src/parquet-loader-options.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAC,mBAAmB,EAAC,oCAAiC;AAElE,mEAAmE;AACnE,KAAK,0BAA0B,GAAG;IAChC,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF,sEAAsE;AACtE,KAAK,wBAAwB,GAAG,0BAA0B,GAAG;IAC3D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,kBAAkB,GAAG,aAAa,CAAC;KAC5C,GAAG;SACD,GAAG,IAAI,MAAM,wBAAwB,CAAC,CAAC,EAAE,wBAAwB,CAAC,GAAG,CAAC;KACxE,CAAC;CACH,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG;IACnD,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,kBAAkB,GAAG,aAAa,CAAC;QAC3C,0EAA0E;QAC1E,YAAY,CAAC,EAAE,mBAAmB,CAAC;QACnC,+EAA+E;QAC/E,SAAS,CAAC,EAAE,UAAU,CAAC;QACvB,2EAA2E;QAC3E,qBAAqB,CAAC,EAAE,OAAO,CAAC;KACjC,GAAG;SACD,GAAG,IAAI,MAAM,0BAA0B,CAAC,CAAC,EAAE,0BAA0B,CAAC,GAAG,CAAC;KAC5E,CAAC;CACH,CAAC;AAEF,qEAAqE;AACrE,eAAO,MAAM,8BAA8B;;;;CAIjC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-loader-options.js","sourceRoot":"","sources":["../src/parquet-loader-options.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;
|
|
1
|
+
{"version":3,"file":"parquet-loader-options.js","sourceRoot":"","sources":["../src/parquet-loader-options.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAsDpC,qEAAqE;AACrE,MAAM,CAAC,MAAM,8BAA8B,GAAG;IAC5C,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,KAAK;IACrB,KAAK,EAAE,kBAAkB;CACjB,CAAC"}
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import type { TableQueryCapabilities } from '@loaders.gl/loader-utils';
|
|
2
|
+
/** Portable table-query capabilities of Parquet-backed scans. */
|
|
3
|
+
export declare const PARQUET_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities;
|
|
1
4
|
/** Feature support advertised by a {@link ParquetSource}. */
|
|
2
5
|
export type ParquetSourceCapabilities = Readonly<{
|
|
6
|
+
/** Common portable query contract supported by this source. */
|
|
7
|
+
tableQuery: TableQueryCapabilities;
|
|
3
8
|
/** Schema and footer metadata are cached for the lifetime of the source. */
|
|
4
9
|
supportsCachedMetadata: boolean;
|
|
5
10
|
/** Reads can select explicit Parquet row groups. */
|
|
@@ -16,8 +21,10 @@ export type ParquetSourceCapabilities = Readonly<{
|
|
|
16
21
|
supportsColumnStatistics: boolean;
|
|
17
22
|
/** Serializable predicates can prune impossible row groups using column statistics. */
|
|
18
23
|
supportsPredicatePushdown: boolean;
|
|
19
|
-
/** Column and offset indexes can avoid irrelevant data-page
|
|
24
|
+
/** Column and offset indexes can avoid irrelevant data-page reads for non-repeated columns. */
|
|
20
25
|
supportsPageIndexPruning: boolean;
|
|
26
|
+
/** Native geospatial statistics and GeoParquet 1.1 coverings drive spatial pruning. */
|
|
27
|
+
supportsGeoParquetSpatialPruning: boolean;
|
|
21
28
|
/** Surviving decoded rows are filtered exactly on the caller thread or worker. */
|
|
22
29
|
supportsExactPredicateFiltering: boolean;
|
|
23
30
|
/** Callers can supply the random-access transport used by the source. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-capabilities.d.ts","sourceRoot":"","sources":["../src/parquet-source-capabilities.ts"],"names":[],"mappings":"AAIA,6DAA6D;AAC7D,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,4EAA4E;IAC5E,sBAAsB,EAAE,OAAO,CAAC;IAChC,oDAAoD;IACpD,yBAAyB,EAAE,OAAO,CAAC;IACnC,uDAAuD;IACvD,wBAAwB,EAAE,OAAO,CAAC;IAClC,2EAA2E;IAC3E,uBAAuB,EAAE,OAAO,CAAC;IACjC,+EAA+E;IAC/E,mCAAmC,EAAE,OAAO,CAAC;IAC7C,iFAAiF;IACjF,sBAAsB,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,wBAAwB,EAAE,OAAO,CAAC;IAClC,uFAAuF;IACvF,yBAAyB,EAAE,OAAO,CAAC;IACnC
|
|
1
|
+
{"version":3,"file":"parquet-source-capabilities.d.ts","sourceRoot":"","sources":["../src/parquet-source-capabilities.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,0BAA0B,CAAC;AAErE,iEAAiE;AACjE,eAAO,MAAM,gCAAgC,EAAE,sBAM7C,CAAC;AAEH,6DAA6D;AAC7D,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,+DAA+D;IAC/D,UAAU,EAAE,sBAAsB,CAAC;IACnC,4EAA4E;IAC5E,sBAAsB,EAAE,OAAO,CAAC;IAChC,oDAAoD;IACpD,yBAAyB,EAAE,OAAO,CAAC;IACnC,uDAAuD;IACvD,wBAAwB,EAAE,OAAO,CAAC;IAClC,2EAA2E;IAC3E,uBAAuB,EAAE,OAAO,CAAC;IACjC,+EAA+E;IAC/E,mCAAmC,EAAE,OAAO,CAAC;IAC7C,iFAAiF;IACjF,sBAAsB,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,wBAAwB,EAAE,OAAO,CAAC;IAClC,uFAAuF;IACvF,yBAAyB,EAAE,OAAO,CAAC;IACnC,+FAA+F;IAC/F,wBAAwB,EAAE,OAAO,CAAC;IAClC,uFAAuF;IACvF,gCAAgC,EAAE,OAAO,CAAC;IAC1C,kFAAkF;IAClF,+BAA+B,EAAE,OAAO,CAAC;IACzC,yEAAyE;IACzE,4BAA4B,EAAE,OAAO,CAAC;IACtC,gFAAgF;IAChF,+BAA+B,EAAE,OAAO,CAAC;IACzC,wFAAwF;IACxF,wBAAwB,EAAE,OAAO,CAAC;IAClC,kFAAkF;IAClF,uBAAuB,EAAE,OAAO,CAAC;IACjC,sEAAsE;IACtE,sBAAsB,EAAE,OAAO,CAAC;CACjC,CAAC,CAAC;AAEH,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,EAAE,yBAkBxC,CAAC"}
|
|
@@ -1,8 +1,17 @@
|
|
|
1
1
|
// loaders.gl
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
|
+
/** Portable table-query capabilities of Parquet-backed scans. */
|
|
5
|
+
export const PARQUET_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
6
|
+
projection: 'pushdown',
|
|
7
|
+
predicate: 'pushdown+residual',
|
|
8
|
+
limit: 'pushdown',
|
|
9
|
+
streaming: true,
|
|
10
|
+
cancellation: true
|
|
11
|
+
});
|
|
4
12
|
/** Capabilities of the current range-backed TypeScript {@link ParquetSource}. */
|
|
5
13
|
export const PARQUET_SOURCE_CAPABILITIES = Object.freeze({
|
|
14
|
+
tableQuery: PARQUET_TABLE_QUERY_CAPABILITIES,
|
|
6
15
|
supportsCachedMetadata: true,
|
|
7
16
|
supportsRowGroupSelection: true,
|
|
8
17
|
supportsColumnProjection: true,
|
|
@@ -12,6 +21,7 @@ export const PARQUET_SOURCE_CAPABILITIES = Object.freeze({
|
|
|
12
21
|
supportsColumnStatistics: true,
|
|
13
22
|
supportsPredicatePushdown: true,
|
|
14
23
|
supportsPageIndexPruning: true,
|
|
24
|
+
supportsGeoParquetSpatialPruning: true,
|
|
15
25
|
supportsExactPredicateFiltering: true,
|
|
16
26
|
supportsCustomRangeTransport: true,
|
|
17
27
|
supportsObjectVersionValidation: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-capabilities.js","sourceRoot":"","sources":["../src/parquet-source-capabilities.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;
|
|
1
|
+
{"version":3,"file":"parquet-source-capabilities.js","sourceRoot":"","sources":["../src/parquet-source-capabilities.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,iEAAiE;AACjE,MAAM,CAAC,MAAM,gCAAgC,GAA2B,MAAM,CAAC,MAAM,CAAC;IACpF,UAAU,EAAE,UAAU;IACtB,SAAS,EAAE,mBAAmB;IAC9B,KAAK,EAAE,UAAU;IACjB,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,IAAI;CACnB,CAAC,CAAC;AAwCH,iFAAiF;AACjF,MAAM,CAAC,MAAM,2BAA2B,GAA8B,MAAM,CAAC,MAAM,CAAC;IAClF,UAAU,EAAE,gCAAgC;IAC5C,sBAAsB,EAAE,IAAI;IAC5B,yBAAyB,EAAE,IAAI;IAC/B,wBAAwB,EAAE,IAAI;IAC9B,uBAAuB,EAAE,IAAI;IAC7B,mCAAmC,EAAE,IAAI;IACzC,sBAAsB,EAAE,IAAI;IAC5B,wBAAwB,EAAE,IAAI;IAC9B,yBAAyB,EAAE,IAAI;IAC/B,wBAAwB,EAAE,IAAI;IAC9B,gCAAgC,EAAE,IAAI;IACtC,+BAA+B,EAAE,IAAI;IACrC,4BAA4B,EAAE,IAAI;IAClC,+BAA+B,EAAE,IAAI;IACrC,wBAAwB,EAAE,IAAI;IAC9B,uBAAuB,EAAE,IAAI;IAC7B,sBAAsB,EAAE,IAAI;CAC7B,CAAC,CAAC"}
|
|
@@ -26,8 +26,13 @@ export declare const ParquetSourceLoader: {
|
|
|
26
26
|
readonly batchSize: undefined;
|
|
27
27
|
readonly concurrency: undefined;
|
|
28
28
|
readonly rowGroupFilter: undefined;
|
|
29
|
+
readonly predicate: undefined;
|
|
30
|
+
readonly bbox: undefined;
|
|
31
|
+
readonly geometryColumn: undefined;
|
|
29
32
|
readonly headers: undefined;
|
|
30
33
|
readonly preserveBinary: false;
|
|
34
|
+
readonly verifyPageChecksums: false;
|
|
35
|
+
readonly verifyFooterSignature: true;
|
|
31
36
|
readonly onTelemetry: undefined;
|
|
32
37
|
readonly workerUrl: undefined;
|
|
33
38
|
readonly wasmUrl: undefined;
|
|
@@ -45,8 +50,13 @@ export declare const ParquetSourceLoader: {
|
|
|
45
50
|
readonly batchSize: never;
|
|
46
51
|
readonly concurrency: never;
|
|
47
52
|
readonly rowGroupFilter: never;
|
|
53
|
+
readonly predicate: never;
|
|
54
|
+
readonly bbox: never;
|
|
55
|
+
readonly geometryColumn: never;
|
|
48
56
|
readonly headers: never;
|
|
49
57
|
readonly preserveBinary: false;
|
|
58
|
+
readonly verifyPageChecksums: false;
|
|
59
|
+
readonly verifyFooterSignature: true;
|
|
50
60
|
readonly onTelemetry: never;
|
|
51
61
|
readonly workerUrl: never;
|
|
52
62
|
readonly wasmUrl: never;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-loader-types.d.ts","sourceRoot":"","sources":["../src/parquet-source-loader-types.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,OAAO,EAAE,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAGpE,OAAO,KAAK,EAAC,aAAa,EAAC,mCAAgC;AAC3D,OAAO,KAAK,EAAC,0BAA0B,EAAC,kCAA+B;AAMvE,yFAAyF;AACzF,iBAAe,0BAA0B,IAAI,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAGhF;AAED;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB;uBAED,aAAa;wBACvB,KAAK
|
|
1
|
+
{"version":3,"file":"parquet-source-loader-types.d.ts","sourceRoot":"","sources":["../src/parquet-source-loader-types.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,OAAO,EAAE,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAGpE,OAAO,KAAK,EAAC,aAAa,EAAC,mCAAgC;AAC3D,OAAO,KAAK,EAAC,0BAA0B,EAAC,kCAA+B;AAMvE,yFAAyF;AACzF,iBAAe,0BAA0B,IAAI,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAGhF;AAED;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB;uBAED,aAAa;wBACvB,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAwDT,MAAM,KAAG,OAAO;;uCAGtB,MAAM,GAAG,IAAI,YACV,0BAA0B,aACzB,OAAO,KACjB,aAAa;;;;;;;;CAK8B,CAAC"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { ParquetFormat } from "./parquet-format.js";
|
|
5
5
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
6
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
7
|
+
const VERSION = typeof "5.0.0-alpha.4" !== 'undefined' ? "5.0.0-alpha.4" : 'latest';
|
|
8
8
|
/** Loads the runtime Parquet source implementation from its explicit package subpath. */
|
|
9
9
|
async function preloadParquetSourceLoader() {
|
|
10
10
|
const { ParquetSourceLoaderWithParser } = await import('@loaders.gl/parquet/parquet-source-loader');
|
|
@@ -34,8 +34,13 @@ export const ParquetSourceLoader = {
|
|
|
34
34
|
batchSize: undefined,
|
|
35
35
|
concurrency: undefined,
|
|
36
36
|
rowGroupFilter: undefined,
|
|
37
|
+
predicate: undefined,
|
|
38
|
+
bbox: undefined,
|
|
39
|
+
geometryColumn: undefined,
|
|
37
40
|
headers: undefined,
|
|
38
41
|
preserveBinary: false,
|
|
42
|
+
verifyPageChecksums: false,
|
|
43
|
+
verifyFooterSignature: true,
|
|
39
44
|
onTelemetry: undefined,
|
|
40
45
|
workerUrl: undefined,
|
|
41
46
|
wasmUrl: undefined
|
|
@@ -53,8 +58,13 @@ export const ParquetSourceLoader = {
|
|
|
53
58
|
batchSize: undefined,
|
|
54
59
|
concurrency: undefined,
|
|
55
60
|
rowGroupFilter: undefined,
|
|
61
|
+
predicate: undefined,
|
|
62
|
+
bbox: undefined,
|
|
63
|
+
geometryColumn: undefined,
|
|
56
64
|
headers: undefined,
|
|
57
65
|
preserveBinary: false,
|
|
66
|
+
verifyPageChecksums: false,
|
|
67
|
+
verifyFooterSignature: true,
|
|
58
68
|
onTelemetry: undefined,
|
|
59
69
|
workerUrl: undefined,
|
|
60
70
|
wasmUrl: undefined
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-loader-types.js","sourceRoot":"","sources":["../src/parquet-source-loader-types.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAI/C,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAE5E,yFAAyF;AACzF,KAAK,UAAU,0BAA0B;IACvC,MAAM,EAAC,6BAA6B,EAAC,GAAG,MAAM,MAAM,CAAC,2CAA2C,CAAC,CAAC;IAClG,OAAO,6BAA6B,CAAC;AACvC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,GAAG,aAAa;IAChB,QAAQ,EAAE,IAAgC;IAC1C,SAAS,EAAE,IAAa;IACxB,IAAI,EAAE,qBAAqB;IAC3B,EAAE,EAAE,gBAAgB;IACpB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,IAAI,EAAE,SAAS;IACf,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE;QACP,OAAO,EAAE;YACP,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,SAAS;YACpB,WAAW,EAAE,SAAS;YACtB,cAAc,EAAE,SAAS;YACzB,OAAO,EAAE,SAAS;YAClB,cAAc,EAAE,KAAK;YACrB,WAAW,EAAE,SAAS;YACtB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,SAAS;SACnB;QACD,aAAa,EAAE;YACb,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,CAAC;SACvB;KACF;IACD,cAAc,EAAE;QACd,OAAO,EAAE;YACP,SAAS,EAAE,SAAU;YACrB,OAAO,EAAE,SAAU;YACnB,SAAS,EAAE,SAAU;YACrB,WAAW,EAAE,SAAU;YACvB,cAAc,EAAE,SAAU;YAC1B,OAAO,EAAE,SAAU;YACnB,cAAc,EAAE,KAAK;YACrB,WAAW,EAAE,SAAU;YACvB,SAAS,EAAE,SAAU;YACrB,OAAO,EAAE,SAAU;SACpB;QACD,aAAa,EAAE;YACb,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,CAAC;SACvB;KACF;IACD,OAAO,EAAE,CAAC,GAAW,EAAW,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;IACnE,OAAO,EAAE,0BAA0B;IACnC,gBAAgB,CACd,KAAoB,EACpB,QAAoC,EACpC,QAAkB;QAElB,MAAM,IAAI,KAAK,CACb,2GAA2G,CAC5G,CAAC;IACJ,CAAC;CAC6C,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet-source-loader-types.js","sourceRoot":"","sources":["../src/parquet-source-loader-types.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAI/C,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAE5E,yFAAyF;AACzF,KAAK,UAAU,0BAA0B;IACvC,MAAM,EAAC,6BAA6B,EAAC,GAAG,MAAM,MAAM,CAAC,2CAA2C,CAAC,CAAC;IAClG,OAAO,6BAA6B,CAAC;AACvC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,GAAG,aAAa;IAChB,QAAQ,EAAE,IAAgC;IAC1C,SAAS,EAAE,IAAa;IACxB,IAAI,EAAE,qBAAqB;IAC3B,EAAE,EAAE,gBAAgB;IACpB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,OAAO;IAChB,IAAI,EAAE,SAAS;IACf,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE;QACP,OAAO,EAAE;YACP,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,SAAS;YACpB,WAAW,EAAE,SAAS;YACtB,cAAc,EAAE,SAAS;YACzB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,SAAS;YACf,cAAc,EAAE,SAAS;YACzB,OAAO,EAAE,SAAS;YAClB,cAAc,EAAE,KAAK;YACrB,mBAAmB,EAAE,KAAK;YAC1B,qBAAqB,EAAE,IAAI;YAC3B,WAAW,EAAE,SAAS;YACtB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,SAAS;SACnB;QACD,aAAa,EAAE;YACb,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,CAAC;SACvB;KACF;IACD,cAAc,EAAE;QACd,OAAO,EAAE;YACP,SAAS,EAAE,SAAU;YACrB,OAAO,EAAE,SAAU;YACnB,SAAS,EAAE,SAAU;YACrB,WAAW,EAAE,SAAU;YACvB,cAAc,EAAE,SAAU;YAC1B,SAAS,EAAE,SAAU;YACrB,IAAI,EAAE,SAAU;YAChB,cAAc,EAAE,SAAU;YAC1B,OAAO,EAAE,SAAU;YACnB,cAAc,EAAE,KAAK;YACrB,mBAAmB,EAAE,KAAK;YAC1B,qBAAqB,EAAE,IAAI;YAC3B,WAAW,EAAE,SAAU;YACvB,SAAS,EAAE,SAAU;YACrB,OAAO,EAAE,SAAU;SACpB;QACD,aAAa,EAAE;YACb,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,CAAC;SACvB;KACF;IACD,OAAO,EAAE,CAAC,GAAW,EAAW,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;IACnE,OAAO,EAAE,0BAA0B;IACnC,gBAAgB,CACd,KAAoB,EACpB,QAAoC,EACpC,QAAkB;QAElB,MAAM,IAAI,KAAK,CACb,2GAA2G,CAC5G,CAAC;IACJ,CAAC;CAC6C,CAAC"}
|