@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15841 -3306
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +16528 -4928
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-encoded-page-types.d.ts +108 -0
- package/dist/parquet-encoded-page-types.d.ts.map +1 -0
- package/dist/parquet-encoded-page-types.js +5 -0
- package/dist/parquet-encoded-page-types.js.map +1 -0
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14121 -2796
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +51 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +651 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7378 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8501 -1045
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
- package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
- package/dist/parquetjs/parser/encoded-pages.js +206 -0
- package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts +88 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +713 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -13
- package/src/bundled.ts +10 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +59 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-encoded-page-types.ts +122 -0
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +1024 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/encoded-pages.ts +278 -0
- package/src/parquetjs/parser/parquet-reader.ts +1112 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read-utils.js","sourceRoot":"","sources":["../../../src/parquetjs/utils/read-utils.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,OAAO,
|
|
1
|
+
{"version":3,"file":"read-utils.js","sourceRoot":"","sources":["../../../src/parquetjs/utils/read-utils.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,YAAY,EACZ,UAAU,EACX,mCAAgC;AACjC,OAAO,EAAC,yBAAyB,EAAC,0CAAuC;AACzE,OAAO,EAAC,+BAA+B,EAAC,iDAA8C;AACtF,OAAO,EAAC,mBAAmB,EAAC,mCAAgC;AAE5D;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAQ;IACtC,MAAM,QAAQ,GAAG,IAAI,+BAA+B,EAAE,CAAC;IACvD,GAAG,CAAC,KAAK,CAAC,QAAe,CAAC,CAAC;IAC3B,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAQ,EAAE,GAAe,EAAE,MAAe;IACrE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,qDAAqD;QACrD,MAAM,GAAG,CAAC,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC/C,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC;IAC3B,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC;IAC1D,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnB,OAAO,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,UAAkB,EAAE,KAAsB;IACtE,MAAM,UAAU,GAAG,UAAsD,CAAC;IAC1E,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;YACnC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAe,EAAE,MAAe;IACjE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,qDAAqD;QACrD,MAAM,GAAG,CAAC,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC/C,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC;IAC3B,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,QAAe,CAAC,CAAC;IACpD,OAAO,EAAC,MAAM,EAAE,SAAS,CAAC,OAAO,GAAG,MAAM,EAAE,QAAQ,EAAC,CAAC;AACxD,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,wBAAwB,CAAC,GAAe,EAAE,MAAe;IACvE,IAAI,CAAC,MAAM;QAAE,MAAM,GAAG,CAAC,CAAC;IACxB,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC/C,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC;IAC3B,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAe,CAAC,CAAC;IAC1D,OAAO,EAAC,MAAM,EAAE,SAAS,CAAC,OAAO,GAAG,MAAM,EAAE,QAAQ,EAAC,CAAC;AACxD,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,oBAAoB,CAAC,GAAe,EAAE,MAAe;IACnE,IAAI,CAAC,MAAM;QAAE,MAAM,GAAG,CAAC,CAAC;IACxB,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC/C,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC;IAC3B,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,QAAe,CAAC,CAAC;IACtD,OAAO,EAAC,MAAM,EAAE,SAAS,CAAC,OAAO,GAAG,MAAM,EAAE,QAAQ,EAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAe,EAAE,MAAe;IAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,qDAAqD;QACrD,MAAM,GAAG,CAAC,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC/C,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC;IAC3B,MAAM,QAAQ,GAAG,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,QAAe,CAAC,CAAC;IACpD,OAAO,EAAC,MAAM,EAAE,SAAS,CAAC,OAAO,GAAG,MAAM,EAAE,UAAU,EAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;QACT,gDAAgD;IAClD,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe,EAAE,IAAc;IAC1D,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QAC/D,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YACrE,MAAM,eAAe,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAChD,IAAI,eAAe,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,UAAU,CAAC;YACpB,CAAC;YACD,IACE,SAAS,IAAI,IAAI,CAAC,MAAM;gBACxB,CAAC,eAAe,KAAK,GAAG,IAAI,eAAe,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAChE,CAAC;gBACD,OAAO,GAAG,KAAK,CAAC;gBAChB,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC"}
|
package/dist/unbundled.d.ts
CHANGED
|
@@ -3,7 +3,8 @@ export type { GeoParquetLoaderOptions } from "./geoparquet-loader.js";
|
|
|
3
3
|
export { ParquetLoader } from "./parquet-loader-types.js";
|
|
4
4
|
export { GeoParquetLoader } from "./geoparquet-loader.js";
|
|
5
5
|
export { ParquetJSLoader } from "./parquet-js-loader-types.js";
|
|
6
|
-
export { ParquetSourceLoader, ParquetSource, type ParquetSourceLoaderOptions, type ParquetSourceReadOptions, type ParquetSourceMetadata, type ParquetRowGroupMetadata, type ParquetColumnChunkMetadata, type ParquetBatchMetadata, type ParquetSourceBatch } from "./parquet-source-loader.js";
|
|
6
|
+
export { ParquetSourceLoader, ParquetSource, type ParquetSourceLoaderOptions, type ParquetSourceReadOptions, type ParquetSourceMetadata, type ParquetRowGroupMetadata, type ParquetSortingColumn, type ParquetColumnChunkMetadata, type ParquetColumnChunkSizeStatistics, type ParquetGeospatialBoundingBox, type ParquetGeospatialStatistics, type ParquetBatchMetadata, type ParquetSourceBatch } from "./parquet-source-loader.js";
|
|
7
|
+
export { createParquetModuleAad, decryptParquetModule, encryptParquetModule, createParquetFooterSignature, type ParquetDecryptModuleOptions, type ParquetEncryptionAlgorithm, type ParquetKeyRetriever, type ParquetWriterFooterSignatureOptions } from "./lib/parquet-encryption.js";
|
|
7
8
|
export { ParquetDatasetSource, type ParquetDatasetBatch, type ParquetDatasetFile, type ParquetDatasetFileProvider, type ParquetDatasetReadOptions, type ParquetDatasetSourceOptions, type ParquetDatasetTelemetry } from "./parquet-dataset-source.js";
|
|
8
9
|
export { PARQUET_SOURCE_CAPABILITIES, type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
|
|
9
10
|
//# sourceMappingURL=unbundled.d.ts.map
|
package/dist/unbundled.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unbundled.d.ts","sourceRoot":"","sources":["../src/unbundled.ts"],"names":[],"mappings":"AAIA,YAAY,EAAC,oBAAoB,EAAC,kCAA+B;AACjE,YAAY,EAAC,uBAAuB,EAAC,+BAA4B;AACjE,OAAO,EAAC,aAAa,EAAC,kCAA+B;AACrD,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,qCAAkC;AAC1D,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,KAAK,0BAA0B,EAC/B,KAAK,wBAAwB,EAC7B,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,EAC5B,KAAK,0BAA0B,EAC/B,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACxB,mCAAgC;AACjC,OAAO,EACL,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC7B,oCAAiC;AAClC,OAAO,EACL,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC"}
|
|
1
|
+
{"version":3,"file":"unbundled.d.ts","sourceRoot":"","sources":["../src/unbundled.ts"],"names":[],"mappings":"AAIA,YAAY,EAAC,oBAAoB,EAAC,kCAA+B;AACjE,YAAY,EAAC,uBAAuB,EAAC,+BAA4B;AACjE,OAAO,EAAC,aAAa,EAAC,kCAA+B;AACrD,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,qCAAkC;AAC1D,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,KAAK,0BAA0B,EAC/B,KAAK,wBAAwB,EAC7B,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,gCAAgC,EACrC,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACxB,mCAAgC;AACjC,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,4BAA4B,EAC5B,KAAK,2BAA2B,EAChC,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EACxB,KAAK,mCAAmC,EACzC,oCAAiC;AAClC,OAAO,EACL,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC7B,oCAAiC;AAClC,OAAO,EACL,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC"}
|
package/dist/unbundled.js
CHANGED
|
@@ -5,6 +5,7 @@ export { ParquetLoader } from "./parquet-loader-types.js";
|
|
|
5
5
|
export { GeoParquetLoader } from "./geoparquet-loader.js";
|
|
6
6
|
export { ParquetJSLoader } from "./parquet-js-loader-types.js";
|
|
7
7
|
export { ParquetSourceLoader, ParquetSource } from "./parquet-source-loader.js";
|
|
8
|
+
export { createParquetModuleAad, decryptParquetModule, encryptParquetModule, createParquetFooterSignature } from "./lib/parquet-encryption.js";
|
|
8
9
|
export { ParquetDatasetSource } from "./parquet-dataset-source.js";
|
|
9
10
|
export { PARQUET_SOURCE_CAPABILITIES } from "./parquet-source-capabilities.js";
|
|
10
11
|
//# sourceMappingURL=unbundled.js.map
|
package/dist/unbundled.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unbundled.js","sourceRoot":"","sources":["../src/unbundled.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,aAAa,EAAC,kCAA+B;AACrD,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,qCAAkC;AAC1D,OAAO,EACL,mBAAmB,EACnB,aAAa,
|
|
1
|
+
{"version":3,"file":"unbundled.js","sourceRoot":"","sources":["../src/unbundled.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,aAAa,EAAC,kCAA+B;AACrD,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,qCAAkC;AAC1D,OAAO,EACL,mBAAmB,EACnB,aAAa,EAYd,mCAAgC;AACjC,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,4BAA4B,EAK7B,oCAAiC;AAClC,OAAO,EACL,oBAAoB,EAOrB,oCAAiC;AAClC,OAAO,EACL,2BAA2B,EAE5B,yCAAsC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/parquet",
|
|
3
|
-
"version": "5.0.0-alpha.
|
|
3
|
+
"version": "5.0.0-alpha.4",
|
|
4
4
|
"description": "Framework-independent loader for Apache Parquet files",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -43,6 +43,30 @@
|
|
|
43
43
|
"types": "./dist/parquet-loader.d.ts",
|
|
44
44
|
"import": "./dist/parquet-loader.js"
|
|
45
45
|
},
|
|
46
|
+
"./iceberg-table-source": {
|
|
47
|
+
"types": "./dist/iceberg-table-source.d.ts",
|
|
48
|
+
"import": "./dist/iceberg-table-source.js"
|
|
49
|
+
},
|
|
50
|
+
"./iceberg-types": {
|
|
51
|
+
"types": "./dist/iceberg-types.d.ts",
|
|
52
|
+
"import": "./dist/iceberg-types.js"
|
|
53
|
+
},
|
|
54
|
+
"./iceberg-rest-catalog": {
|
|
55
|
+
"types": "./dist/iceberg-rest-catalog.d.ts",
|
|
56
|
+
"import": "./dist/iceberg-rest-catalog.js"
|
|
57
|
+
},
|
|
58
|
+
"./delta-source": {
|
|
59
|
+
"types": "./dist/delta-source.d.ts",
|
|
60
|
+
"import": "./dist/delta-source.js"
|
|
61
|
+
},
|
|
62
|
+
"./delta-source-loader-types": {
|
|
63
|
+
"types": "./dist/delta-source-loader-types.d.ts",
|
|
64
|
+
"import": "./dist/delta-source-loader-types.js"
|
|
65
|
+
},
|
|
66
|
+
"./delta-types": {
|
|
67
|
+
"types": "./dist/delta-types.d.ts",
|
|
68
|
+
"import": "./dist/delta-types.js"
|
|
69
|
+
},
|
|
46
70
|
"./parquet-js-loader": {
|
|
47
71
|
"types": "./dist/parquet-js-loader.d.ts",
|
|
48
72
|
"import": "./dist/parquet-js-loader.js"
|
|
@@ -98,22 +122,21 @@
|
|
|
98
122
|
"util": false
|
|
99
123
|
},
|
|
100
124
|
"dependencies": {
|
|
101
|
-
"@loaders.gl/arrow": "5.0.0-alpha.
|
|
102
|
-
"@loaders.gl/
|
|
103
|
-
"@loaders.gl/
|
|
104
|
-
"@loaders.gl/
|
|
105
|
-
"@loaders.gl/
|
|
106
|
-
"@loaders.gl/
|
|
107
|
-
"@loaders.gl/
|
|
108
|
-
"@loaders.gl/schema
|
|
109
|
-
"@loaders.gl/
|
|
125
|
+
"@loaders.gl/arrow": "5.0.0-alpha.4",
|
|
126
|
+
"@loaders.gl/avro": "5.0.0-alpha.4",
|
|
127
|
+
"@loaders.gl/bson": "5.0.0-alpha.4",
|
|
128
|
+
"@loaders.gl/compression": "5.0.0-alpha.4",
|
|
129
|
+
"@loaders.gl/geoarrow": "5.0.0-alpha.4",
|
|
130
|
+
"@loaders.gl/gis": "5.0.0-alpha.4",
|
|
131
|
+
"@loaders.gl/loader-utils": "5.0.0-alpha.4",
|
|
132
|
+
"@loaders.gl/schema": "5.0.0-alpha.4",
|
|
133
|
+
"@loaders.gl/schema-utils": "5.0.0-alpha.4",
|
|
134
|
+
"@loaders.gl/wkt": "5.0.0-alpha.4",
|
|
110
135
|
"@probe.gl/log": "^4.1.1",
|
|
111
136
|
"async-mutex": "^0.2.2",
|
|
112
137
|
"isomorphic-ws": "^5.0.0",
|
|
113
|
-
"lz4js": "^0.2.0",
|
|
114
138
|
"object-stream": "0.0.1",
|
|
115
139
|
"parquet-wasm": "0.7.2",
|
|
116
|
-
"snappyjs": "^0.6.0",
|
|
117
140
|
"varint": "^6.0.0"
|
|
118
141
|
},
|
|
119
142
|
"devDependencies": {
|
|
@@ -127,5 +150,5 @@
|
|
|
127
150
|
"@loaders.gl/core": "~5.0.0-alpha.0",
|
|
128
151
|
"apache-arrow": ">= 17.0.0"
|
|
129
152
|
},
|
|
130
|
-
"gitHead": "
|
|
153
|
+
"gitHead": "470a9cf1956b8326c92015b314f4ad8a10a3517c"
|
|
131
154
|
}
|
package/src/bundled.ts
CHANGED
|
@@ -10,11 +10,21 @@ export {ParquetJSLoaderWithParser as ParquetJSLoader} from './parquet-js-loader'
|
|
|
10
10
|
export {
|
|
11
11
|
ParquetSourceLoader,
|
|
12
12
|
ParquetSource,
|
|
13
|
+
type ParquetDeferredPageFilter,
|
|
14
|
+
type ParquetEncodedColumnChunk,
|
|
15
|
+
type ParquetEncodedPage,
|
|
16
|
+
type ParquetEncodedPageBatch,
|
|
17
|
+
type ParquetEncodedPageReadOptions,
|
|
18
|
+
type ParquetEncodedPageSection,
|
|
19
|
+
type ParquetPageCompressionState,
|
|
13
20
|
type ParquetSourceLoaderOptions,
|
|
14
21
|
type ParquetSourceReadOptions,
|
|
15
22
|
type ParquetSourceMetadata,
|
|
16
23
|
type ParquetRowGroupMetadata,
|
|
24
|
+
type ParquetSortingColumn,
|
|
17
25
|
type ParquetColumnChunkMetadata,
|
|
26
|
+
type ParquetGeospatialBoundingBox,
|
|
27
|
+
type ParquetGeospatialStatistics,
|
|
18
28
|
type ParquetBatchMetadata,
|
|
19
29
|
type ParquetSourceBatch
|
|
20
30
|
} from './parquet-source-loader';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
/** Metadata describing the Delta Lake transaction-log format. */
|
|
6
|
+
export const DeltaFormat = {
|
|
7
|
+
name: 'Delta Lake',
|
|
8
|
+
id: 'delta',
|
|
9
|
+
module: 'parquet',
|
|
10
|
+
extensions: ['json', 'checkpoint.parquet'],
|
|
11
|
+
mimeTypes: ['application/json']
|
|
12
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {SourceLoader} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {DeltaFormat} from './delta-format';
|
|
7
|
+
import type {DeltaSourceOptions} from './delta-types';
|
|
8
|
+
import type {DeltaTableSource} from './delta-source';
|
|
9
|
+
|
|
10
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
11
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
12
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
13
|
+
|
|
14
|
+
/** Metadata-only Delta source loader options. */
|
|
15
|
+
export type DeltaSourceLoaderOptions = DeltaSourceOptions;
|
|
16
|
+
|
|
17
|
+
/** Loads the parser-bearing Delta source implementation on demand. */
|
|
18
|
+
async function preloadDeltaSourceLoader(): Promise<SourceLoader<DeltaTableSource>> {
|
|
19
|
+
const {DeltaSourceLoaderWithParser} = await import('@loaders.gl/parquet/delta-source');
|
|
20
|
+
return DeltaSourceLoaderWithParser;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Metadata-only Delta source loader; runtime code is available through `preload()`. */
|
|
24
|
+
export const DeltaSourceLoader = {
|
|
25
|
+
...DeltaFormat,
|
|
26
|
+
dataType: null as unknown as DeltaTableSource,
|
|
27
|
+
batchType: null as never,
|
|
28
|
+
name: 'DeltaSourceLoader',
|
|
29
|
+
version: VERSION,
|
|
30
|
+
type: 'delta',
|
|
31
|
+
fromUrl: true,
|
|
32
|
+
fromBlob: true,
|
|
33
|
+
options: {},
|
|
34
|
+
defaultOptions: {},
|
|
35
|
+
testURL: (url: string): boolean =>
|
|
36
|
+
/\/_delta_log\/(?:\d{20}\.json|_last_checkpoint)(?:$|[?#])/i.test(url),
|
|
37
|
+
preload: preloadDeltaSourceLoader
|
|
38
|
+
} as const satisfies Omit<SourceLoader<DeltaTableSource>, 'createDataSource'> & {
|
|
39
|
+
preload: () => Promise<SourceLoader<DeltaTableSource>>;
|
|
40
|
+
};
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
ScanFragment,
|
|
7
|
+
ScanFragmentProvider,
|
|
8
|
+
ScanQueryMetadata,
|
|
9
|
+
ScanQueryMetadataOptions,
|
|
10
|
+
TableScanSource,
|
|
11
|
+
SourceLoader
|
|
12
|
+
} from '@loaders.gl/loader-utils';
|
|
13
|
+
import {createScanQueryMetadata, DataSource} from '@loaders.gl/loader-utils';
|
|
14
|
+
import {ParquetDatasetSource} from './parquet-dataset-source';
|
|
15
|
+
import {PARQUET_TABLE_QUERY_CAPABILITIES} from './parquet-source-capabilities';
|
|
16
|
+
import type {DeltaAction, DeltaScanOptions, DeltaSourceOptions} from './delta-types';
|
|
17
|
+
import type {
|
|
18
|
+
ParquetDatasetBatch,
|
|
19
|
+
ParquetDatasetExplain,
|
|
20
|
+
ParquetPredicate
|
|
21
|
+
} from './parquet-source-types';
|
|
22
|
+
import {DeltaFormat} from './delta-format';
|
|
23
|
+
|
|
24
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
25
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
26
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
27
|
+
|
|
28
|
+
/** Lightweight Delta Lake snapshot source backed by a newline-delimited commit log. */
|
|
29
|
+
export class DeltaTableSource
|
|
30
|
+
extends DataSource<string | Blob, DeltaSourceOptions>
|
|
31
|
+
implements
|
|
32
|
+
TableScanSource<ParquetDatasetBatch, ParquetPredicate>,
|
|
33
|
+
ScanFragmentProvider<ParquetPredicate>
|
|
34
|
+
{
|
|
35
|
+
private readonly actionsPromises = new Map<string, Promise<readonly DeltaAction[]>>();
|
|
36
|
+
|
|
37
|
+
/** Creates a source from a Delta commit JSON URL or Blob. */
|
|
38
|
+
constructor(data: string | Blob, options: DeltaSourceOptions = {}) {
|
|
39
|
+
super(data, options);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Discovers the active Parquet files represented by the commit log. */
|
|
43
|
+
async getScanFragments(options: DeltaScanOptions = {}): Promise<readonly ScanFragment[]> {
|
|
44
|
+
const actions = await this.getActions(options.version, options.signal);
|
|
45
|
+
const activeFiles = selectActiveDeltaFiles(actions);
|
|
46
|
+
return Object.freeze(
|
|
47
|
+
activeFiles.map(file =>
|
|
48
|
+
Object.freeze({
|
|
49
|
+
id: file.path,
|
|
50
|
+
uri: this.resolveDataFile(file.path),
|
|
51
|
+
partitionValues: file.partitionValues,
|
|
52
|
+
byteLength: file.size,
|
|
53
|
+
rowCount: getDeltaRecordCount(file.stats),
|
|
54
|
+
metadata: {stats: file.stats}
|
|
55
|
+
})
|
|
56
|
+
)
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Discovers schema and snapshot statistics without decoding all rows. */
|
|
61
|
+
async getQueryMetadata(options: ScanQueryMetadataOptions = {}): Promise<ScanQueryMetadata> {
|
|
62
|
+
const fragments = await this.getScanFragments(options);
|
|
63
|
+
if (!fragments.length) {
|
|
64
|
+
throw new Error('Delta snapshot contains no active Parquet files');
|
|
65
|
+
}
|
|
66
|
+
const dataset = this.createDataset(fragments);
|
|
67
|
+
const schema = await dataset.getSchema({signal: options.signal});
|
|
68
|
+
return createScanQueryMetadata({
|
|
69
|
+
sourceType: 'delta',
|
|
70
|
+
queryType: 'table',
|
|
71
|
+
execution: {status: 'supported', method: 'read'},
|
|
72
|
+
name: this.data instanceof Blob ? undefined : this.data,
|
|
73
|
+
schema,
|
|
74
|
+
capabilities: {table: PARQUET_TABLE_QUERY_CAPABILITIES},
|
|
75
|
+
statistics: {
|
|
76
|
+
rowCount: fragments.reduce((total, fragment) => total + Number(fragment.rowCount || 0), 0),
|
|
77
|
+
byteLength: fragments.reduce(
|
|
78
|
+
(total, fragment) => total + Number(fragment.byteLength || 0),
|
|
79
|
+
0
|
|
80
|
+
)
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Explains the Delta snapshot and delegated Parquet physical plans. */
|
|
86
|
+
async getScanPlan(options: DeltaScanOptions = {}): Promise<ParquetDatasetExplain> {
|
|
87
|
+
return this.createDataset(await this.getScanFragments(options)).getScanPlan(options);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Reads the active snapshot through the shared Parquet dataset executor. */
|
|
91
|
+
read(options: DeltaScanOptions = {}): AsyncIterable<ParquetDatasetBatch> {
|
|
92
|
+
return this.createDatasetFromOptions(options).read(options);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Alias for `read()` used by scan-aware consumers. */
|
|
96
|
+
scan(options: DeltaScanOptions = {}): AsyncIterable<ParquetDatasetBatch> {
|
|
97
|
+
return this.read(options);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private async getActions(
|
|
101
|
+
requestedVersion?: number,
|
|
102
|
+
signal?: AbortSignal
|
|
103
|
+
): Promise<readonly DeltaAction[]> {
|
|
104
|
+
const version = this.getSnapshotVersion(requestedVersion);
|
|
105
|
+
const cacheKey = version === undefined ? 'single-commit' : String(version);
|
|
106
|
+
let actionsPromise = this.actionsPromises.get(cacheKey);
|
|
107
|
+
if (!actionsPromise) {
|
|
108
|
+
actionsPromise = this.loadActions(version, signal);
|
|
109
|
+
this.actionsPromises.set(cacheKey, actionsPromise);
|
|
110
|
+
}
|
|
111
|
+
return actionsPromise;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Returns the configured or URL-derived snapshot version, when available. */
|
|
115
|
+
private getSnapshotVersion(version?: number): number | undefined {
|
|
116
|
+
const selectedVersion = version ?? this.options.delta?.version;
|
|
117
|
+
if (selectedVersion !== undefined) {
|
|
118
|
+
if (!Number.isSafeInteger(selectedVersion) || selectedVersion < 0) {
|
|
119
|
+
throw new Error('Delta snapshot version must be a non-negative safe integer');
|
|
120
|
+
}
|
|
121
|
+
return selectedVersion;
|
|
122
|
+
}
|
|
123
|
+
if (typeof this.data !== 'string') return undefined;
|
|
124
|
+
const match = /\/_delta_log\/(\d{20})\.json(?:$|[?#])/i.exec(this.data);
|
|
125
|
+
return match ? Number(match[1]) : undefined;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Reads one commit log or replays all commits through the requested version. */
|
|
129
|
+
private async loadActions(
|
|
130
|
+
version: number | undefined,
|
|
131
|
+
signal?: AbortSignal
|
|
132
|
+
): Promise<readonly DeltaAction[]> {
|
|
133
|
+
if (typeof this.data !== 'string' || version === undefined) {
|
|
134
|
+
const text =
|
|
135
|
+
typeof this.data === 'string'
|
|
136
|
+
? await this.readResponse(
|
|
137
|
+
await this.fetch(this.url, {
|
|
138
|
+
headers: this.options.delta?.headers,
|
|
139
|
+
signal
|
|
140
|
+
})
|
|
141
|
+
)
|
|
142
|
+
: await this.data.text();
|
|
143
|
+
const actions = parseDeltaActions(text);
|
|
144
|
+
validateDeltaActions(actions);
|
|
145
|
+
return actions;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const tableRoot = this.data.replace(/\/_delta_log\/[^/?#]+(?:[?#].*)?$/i, '');
|
|
149
|
+
const actions: DeltaAction[] = [];
|
|
150
|
+
for (let currentVersion = 0; currentVersion <= version; currentVersion++) {
|
|
151
|
+
const commitURL = `${tableRoot}/_delta_log/${String(currentVersion).padStart(20, '0')}.json`;
|
|
152
|
+
const text = await this.readResponse(
|
|
153
|
+
await this.fetch(commitURL, {
|
|
154
|
+
headers: this.options.delta?.headers,
|
|
155
|
+
signal
|
|
156
|
+
})
|
|
157
|
+
);
|
|
158
|
+
actions.push(...parseDeltaActions(text));
|
|
159
|
+
}
|
|
160
|
+
const replayedActions = Object.freeze(actions);
|
|
161
|
+
validateDeltaActions(replayedActions);
|
|
162
|
+
return replayedActions;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private async readResponse(response: Response): Promise<string> {
|
|
166
|
+
if (!response.ok)
|
|
167
|
+
throw new Error(`Delta commit log request failed with status ${response.status}`);
|
|
168
|
+
return response.text();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private createDataset(fragments: readonly ScanFragment[]): ParquetDatasetSource {
|
|
172
|
+
return new ParquetDatasetSource(
|
|
173
|
+
fragments.map(fragment => ({
|
|
174
|
+
data: fragment.uri!,
|
|
175
|
+
id: fragment.id,
|
|
176
|
+
partitions: fragment.partitionValues as never,
|
|
177
|
+
metadata: fragment.metadata
|
|
178
|
+
})),
|
|
179
|
+
this.options
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private createDatasetFromOptions(options: DeltaScanOptions): ParquetDatasetSource {
|
|
184
|
+
return new ParquetDatasetSource(
|
|
185
|
+
() =>
|
|
186
|
+
this.getScanFragments(options).then(fragments =>
|
|
187
|
+
fragments.map(fragment => ({
|
|
188
|
+
data: fragment.uri!,
|
|
189
|
+
id: fragment.id,
|
|
190
|
+
partitions: fragment.partitionValues as never
|
|
191
|
+
}))
|
|
192
|
+
),
|
|
193
|
+
this.options
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
private resolveDataFile(path: string): string {
|
|
198
|
+
if (this.options.delta?.baseUrl) return new URL(path, this.options.delta.baseUrl).toString();
|
|
199
|
+
if (typeof this.data === 'string') {
|
|
200
|
+
const tableRoot = this.data.replace(/\/_delta_log\/[^/]*$/i, '/');
|
|
201
|
+
return new URL(path, tableRoot).toString();
|
|
202
|
+
}
|
|
203
|
+
return path;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Parser-bearing Delta source loader exposed through the explicit source subpath. */
|
|
208
|
+
export const DeltaSourceLoaderWithParser = {
|
|
209
|
+
...DeltaFormat,
|
|
210
|
+
dataType: null as unknown as DeltaTableSource,
|
|
211
|
+
batchType: null as never,
|
|
212
|
+
name: 'DeltaSourceLoader',
|
|
213
|
+
version: VERSION,
|
|
214
|
+
type: 'delta',
|
|
215
|
+
fromUrl: true,
|
|
216
|
+
fromBlob: true,
|
|
217
|
+
options: {},
|
|
218
|
+
defaultOptions: {},
|
|
219
|
+
testURL: (url: string): boolean =>
|
|
220
|
+
/\/_delta_log\/(?:\d{20}\.json|_last_checkpoint)(?:$|[?#])/i.test(url),
|
|
221
|
+
createDataSource: (data: string | Blob, options: DeltaSourceOptions = {}): DeltaTableSource =>
|
|
222
|
+
new DeltaTableSource(data, options)
|
|
223
|
+
} as const satisfies SourceLoader<DeltaTableSource>;
|
|
224
|
+
|
|
225
|
+
/** Extracts Delta's optional `numRecords` statistic from an add action. */
|
|
226
|
+
function getDeltaRecordCount(
|
|
227
|
+
stats: string | Readonly<Record<string, unknown>> | undefined
|
|
228
|
+
): number | undefined {
|
|
229
|
+
const parsed = typeof stats === 'string' ? parseDeltaStats(stats) : stats;
|
|
230
|
+
const value = parsed?.numRecords;
|
|
231
|
+
return typeof value === 'number' && Number.isSafeInteger(value) ? value : undefined;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function parseDeltaStats(stats: string): Readonly<Record<string, unknown>> | undefined {
|
|
235
|
+
try {
|
|
236
|
+
const value: unknown = JSON.parse(stats);
|
|
237
|
+
return value && typeof value === 'object'
|
|
238
|
+
? (value as Readonly<Record<string, unknown>>)
|
|
239
|
+
: undefined;
|
|
240
|
+
} catch {
|
|
241
|
+
return undefined;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function selectActiveDeltaFiles(
|
|
246
|
+
actions: readonly DeltaAction[]
|
|
247
|
+
): Array<NonNullable<DeltaAction['add']>> {
|
|
248
|
+
const files = new Map<string, NonNullable<DeltaAction['add']>>();
|
|
249
|
+
for (const action of actions) {
|
|
250
|
+
if (action.add) {
|
|
251
|
+
files.set(action.add.path, action.add);
|
|
252
|
+
}
|
|
253
|
+
if (action.remove) files.delete(action.remove.path);
|
|
254
|
+
}
|
|
255
|
+
const activeFiles = [...files.values()];
|
|
256
|
+
const deletionVectorFile = activeFiles.find(file => file.deletionVector !== undefined);
|
|
257
|
+
if (deletionVectorFile) {
|
|
258
|
+
throw new Error(
|
|
259
|
+
`Delta deletion vectors are not supported for active file ${deletionVectorFile.path}`
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
return activeFiles;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** Parses a newline-delimited Delta commit into typed actions. */
|
|
266
|
+
function parseDeltaActions(text: string): readonly DeltaAction[] {
|
|
267
|
+
return Object.freeze(
|
|
268
|
+
text
|
|
269
|
+
.split(/\r?\n/)
|
|
270
|
+
.filter(line => line.trim().length > 0)
|
|
271
|
+
.map(line => JSON.parse(line) as DeltaAction)
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** Rejects Delta protocol features that this Parquet snapshot adapter cannot interpret safely. */
|
|
276
|
+
function validateDeltaActions(actions: readonly DeltaAction[]): void {
|
|
277
|
+
for (const action of actions) {
|
|
278
|
+
const protocol = action.protocol;
|
|
279
|
+
if (protocol) {
|
|
280
|
+
if (
|
|
281
|
+
protocol.minReaderVersion !== undefined &&
|
|
282
|
+
(!Number.isSafeInteger(protocol.minReaderVersion) || protocol.minReaderVersion < 1)
|
|
283
|
+
) {
|
|
284
|
+
throw new Error('Delta protocol has an invalid minReaderVersion');
|
|
285
|
+
}
|
|
286
|
+
if ((protocol.minReaderVersion ?? 1) > 1) {
|
|
287
|
+
throw new Error(
|
|
288
|
+
`Delta reader protocol ${protocol.minReaderVersion} is not supported by this source`
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
if (protocol.readerFeatures?.length) {
|
|
292
|
+
throw new Error(
|
|
293
|
+
`Delta reader features are not supported: ${protocol.readerFeatures.join(', ')}`
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const columnMappingMode = action.metaData?.configuration?.['delta.columnMapping.mode'];
|
|
299
|
+
if (columnMappingMode && columnMappingMode !== 'none') {
|
|
300
|
+
throw new Error(
|
|
301
|
+
`Delta column mapping mode "${columnMappingMode}" is not supported by this source`
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {ParquetDatasetSourceOptions, ParquetDatasetReadOptions} from './parquet-source-types';
|
|
6
|
+
|
|
7
|
+
/** One Delta Lake transaction action from a commit log or checkpoint projection. */
|
|
8
|
+
export type DeltaAction = Readonly<{
|
|
9
|
+
protocol?: Readonly<{
|
|
10
|
+
minReaderVersion?: number;
|
|
11
|
+
readerFeatures?: readonly string[];
|
|
12
|
+
}>;
|
|
13
|
+
metaData?: Readonly<{
|
|
14
|
+
configuration?: Readonly<Record<string, string | null>>;
|
|
15
|
+
}>;
|
|
16
|
+
add?: Readonly<{
|
|
17
|
+
path: string;
|
|
18
|
+
size?: number;
|
|
19
|
+
partitionValues?: Readonly<Record<string, string | null>>;
|
|
20
|
+
stats?: string | Readonly<Record<string, unknown>>;
|
|
21
|
+
/** Delta deletion-vector descriptor, which is not decoded by this source. */
|
|
22
|
+
deletionVector?: unknown;
|
|
23
|
+
}>;
|
|
24
|
+
remove?: Readonly<{path: string}>;
|
|
25
|
+
}>;
|
|
26
|
+
|
|
27
|
+
/** Options for reading a Delta snapshot from its newline-delimited commit log. */
|
|
28
|
+
export type DeltaSourceOptions = ParquetDatasetSourceOptions & {
|
|
29
|
+
delta?: {
|
|
30
|
+
/** Snapshot version to replay when the source is a commit-log URL. */
|
|
31
|
+
version?: number;
|
|
32
|
+
/** Table root used to resolve relative Parquet add-file paths. */
|
|
33
|
+
baseUrl?: string;
|
|
34
|
+
/** Headers sent while reading commit-log files. */
|
|
35
|
+
headers?: HeadersInit;
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/** Query options accepted by a Delta snapshot scan. */
|
|
40
|
+
export type DeltaScanOptions = ParquetDatasetReadOptions & {version?: number};
|