@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15005 -3143
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +15766 -4896
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14911 -4316
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +42 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +585 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7055 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8505 -1041
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +78 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +605 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -11
- package/src/bundled.ts +3 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +50 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +929 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/parquet-reader.ts +935 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { ArrayType } from '@loaders.gl/schema';
|
|
2
|
-
import type { ParquetColumnChunkStatistics, ParquetComparisonPredicate, ParquetInPredicate, ParquetNullPredicate, ParquetPredicate, ParquetRowGroupMetadata } from "../parquet-source-types.js";
|
|
2
|
+
import type { ParquetColumnChunkStatistics, ParquetComparisonPredicate, ParquetInPredicate, ParquetNullPredicate, ParquetPredicate, ParquetPredicateProperty, ParquetRowGroupMetadata } from "../parquet-source-types.js";
|
|
3
3
|
/** Returns the unique top-level columns referenced by a predicate. */
|
|
4
4
|
export declare function getParquetPredicateColumns(predicate: ParquetPredicate): string[];
|
|
5
|
+
/** Returns the unique physical schema paths referenced by a predicate. */
|
|
6
|
+
export declare function getParquetPredicatePaths(predicate: ParquetPredicate): string[][];
|
|
5
7
|
/** Copies a predicate tree and mutable scalar values for read-scoped option snapshots. */
|
|
6
8
|
export declare function copyParquetPredicate(predicate: ParquetPredicate): ParquetPredicate;
|
|
7
9
|
/** Validates a predicate and every referenced top-level column. */
|
|
@@ -14,4 +16,6 @@ export declare function canParquetStatisticsMatch(predicate: ParquetComparisonPr
|
|
|
14
16
|
export declare function filterParquetRowIndices(predicate: ParquetPredicate | undefined, columns: Record<string, ArrayType>, rowCount: number): number[];
|
|
15
17
|
/** Gathers selected source rows without constructing row objects. */
|
|
16
18
|
export declare function gatherParquetColumns(columns: Record<string, ArrayType>, rowIndices: readonly number[], selectedColumns?: ReadonlySet<string>): Record<string, ArrayType>;
|
|
19
|
+
/** Returns one predicate property as an explicit Parquet schema path. */
|
|
20
|
+
export declare function getParquetPredicatePath(property: ParquetPredicateProperty): readonly string[];
|
|
17
21
|
//# sourceMappingURL=parquet-predicate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-predicate.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-predicate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAElD,OAAO,KAAK,EACV,4BAA4B,EAC5B,0BAA0B,EAC1B,kBAAkB,EAGlB,oBAAoB,EACpB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"parquet-predicate.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-predicate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAElD,OAAO,KAAK,EACV,4BAA4B,EAC5B,0BAA0B,EAC1B,kBAAkB,EAGlB,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EAExB,uBAAuB,EACxB,mCAAgC;AAGjC,sEAAsE;AACtE,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAEhF;AAED,0EAA0E;AAC1E,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,gBAAgB,GAAG,MAAM,EAAE,EAAE,CAUhF;AAED,0FAA0F;AAC1F,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,gBAAgB,GAAG,gBAAgB,CA2BlF;AAED,mEAAmE;AACnE,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,gBAAgB,EAC3B,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAAC,GACpC,IAAI,CAuBN;AAED,gFAAgF;AAChF,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,gBAAgB,EAC3B,QAAQ,EAAE,uBAAuB,GAChC,OAAO,CAqBT;AAED,0FAA0F;AAC1F,wBAAgB,yBAAyB,CACvC,SAAS,EAAE,0BAA0B,GAAG,kBAAkB,GAAG,oBAAoB,EACjF,UAAU,EAAE,4BAA4B,EACxC,QAAQ,EAAE,MAAM,GACf,OAAO,CAaT;AAED,6DAA6D;AAC7D,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,gBAAgB,GAAG,SAAS,EACvC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAClC,QAAQ,EAAE,MAAM,GACf,MAAM,EAAE,CAWV;AAED,qEAAqE;AACrE,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAClC,UAAU,EAAE,SAAS,MAAM,EAAE,EAC7B,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAS3B;AA6LD,yEAAyE;AACzE,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,wBAAwB,GAAG,SAAS,MAAM,EAAE,CAE7F"}
|
|
@@ -4,14 +4,19 @@
|
|
|
4
4
|
import { encodeUtf8 } from "../parquetjs/utils/binary-utils.js";
|
|
5
5
|
/** Returns the unique top-level columns referenced by a predicate. */
|
|
6
6
|
export function getParquetPredicateColumns(predicate) {
|
|
7
|
-
|
|
7
|
+
return [...new Set(getParquetPredicatePaths(predicate).map(path => path[0]))];
|
|
8
|
+
}
|
|
9
|
+
/** Returns the unique physical schema paths referenced by a predicate. */
|
|
10
|
+
export function getParquetPredicatePaths(predicate) {
|
|
11
|
+
const paths = new Map();
|
|
8
12
|
visitParquetPredicate(predicate, child => {
|
|
9
13
|
const property = getPredicateProperty(child);
|
|
10
14
|
if (property) {
|
|
11
|
-
|
|
15
|
+
const path = [...getParquetPredicatePath(property)];
|
|
16
|
+
paths.set(path.join('\0'), path);
|
|
12
17
|
}
|
|
13
18
|
});
|
|
14
|
-
return [...
|
|
19
|
+
return [...paths.values()];
|
|
15
20
|
}
|
|
16
21
|
/** Copies a predicate tree and mutable scalar values for read-scoped option snapshots. */
|
|
17
22
|
export function copyParquetPredicate(predicate) {
|
|
@@ -55,8 +60,12 @@ export function validateParquetPredicate(predicate, availableColumns) {
|
|
|
55
60
|
return;
|
|
56
61
|
}
|
|
57
62
|
const property = child.args[0];
|
|
58
|
-
|
|
59
|
-
|
|
63
|
+
const path = getParquetPredicatePath(property);
|
|
64
|
+
if (!path.length || path.some(pathComponent => !pathComponent)) {
|
|
65
|
+
throw new Error('Parquet predicate property path must contain non-empty strings');
|
|
66
|
+
}
|
|
67
|
+
if (!availableColumns.has(path[0])) {
|
|
68
|
+
throw new Error(`Parquet predicate column not found: ${path.join('.')}`);
|
|
60
69
|
}
|
|
61
70
|
if (child.op === 'in' && child.args[1].length === 0) {
|
|
62
71
|
throw new Error('Parquet predicate in requires at least one value');
|
|
@@ -75,10 +84,10 @@ export function canParquetRowGroupMatch(predicate, rowGroup) {
|
|
|
75
84
|
if (child.op !== 'isNull') {
|
|
76
85
|
return true;
|
|
77
86
|
}
|
|
78
|
-
const column = getRowGroupColumn(rowGroup, child.args[0]
|
|
87
|
+
const column = getRowGroupColumn(rowGroup, getParquetPredicatePath(child.args[0]));
|
|
79
88
|
return !column?.statistics || column.statistics.nullCount !== rowGroup.rowCount;
|
|
80
89
|
}
|
|
81
|
-
const column = getRowGroupColumn(rowGroup, predicate.args[0]
|
|
90
|
+
const column = getRowGroupColumn(rowGroup, getParquetPredicatePath(predicate.args[0]));
|
|
82
91
|
const statistics = column?.statistics;
|
|
83
92
|
if (!statistics) {
|
|
84
93
|
return true;
|
|
@@ -120,7 +129,7 @@ export function gatherParquetColumns(columns, rowIndices, selectedColumns) {
|
|
|
120
129
|
if (selectedColumns && !selectedColumns.has(name)) {
|
|
121
130
|
continue;
|
|
122
131
|
}
|
|
123
|
-
gatheredColumns[name] = rowIndices.map(rowIndex =>
|
|
132
|
+
gatheredColumns[name] = rowIndices.map(rowIndex => column[rowIndex]);
|
|
124
133
|
}
|
|
125
134
|
return gatheredColumns;
|
|
126
135
|
}
|
|
@@ -137,7 +146,7 @@ function evaluateParquetPredicate(predicate, columns, rowIndex) {
|
|
|
137
146
|
const result = evaluateParquetPredicate(predicate.args[0], columns, rowIndex);
|
|
138
147
|
return result === null ? null : !result;
|
|
139
148
|
}
|
|
140
|
-
const value = getColumnValue(columns
|
|
149
|
+
const value = getColumnValue(columns, getParquetPredicatePath(predicate.args[0]), rowIndex);
|
|
141
150
|
if (predicate.op === 'isNull') {
|
|
142
151
|
return value === null || value === undefined;
|
|
143
152
|
}
|
|
@@ -250,11 +259,23 @@ function compareUint8Arrays(left, right) {
|
|
|
250
259
|
return Math.sign(left.length - right.length);
|
|
251
260
|
}
|
|
252
261
|
/** Reads one value from an array-like materialized Parquet column. */
|
|
253
|
-
function getColumnValue(
|
|
262
|
+
function getColumnValue(columns, path, rowIndex) {
|
|
263
|
+
const column = columns[path[0]];
|
|
254
264
|
if (!column) {
|
|
255
265
|
throw new Error('Parquet predicate column was not decoded');
|
|
256
266
|
}
|
|
257
|
-
|
|
267
|
+
let value = column[rowIndex];
|
|
268
|
+
for (let pathIndex = 1; pathIndex < path.length; pathIndex++) {
|
|
269
|
+
if (value === null || value === undefined || typeof value !== 'object') {
|
|
270
|
+
return undefined;
|
|
271
|
+
}
|
|
272
|
+
const property = path[pathIndex];
|
|
273
|
+
value =
|
|
274
|
+
typeof value.get === 'function'
|
|
275
|
+
? value.get(property)
|
|
276
|
+
: value[property];
|
|
277
|
+
}
|
|
278
|
+
return value;
|
|
258
279
|
}
|
|
259
280
|
/** Copies mutable binary and date predicate values. */
|
|
260
281
|
function copyPredicateValue(value) {
|
|
@@ -268,7 +289,13 @@ function copyPredicateValue(value) {
|
|
|
268
289
|
}
|
|
269
290
|
/** Copies a predicate property reference. */
|
|
270
291
|
function copyPredicateProperty(property) {
|
|
271
|
-
return {
|
|
292
|
+
return {
|
|
293
|
+
property: Array.isArray(property.property) ? [...property.property] : property.property
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
/** Returns one predicate property as an explicit Parquet schema path. */
|
|
297
|
+
export function getParquetPredicatePath(property) {
|
|
298
|
+
return Array.isArray(property.property) ? property.property : [property.property];
|
|
272
299
|
}
|
|
273
300
|
/** Returns the property reference from a leaf predicate. */
|
|
274
301
|
function getPredicateProperty(predicate) {
|
|
@@ -277,8 +304,9 @@ function getPredicateProperty(predicate) {
|
|
|
277
304
|
: predicate.args[0];
|
|
278
305
|
}
|
|
279
306
|
/** Returns one top-level column chunk from normalized row-group metadata. */
|
|
280
|
-
function getRowGroupColumn(rowGroup,
|
|
281
|
-
return rowGroup.columns.find(columnChunk => columnChunk.path.length ===
|
|
307
|
+
function getRowGroupColumn(rowGroup, path) {
|
|
308
|
+
return rowGroup.columns.find(columnChunk => columnChunk.path.length === path.length &&
|
|
309
|
+
columnChunk.path.every((pathComponent, index) => pathComponent === path[index]));
|
|
282
310
|
}
|
|
283
311
|
/** Visits every predicate node depth-first. */
|
|
284
312
|
function visitParquetPredicate(predicate, visit) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-predicate.js","sourceRoot":"","sources":["../../src/lib/parquet-predicate.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAepC,OAAO,EAAC,UAAU,EAAC,2CAAwC;AAE3D,sEAAsE;AACtE,MAAM,UAAU,0BAA0B,CAAC,SAA2B;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,oBAAoB,CAAC,SAA2B;IAC9D,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC;SAC/C,CAAC;IACJ,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,CAAC,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;SAChD,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAC,CAAC;IAC1C,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO;YACL,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SAC5D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,wBAAwB,CACtC,SAA2B,EAC3B,gBAAqC;IAErC,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,IAAI,yBAAyB,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,CAAC,EAAE,yCAAyC,CAAC,CAAC;YAC1F,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,uBAAuB,CACrC,SAA2B,EAC3B,QAAiC;IAEjC,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK;YAC3B,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,KAAK,QAAQ,CAAC,QAAQ,CAAC;IAClF,CAAC;IAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACvE,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,yBAAyB,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,yBAAyB,CACvC,SAAiF,EACjF,UAAwC,EACxC,QAAgB;IAEhB,IAAI,UAAU,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC;IACnC,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,UAAU,CAAC,SAAS,KAAK,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,kBAAkB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,uBAAuB,CACrC,SAAuC,EACvC,OAAkC,EAClC,QAAgB;IAEhB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;QACvD,IAAI,wBAAwB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YACpE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,oBAAoB,CAClC,OAAkC,EAClC,UAA6B,EAC7B,eAAqC;IAErC,MAAM,eAAe,GAA8B,EAAE,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QACD,eAAe,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,0DAA0D;AAC1D,SAAS,wBAAwB,CAC/B,SAA2B,EAC3B,OAAkC,EAClC,QAAgB;IAEhB,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChG,IAAI,SAAS,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAChF,CAAC;QACD,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9E,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5E,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;IAC/C,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,QAAQ,SAAS,CAAC,EAAE,EAAE,CAAC;QACrB,KAAK,GAAG;YACN,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,IAAI;YACP,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;QACzB,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,kFAAkF;AAClF,SAAS,kBAAkB,CACzB,QAA8C,EAC9C,KAA4B,EAC5B,OAAgB,EAChB,OAAgB;IAEhB,IAAI,CAAC;QACH,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,GAAG;gBACN,OAAO,CAAC,CACN,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtE,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;YACJ,KAAK,IAAI;gBACP,OAAO,CAAC,CACN,OAAO,KAAK,SAAS;oBACrB,OAAO,KAAK,SAAS;oBACrB,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;oBAC7C,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAC9C,CAAC;YACJ,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/E,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,SAAS,uBAAuB,CAAC,IAAa,EAAE,KAAc;IAC5D,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,sBAAsB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAED,kFAAkF;AAClF,SAAS,sBAAsB,CAAC,IAAa,EAAE,KAAc;IAC3D,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,OAAO,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,IAAI,YAAY,IAAI,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,YAAY,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC;IAC7B,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC;IAC/B,MAAM,WAAW,GACf,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;QAChD,CAAC,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,SAAS,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,SAAS,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,cAAc,GAAG,IAA0C,CAAC;IAClE,MAAM,eAAe,GAAG,KAA2C,CAAC;IACpE,IACE,CAAC,OAAO,cAAc,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QACxE,CAAC,OAAO,eAAe,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,EAC1E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC,CAAC;IACZ,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,yEAAyE;AACzE,SAAS,kBAAkB,CAAC,IAAgB,EAAE,KAAiB;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,sEAAsE;AACtE,SAAS,cAAc,CAAC,MAA6B,EAAE,QAAgB;IACrE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,OAAQ,MAA6B,CAAC,QAAQ,CAAC,CAAC;AAClD,CAAC;AAED,uDAAuD;AACvD,SAAS,kBAAkB,CAAC,KAA4B;IACtD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6CAA6C;AAC7C,SAAS,qBAAqB,CAAC,QAA4B;IACzD,OAAO,EAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAC,CAAC;AACvC,CAAC;AAED,4DAA4D;AAC5D,SAAS,oBAAoB,CAAC,SAA2B;IACvD,OAAO,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAC7E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACxB,CAAC;AAED,6EAA6E;AAC7E,SAAS,iBAAiB,CAAC,QAAiC,EAAE,QAAgB;IAC5E,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAC1B,WAAW,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,CACjF,CAAC;AACJ,CAAC;AAED,+CAA+C;AAC/C,SAAS,qBAAqB,CAC5B,SAA2B,EAC3B,KAA4C;IAE5C,KAAK,CAAC,SAAS,CAAC,CAAC;IACjB,IAAI,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,qBAAqB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,SAAS,yBAAyB,CAChC,SAA2B;IAE3B,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC;AACzD,CAAC;AAED,+DAA+D;AAC/D,SAAS,qBAAqB,CAAC,SAA2B;IACxD,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,CAAC;AAChC,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet-predicate.js","sourceRoot":"","sources":["../../src/lib/parquet-predicate.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAgBpC,OAAO,EAAC,UAAU,EAAC,2CAAwC;AAE3D,sEAAsE;AACtE,MAAM,UAAU,0BAA0B,CAAC,SAA2B;IACpE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,wBAAwB,CAAC,SAA2B;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,CAAC,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpD,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,oBAAoB,CAAC,SAA2B;IAC9D,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC;SAC/C,CAAC;IACJ,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,CAAC,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;SAChD,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAC,CAAC;IAC1C,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO;YACL,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SAC5D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,wBAAwB,CACtC,SAA2B,EAC3B,gBAAqC;IAErC,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE;QACvC,IAAI,yBAAyB,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,CAAC,EAAE,yCAAyC,CAAC,CAAC;YAC1F,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,uBAAuB,CACrC,SAA2B,EAC3B,QAAiC;IAEjC,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK;YAC3B,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,OAAO,CAAC,MAAM,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,KAAK,QAAQ,CAAC,QAAQ,CAAC;IAClF,CAAC;IAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,yBAAyB,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,yBAAyB,CACvC,SAAiF,EACjF,UAAwC,EACxC,QAAgB;IAEhB,IAAI,UAAU,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC;IACnC,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,UAAU,CAAC,SAAS,KAAK,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7E,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,kBAAkB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,uBAAuB,CACrC,SAAuC,EACvC,OAAkC,EAClC,QAAgB;IAEhB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;QACvD,IAAI,wBAAwB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YACpE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,oBAAoB,CAClC,OAAkC,EAClC,UAA6B,EAC7B,eAAqC;IAErC,MAAM,eAAe,GAA8B,EAAE,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QACD,eAAe,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAE,MAA6B,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,0DAA0D;AAC1D,SAAS,wBAAwB,CAC/B,SAA2B,EAC3B,OAAkC,EAClC,QAAgB;IAEhB,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChG,IAAI,SAAS,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAChF,CAAC;QACD,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/E,CAAC;IACD,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9E,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5F,IAAI,SAAS,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;IAC/C,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,QAAQ,SAAS,CAAC,EAAE,EAAE,CAAC;QACrB,KAAK,GAAG;YACN,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,IAAI;YACP,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;QACzB,KAAK,GAAG;YACN,OAAO,UAAU,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI;YACP,OAAO,UAAU,IAAI,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,kFAAkF;AAClF,SAAS,kBAAkB,CACzB,QAA8C,EAC9C,KAA4B,EAC5B,OAAgB,EAChB,OAAgB;IAEhB,IAAI,CAAC;QACH,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,GAAG;gBACN,OAAO,CAAC,CACN,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtE,CAAC,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;YACJ,KAAK,IAAI;gBACP,OAAO,CAAC,CACN,OAAO,KAAK,SAAS;oBACrB,OAAO,KAAK,SAAS;oBACrB,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;oBAC7C,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAC9C,CAAC;YACJ,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/E,KAAK,GAAG;gBACN,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9E,KAAK,IAAI;gBACP,OAAO,OAAO,KAAK,SAAS,IAAI,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,SAAS,uBAAuB,CAAC,IAAa,EAAE,KAAc;IAC5D,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,sBAAsB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAED,kFAAkF;AAClF,SAAS,sBAAsB,CAAC,IAAa,EAAE,KAAc;IAC3D,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,OAAO,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,IAAI,YAAY,UAAU,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,IAAI,YAAY,IAAI,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,YAAY,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC;IAC7B,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC;IAC/B,MAAM,WAAW,GACf,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;QAChD,CAAC,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,SAAS,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,SAAS,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,cAAc,GAAG,IAA0C,CAAC;IAClE,MAAM,eAAe,GAAG,KAA2C,CAAC;IACpE,IACE,CAAC,OAAO,cAAc,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QACxE,CAAC,OAAO,eAAe,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,EAC1E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC,CAAC;IACZ,CAAC;IACD,IAAK,cAAwB,GAAI,eAAyB,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,yEAAyE;AACzE,SAAS,kBAAkB,CAAC,IAAgB,EAAE,KAAiB;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,sEAAsE;AACtE,SAAS,cAAc,CACrB,OAAkC,EAClC,IAAuB,EACvB,QAAgB;IAEhB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,KAAK,GAAI,MAA6B,CAAC,QAAQ,CAAC,CAAC;IACrD,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QAC7D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QACjC,KAAK;YACH,OAAQ,KAAyB,CAAC,GAAG,KAAK,UAAU;gBAClD,CAAC,CAAE,KAA0C,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC3D,CAAC,CAAE,KAAiC,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,uDAAuD;AACvD,SAAS,kBAAkB,CAAC,KAA4B;IACtD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6CAA6C;AAC7C,SAAS,qBAAqB,CAAC,QAAkC;IAC/D,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ;KACxF,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,uBAAuB,CAAC,QAAkC;IACxE,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAkB,CAAC,CAAC;AAC9F,CAAC;AAED,4DAA4D;AAC5D,SAAS,oBAAoB,CAAC,SAA2B;IACvD,OAAO,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAC7E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACxB,CAAC;AAED,6EAA6E;AAC7E,SAAS,iBAAiB,CAAC,QAAiC,EAAE,IAAuB;IACnF,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAC1B,WAAW,CAAC,EAAE,CACZ,WAAW,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;QACvC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAClF,CAAC;AACJ,CAAC;AAED,+CAA+C;AAC/C,SAAS,qBAAqB,CAC5B,SAA2B,EAC3B,KAA4C;IAE5C,KAAK,CAAC,SAAS,CAAC,CAAC;IACjB,IAAI,yBAAyB,CAAC,SAAS,CAAC,IAAI,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,qBAAqB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,SAAS,yBAAyB,CAChC,SAA2B;IAE3B,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC;AACzD,CAAC;AAED,+DAA+D;AAC/D,SAAS,qBAAqB,CAAC,SAA2B;IACxD,OAAO,SAAS,CAAC,EAAE,KAAK,KAAK,CAAC;AAChC,CAAC"}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ParquetKeyRetriever } from "./parquet-encryption.js";
|
|
2
|
+
import type { ParquetSourceWorkerColumnChunk, ParquetSourceWorkerInput, ParquetSourceWorkerResult } from "./parquet-source-worker-types.js";
|
|
2
3
|
/** Decodes one transferred row group entirely inside the worker. */
|
|
3
4
|
export declare function decodeParquetSourceWorkerInput(input: ParquetSourceWorkerInput): Promise<ParquetSourceWorkerResult>;
|
|
5
|
+
/** Creates a worker-local key retriever from caller-resolved transferable keys. */
|
|
6
|
+
export declare function createWorkerKeyRetriever(columnChunks: readonly ParquetSourceWorkerColumnChunk[], aadPrefixByteLength: number, fileUniqueByteLength: number): ParquetKeyRetriever;
|
|
4
7
|
//# sourceMappingURL=parquet-source-worker-decoder.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-worker-decoder.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAEV,wBAAwB,EACxB,yBAAyB,EAC1B,yCAAsC;AAEvC,oEAAoE;AACpE,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,yBAAyB,CAAC,
|
|
1
|
+
{"version":3,"file":"parquet-source-worker-decoder.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAC,mBAAmB,EAAC,gCAA6B;AAC9D,OAAO,KAAK,EAEV,8BAA8B,EAC9B,wBAAwB,EACxB,yBAAyB,EAC1B,yCAAsC;AAEvC,oEAAoE;AACpE,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,yBAAyB,CAAC,CAsGpC;AAyED,mFAAmF;AACnF,wBAAgB,wBAAwB,CACtC,YAAY,EAAE,SAAS,8BAA8B,EAAE,EACvD,mBAAmB,EAAE,MAAM,EAC3B,oBAAoB,EAAE,MAAM,GAC3B,mBAAmB,CAkBrB"}
|
|
@@ -13,12 +13,35 @@ export async function decodeParquetSourceWorkerInput(input) {
|
|
|
13
13
|
await preloadCompressions();
|
|
14
14
|
const file = new ParquetSourceWorkerFile(input.fileByteLength, input.ranges);
|
|
15
15
|
const schema = new ParquetSchema(input.schemaDefinition);
|
|
16
|
-
const
|
|
16
|
+
const keyRetriever = input.encryption
|
|
17
|
+
? createWorkerKeyRetriever(input.columnChunks, input.encryption.aadPrefix?.byteLength ?? 0, input.encryption.fileUnique.byteLength)
|
|
18
|
+
: undefined;
|
|
19
|
+
const reader = new ParquetReader(file, {
|
|
20
|
+
preserveBinary: input.preserveBinary,
|
|
21
|
+
int96AsTimestamp: input.int96AsTimestamp,
|
|
22
|
+
verifyPageChecksums: input.verifyPageChecksums,
|
|
23
|
+
encryptionContext: input.encryption
|
|
24
|
+
? {
|
|
25
|
+
algorithm: input.encryption.algorithm,
|
|
26
|
+
aadPrefix: input.encryption.aadPrefix
|
|
27
|
+
? new Uint8Array(input.encryption.aadPrefix)
|
|
28
|
+
: undefined,
|
|
29
|
+
fileUnique: new Uint8Array(input.encryption.fileUnique)
|
|
30
|
+
}
|
|
31
|
+
: undefined,
|
|
32
|
+
keyRetriever
|
|
33
|
+
});
|
|
17
34
|
const rowGroup = createWorkerRowGroup(input);
|
|
18
35
|
const decodeStartTime = getCurrentTime();
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
36
|
+
let decodedRowGroups;
|
|
37
|
+
try {
|
|
38
|
+
decodedRowGroups = input.pagePlan
|
|
39
|
+
? await Promise.all(input.pagePlan.rowRanges.map(rowRange => reader.readRowGroupRange(schema, rowGroup, [], rowRange, input.pagePlan.pageLocations)))
|
|
40
|
+
: [await reader.readRowGroup(schema, rowGroup, [])];
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw new Error(`Parquet worker decode failed: ${error instanceof Error ? `${error.name}: ${error.message}` : String(error)}`);
|
|
44
|
+
}
|
|
22
45
|
const columns = concatenateMaterializedColumns(decodedRowGroups.map(decodedRowGroup => schema.materializeColumns(decodedRowGroup)));
|
|
23
46
|
const sourceRowIndices = input.pagePlan
|
|
24
47
|
? input.pagePlan.rowRanges.flatMap(rowRange => Array.from({ length: rowRange.end - rowRange.start }, (_, index) => rowRange.start + index))
|
|
@@ -92,22 +115,65 @@ function createWorkerRowGroup(input) {
|
|
|
92
115
|
return {
|
|
93
116
|
num_rows: input.rowCount,
|
|
94
117
|
total_byte_size: input.uncompressedByteLength,
|
|
95
|
-
columns: input.columnChunks.map(columnChunk =>
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
118
|
+
columns: input.columnChunks.map((columnChunk, index) => {
|
|
119
|
+
const result = {
|
|
120
|
+
file_path: columnChunk.filePath,
|
|
121
|
+
parquetColumnOrdinal: columnChunk.columnOrdinal ?? index,
|
|
122
|
+
meta_data: {
|
|
123
|
+
type: columnChunk.physicalType,
|
|
124
|
+
codec: columnChunk.compressionCodec,
|
|
125
|
+
path_in_schema: columnChunk.path,
|
|
126
|
+
num_values: columnChunk.valueCount,
|
|
127
|
+
total_compressed_size: columnChunk.compressedByteLength,
|
|
128
|
+
total_uncompressed_size: columnChunk.uncompressedByteLength,
|
|
129
|
+
data_page_offset: columnChunk.dataPageOffset,
|
|
130
|
+
dictionary_page_offset: columnChunk.dictionaryPageOffset,
|
|
131
|
+
encodings: []
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
if (columnChunk.encrypted) {
|
|
135
|
+
result.crypto_metadata = (columnChunk.keyMetadata
|
|
136
|
+
? {
|
|
137
|
+
ENCRYPTION_WITH_COLUMN_KEY: {
|
|
138
|
+
path_in_schema: columnChunk.path,
|
|
139
|
+
key_metadata: new Uint8Array(columnChunk.keyMetadata)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
: {
|
|
143
|
+
ENCRYPTION_WITH_FOOTER_KEY: {}
|
|
144
|
+
});
|
|
107
145
|
}
|
|
108
|
-
|
|
146
|
+
return result;
|
|
147
|
+
})
|
|
109
148
|
};
|
|
110
149
|
}
|
|
150
|
+
/** Creates a worker-local key retriever from caller-resolved transferable keys. */
|
|
151
|
+
export function createWorkerKeyRetriever(columnChunks, aadPrefixByteLength, fileUniqueByteLength) {
|
|
152
|
+
return (keyMetadata, context) => {
|
|
153
|
+
const columnOrdinal = getColumnOrdinalFromAad(context.aad, aadPrefixByteLength, fileUniqueByteLength);
|
|
154
|
+
const metadataMatches = columnChunks.filter(columnChunk => equalBytes(keyMetadata, columnChunk.keyMetadata && new Uint8Array(columnChunk.keyMetadata)));
|
|
155
|
+
const matchingColumn = metadataMatches.find(columnChunk => columnChunk.columnOrdinal === columnOrdinal) ??
|
|
156
|
+
(metadataMatches.length === 1 ? metadataMatches[0] : undefined);
|
|
157
|
+
if (!matchingColumn?.keyMaterial) {
|
|
158
|
+
throw new Error('Encrypted Parquet worker key was not transferred for the selected column');
|
|
159
|
+
}
|
|
160
|
+
return new Uint8Array(matchingColumn.keyMaterial);
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
/** Extracts the physical column ordinal encoded in a page-module AAD suffix. */
|
|
164
|
+
function getColumnOrdinalFromAad(aad, aadPrefixByteLength, fileUniqueByteLength) {
|
|
165
|
+
const columnOrdinalOffset = aadPrefixByteLength + fileUniqueByteLength + 3;
|
|
166
|
+
if (columnOrdinalOffset + 2 > aad.byteLength)
|
|
167
|
+
return undefined;
|
|
168
|
+
return new DataView(aad.buffer, aad.byteOffset + columnOrdinalOffset, 2).getInt16(0, true);
|
|
169
|
+
}
|
|
170
|
+
/** Compares optional key metadata without exposing key material in structured-clone state. */
|
|
171
|
+
function equalBytes(firstBytes, secondBytes) {
|
|
172
|
+
if (!firstBytes || !secondBytes)
|
|
173
|
+
return !firstBytes && !secondBytes;
|
|
174
|
+
return (firstBytes.length === secondBytes.length &&
|
|
175
|
+
firstBytes.every((value, index) => value === secondBytes[index]));
|
|
176
|
+
}
|
|
111
177
|
/** Returns a monotonic timestamp when available and falls back to wall-clock time. */
|
|
112
178
|
function getCurrentTime() {
|
|
113
179
|
return globalThis.performance?.now() ?? Date.now();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-worker-decoder.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,mBAAmB,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAC,mBAAmB,EAAC,oCAAiC;AAC7D,OAAO,EAAC,uBAAuB,EAAE,oBAAoB,EAAC,+BAA4B;AAElF,OAAO,EAAC,aAAa,EAAC,8CAA2C;AACjE,OAAO,EAAC,aAAa,EAAC,sCAAmC;AACzD,OAAO,EAAC,uBAAuB,EAAC,wCAAqC;
|
|
1
|
+
{"version":3,"file":"parquet-source-worker-decoder.js","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-decoder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,mBAAmB,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAC,mBAAmB,EAAC,oCAAiC;AAC7D,OAAO,EAAC,uBAAuB,EAAE,oBAAoB,EAAC,+BAA4B;AAElF,OAAO,EAAC,aAAa,EAAC,8CAA2C;AACjE,OAAO,EAAC,aAAa,EAAC,sCAAmC;AACzD,OAAO,EAAC,uBAAuB,EAAC,wCAAqC;AASrE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,KAA+B;IAE/B,MAAM,mBAAmB,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,uBAAuB,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,YAAY,GAAG,KAAK,CAAC,UAAU;QACnC,CAAC,CAAC,wBAAwB,CACtB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,EAC3C,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,CACvC;QACH,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE;QACrC,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,iBAAiB,EAAE,KAAK,CAAC,UAAU;YACjC,CAAC,CAAC;gBACE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS;gBACrC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS;oBACnC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC5C,CAAC,CAAC,SAAS;gBACb,UAAU,EAAE,IAAI,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;aACxD;YACH,CAAC,CAAC,SAAS;QACb,YAAY;KACb,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,cAAc,EAAE,CAAC;IACzC,IAAI,gBAAgB,CAAC;IACrB,IAAI,CAAC;QACH,gBAAgB,GAAG,KAAK,CAAC,QAAQ;YAC/B,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACtC,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAS,CAAC,aAAa,CAAC,CACxF,CACF;YACH,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,iCACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAC3E,EAAE,CACH,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,8BAA8B,CAC5C,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC,CACpF,CAAC;IACF,MAAM,gBAAgB,GAAG,KAAK,CAAC,QAAQ;QACrC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAC1C,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,CAC1F;QACH,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,GAAG,gBAAgB,EAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC;IAClE,MAAM,eAAe,GAAG,KAAK,CAAC,SAAS;QACrC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,cAAc,CAAC;QACnE,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,UAAU,GAAG,eAAe,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,CAAC;IAC9F,MAAM,gBAAgB,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC;IAE5D,MAAM,mBAAmB,GAAG,cAAc,EAAE,CAAC;IAC7C,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,UAAU,EAAE,MAAM,IAAI,cAAc,CAAC;IAC5D,KACE,IAAI,eAAe,GAAG,CAAC,EACvB,eAAe,GAAG,cAAc,EAChC,eAAe,IAAI,KAAK,CAAC,SAAS,EAClC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC;QAC7E,MAAM,eAAe,GAAG,UAAU,EAAE,KAAK,CAAC,eAAe,EAAE,eAAe,GAAG,QAAQ,CAAC,CAAC;QACvF,MAAM,oBAAoB,GAAG,eAAe,EAAE,KAAK,CACjD,eAAe,EACf,eAAe,GAAG,QAAQ,CAC3B,CAAC;QACF,MAAM,iBAAiB,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC;QAClE,MAAM,YAAY,GAAG,oBAAoB;YACvC,CAAC,CAAC,oBAAoB,CAAC,OAAO,EAAE,oBAAoB,EAAE,aAAa,CAAC;YACpE,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,eAAe,EAAE,eAAe,GAAG,QAAQ,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,YAAY,CAC7B;YACE,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,KAAK,CAAC,eAAe;YAC7B,IAAI,EAAE,YAAY;SACnB,EACD,aAAa,CACd,CAAC;QACF,OAAO,CAAC,IAAI,CAAC;YACX,iBAAiB;YACjB,QAAQ;YACR,kBAAkB,EAAE,eAAe;YACnC,UAAU,EAAE,mBAAmB,CAAC,UAAU,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,yBAAyB,GAAG,cAAc,EAAE,GAAG,mBAAmB,CAAC;IACzE,OAAO;QACL,cAAc;QACd,QAAQ,EAAE,cAAc;QACxB,OAAO;QACP,gBAAgB;QAChB,yBAAyB;KAC1B,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,SAAS,8BAA8B,CACrC,SAA+C;IAE/C,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAClC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBACnD,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAoC,CAAC;AAC9C,CAAC;AAED,+FAA+F;AAC/F,SAAS,YAAY,CACnB,OAAkC,EAClC,KAAa,EACb,GAAW;IAEX,MAAM,aAAa,GAA8B,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAI,MAA0E,CAAC,KAAK,CAAC;QAChG,aAAa,CAAC,IAAI,CAAC,GAAG,KAAK;YACzB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC;YAChC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,uFAAuF;AACvF,SAAS,oBAAoB,CAAC,KAA+B;IAC3D,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,eAAe,EAAE,KAAK,CAAC,sBAAsB;QAC7C,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE;YACrD,MAAM,MAAM,GAAG;gBACb,SAAS,EAAE,WAAW,CAAC,QAAQ;gBAC/B,oBAAoB,EAAE,WAAW,CAAC,aAAa,IAAI,KAAK;gBACxD,SAAS,EAAE;oBACT,IAAI,EAAE,WAAW,CAAC,YAAY;oBAC9B,KAAK,EAAE,WAAW,CAAC,gBAAgB;oBACnC,cAAc,EAAE,WAAW,CAAC,IAAI;oBAChC,UAAU,EAAE,WAAW,CAAC,UAAU;oBAClC,qBAAqB,EAAE,WAAW,CAAC,oBAAoB;oBACvD,uBAAuB,EAAE,WAAW,CAAC,sBAAsB;oBAC3D,gBAAgB,EAAE,WAAW,CAAC,cAAc;oBAC5C,sBAAsB,EAAE,WAAW,CAAC,oBAAoB;oBACxD,SAAS,EAAE,EAAE;iBACd;aACwC,CAAC;YAC5C,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1B,MAAM,CAAC,eAAe,GAAG,CAAC,WAAW,CAAC,WAAW;oBAC/C,CAAC,CAAC;wBACE,0BAA0B,EAAE;4BAC1B,cAAc,EAAE,WAAW,CAAC,IAAI;4BAChC,YAAY,EAAE,IAAI,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC;yBACtD;qBACF;oBACH,CAAC,CAAC;wBACE,0BAA0B,EAAE,EAAE;qBAC/B,CAA8D,CAAC;YACtE,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC;KACoB,CAAC;AAC3B,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,wBAAwB,CACtC,YAAuD,EACvD,mBAA2B,EAC3B,oBAA4B;IAE5B,OAAO,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE;QAC9B,MAAM,aAAa,GAAG,uBAAuB,CAC3C,OAAO,CAAC,GAAG,EACX,mBAAmB,EACnB,oBAAoB,CACrB,CAAC;QACF,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CACxD,UAAU,CAAC,WAAW,EAAE,WAAW,CAAC,WAAW,IAAI,IAAI,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAC5F,CAAC;QACF,MAAM,cAAc,GAClB,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,WAAW,CAAC,aAAa,KAAK,aAAa,CAAC;YAChF,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,IAAI,UAAU,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACpD,CAAC,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,SAAS,uBAAuB,CAC9B,GAAe,EACf,mBAA2B,EAC3B,oBAA4B;IAE5B,MAAM,mBAAmB,GAAG,mBAAmB,GAAG,oBAAoB,GAAG,CAAC,CAAC;IAC3E,IAAI,mBAAmB,GAAG,CAAC,GAAG,GAAG,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAC/D,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,GAAG,mBAAmB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,8FAA8F;AAC9F,SAAS,UAAU,CACjB,UAAkC,EAClC,WAAmC;IAEnC,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;QAAE,OAAO,CAAC,UAAU,IAAI,CAAC,WAAW,CAAC;IACpE,OAAO,CACL,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM;QACxC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,WAAW,CAAC,KAAK,CAAC,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,SAAS,cAAc;IACrB,OAAO,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AACrD,CAAC"}
|
|
@@ -2,6 +2,7 @@ import type { DehydratedArrowTable } from '@loaders.gl/arrow';
|
|
|
2
2
|
import type { StrictLoaderOptions } from '@loaders.gl/loader-utils';
|
|
3
3
|
import type { Schema } from '@loaders.gl/schema';
|
|
4
4
|
import type { ParquetPredicate } from "../parquet-source-types.js";
|
|
5
|
+
import type { ParquetEncryptionAlgorithm } from "./parquet-encryption.js";
|
|
5
6
|
import type { ParquetPagePruningPlan } from "./parquet-page-index.js";
|
|
6
7
|
import type { SchemaDefinition } from "../parquetjs/schema/declare.js";
|
|
7
8
|
/** One transferable range containing a selected Parquet column chunk. */
|
|
@@ -31,6 +32,23 @@ export type ParquetSourceWorkerColumnChunk = {
|
|
|
31
32
|
dataPageOffset: number;
|
|
32
33
|
/** Absolute offset of the dictionary page, when present. */
|
|
33
34
|
dictionaryPageOffset?: number;
|
|
35
|
+
/** Physical column ordinal in the original Parquet row group. */
|
|
36
|
+
columnOrdinal?: number;
|
|
37
|
+
/** Whether this column chunk's page modules are encrypted. */
|
|
38
|
+
encrypted?: boolean;
|
|
39
|
+
/** Key metadata identifying the column key, when present. */
|
|
40
|
+
keyMetadata?: ArrayBuffer;
|
|
41
|
+
/** Raw data key resolved by the caller before worker transfer. */
|
|
42
|
+
keyMaterial?: ArrayBuffer;
|
|
43
|
+
};
|
|
44
|
+
/** File-level encryption context transferred to a worker for encrypted page reads. */
|
|
45
|
+
export type ParquetSourceWorkerEncryption = {
|
|
46
|
+
/** Modular-encryption algorithm selected by the file. */
|
|
47
|
+
algorithm: ParquetEncryptionAlgorithm;
|
|
48
|
+
/** Optional caller-supplied AAD prefix. */
|
|
49
|
+
aadPrefix?: ArrayBuffer;
|
|
50
|
+
/** Random per-file AAD suffix component. */
|
|
51
|
+
fileUnique: ArrayBuffer;
|
|
34
52
|
};
|
|
35
53
|
/** Transferable input for one worker-backed Parquet source row-group decode. */
|
|
36
54
|
export type ParquetSourceWorkerInput = {
|
|
@@ -54,8 +72,14 @@ export type ParquetSourceWorkerInput = {
|
|
|
54
72
|
predicate?: ParquetPredicate;
|
|
55
73
|
/** Selective data-page plan prepared from column and offset indexes on the caller thread. */
|
|
56
74
|
pagePlan?: ParquetPagePruningPlan;
|
|
75
|
+
/** Encryption context and resolved keys needed for encrypted worker decoding. */
|
|
76
|
+
encryption?: ParquetSourceWorkerEncryption;
|
|
57
77
|
/** Whether BYTE_ARRAY values stay binary during logical conversion. */
|
|
58
78
|
preserveBinary: boolean;
|
|
79
|
+
/** Whether legacy INT96 values decode as epoch-nanosecond timestamps. */
|
|
80
|
+
int96AsTimestamp: boolean;
|
|
81
|
+
/** Whether page CRC values are verified while decoding in the worker. */
|
|
82
|
+
verifyPageChecksums: boolean;
|
|
59
83
|
};
|
|
60
84
|
/** One directly transferable Arrow batch decoded by a Parquet source worker. */
|
|
61
85
|
export type ParquetSourceWorkerBatch = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-source-worker-types.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-types.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,0BAA0B,CAAC;AAClE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAE/C,OAAO,KAAK,EAAC,gBAAgB,EAAC,mCAAgC;AAC9D,OAAO,KAAK,EAAC,sBAAsB,EAAC,gCAA6B;AACjE,OAAO,KAAK,EAAC,gBAAgB,EAAC,uCAAoC;AAElE,yEAAyE;AACzE,MAAM,MAAM,wBAAwB,GAAG;IACrC,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AAEF,kFAAkF;AAClF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,YAAY,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,gBAAgB,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,oBAAoB,EAAE,MAAM,CAAC;IAC7B,6DAA6D;IAC7D,sBAAsB,EAAE,MAAM,CAAC;IAC/B,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,oBAAoB,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"parquet-source-worker-types.d.ts","sourceRoot":"","sources":["../../src/lib/parquet-source-worker-types.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,0BAA0B,CAAC;AAClE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAE/C,OAAO,KAAK,EAAC,gBAAgB,EAAC,mCAAgC;AAC9D,OAAO,KAAK,EAAC,0BAA0B,EAAC,gCAA6B;AACrE,OAAO,KAAK,EAAC,sBAAsB,EAAC,gCAA6B;AACjE,OAAO,KAAK,EAAC,gBAAgB,EAAC,uCAAoC;AAElE,yEAAyE;AACzE,MAAM,MAAM,wBAAwB,GAAG;IACrC,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AAEF,kFAAkF;AAClF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,YAAY,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,gBAAgB,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,oBAAoB,EAAE,MAAM,CAAC;IAC7B,6DAA6D;IAC7D,sBAAsB,EAAE,MAAM,CAAC;IAC/B,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,iEAAiE;IACjE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6DAA6D;IAC7D,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,kEAAkE;IAClE,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,sFAAsF;AACtF,MAAM,MAAM,6BAA6B,GAAG;IAC1C,yDAAyD;IACzD,SAAS,EAAE,0BAA0B,CAAC;IACtC,2CAA2C;IAC3C,SAAS,CAAC,EAAE,WAAW,CAAC;IACxB,4CAA4C;IAC5C,UAAU,EAAE,WAAW,CAAC;CACzB,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,wBAAwB,GAAG;IACrC,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;IACvB,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,sBAAsB,EAAE,MAAM,CAAC;IAC/B,mEAAmE;IACnE,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,6DAA6D;IAC7D,eAAe,EAAE,MAAM,CAAC;IACxB,yEAAyE;IACzE,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,mEAAmE;IACnE,MAAM,EAAE,wBAAwB,EAAE,CAAC;IACnC,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,iFAAiF;IACjF,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,uEAAuE;IACvE,cAAc,EAAE,OAAO,CAAC;IACxB,yEAAyE;IACzE,gBAAgB,EAAE,OAAO,CAAC;IAC1B,yEAAyE;IACzE,mBAAmB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,wBAAwB,GAAG;IACrC,yDAAyD;IACzD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gFAAgF;IAChF,UAAU,EAAE,oBAAoB,CAAC;CAClC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,yBAAyB,GAAG;IACtC,0EAA0E;IAC1E,cAAc,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,OAAO,EAAE,wBAAwB,EAAE,CAAC;IACpC,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;IACzB,qFAAqF;IACrF,yBAAyB,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,4DAA4D;AAC5D,MAAM,MAAM,0BAA0B,GAAG,mBAAmB,GAAG;IAC7D,mEAAmE;IACnE,gBAAgB,CAAC,EAAE;QACjB,kEAAkE;QAClE,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,qCAAqC;IACrC,OAAO,CAAC,EAAE;QACR,kDAAkD;QAClD,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,CAAC;CACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-parquet-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAE1C,OAAO,EAAC,aAAa,EAAC,iDAA8C;AAIpE,wBAAsB,0BAA0B,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"get-parquet-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAE1C,OAAO,EAAC,aAAa,EAAC,iDAA8C;AAIpE,wBAAsB,0BAA0B,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAUvF"}
|
|
@@ -7,7 +7,9 @@ import { applyGeoParquetToFieldMetadata } from "../geo/geospatial-metadata.js";
|
|
|
7
7
|
export async function getSchemaFromParquetReader(reader) {
|
|
8
8
|
const parquetSchema = await reader.getSchema();
|
|
9
9
|
const parquetMetadata = await reader.getFileMetadata();
|
|
10
|
-
const schema = convertParquetSchema(parquetSchema, parquetMetadata
|
|
10
|
+
const schema = convertParquetSchema(parquetSchema, parquetMetadata, {
|
|
11
|
+
int96AsTimestamp: reader.int96AsTimestamp
|
|
12
|
+
});
|
|
11
13
|
applyGeoParquetToFieldMetadata(schema);
|
|
12
14
|
unpackGeoMetadata(schema.metadata);
|
|
13
15
|
unpackJSONStringMetadata(schema.metadata, 'pandas');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-parquet-schema.js","sourceRoot":"","sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,iBAAiB,EAAE,wBAAwB,EAAC,MAAM,iBAAiB,CAAC;AAE5E,OAAO,EAAC,oBAAoB,EAAC,gDAA6C;AAC1E,OAAO,EAAC,8BAA8B,EAAC,sCAAmC;AAE1E,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,MAAqB;IACpE,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAC/C,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;IACvD,MAAM,MAAM,GAAG,oBAAoB,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"get-parquet-schema.js","sourceRoot":"","sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,iBAAiB,EAAE,wBAAwB,EAAC,MAAM,iBAAiB,CAAC;AAE5E,OAAO,EAAC,oBAAoB,EAAC,gDAA6C;AAC1E,OAAO,EAAC,8BAA8B,EAAC,sCAAmC;AAE1E,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,MAAqB;IACpE,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAC/C,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;IACvD,MAAM,MAAM,GAAG,oBAAoB,CAAC,aAAa,EAAE,eAAe,EAAE;QAClE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;KAC1C,CAAC,CAAC;IACH,8BAA8B,CAAC,MAAM,CAAC,CAAC;IACvC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,wBAAwB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-parquet-to-arrow-js.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-arrow-js.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAEV,UAAU,EACV,eAAe,EAMhB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,KAAK,EAAC,sBAAsB,EAAC,wCAAqC;AAgFzE;;;;;GAKG;AACH,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,UAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"parse-parquet-to-arrow-js.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-arrow-js.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAEV,UAAU,EACV,eAAe,EAMhB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,KAAK,EAAC,sBAAsB,EAAC,wCAAqC;AAgFzE;;;;;GAKG;AACH,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,UAAU,CAAC,CAqBrB;AAgED;;;;;;GAMG;AACH,wBAAuB,sCAAsC,CAC3D,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,aAAa,CAAC,eAAe,CAAC,CA2EhC"}
|