@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15005 -3143
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +15766 -4896
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14911 -4316
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +42 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +585 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7055 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8505 -1041
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +78 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +605 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -11
- package/src/bundled.ts +3 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +50 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +929 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/parquet-reader.ts +935 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -3,6 +3,7 @@ import { ParquetSchema } from "../schema/schema.js";
|
|
|
3
3
|
import { ColumnChunk, FileMetaData, RowGroup } from "../parquet-thrift/index.js";
|
|
4
4
|
import { ParquetRowGroup, ParquetColumnChunk, ParquetReaderContext } from "../schema/declare.js";
|
|
5
5
|
import type { ParquetDataPageLocation, ParquetPageLocations, ParquetRowRange } from "../../lib/parquet-page-index.js";
|
|
6
|
+
import { type ParquetEncryptionAlgorithm, type ParquetKeyRetriever } from "../../lib/parquet-encryption.js";
|
|
6
7
|
export type ParquetReaderProps = {
|
|
7
8
|
/** Maximum dictionary-page read size. */
|
|
8
9
|
defaultDictionarySize?: number;
|
|
@@ -14,8 +15,22 @@ export type ParquetReaderProps = {
|
|
|
14
15
|
useTypedValueBuffers?: boolean;
|
|
15
16
|
/** Decode repetition and definition levels into compact unsigned typed arrays. */
|
|
16
17
|
useTypedLevelBuffers?: boolean;
|
|
18
|
+
/** Decode eligible PLAIN BYTE_ARRAY columns into Arrow-compatible contiguous buffers. */
|
|
19
|
+
useArrowByteArrayBuffers?: boolean;
|
|
20
|
+
/** Verify page-header CRC values when present. Disabled by default for throughput. */
|
|
21
|
+
verifyPageChecksums?: boolean;
|
|
22
|
+
/** Decode legacy INT96 values as epoch nanoseconds for Arrow timestamp columns. */
|
|
23
|
+
int96AsTimestamp?: boolean;
|
|
24
|
+
/** Verify plaintext-footer signatures when present. Enabled by default. */
|
|
25
|
+
verifyFooterSignature?: boolean;
|
|
17
26
|
/** Abort signal forwarded to every underlying random-access read. */
|
|
18
27
|
signal?: AbortSignal;
|
|
28
|
+
/** Resolves modular-encryption keys from the file's key metadata. */
|
|
29
|
+
keyRetriever?: ParquetKeyRetriever;
|
|
30
|
+
/** AAD prefix for files that intentionally omit it from FileCryptoMetaData. */
|
|
31
|
+
aadPrefix?: Uint8Array;
|
|
32
|
+
/** Pre-resolved encryption context used by worker-local readers. */
|
|
33
|
+
encryptionContext?: ParquetReaderEncryptionContext;
|
|
19
34
|
};
|
|
20
35
|
/** Properties for initializing a ParquetRowGroupReader */
|
|
21
36
|
export type ParquetIterationProps = {
|
|
@@ -26,6 +41,17 @@ export type ParquetIterationProps = {
|
|
|
26
41
|
/** Abort signal forwarded to row-group and column-chunk reads. */
|
|
27
42
|
signal?: AbortSignal;
|
|
28
43
|
};
|
|
44
|
+
type NormalizedParquetReaderProps = Required<Omit<ParquetReaderProps, 'signal' | 'keyRetriever' | 'aadPrefix' | 'encryptionContext'>> & Pick<ParquetReaderProps, 'signal' | 'keyRetriever' | 'aadPrefix' | 'encryptionContext'>;
|
|
45
|
+
/** File-level parameters needed to authenticate modular-encrypted modules. */
|
|
46
|
+
type ParquetEncryptionContext = {
|
|
47
|
+
algorithm: ParquetEncryptionAlgorithm;
|
|
48
|
+
aadPrefix?: Uint8Array;
|
|
49
|
+
fileUnique: Uint8Array;
|
|
50
|
+
footerKeyMetadata?: Uint8Array;
|
|
51
|
+
footerSigningKeyMetadata?: Uint8Array;
|
|
52
|
+
};
|
|
53
|
+
/** File-level encryption parameters needed to decrypt worker-transferred page modules. */
|
|
54
|
+
export type ParquetReaderEncryptionContext = Pick<ParquetEncryptionContext, 'algorithm' | 'aadPrefix' | 'fileUnique' | 'footerKeyMetadata'>;
|
|
29
55
|
/**
|
|
30
56
|
* The parquet envelope reader allows direct, unbuffered access to the individual
|
|
31
57
|
* sections of the parquet file, namely the header, footer and the row groups.
|
|
@@ -33,18 +59,37 @@ export type ParquetIterationProps = {
|
|
|
33
59
|
* rows from a parquet file use the ParquetReader instead
|
|
34
60
|
*/
|
|
35
61
|
export declare class ParquetReader {
|
|
36
|
-
static defaultProps:
|
|
37
|
-
|
|
38
|
-
};
|
|
39
|
-
props: Required<Omit<ParquetReaderProps, 'signal'>> & {
|
|
40
|
-
signal?: AbortSignal;
|
|
41
|
-
};
|
|
62
|
+
static defaultProps: NormalizedParquetReaderProps;
|
|
63
|
+
props: NormalizedParquetReaderProps;
|
|
42
64
|
file: ReadableFile;
|
|
43
65
|
metadata: Promise<FileMetaData> | null;
|
|
44
66
|
/** Parsed Parquet schema shared by metadata, iteration, and materialization paths. */
|
|
45
67
|
private schema;
|
|
68
|
+
/** Encryption parameters decoded from plaintext or encrypted footer metadata. */
|
|
69
|
+
private encryptionContext?;
|
|
70
|
+
/** Resolved keys shared by encrypted modules in this reader instance. */
|
|
71
|
+
private readonly encryptionKeyCache;
|
|
72
|
+
/** Compression backends selected once and shared by every page in this reader. */
|
|
73
|
+
private readonly pageDecompressors;
|
|
46
74
|
constructor(file: ReadableFile, props?: ParquetReaderProps);
|
|
75
|
+
/** Whether the decoded footer describes an encrypted Parquet file. */
|
|
76
|
+
get encrypted(): boolean;
|
|
77
|
+
/** Whether this reader decodes legacy INT96 values as epoch nanoseconds. */
|
|
78
|
+
get int96AsTimestamp(): boolean;
|
|
79
|
+
/** Returns a cloneable snapshot of the file encryption context for worker decoding. */
|
|
80
|
+
getEncryptionContextForWorker(): ParquetReaderEncryptionContext | undefined;
|
|
81
|
+
/** Resolves one column key on the caller thread before transferring encrypted ranges. */
|
|
82
|
+
getColumnKeyForWorker(columnChunk: ColumnChunk, rowGroupOrdinal: number, columnOrdinal: number): Promise<{
|
|
83
|
+
keyMetadata?: Uint8Array;
|
|
84
|
+
keyMaterial: Uint8Array;
|
|
85
|
+
}>;
|
|
47
86
|
close(): void;
|
|
87
|
+
/** Returns the reader-scoped decoder for one Parquet compression method. */
|
|
88
|
+
private getPageDecompressor;
|
|
89
|
+
/** Reads a byte range, retaining a zero-copy view when the complete file is already in memory. */
|
|
90
|
+
private readBytes;
|
|
91
|
+
/** Returns a reader-scoped key resolver that avoids repeating key-provider work per module. */
|
|
92
|
+
private getCachedKeyRetriever;
|
|
48
93
|
/** Yield one row at a time */
|
|
49
94
|
rowIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRow, void, unknown>;
|
|
50
95
|
/** Yield one batch of rows at a time */
|
|
@@ -63,16 +108,36 @@ export declare class ParquetReader {
|
|
|
63
108
|
readHeader(signal?: AbortSignal): Promise<void>;
|
|
64
109
|
/** Metadata is stored in the footer */
|
|
65
110
|
readFooter(signal?: AbortSignal): Promise<FileMetaData>;
|
|
111
|
+
/** Reads and decrypts an encrypted footer after the standard trailer. */
|
|
112
|
+
private readEncryptedFooter;
|
|
113
|
+
/** Records the encryption parameters exposed by plaintext-footer files. */
|
|
114
|
+
private setEncryptionContext;
|
|
115
|
+
/** Resolves only the selected encrypted columns, using crypto metadata paths before decryption. */
|
|
116
|
+
resolveColumnMetadata(rowGroup: RowGroup, rowGroupOrdinal: number, selectedColumnPaths?: readonly string[][]): Promise<void>;
|
|
117
|
+
/** Decrypts one encrypted column index or offset index module for selective planning. */
|
|
118
|
+
decryptIndexModule(bytes: Uint8Array, module: 'column-index' | 'offset-index', rowGroupOrdinal: number, columnOrdinal: number, columnChunk: ColumnChunk): Promise<Uint8Array>;
|
|
119
|
+
/** Decrypts the encrypted Bloom-filter header and bitset modules for one column chunk. */
|
|
120
|
+
decryptBloomFilter(bytes: Uint8Array, rowGroupOrdinal: number, columnOrdinal: number, columnChunk: ColumnChunk): Promise<Uint8Array>;
|
|
121
|
+
/** Resolves a column's plaintext metadata, decrypting column-specific metadata when present. */
|
|
122
|
+
private getColumnMetadata;
|
|
66
123
|
/** Data is stored in row groups (similar to Apache Arrow record batches) */
|
|
67
|
-
readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], signal?: AbortSignal): Promise<ParquetRowGroup>;
|
|
68
|
-
/** Reads independently materializable
|
|
69
|
-
readRowGroupRange(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], rowRange: ParquetRowRange, pageLocations: ParquetPageLocations, signal?: AbortSignal): Promise<ParquetRowGroup>;
|
|
124
|
+
readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], signal?: AbortSignal, rowGroupOrdinal?: number): Promise<ParquetRowGroup>;
|
|
125
|
+
/** Reads independently materializable non-repeated columns for one row range using page indexes. */
|
|
126
|
+
readRowGroupRange(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], rowRange: ParquetRowRange, pageLocations: ParquetPageLocations, signal?: AbortSignal, rowGroupOrdinal?: number): Promise<ParquetRowGroup>;
|
|
127
|
+
/**
|
|
128
|
+
* Each row group contains column chunks for all the columns.
|
|
129
|
+
*/
|
|
130
|
+
private readUncompressedColumnChunkFromMemory;
|
|
70
131
|
/**
|
|
71
132
|
* Each row group contains column chunks for all the columns.
|
|
72
133
|
*/
|
|
73
|
-
readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk, signal?: AbortSignal): Promise<ParquetColumnChunk>;
|
|
74
|
-
/**
|
|
75
|
-
|
|
134
|
+
readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk, signal?: AbortSignal, rowGroupOrdinal?: number, columnOrdinal?: number, resolvedMetadata?: NonNullable<ColumnChunk['meta_data']>): Promise<ParquetColumnChunk>;
|
|
135
|
+
/** Decrypts the length-prefixed page header and data modules in one column chunk. */
|
|
136
|
+
private decryptColumnPages;
|
|
137
|
+
/** Resolves the key metadata reference for an encrypted column. */
|
|
138
|
+
private getColumnKeyMetadata;
|
|
139
|
+
/** Reads and decodes contiguous data pages that begin and end on complete row boundaries. */
|
|
140
|
+
readColumnChunkRange(schema: ParquetSchema, columnChunk: ColumnChunk, rowRange: ParquetRowRange, pages: readonly ParquetDataPageLocation[], signal?: AbortSignal, rowGroupOrdinal?: number, columnOrdinal?: number, resolvedMetadata?: NonNullable<ColumnChunk['meta_data']>): Promise<ParquetColumnChunk>;
|
|
76
141
|
/**
|
|
77
142
|
* Getting dictionary for allows to flatten values by indices.
|
|
78
143
|
* @param dictionaryPageOffset
|
|
@@ -82,4 +147,5 @@ export declare class ParquetReader {
|
|
|
82
147
|
*/
|
|
83
148
|
getDictionary(dictionaryPageOffset: number, context: ParquetReaderContext, pagesOffset: number, signal?: AbortSignal): Promise<any[]>;
|
|
84
149
|
}
|
|
150
|
+
export {};
|
|
85
151
|
//# sourceMappingURL=parquet-reader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;
|
|
1
|
+
{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAY/C,OAAO,EACL,WAAW,EAGX,YAAY,EAEZ,QAAQ,EAET,mCAAgC;AACjC,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EAErB,6BAA0B;AAY3B,OAAO,KAAK,EACV,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EAChB,wCAAqC;AACtC,OAAO,EAKL,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EACzB,wCAAqC;AA0CtC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,yCAAyC;IACzC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sEAAsE;IACtE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wFAAwF;IACxF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,gGAAgG;IAChG,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,kFAAkF;IAClF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,yFAAyF;IACzF,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,sFAAsF;IACtF,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,2EAA2E;IAC3E,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,qEAAqE;IACrE,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,8BAA8B,CAAC;CACpD,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;IACnC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,KAAK,4BAA4B,GAAG,QAAQ,CAC1C,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,GAAG,mBAAmB,CAAC,CACxF,GACC,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,GAAG,mBAAmB,CAAC,CAAC;AAE1F,8EAA8E;AAC9E,KAAK,wBAAwB,GAAG;IAC9B,SAAS,EAAE,0BAA0B,CAAC;IACtC,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,UAAU,EAAE,UAAU,CAAC;IACvB,iBAAiB,CAAC,EAAE,UAAU,CAAC;IAC/B,wBAAwB,CAAC,EAAE,UAAU,CAAC;CACvC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,MAAM,8BAA8B,GAAG,IAAI,CAC/C,wBAAwB,EACxB,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,mBAAmB,CAC/D,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,MAAM,CAAC,YAAY,EAAE,4BAA4B,CAe/C;IAEF,KAAK,EAAE,4BAA4B,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;IAC9C,sFAAsF;IACtF,OAAO,CAAC,MAAM,CAAuC;IACrD,iFAAiF;IACjF,OAAO,CAAC,iBAAiB,CAAC,CAA2B;IACrD,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA6D;IAChG,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA0D;gBAEhF,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAM1D,sEAAsE;IACtE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,4EAA4E;IAC5E,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,uFAAuF;IACvF,6BAA6B,IAAI,8BAA8B,GAAG,SAAS;IAY3E,yFAAyF;IACnF,qBAAqB,CACzB,WAAW,EAAE,WAAW,EACxB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,UAAU,CAAC;QAAC,WAAW,EAAE,UAAU,CAAA;KAAC,CAAC;IAuB/D,KAAK,IAAI,IAAI;IAOb,4EAA4E;IAC5E,OAAO,CAAC,mBAAmB;IAS3B,kGAAkG;YACpF,SAAS;IAevB,+FAA+F;IAC/F,OAAO,CAAC,qBAAqB;IA8B7B,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IA0B/C,WAAW,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAKlD,SAAS,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC;IAS7D;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAUlE,uCAAuC;IACjC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYrD,uCAAuC;IACjC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAqD7D,yEAAyE;YAC3D,mBAAmB;IAmCjC,2EAA2E;IAC3E,OAAO,CAAC,oBAAoB;IAgB5B,mGAAmG;IAC7F,qBAAqB,CACzB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,MAAM,EACvB,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAuBhB,yFAAyF;IACnF,kBAAkB,CACtB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,cAAc,GAAG,cAAc,EACvC,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,UAAU,CAAC;IAqBtB,0FAA0F;IACpF,kBAAkB,CACtB,KAAK,EAAE,UAAU,EACjB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,UAAU,CAAC;IAuCtB,gGAAgG;YAClF,iBAAiB;IAiD/B,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,GAClB,OAAO,CAAC,eAAe,CAAC;IAmF3B,oGAAoG;IAC9F,iBAAiB,CACrB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,QAAQ,EAAE,eAAe,EACzB,aAAa,EAAE,oBAAoB,EACnC,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,GAClB,OAAO,CAAC,eAAe,CAAC;IA+C3B;;OAEG;IACH,OAAO,CAAC,qCAAqC;IAuE7C;;OAEG;IACG,eAAe,CACnB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,WAAW,EACrB,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,EACnB,aAAa,SAAI,EACjB,gBAAgB,CAAC,EAAE,WAAW,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,GACvD,OAAO,CAAC,kBAAkB,CAAC;IAkG9B,qFAAqF;YACvE,kBAAkB;IAsEhC,mEAAmE;IACnE,OAAO,CAAC,oBAAoB;IAQ5B,6FAA6F;IACvF,oBAAoB,CACxB,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,SAAS,uBAAuB,EAAE,EACzC,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,EACnB,aAAa,SAAI,EACjB,gBAAgB,CAAC,EAAE,WAAW,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,GACvD,OAAO,CAAC,kBAAkB,CAAC;IA0H9B;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,oBAAoB,EAC7B,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,EAAE,CAAC;CAoBlB"}
|