@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15005 -3143
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +15766 -4896
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14911 -4316
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +42 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +585 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7055 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8505 -1041
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +78 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +605 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -11
- package/src/bundled.ts +3 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +50 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +929 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/parquet-reader.ts +935 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
/* eslint-disable camelcase */
|
|
8
8
|
import {stream} from '@loaders.gl/loader-utils';
|
|
9
|
+
import {getWKBGeometryStatistics, type WKBGeometryBoundingBox} from '@loaders.gl/gis';
|
|
9
10
|
import {ParquetCodecOptions, PARQUET_CODECS} from '../codecs/index';
|
|
10
11
|
import * as Compression from '../compression';
|
|
11
|
-
import {
|
|
12
|
+
import type {
|
|
12
13
|
ParquetRowGroup,
|
|
13
14
|
ParquetCodec,
|
|
14
15
|
ParquetColumnChunk,
|
|
@@ -21,8 +22,14 @@ import {ParquetSchema} from '../schema/schema';
|
|
|
21
22
|
import * as Shred from '../schema/shred';
|
|
22
23
|
import {
|
|
23
24
|
BsonType,
|
|
25
|
+
BoundingBox,
|
|
24
26
|
ColumnChunk,
|
|
27
|
+
ColumnCryptoMetaData,
|
|
28
|
+
ColumnIndex,
|
|
25
29
|
ColumnMetaData,
|
|
30
|
+
SizeStatistics,
|
|
31
|
+
Statistics,
|
|
32
|
+
SortingColumn,
|
|
26
33
|
CompressionCodec,
|
|
27
34
|
ConvertedType,
|
|
28
35
|
DataPageHeader,
|
|
@@ -37,6 +44,7 @@ import {
|
|
|
37
44
|
FileMetaData,
|
|
38
45
|
Float16Type,
|
|
39
46
|
GeographyType,
|
|
47
|
+
GeospatialStatistics,
|
|
40
48
|
GeometryType,
|
|
41
49
|
IntType,
|
|
42
50
|
JsonType,
|
|
@@ -51,6 +59,7 @@ import {
|
|
|
51
59
|
PageHeader,
|
|
52
60
|
PageEncodingStats,
|
|
53
61
|
PageType,
|
|
62
|
+
PageLocation,
|
|
54
63
|
RowGroup,
|
|
55
64
|
SchemaElement,
|
|
56
65
|
StringType,
|
|
@@ -59,20 +68,43 @@ import {
|
|
|
59
68
|
TimestampType,
|
|
60
69
|
Type,
|
|
61
70
|
UUIDType,
|
|
62
|
-
VariantType
|
|
71
|
+
VariantType,
|
|
72
|
+
OffsetIndex
|
|
63
73
|
} from '../parquet-thrift/index';
|
|
64
74
|
import {osopen, oswrite, osclose} from '../utils/file-utils';
|
|
65
75
|
import {getBitWidth, serializeThrift} from '../utils/read-utils';
|
|
66
76
|
import {concatUint8Arrays, encodeUtf8, writeUInt32LE} from '../utils/binary-utils';
|
|
77
|
+
import {crc32} from '../utils/crc32';
|
|
67
78
|
import {CompactInt64} from '../utils/uint8-array-compact-protocol';
|
|
68
79
|
import {planColumnPages} from './page-planner';
|
|
69
80
|
import {planDictionary, type ParquetDictionaryPolicy} from './dictionary-planner';
|
|
81
|
+
import {
|
|
82
|
+
decodeParquetSplitBlockBloomFilter,
|
|
83
|
+
encodeParquetBloomFilterValue,
|
|
84
|
+
encodeParquetSplitBlockBloomFilter
|
|
85
|
+
} from '../../lib/parquet-bloom-filter';
|
|
86
|
+
import {BoundaryOrder} from '../parquet-thrift/BoundaryOrder';
|
|
87
|
+
import {
|
|
88
|
+
EncryptionWithColumnKey,
|
|
89
|
+
EncryptionWithFooterKey
|
|
90
|
+
} from '../parquet-thrift/ColumnCryptoMetaData';
|
|
91
|
+
import {EncryptionAlgorithm} from '../parquet-thrift/EncryptionAlgorithm';
|
|
92
|
+
import {FileCryptoMetaData} from '../parquet-thrift/FileCryptoMetaData';
|
|
93
|
+
import {PARQUET_MAGIC_ENCRYPTED} from '../../lib/constants';
|
|
94
|
+
import {
|
|
95
|
+
createParquetModuleAad,
|
|
96
|
+
createParquetFooterSignature,
|
|
97
|
+
encryptParquetModule,
|
|
98
|
+
type ParquetWriterEncryptionOptions,
|
|
99
|
+
type ParquetWriterFooterSignatureOptions
|
|
100
|
+
} from '../../lib/parquet-encryption';
|
|
70
101
|
|
|
71
102
|
/**
|
|
72
103
|
* Parquet File Magic String
|
|
73
104
|
*/
|
|
74
105
|
const PARQUET_MAGIC = 'PAR1';
|
|
75
106
|
const PARQUET_MAGIC_BYTES = encodeUtf8(PARQUET_MAGIC);
|
|
107
|
+
const PARQUET_MAGIC_ENCRYPTED_BYTES = encodeUtf8(PARQUET_MAGIC_ENCRYPTED);
|
|
76
108
|
|
|
77
109
|
/**
|
|
78
110
|
* Parquet File Format Version
|
|
@@ -99,6 +131,26 @@ export interface ParquetEncoderOptions {
|
|
|
99
131
|
dictionary?: ParquetDictionaryPolicy;
|
|
100
132
|
columnDictionaries?: Record<string, ParquetDictionaryPolicy>;
|
|
101
133
|
dictionaryPageSizeLimit?: number;
|
|
134
|
+
bloomFilter?: boolean | Record<string, boolean>;
|
|
135
|
+
pageIndex?: boolean | Record<string, boolean>;
|
|
136
|
+
/** Emit CRC-32 checksums for page bodies. */
|
|
137
|
+
writePageChecksums?: boolean;
|
|
138
|
+
/** Emit optional SizeStatistics metadata for each column chunk. */
|
|
139
|
+
writeSizeStatistics?: boolean;
|
|
140
|
+
/** Emit optional min/max/null-count statistics for each column chunk. */
|
|
141
|
+
writeStatistics?: boolean | Record<string, boolean>;
|
|
142
|
+
/** Declare the row-group sort order using top-level or dotted leaf column names. */
|
|
143
|
+
sortingColumns?: readonly ParquetSortingColumnOption[];
|
|
144
|
+
/** Encode INT96 input values as epoch nanoseconds using the canonical Julian-day layout. */
|
|
145
|
+
int96AsTimestamp?: boolean;
|
|
146
|
+
/** Encrypt the footer using Parquet modular encryption. */
|
|
147
|
+
encryption?: ParquetWriterEncryptionOptions;
|
|
148
|
+
/** Authenticate a plaintext footer with a Parquet modular-encryption signature. */
|
|
149
|
+
footerSignature?: ParquetWriterFooterSignatureOptions;
|
|
150
|
+
/** Stable file identifier shared by encrypted column modules and the footer. */
|
|
151
|
+
encryptionFileUnique?: Uint8Array;
|
|
152
|
+
/** Zero-based row-group ordinal used by encrypted module AAD. */
|
|
153
|
+
rowGroupOrdinal?: number;
|
|
102
154
|
|
|
103
155
|
// Write Stream Options
|
|
104
156
|
flags?: string;
|
|
@@ -109,6 +161,16 @@ export interface ParquetEncoderOptions {
|
|
|
109
161
|
start?: number;
|
|
110
162
|
}
|
|
111
163
|
|
|
164
|
+
/** Writer-facing declaration for one row-group sort key. */
|
|
165
|
+
export type ParquetSortingColumnOption = {
|
|
166
|
+
/** Top-level field name or dotted nested leaf path. */
|
|
167
|
+
column: string;
|
|
168
|
+
/** Sort direction; defaults to ascending. */
|
|
169
|
+
descending?: boolean;
|
|
170
|
+
/** Whether null values sort before non-null values; defaults to false. */
|
|
171
|
+
nullsFirst?: boolean;
|
|
172
|
+
};
|
|
173
|
+
|
|
112
174
|
/**
|
|
113
175
|
* Write a parquet file to an output stream. The ParquetEncoder will perform
|
|
114
176
|
* buffering/batching for performance, so close() must be called after all rows
|
|
@@ -278,6 +340,18 @@ export class ParquetEnvelopeWriter {
|
|
|
278
340
|
public dictionary: ParquetDictionaryPolicy;
|
|
279
341
|
public columnDictionaries: Record<string, ParquetDictionaryPolicy>;
|
|
280
342
|
public dictionaryPageSizeLimit: number;
|
|
343
|
+
public bloomFilter?: boolean | Record<string, boolean>;
|
|
344
|
+
public pageIndex?: boolean | Record<string, boolean>;
|
|
345
|
+
public writePageChecksums: boolean;
|
|
346
|
+
public writeSizeStatistics: boolean;
|
|
347
|
+
public writeStatistics?: boolean | Record<string, boolean>;
|
|
348
|
+
public sortingColumns?: readonly ParquetSortingColumnOption[];
|
|
349
|
+
/** Optional modular-encryption configuration for the footer. */
|
|
350
|
+
public encryption?: ParquetWriterEncryptionOptions;
|
|
351
|
+
/** Optional plaintext-footer signature configuration. */
|
|
352
|
+
public footerSignature?: ParquetWriterFooterSignatureOptions;
|
|
353
|
+
/** File identifier used for encrypted column-module AAD. */
|
|
354
|
+
public encryptionFileUnique?: Uint8Array;
|
|
281
355
|
|
|
282
356
|
constructor(
|
|
283
357
|
schema: ParquetSchema,
|
|
@@ -297,6 +371,43 @@ export class ParquetEnvelopeWriter {
|
|
|
297
371
|
this.dictionary = opts.dictionary ?? 'auto';
|
|
298
372
|
this.columnDictionaries = opts.columnDictionaries || {};
|
|
299
373
|
this.dictionaryPageSizeLimit = opts.dictionaryPageSizeLimit ?? 1024 * 1024;
|
|
374
|
+
this.bloomFilter = opts.bloomFilter;
|
|
375
|
+
this.pageIndex = opts.pageIndex;
|
|
376
|
+
this.writePageChecksums = Boolean(opts.writePageChecksums);
|
|
377
|
+
this.writeSizeStatistics = Boolean(opts.writeSizeStatistics);
|
|
378
|
+
this.writeStatistics = opts.writeStatistics;
|
|
379
|
+
this.sortingColumns = opts.sortingColumns;
|
|
380
|
+
const encryptionFileUnique = opts.encryption?.fileUnique
|
|
381
|
+
? new Uint8Array(opts.encryption.fileUnique)
|
|
382
|
+
: undefined;
|
|
383
|
+
const footerSignatureFileUnique = opts.footerSignature?.fileUnique
|
|
384
|
+
? new Uint8Array(opts.footerSignature.fileUnique)
|
|
385
|
+
: undefined;
|
|
386
|
+
if (
|
|
387
|
+
encryptionFileUnique &&
|
|
388
|
+
footerSignatureFileUnique &&
|
|
389
|
+
!encryptionFileUnique.every((value, index) => value === footerSignatureFileUnique[index])
|
|
390
|
+
) {
|
|
391
|
+
throw new Error('Parquet encryption and footer signature file_unique values must match');
|
|
392
|
+
}
|
|
393
|
+
const sharedFileUnique =
|
|
394
|
+
encryptionFileUnique ??
|
|
395
|
+
footerSignatureFileUnique ??
|
|
396
|
+
(opts.encryption || opts.footerSignature ? createFileUnique() : undefined);
|
|
397
|
+
this.encryption = opts.encryption
|
|
398
|
+
? {...opts.encryption, fileUnique: new Uint8Array(sharedFileUnique!)}
|
|
399
|
+
: undefined;
|
|
400
|
+
this.footerSignature = opts.footerSignature
|
|
401
|
+
? {
|
|
402
|
+
...opts.footerSignature,
|
|
403
|
+
keyMetadata: new Uint8Array(opts.footerSignature.keyMetadata),
|
|
404
|
+
aadPrefix: opts.footerSignature.aadPrefix
|
|
405
|
+
? new Uint8Array(opts.footerSignature.aadPrefix)
|
|
406
|
+
: undefined,
|
|
407
|
+
fileUnique: new Uint8Array(sharedFileUnique!)
|
|
408
|
+
}
|
|
409
|
+
: undefined;
|
|
410
|
+
this.encryptionFileUnique = sharedFileUnique;
|
|
300
411
|
}
|
|
301
412
|
|
|
302
413
|
writeSection(buf: Uint8Array): Promise<void> {
|
|
@@ -308,7 +419,9 @@ export class ParquetEnvelopeWriter {
|
|
|
308
419
|
* Encode the parquet file header
|
|
309
420
|
*/
|
|
310
421
|
writeHeader(): Promise<void> {
|
|
311
|
-
return this.writeSection(
|
|
422
|
+
return this.writeSection(
|
|
423
|
+
this.encryption && !this.footerSignature ? PARQUET_MAGIC_ENCRYPTED_BYTES : PARQUET_MAGIC_BYTES
|
|
424
|
+
);
|
|
312
425
|
}
|
|
313
426
|
|
|
314
427
|
/**
|
|
@@ -322,7 +435,16 @@ export class ParquetEnvelopeWriter {
|
|
|
322
435
|
useDataPageV2: this.useDataPageV2,
|
|
323
436
|
dictionary: this.dictionary,
|
|
324
437
|
columnDictionaries: this.columnDictionaries,
|
|
325
|
-
dictionaryPageSizeLimit: this.dictionaryPageSizeLimit
|
|
438
|
+
dictionaryPageSizeLimit: this.dictionaryPageSizeLimit,
|
|
439
|
+
bloomFilter: this.bloomFilter,
|
|
440
|
+
pageIndex: this.pageIndex,
|
|
441
|
+
writePageChecksums: this.writePageChecksums,
|
|
442
|
+
writeSizeStatistics: this.writeSizeStatistics,
|
|
443
|
+
writeStatistics: this.writeStatistics,
|
|
444
|
+
sortingColumns: this.sortingColumns,
|
|
445
|
+
encryption: this.encryption,
|
|
446
|
+
encryptionFileUnique: this.encryptionFileUnique,
|
|
447
|
+
rowGroupOrdinal: this.rowGroups.length
|
|
326
448
|
});
|
|
327
449
|
|
|
328
450
|
this.rowCount += records.rowCount;
|
|
@@ -333,15 +455,30 @@ export class ParquetEnvelopeWriter {
|
|
|
333
455
|
/**
|
|
334
456
|
* Write the parquet file footer
|
|
335
457
|
*/
|
|
336
|
-
writeFooter(userMetadata: Record<string, string>): Promise<void> {
|
|
458
|
+
async writeFooter(userMetadata: Record<string, string>): Promise<void> {
|
|
337
459
|
if (!userMetadata) {
|
|
338
460
|
// tslint:disable-next-line:no-parameter-reassignment
|
|
339
461
|
userMetadata = {};
|
|
340
462
|
}
|
|
341
463
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
464
|
+
if (this.footerSignature) {
|
|
465
|
+
await this.writeSection(
|
|
466
|
+
await encodeSignedFooter(
|
|
467
|
+
this.schema,
|
|
468
|
+
this.rowCount,
|
|
469
|
+
this.rowGroups,
|
|
470
|
+
userMetadata,
|
|
471
|
+
this.footerSignature
|
|
472
|
+
)
|
|
473
|
+
);
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
const footer = encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata);
|
|
477
|
+
if (!this.encryption) {
|
|
478
|
+
await this.writeSection(footer);
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
await this.writeSection(await encodeEncryptedFooter(footer, this.encryption));
|
|
345
482
|
}
|
|
346
483
|
|
|
347
484
|
/**
|
|
@@ -412,7 +549,10 @@ async function encodeDataPage(
|
|
|
412
549
|
column: ParquetField,
|
|
413
550
|
data: ParquetColumnChunk,
|
|
414
551
|
valueEncoding: ParquetCodec = column.encoding!,
|
|
415
|
-
valueBitWidth?: number
|
|
552
|
+
valueBitWidth?: number,
|
|
553
|
+
writePageChecksums = false,
|
|
554
|
+
statistics?: Statistics,
|
|
555
|
+
int96AsTimestamp = false
|
|
416
556
|
): Promise<{
|
|
417
557
|
header: PageHeader;
|
|
418
558
|
headerSize: number;
|
|
@@ -448,7 +588,8 @@ async function encodeDataPage(
|
|
|
448
588
|
/* encode values */
|
|
449
589
|
const valuesBuf = encodeValues(column.primitiveType!, valueEncoding, data.values as any[], {
|
|
450
590
|
typeLength: column.typeLength,
|
|
451
|
-
bitWidth: valueBitWidth ?? column.typeLength
|
|
591
|
+
bitWidth: valueBitWidth ?? column.typeLength,
|
|
592
|
+
int96AsTimestamp
|
|
452
593
|
});
|
|
453
594
|
|
|
454
595
|
const dataBuf = concatUint8Arrays([rLevelsBuf, dLevelsBuf, valuesBuf]);
|
|
@@ -463,10 +604,12 @@ async function encodeDataPage(
|
|
|
463
604
|
num_values: data.count,
|
|
464
605
|
encoding: Encoding[valueEncoding] as any,
|
|
465
606
|
definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],
|
|
466
|
-
repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]
|
|
607
|
+
repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING]
|
|
608
|
+
statistics
|
|
467
609
|
}),
|
|
468
610
|
uncompressed_page_size: dataBuf.length,
|
|
469
|
-
compressed_page_size: compressedBuf.length
|
|
611
|
+
compressed_page_size: compressedBuf.length,
|
|
612
|
+
crc: writePageChecksums ? crc32(compressedBuf) : undefined
|
|
470
613
|
});
|
|
471
614
|
|
|
472
615
|
/* concat page header, repetition and definition levels and values */
|
|
@@ -484,7 +627,10 @@ async function encodeDataPageV2(
|
|
|
484
627
|
data: ParquetColumnChunk,
|
|
485
628
|
rowCount: number,
|
|
486
629
|
valueEncoding: ParquetCodec = column.encoding!,
|
|
487
|
-
valueBitWidth?: number
|
|
630
|
+
valueBitWidth?: number,
|
|
631
|
+
writePageChecksums = false,
|
|
632
|
+
statistics?: Statistics,
|
|
633
|
+
int96AsTimestamp = false
|
|
488
634
|
): Promise<{
|
|
489
635
|
header: PageHeader;
|
|
490
636
|
headerSize: number;
|
|
@@ -493,7 +639,8 @@ async function encodeDataPageV2(
|
|
|
493
639
|
/* encode values */
|
|
494
640
|
const valuesBuf = encodeValues(column.primitiveType!, valueEncoding, data.values as any[], {
|
|
495
641
|
typeLength: column.typeLength,
|
|
496
|
-
bitWidth: valueBitWidth ?? column.typeLength
|
|
642
|
+
bitWidth: valueBitWidth ?? column.typeLength,
|
|
643
|
+
int96AsTimestamp
|
|
497
644
|
});
|
|
498
645
|
|
|
499
646
|
// compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
|
|
@@ -536,10 +683,14 @@ async function encodeDataPageV2(
|
|
|
536
683
|
encoding: Encoding[valueEncoding] as any,
|
|
537
684
|
definition_levels_byte_length: dLevelsBuf.length,
|
|
538
685
|
repetition_levels_byte_length: rLevelsBuf.length,
|
|
539
|
-
is_compressed: column.compression !== 'UNCOMPRESSED'
|
|
686
|
+
is_compressed: column.compression !== 'UNCOMPRESSED',
|
|
687
|
+
statistics
|
|
540
688
|
}),
|
|
541
689
|
uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,
|
|
542
|
-
compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length
|
|
690
|
+
compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length,
|
|
691
|
+
crc: writePageChecksums
|
|
692
|
+
? crc32(concatUint8Arrays([rLevelsBuf, dLevelsBuf, compressedBuf]))
|
|
693
|
+
: undefined
|
|
543
694
|
});
|
|
544
695
|
|
|
545
696
|
/* concat page header, repetition and definition levels and values */
|
|
@@ -551,10 +702,13 @@ async function encodeDataPageV2(
|
|
|
551
702
|
/** Encodes one chunk-wide PLAIN dictionary page. */
|
|
552
703
|
async function encodeDictionaryPage(
|
|
553
704
|
column: ParquetField,
|
|
554
|
-
dictionaryValues: unknown[]
|
|
705
|
+
dictionaryValues: unknown[],
|
|
706
|
+
writePageChecksums = false,
|
|
707
|
+
int96AsTimestamp = false
|
|
555
708
|
): Promise<{header: PageHeader; headerSize: number; page: Uint8Array}> {
|
|
556
709
|
const valuesBuffer = encodeValues(column.primitiveType!, 'PLAIN', dictionaryValues as any[], {
|
|
557
|
-
typeLength: column.typeLength
|
|
710
|
+
typeLength: column.typeLength,
|
|
711
|
+
int96AsTimestamp
|
|
558
712
|
});
|
|
559
713
|
const compressedBuffer = await Compression.deflate(column.compression!, valuesBuffer);
|
|
560
714
|
const header = new PageHeader({
|
|
@@ -564,7 +718,8 @@ async function encodeDictionaryPage(
|
|
|
564
718
|
encoding: Encoding.PLAIN
|
|
565
719
|
}),
|
|
566
720
|
uncompressed_page_size: valuesBuffer.length,
|
|
567
|
-
compressed_page_size: compressedBuffer.length
|
|
721
|
+
compressed_page_size: compressedBuffer.length,
|
|
722
|
+
crc: writePageChecksums ? crc32(compressedBuffer) : undefined
|
|
568
723
|
});
|
|
569
724
|
const headerBuffer = serializeThrift(header);
|
|
570
725
|
return {
|
|
@@ -574,6 +729,65 @@ async function encodeDictionaryPage(
|
|
|
574
729
|
};
|
|
575
730
|
}
|
|
576
731
|
|
|
732
|
+
/** Computes native Parquet geospatial statistics for a GEOMETRY or GEOGRAPHY column chunk. */
|
|
733
|
+
function createGeospatialStatistics(
|
|
734
|
+
column: ParquetField,
|
|
735
|
+
values: ParquetColumnChunk['values']
|
|
736
|
+
): GeospatialStatistics | undefined {
|
|
737
|
+
const logicalType = column.logicalType?.type || column.originalType;
|
|
738
|
+
if (logicalType !== 'GEOMETRY' && logicalType !== 'GEOGRAPHY') return undefined;
|
|
739
|
+
const geometryTypes = new Set<number>();
|
|
740
|
+
let bbox: WKBGeometryBoundingBox | undefined;
|
|
741
|
+
// Vertex bounds are not conservative for GEOGRAPHY: non-linear edge algorithms can reach
|
|
742
|
+
// extrema between vertices. Until edge-aware geodesic bounds are available, omit the bbox while
|
|
743
|
+
// retaining the useful geometry-type statistics.
|
|
744
|
+
const canWriteBoundingBox = logicalType === 'GEOMETRY';
|
|
745
|
+
for (const value of values) {
|
|
746
|
+
if (!(value instanceof ArrayBuffer) && !ArrayBuffer.isView(value)) {
|
|
747
|
+
throw new Error(`${logicalType} columns must contain WKB binary values`);
|
|
748
|
+
}
|
|
749
|
+
const statistics = getWKBGeometryStatistics(value);
|
|
750
|
+
geometryTypes.add(statistics.geometryType);
|
|
751
|
+
if (canWriteBoundingBox) {
|
|
752
|
+
bbox = mergeGeometryBoundingBoxes(bbox, statistics.bbox);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return new GeospatialStatistics({
|
|
756
|
+
bbox: bbox ? new BoundingBox(bbox) : undefined,
|
|
757
|
+
geospatial_types: [...geometryTypes].sort((left, right) => left - right)
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/** Merges finite per-dimension bounds without inventing absent Z or M dimensions. */
|
|
762
|
+
function mergeGeometryBoundingBoxes(
|
|
763
|
+
target: WKBGeometryBoundingBox | undefined,
|
|
764
|
+
source: WKBGeometryBoundingBox | undefined
|
|
765
|
+
): WKBGeometryBoundingBox | undefined {
|
|
766
|
+
if (!source) return target;
|
|
767
|
+
if (!target) return {...source};
|
|
768
|
+
target.xmin = Math.min(target.xmin, source.xmin);
|
|
769
|
+
target.xmax = Math.max(target.xmax, source.xmax);
|
|
770
|
+
target.ymin = Math.min(target.ymin, source.ymin);
|
|
771
|
+
target.ymax = Math.max(target.ymax, source.ymax);
|
|
772
|
+
mergeOptionalBounds(target, source, 'zmin', Math.min);
|
|
773
|
+
mergeOptionalBounds(target, source, 'zmax', Math.max);
|
|
774
|
+
mergeOptionalBounds(target, source, 'mmin', Math.min);
|
|
775
|
+
mergeOptionalBounds(target, source, 'mmax', Math.max);
|
|
776
|
+
return target;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/** Merges one optional bound while preserving a dimension absent from every value. */
|
|
780
|
+
function mergeOptionalBounds(
|
|
781
|
+
target: WKBGeometryBoundingBox,
|
|
782
|
+
source: WKBGeometryBoundingBox,
|
|
783
|
+
key: 'zmin' | 'zmax' | 'mmin' | 'mmax',
|
|
784
|
+
merge: (left: number, right: number) => number
|
|
785
|
+
): void {
|
|
786
|
+
const sourceValue = source[key];
|
|
787
|
+
if (sourceValue === undefined) return;
|
|
788
|
+
target[key] = target[key] === undefined ? sourceValue : merge(target[key]!, sourceValue);
|
|
789
|
+
}
|
|
790
|
+
|
|
577
791
|
/**
|
|
578
792
|
* Encode an array of values into a parquet column chunk
|
|
579
793
|
*/
|
|
@@ -581,16 +795,29 @@ async function encodeColumnChunk(
|
|
|
581
795
|
column: ParquetField,
|
|
582
796
|
buffer: ParquetRowGroup,
|
|
583
797
|
offset: number,
|
|
584
|
-
opts: ParquetEncoderOptions
|
|
798
|
+
opts: ParquetEncoderOptions,
|
|
799
|
+
rowGroupOrdinal: number,
|
|
800
|
+
columnOrdinal: number
|
|
585
801
|
): Promise<{
|
|
586
802
|
body: Uint8Array;
|
|
587
803
|
metadata: ColumnMetaData;
|
|
804
|
+
encryptedColumnMetadata?: Uint8Array;
|
|
805
|
+
cryptoMetadata?: ColumnCryptoMetaData;
|
|
588
806
|
metadataOffset: number;
|
|
807
|
+
offsetIndexOffset?: number;
|
|
808
|
+
offsetIndexLength?: number;
|
|
809
|
+
columnIndexOffset?: number;
|
|
810
|
+
columnIndexLength?: number;
|
|
589
811
|
}> {
|
|
590
812
|
const data = buffer.columnData[column.path.join()];
|
|
591
813
|
const baseOffset = (opts.baseOffset || 0) + offset;
|
|
592
814
|
/* encode data page(s) */
|
|
593
|
-
const
|
|
815
|
+
const pageRecords: Array<{
|
|
816
|
+
page: Uint8Array;
|
|
817
|
+
headerSize: number;
|
|
818
|
+
dictionary: boolean;
|
|
819
|
+
}> = [];
|
|
820
|
+
const pageLocations: PageLocation[] = [];
|
|
594
821
|
// tslint:disable-next-line:variable-name
|
|
595
822
|
let total_uncompressed_size = 0;
|
|
596
823
|
// tslint:disable-next-line:variable-name
|
|
@@ -610,13 +837,24 @@ async function encodeColumnChunk(
|
|
|
610
837
|
opts.dictionaryPageSizeLimit ?? 1024 * 1024
|
|
611
838
|
);
|
|
612
839
|
if (dictionaryPlan) {
|
|
613
|
-
const result = await encodeDictionaryPage(
|
|
614
|
-
|
|
840
|
+
const result = await encodeDictionaryPage(
|
|
841
|
+
column,
|
|
842
|
+
dictionaryPlan.values,
|
|
843
|
+
opts.writePageChecksums,
|
|
844
|
+
opts.int96AsTimestamp
|
|
845
|
+
);
|
|
846
|
+
pageRecords.push({page: result.page, headerSize: result.headerSize, dictionary: true});
|
|
615
847
|
total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
|
|
616
848
|
total_compressed_size += result.header.compressed_page_size + result.headerSize;
|
|
617
849
|
}
|
|
618
850
|
|
|
619
|
-
const valueEncoding: ParquetCodec = dictionaryPlan
|
|
851
|
+
const valueEncoding: ParquetCodec = dictionaryPlan
|
|
852
|
+
? column.encoding === 'PLAIN_DICTIONARY'
|
|
853
|
+
? 'PLAIN_DICTIONARY'
|
|
854
|
+
: 'RLE_DICTIONARY'
|
|
855
|
+
: column.encoding === 'PLAIN_DICTIONARY' || column.encoding === 'RLE_DICTIONARY'
|
|
856
|
+
? 'PLAIN'
|
|
857
|
+
: column.encoding!;
|
|
620
858
|
let dictionaryIndexOffset = 0;
|
|
621
859
|
for (const plannedPage of plannedPages) {
|
|
622
860
|
const pageData = dictionaryPlan
|
|
@@ -635,16 +873,90 @@ async function encodeColumnChunk(
|
|
|
635
873
|
pageData,
|
|
636
874
|
plannedPage.rowCount,
|
|
637
875
|
valueEncoding,
|
|
638
|
-
dictionaryPlan?.bitWidth
|
|
876
|
+
dictionaryPlan?.bitWidth,
|
|
877
|
+
opts.writePageChecksums,
|
|
878
|
+
isStatisticsEnabled(opts.writeStatistics, column)
|
|
879
|
+
? createColumnStatistics(column, plannedPage.data)
|
|
880
|
+
: undefined,
|
|
881
|
+
opts.int96AsTimestamp
|
|
639
882
|
)
|
|
640
|
-
: await encodeDataPage(
|
|
641
|
-
|
|
883
|
+
: await encodeDataPage(
|
|
884
|
+
column,
|
|
885
|
+
pageData,
|
|
886
|
+
valueEncoding,
|
|
887
|
+
dictionaryPlan?.bitWidth,
|
|
888
|
+
opts.writePageChecksums,
|
|
889
|
+
isStatisticsEnabled(opts.writeStatistics, column)
|
|
890
|
+
? createColumnStatistics(column, plannedPage.data)
|
|
891
|
+
: undefined,
|
|
892
|
+
opts.int96AsTimestamp
|
|
893
|
+
);
|
|
894
|
+
pageRecords.push({page: result.page, headerSize: result.headerSize, dictionary: false});
|
|
642
895
|
total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
|
|
643
896
|
total_compressed_size += result.header.compressed_page_size + result.headerSize;
|
|
644
897
|
}
|
|
645
898
|
|
|
899
|
+
const encryption = opts.encryption;
|
|
900
|
+
const encryptedColumn = Boolean(encryption && shouldEncryptColumn(encryption, column));
|
|
901
|
+
const columnKeyMetadata = encryption ? getColumnKeyMetadata(encryption, column) : undefined;
|
|
902
|
+
const pages = encryptedColumn
|
|
903
|
+
? await encryptParquetColumnPages(
|
|
904
|
+
pageRecords,
|
|
905
|
+
encryption!,
|
|
906
|
+
opts.encryptionFileUnique!,
|
|
907
|
+
opts.rowGroupOrdinal ?? 0,
|
|
908
|
+
columnOrdinal,
|
|
909
|
+
columnKeyMetadata
|
|
910
|
+
)
|
|
911
|
+
: pageRecords.map(record => record.page);
|
|
912
|
+
let pageOffset = 0;
|
|
913
|
+
let dataPageIndex = 0;
|
|
914
|
+
let firstRowIndex = 0;
|
|
915
|
+
for (const [pageIndex, page] of pages.entries()) {
|
|
916
|
+
if (!pageRecords[pageIndex].dictionary) {
|
|
917
|
+
pageLocations.push(
|
|
918
|
+
new PageLocation({
|
|
919
|
+
offset: baseOffset + pageOffset,
|
|
920
|
+
compressed_page_size: page.length,
|
|
921
|
+
first_row_index: firstRowIndex
|
|
922
|
+
})
|
|
923
|
+
);
|
|
924
|
+
firstRowIndex += plannedPages[dataPageIndex++].rowCount;
|
|
925
|
+
}
|
|
926
|
+
pageOffset += page.length;
|
|
927
|
+
}
|
|
928
|
+
total_compressed_size = pages.reduce((total, page) => total + page.length, 0);
|
|
646
929
|
const pagesBuf = concatUint8Arrays(pages);
|
|
647
930
|
// const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;
|
|
931
|
+
const bloomFilterEnabled =
|
|
932
|
+
opts.bloomFilter === true || opts.bloomFilter?.[column.path[0]] === true;
|
|
933
|
+
const bloomFilter = bloomFilterEnabled ? createColumnBloomFilter(column, data.values) : undefined;
|
|
934
|
+
const encodedBloomFilter =
|
|
935
|
+
bloomFilter && encryptedColumn
|
|
936
|
+
? await encryptParquetBloomFilter(
|
|
937
|
+
bloomFilter,
|
|
938
|
+
encryption!,
|
|
939
|
+
opts.encryptionFileUnique!,
|
|
940
|
+
opts.rowGroupOrdinal ?? 0,
|
|
941
|
+
columnOrdinal,
|
|
942
|
+
columnKeyMetadata
|
|
943
|
+
)
|
|
944
|
+
: bloomFilter;
|
|
945
|
+
const pageIndexEnabled = opts.pageIndex === true || opts.pageIndex?.[column.path[0]] === true;
|
|
946
|
+
const pageIndexes = pageIndexEnabled
|
|
947
|
+
? createColumnPageIndexes(column, plannedPages, pageLocations, opts.int96AsTimestamp)
|
|
948
|
+
: undefined;
|
|
949
|
+
const encodedPageIndexes =
|
|
950
|
+
pageIndexes && encryptedColumn
|
|
951
|
+
? await encryptParquetPageIndexes(
|
|
952
|
+
pageIndexes,
|
|
953
|
+
encryption!,
|
|
954
|
+
opts.encryptionFileUnique!,
|
|
955
|
+
opts.rowGroupOrdinal ?? 0,
|
|
956
|
+
columnOrdinal,
|
|
957
|
+
columnKeyMetadata
|
|
958
|
+
)
|
|
959
|
+
: pageIndexes;
|
|
648
960
|
|
|
649
961
|
/* prepare metadata header */
|
|
650
962
|
const metadata = new ColumnMetaData({
|
|
@@ -672,7 +984,14 @@ async function encodeColumnChunk(
|
|
|
672
984
|
total_uncompressed_size: int64(total_uncompressed_size), // : pagesBuf.length,
|
|
673
985
|
total_compressed_size: int64(total_compressed_size),
|
|
674
986
|
type: Type[column.primitiveType!],
|
|
675
|
-
codec: CompressionCodec[column.compression!]
|
|
987
|
+
codec: CompressionCodec[column.compression!],
|
|
988
|
+
bloom_filter_offset: encodedBloomFilter ? int64(baseOffset + pagesBuf.length) : undefined,
|
|
989
|
+
bloom_filter_length: encodedBloomFilter?.length,
|
|
990
|
+
geospatial_statistics: createGeospatialStatistics(column, data.values),
|
|
991
|
+
size_statistics: opts.writeSizeStatistics ? createSizeStatistics(column, data) : undefined,
|
|
992
|
+
statistics: isStatisticsEnabled(opts.writeStatistics, column)
|
|
993
|
+
? createColumnStatistics(column, data)
|
|
994
|
+
: undefined
|
|
676
995
|
});
|
|
677
996
|
|
|
678
997
|
/* list encodings */
|
|
@@ -681,9 +1000,541 @@ async function encodeColumnChunk(
|
|
|
681
1000
|
metadata.encodings.push(Encoding[valueEncoding]);
|
|
682
1001
|
|
|
683
1002
|
/* concat metadata header and data pages */
|
|
684
|
-
const
|
|
685
|
-
const
|
|
686
|
-
|
|
1003
|
+
const metadataEncoded = serializeThrift(metadata);
|
|
1004
|
+
const metadataSection = encryptedColumn ? undefined : metadataEncoded;
|
|
1005
|
+
const metadataOffset = baseOffset + pagesBuf.length + (encodedBloomFilter?.length || 0);
|
|
1006
|
+
const offsetIndexOffset = encodedPageIndexes
|
|
1007
|
+
? metadataOffset + (metadataSection?.length || 0)
|
|
1008
|
+
: undefined;
|
|
1009
|
+
const columnIndexOffset = encodedPageIndexes?.columnIndex
|
|
1010
|
+
? metadataOffset + (metadataSection?.length || 0) + encodedPageIndexes.offsetIndex.length
|
|
1011
|
+
: undefined;
|
|
1012
|
+
const body = concatUint8Arrays([
|
|
1013
|
+
pagesBuf,
|
|
1014
|
+
...(encodedBloomFilter ? [encodedBloomFilter] : []),
|
|
1015
|
+
...(metadataSection ? [metadataSection] : []),
|
|
1016
|
+
...(encodedPageIndexes
|
|
1017
|
+
? [
|
|
1018
|
+
encodedPageIndexes.offsetIndex,
|
|
1019
|
+
...(encodedPageIndexes.columnIndex ? [encodedPageIndexes.columnIndex] : [])
|
|
1020
|
+
]
|
|
1021
|
+
: [])
|
|
1022
|
+
]);
|
|
1023
|
+
return {
|
|
1024
|
+
body,
|
|
1025
|
+
metadata,
|
|
1026
|
+
metadataOffset,
|
|
1027
|
+
offsetIndexOffset,
|
|
1028
|
+
offsetIndexLength: encodedPageIndexes?.offsetIndex.length,
|
|
1029
|
+
columnIndexOffset,
|
|
1030
|
+
columnIndexLength: encodedPageIndexes?.columnIndex?.length,
|
|
1031
|
+
...(encryptedColumn
|
|
1032
|
+
? {
|
|
1033
|
+
encryptedColumnMetadata: await encryptParquetModule(metadataEncoded, {
|
|
1034
|
+
algorithm: encryption!.algorithm ?? 'AES_GCM_V1',
|
|
1035
|
+
aad: createParquetModuleAad(
|
|
1036
|
+
encryption!.aadPrefix,
|
|
1037
|
+
opts.encryptionFileUnique!,
|
|
1038
|
+
'column-metadata',
|
|
1039
|
+
opts.rowGroupOrdinal ?? 0,
|
|
1040
|
+
columnOrdinal
|
|
1041
|
+
),
|
|
1042
|
+
keyMetadata: columnKeyMetadata ?? encryption!.keyMetadata,
|
|
1043
|
+
keyRetriever: encryption!.keyRetriever
|
|
1044
|
+
}),
|
|
1045
|
+
cryptoMetadata: new ColumnCryptoMetaData({
|
|
1046
|
+
...(columnKeyMetadata
|
|
1047
|
+
? {
|
|
1048
|
+
ENCRYPTION_WITH_COLUMN_KEY: new EncryptionWithColumnKey({
|
|
1049
|
+
path_in_schema: [...column.path],
|
|
1050
|
+
key_metadata: new Uint8Array(columnKeyMetadata)
|
|
1051
|
+
})
|
|
1052
|
+
}
|
|
1053
|
+
: {ENCRYPTION_WITH_FOOTER_KEY: new EncryptionWithFooterKey()})
|
|
1054
|
+
})
|
|
1055
|
+
}
|
|
1056
|
+
: {})
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
/** Returns whether a writer column should use modular encryption. */
|
|
1061
|
+
function shouldEncryptColumn(
|
|
1062
|
+
options: ParquetWriterEncryptionOptions,
|
|
1063
|
+
column: ParquetField
|
|
1064
|
+
): boolean {
|
|
1065
|
+
return Boolean(
|
|
1066
|
+
options.columnKeyMetadata?.[column.path[0]] ||
|
|
1067
|
+
options.encryptColumns === true ||
|
|
1068
|
+
options.encryptColumns?.[column.path[0]] === true
|
|
1069
|
+
);
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
/** Returns a defensive copy of the key metadata assigned to one top-level column. */
|
|
1073
|
+
function getColumnKeyMetadata(
|
|
1074
|
+
options: ParquetWriterEncryptionOptions,
|
|
1075
|
+
column: ParquetField
|
|
1076
|
+
): Uint8Array | undefined {
|
|
1077
|
+
const keyMetadata = options.columnKeyMetadata?.[column.path[0]];
|
|
1078
|
+
return keyMetadata ? new Uint8Array(keyMetadata) : undefined;
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
/** Adds the four-byte module length required by encrypted page and Bloom-filter modules. */
|
|
1082
|
+
function prefixEncryptedModule(module: Uint8Array): Uint8Array {
|
|
1083
|
+
const prefixed = new Uint8Array(module.byteLength + 4);
|
|
1084
|
+
writeUInt32LE(prefixed, module.byteLength, 0);
|
|
1085
|
+
prefixed.set(module, 4);
|
|
1086
|
+
return prefixed;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
/** Encrypts one column's dictionary and data pages using the Parquet module AAD rules. */
|
|
1090
|
+
async function encryptParquetColumnPages(
|
|
1091
|
+
pageRecords: readonly {page: Uint8Array; headerSize: number; dictionary: boolean}[],
|
|
1092
|
+
options: ParquetWriterEncryptionOptions,
|
|
1093
|
+
fileUnique: Uint8Array,
|
|
1094
|
+
rowGroupOrdinal: number,
|
|
1095
|
+
columnOrdinal: number,
|
|
1096
|
+
keyMetadata?: Uint8Array
|
|
1097
|
+
): Promise<Uint8Array[]> {
|
|
1098
|
+
const encryptedPages: Uint8Array[] = [];
|
|
1099
|
+
let dataPageOrdinal = 0;
|
|
1100
|
+
const algorithm = options.algorithm ?? 'AES_GCM_V1';
|
|
1101
|
+
for (const pageRecord of pageRecords) {
|
|
1102
|
+
const moduleType = pageRecord.dictionary ? 'dictionary-page' : 'data-page';
|
|
1103
|
+
const headerModuleType = pageRecord.dictionary ? 'dictionary-page-header' : 'data-page-header';
|
|
1104
|
+
const header = await encryptParquetModule(pageRecord.page.subarray(0, pageRecord.headerSize), {
|
|
1105
|
+
algorithm,
|
|
1106
|
+
aad: createParquetModuleAad(
|
|
1107
|
+
options.aadPrefix,
|
|
1108
|
+
fileUnique,
|
|
1109
|
+
headerModuleType,
|
|
1110
|
+
rowGroupOrdinal,
|
|
1111
|
+
columnOrdinal,
|
|
1112
|
+
pageRecord.dictionary ? undefined : dataPageOrdinal
|
|
1113
|
+
),
|
|
1114
|
+
keyMetadata: keyMetadata ?? options.keyMetadata,
|
|
1115
|
+
keyRetriever: options.keyRetriever
|
|
1116
|
+
});
|
|
1117
|
+
const body = await encryptParquetModule(pageRecord.page.subarray(pageRecord.headerSize), {
|
|
1118
|
+
algorithm,
|
|
1119
|
+
aad: createParquetModuleAad(
|
|
1120
|
+
options.aadPrefix,
|
|
1121
|
+
fileUnique,
|
|
1122
|
+
moduleType,
|
|
1123
|
+
rowGroupOrdinal,
|
|
1124
|
+
columnOrdinal,
|
|
1125
|
+
pageRecord.dictionary ? undefined : dataPageOrdinal
|
|
1126
|
+
),
|
|
1127
|
+
keyMetadata: keyMetadata ?? options.keyMetadata,
|
|
1128
|
+
keyRetriever: options.keyRetriever,
|
|
1129
|
+
page: true
|
|
1130
|
+
});
|
|
1131
|
+
encryptedPages.push(
|
|
1132
|
+
concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(body)])
|
|
1133
|
+
);
|
|
1134
|
+
if (!pageRecord.dictionary) dataPageOrdinal++;
|
|
1135
|
+
}
|
|
1136
|
+
return encryptedPages;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
/** Encrypts a split-block Bloom filter header and bitset as independent Parquet modules. */
|
|
1140
|
+
async function encryptParquetBloomFilter(
|
|
1141
|
+
bloomFilter: Uint8Array,
|
|
1142
|
+
options: ParquetWriterEncryptionOptions,
|
|
1143
|
+
fileUnique: Uint8Array,
|
|
1144
|
+
rowGroupOrdinal: number,
|
|
1145
|
+
columnOrdinal: number,
|
|
1146
|
+
keyMetadata?: Uint8Array
|
|
1147
|
+
): Promise<Uint8Array> {
|
|
1148
|
+
const parsed = decodeParquetSplitBlockBloomFilter(bloomFilter);
|
|
1149
|
+
const algorithm = options.algorithm ?? 'AES_GCM_V1';
|
|
1150
|
+
const keyOptions = {
|
|
1151
|
+
algorithm,
|
|
1152
|
+
keyMetadata: keyMetadata ?? options.keyMetadata,
|
|
1153
|
+
keyRetriever: options.keyRetriever
|
|
1154
|
+
};
|
|
1155
|
+
const header = await encryptParquetModule(bloomFilter.subarray(0, parsed.headerByteLength), {
|
|
1156
|
+
...keyOptions,
|
|
1157
|
+
aad: createParquetModuleAad(
|
|
1158
|
+
options.aadPrefix,
|
|
1159
|
+
fileUnique,
|
|
1160
|
+
'bloom-filter-header',
|
|
1161
|
+
rowGroupOrdinal,
|
|
1162
|
+
columnOrdinal
|
|
1163
|
+
)
|
|
1164
|
+
});
|
|
1165
|
+
const bitset = await encryptParquetModule(parsed.bitset, {
|
|
1166
|
+
...keyOptions,
|
|
1167
|
+
aad: createParquetModuleAad(
|
|
1168
|
+
options.aadPrefix,
|
|
1169
|
+
fileUnique,
|
|
1170
|
+
'bloom-filter-bitset',
|
|
1171
|
+
rowGroupOrdinal,
|
|
1172
|
+
columnOrdinal
|
|
1173
|
+
)
|
|
1174
|
+
});
|
|
1175
|
+
return concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(bitset)]);
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
/** Encrypts column and offset indexes independently while preserving their footer offsets. */
|
|
1179
|
+
async function encryptParquetPageIndexes(
|
|
1180
|
+
pageIndexes: {offsetIndex: Uint8Array; columnIndex?: Uint8Array},
|
|
1181
|
+
options: ParquetWriterEncryptionOptions,
|
|
1182
|
+
fileUnique: Uint8Array,
|
|
1183
|
+
rowGroupOrdinal: number,
|
|
1184
|
+
columnOrdinal: number,
|
|
1185
|
+
keyMetadata?: Uint8Array
|
|
1186
|
+
): Promise<{offsetIndex: Uint8Array; columnIndex?: Uint8Array}> {
|
|
1187
|
+
const algorithm = options.algorithm ?? 'AES_GCM_V1';
|
|
1188
|
+
const encryptIndex = (bytes: Uint8Array, module: 'offset-index' | 'column-index') =>
|
|
1189
|
+
encryptParquetModule(bytes, {
|
|
1190
|
+
algorithm,
|
|
1191
|
+
aad: createParquetModuleAad(
|
|
1192
|
+
options.aadPrefix,
|
|
1193
|
+
fileUnique,
|
|
1194
|
+
module,
|
|
1195
|
+
rowGroupOrdinal,
|
|
1196
|
+
columnOrdinal
|
|
1197
|
+
),
|
|
1198
|
+
keyMetadata: keyMetadata ?? options.keyMetadata,
|
|
1199
|
+
keyRetriever: options.keyRetriever
|
|
1200
|
+
});
|
|
1201
|
+
return {
|
|
1202
|
+
offsetIndex: await encryptIndex(pageIndexes.offsetIndex, 'offset-index'),
|
|
1203
|
+
columnIndex: pageIndexes.columnIndex
|
|
1204
|
+
? await encryptIndex(pageIndexes.columnIndex, 'column-index')
|
|
1205
|
+
: undefined
|
|
1206
|
+
};
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/** Builds page indexes for primitive leaves, including nested and repeated leaves. */
|
|
1210
|
+
function createColumnPageIndexes(
|
|
1211
|
+
column: ParquetField,
|
|
1212
|
+
plannedPages: ReturnType<typeof planColumnPages>,
|
|
1213
|
+
pageLocations: PageLocation[],
|
|
1214
|
+
int96AsTimestamp = false
|
|
1215
|
+
): {offsetIndex: Uint8Array; columnIndex?: Uint8Array} | undefined {
|
|
1216
|
+
if (
|
|
1217
|
+
!column.primitiveType ||
|
|
1218
|
+
!isPageIndexPhysicalType(column.primitiveType) ||
|
|
1219
|
+
plannedPages.length !== pageLocations.length ||
|
|
1220
|
+
plannedPages.length === 0
|
|
1221
|
+
) {
|
|
1222
|
+
return undefined;
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
const offsetIndex = serializeThrift(new OffsetIndex({page_locations: pageLocations}));
|
|
1226
|
+
if (!supportsStatisticsSortOrder(column)) {
|
|
1227
|
+
return {offsetIndex};
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
const nullPages: boolean[] = [];
|
|
1231
|
+
const minValues: Uint8Array[] = [];
|
|
1232
|
+
const maxValues: Uint8Array[] = [];
|
|
1233
|
+
// A repeated leaf's definition-level count is not a row-level null count.
|
|
1234
|
+
// Omit null_counts for those leaves so readers do not interpret entry counts
|
|
1235
|
+
// as null counts while still benefiting from min/max page pruning.
|
|
1236
|
+
const nullCounts: number[] | undefined = column.rLevelMax === 0 ? [] : undefined;
|
|
1237
|
+
for (const plannedPage of plannedPages) {
|
|
1238
|
+
const values = plannedPage.data.values;
|
|
1239
|
+
const nullCount = plannedPage.data.count - values.length;
|
|
1240
|
+
nullCounts?.push(nullCount);
|
|
1241
|
+
if (values.length === 0) {
|
|
1242
|
+
nullPages.push(true);
|
|
1243
|
+
minValues.push(new Uint8Array(0));
|
|
1244
|
+
maxValues.push(new Uint8Array(0));
|
|
1245
|
+
continue;
|
|
1246
|
+
}
|
|
1247
|
+
const bounds = getPageBounds(
|
|
1248
|
+
values,
|
|
1249
|
+
column.primitiveType!,
|
|
1250
|
+
column.typeLength,
|
|
1251
|
+
int96AsTimestamp
|
|
1252
|
+
);
|
|
1253
|
+
if (!bounds) return undefined;
|
|
1254
|
+
nullPages.push(false);
|
|
1255
|
+
minValues.push(bounds.min);
|
|
1256
|
+
maxValues.push(bounds.max);
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
return {
|
|
1260
|
+
offsetIndex,
|
|
1261
|
+
columnIndex: serializeThrift(
|
|
1262
|
+
new ColumnIndex({
|
|
1263
|
+
null_pages: nullPages,
|
|
1264
|
+
min_values: minValues,
|
|
1265
|
+
max_values: maxValues,
|
|
1266
|
+
boundary_order: BoundaryOrder.UNORDERED,
|
|
1267
|
+
null_counts: nullCounts
|
|
1268
|
+
})
|
|
1269
|
+
)
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
function isPageIndexPhysicalType(
|
|
1274
|
+
physicalType: ParquetField['primitiveType']
|
|
1275
|
+
): physicalType is PageIndexPhysicalType {
|
|
1276
|
+
return (
|
|
1277
|
+
physicalType === 'BOOLEAN' ||
|
|
1278
|
+
physicalType === 'INT32' ||
|
|
1279
|
+
physicalType === 'INT64' ||
|
|
1280
|
+
physicalType === 'INT96' ||
|
|
1281
|
+
physicalType === 'FLOAT' ||
|
|
1282
|
+
physicalType === 'DOUBLE' ||
|
|
1283
|
+
physicalType === 'BYTE_ARRAY' ||
|
|
1284
|
+
physicalType === 'FIXED_LEN_BYTE_ARRAY'
|
|
1285
|
+
);
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
type PageIndexPhysicalType =
|
|
1289
|
+
| 'BOOLEAN'
|
|
1290
|
+
| 'INT32'
|
|
1291
|
+
| 'INT64'
|
|
1292
|
+
| 'INT96'
|
|
1293
|
+
| 'FLOAT'
|
|
1294
|
+
| 'DOUBLE'
|
|
1295
|
+
| 'BYTE_ARRAY'
|
|
1296
|
+
| 'FIXED_LEN_BYTE_ARRAY';
|
|
1297
|
+
|
|
1298
|
+
function getPageBounds(
|
|
1299
|
+
values: ParquetColumnChunk['values'],
|
|
1300
|
+
physicalType: PageIndexPhysicalType,
|
|
1301
|
+
typeLength?: number,
|
|
1302
|
+
int96AsTimestamp = false
|
|
1303
|
+
): {min: Uint8Array; max: Uint8Array} | undefined {
|
|
1304
|
+
if (physicalType === 'INT96' && !int96AsTimestamp) return undefined;
|
|
1305
|
+
const normalizedValues = Array.from(values as Iterable<unknown>, value =>
|
|
1306
|
+
normalizePageIndexValue(value, physicalType)
|
|
1307
|
+
);
|
|
1308
|
+
if (normalizedValues.some(value => value === undefined)) return undefined;
|
|
1309
|
+
const comparableValues = normalizedValues as Array<
|
|
1310
|
+
boolean | number | bigint | string | Uint8Array
|
|
1311
|
+
>;
|
|
1312
|
+
let min = comparableValues[0];
|
|
1313
|
+
let max = comparableValues[0];
|
|
1314
|
+
for (const value of comparableValues.slice(1)) {
|
|
1315
|
+
if (comparePageIndexValues(value, min, physicalType) < 0) min = value;
|
|
1316
|
+
if (comparePageIndexValues(value, max, physicalType) > 0) max = value;
|
|
1317
|
+
}
|
|
1318
|
+
try {
|
|
1319
|
+
return {
|
|
1320
|
+
min: encodePageIndexValue(min, physicalType, typeLength, int96AsTimestamp),
|
|
1321
|
+
max: encodePageIndexValue(max, physicalType, typeLength, int96AsTimestamp)
|
|
1322
|
+
};
|
|
1323
|
+
} catch {
|
|
1324
|
+
return undefined;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
function encodePageIndexValue(
|
|
1329
|
+
value: boolean | number | bigint | string | Uint8Array,
|
|
1330
|
+
physicalType: PageIndexPhysicalType,
|
|
1331
|
+
typeLength: number | undefined,
|
|
1332
|
+
int96AsTimestamp: boolean
|
|
1333
|
+
): Uint8Array {
|
|
1334
|
+
if (physicalType === 'INT96') {
|
|
1335
|
+
if (!int96AsTimestamp) {
|
|
1336
|
+
throw new Error('INT96 page indexes require timestamp mode');
|
|
1337
|
+
}
|
|
1338
|
+
return PARQUET_CODECS.PLAIN.encodeValues('INT96', [value], {int96AsTimestamp});
|
|
1339
|
+
}
|
|
1340
|
+
return encodeParquetBloomFilterValue(value, physicalType, typeLength);
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
function normalizePageIndexValue(
|
|
1344
|
+
value: unknown,
|
|
1345
|
+
physicalType: PageIndexPhysicalType
|
|
1346
|
+
): boolean | number | bigint | string | Uint8Array | undefined {
|
|
1347
|
+
if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
|
|
1348
|
+
if (typeof value === 'string') return new TextEncoder().encode(value);
|
|
1349
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
1350
|
+
if (ArrayBuffer.isView(value)) {
|
|
1351
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
|
|
1352
|
+
}
|
|
1353
|
+
return undefined;
|
|
1354
|
+
}
|
|
1355
|
+
if (
|
|
1356
|
+
typeof value === 'boolean' ||
|
|
1357
|
+
typeof value === 'number' ||
|
|
1358
|
+
typeof value === 'bigint' ||
|
|
1359
|
+
typeof value === 'string'
|
|
1360
|
+
) {
|
|
1361
|
+
return value;
|
|
1362
|
+
}
|
|
1363
|
+
return undefined;
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
function comparePageIndexValues(
|
|
1367
|
+
left: boolean | number | bigint | string | Uint8Array,
|
|
1368
|
+
right: boolean | number | bigint | string | Uint8Array,
|
|
1369
|
+
physicalType: PageIndexPhysicalType
|
|
1370
|
+
): number {
|
|
1371
|
+
if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
|
|
1372
|
+
const leftBytes = typeof left === 'string' ? new TextEncoder().encode(left) : left;
|
|
1373
|
+
const rightBytes = typeof right === 'string' ? new TextEncoder().encode(right) : right;
|
|
1374
|
+
if (!(leftBytes instanceof Uint8Array) || !(rightBytes instanceof Uint8Array)) return 0;
|
|
1375
|
+
const length = Math.min(leftBytes.length, rightBytes.length);
|
|
1376
|
+
for (let index = 0; index < length; index++) {
|
|
1377
|
+
if (leftBytes[index] !== rightBytes[index]) {
|
|
1378
|
+
return leftBytes[index] < rightBytes[index] ? -1 : 1;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
return leftBytes.length - rightBytes.length;
|
|
1382
|
+
}
|
|
1383
|
+
if (typeof left === 'boolean' && typeof right === 'boolean') return Number(left) - Number(right);
|
|
1384
|
+
if (physicalType !== 'INT64' && physicalType !== 'INT96') {
|
|
1385
|
+
const leftNumber = Number(left);
|
|
1386
|
+
const rightNumber = Number(right);
|
|
1387
|
+
return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
|
|
1388
|
+
}
|
|
1389
|
+
try {
|
|
1390
|
+
const leftNumber = BigInt(left as number | bigint | string);
|
|
1391
|
+
const rightNumber = BigInt(right as number | bigint | string);
|
|
1392
|
+
return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
|
|
1393
|
+
} catch {
|
|
1394
|
+
const leftNumber = Number(left);
|
|
1395
|
+
const rightNumber = Number(right);
|
|
1396
|
+
return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
/** Builds a Bloom filter for writer-supported physical scalar columns. */
|
|
1401
|
+
function createColumnBloomFilter(
|
|
1402
|
+
column: ParquetField,
|
|
1403
|
+
values: ParquetColumnChunk['values']
|
|
1404
|
+
): Uint8Array | undefined {
|
|
1405
|
+
const physicalType = column.primitiveType;
|
|
1406
|
+
if (
|
|
1407
|
+
physicalType !== 'BOOLEAN' &&
|
|
1408
|
+
physicalType !== 'INT32' &&
|
|
1409
|
+
physicalType !== 'INT64' &&
|
|
1410
|
+
physicalType !== 'FLOAT' &&
|
|
1411
|
+
physicalType !== 'DOUBLE' &&
|
|
1412
|
+
physicalType !== 'BYTE_ARRAY' &&
|
|
1413
|
+
physicalType !== 'FIXED_LEN_BYTE_ARRAY'
|
|
1414
|
+
) {
|
|
1415
|
+
return undefined;
|
|
1416
|
+
}
|
|
1417
|
+
const bloomValues = values.map(value => {
|
|
1418
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
1419
|
+
if (ArrayBuffer.isView(value)) {
|
|
1420
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
|
|
1421
|
+
}
|
|
1422
|
+
return value;
|
|
1423
|
+
});
|
|
1424
|
+
return encodeParquetSplitBlockBloomFilter(
|
|
1425
|
+
bloomValues as Parameters<typeof encodeParquetSplitBlockBloomFilter>[0],
|
|
1426
|
+
physicalType,
|
|
1427
|
+
column.typeLength
|
|
1428
|
+
);
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
/** Builds optional size statistics from one shredded column chunk. */
|
|
1432
|
+
function createSizeStatistics(column: ParquetField, data: ParquetColumnChunk): SizeStatistics {
|
|
1433
|
+
const repetitionLevelHistogram = createLevelHistogram(data.rlevels, column.rLevelMax);
|
|
1434
|
+
const definitionLevelHistogram = createLevelHistogram(data.dlevels, column.dLevelMax);
|
|
1435
|
+
let unencodedByteArrayDataBytes = 0;
|
|
1436
|
+
if (column.primitiveType === 'BYTE_ARRAY') {
|
|
1437
|
+
for (const value of data.values) {
|
|
1438
|
+
if (typeof value === 'string') {
|
|
1439
|
+
unencodedByteArrayDataBytes += encodeUtf8(value).byteLength;
|
|
1440
|
+
} else if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
|
|
1441
|
+
unencodedByteArrayDataBytes += value.byteLength;
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
return new SizeStatistics({
|
|
1446
|
+
unencoded_byte_array_data_bytes:
|
|
1447
|
+
column.primitiveType === 'BYTE_ARRAY' ? unencodedByteArrayDataBytes : undefined,
|
|
1448
|
+
repetition_level_histogram: repetitionLevelHistogram,
|
|
1449
|
+
definition_level_histogram: definitionLevelHistogram
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
/** Returns whether standard column statistics are enabled for one top-level field. */
|
|
1454
|
+
function isStatisticsEnabled(
|
|
1455
|
+
option: boolean | Record<string, boolean> | undefined,
|
|
1456
|
+
column: ParquetField
|
|
1457
|
+
): boolean {
|
|
1458
|
+
return option === true || option?.[column.path[0]] === true;
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
/** Builds conservative standard min/max/null-count statistics for one column chunk. */
|
|
1462
|
+
function createColumnStatistics(
|
|
1463
|
+
column: ParquetField,
|
|
1464
|
+
data: ParquetColumnChunk
|
|
1465
|
+
): Statistics | undefined {
|
|
1466
|
+
if (
|
|
1467
|
+
!column.primitiveType ||
|
|
1468
|
+
!isPageIndexPhysicalType(column.primitiveType) ||
|
|
1469
|
+
!supportsStatisticsSortOrder(column)
|
|
1470
|
+
) {
|
|
1471
|
+
return undefined;
|
|
1472
|
+
}
|
|
1473
|
+
const statistics = new Statistics({
|
|
1474
|
+
null_count: column.rLevelMax === 0 ? data.count - data.values.length : undefined,
|
|
1475
|
+
is_min_value_exact: true,
|
|
1476
|
+
is_max_value_exact: true
|
|
1477
|
+
});
|
|
1478
|
+
if (data.values.length === 0) return statistics;
|
|
1479
|
+
const bounds = getPageBounds(data.values, column.primitiveType, column.typeLength);
|
|
1480
|
+
if (!bounds) return undefined;
|
|
1481
|
+
statistics.min_value = bounds.min;
|
|
1482
|
+
statistics.max_value = bounds.max;
|
|
1483
|
+
return statistics;
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
/** Returns whether the physical representation preserves the logical sort order. */
|
|
1487
|
+
function supportsStatisticsSortOrder(column: ParquetField): boolean {
|
|
1488
|
+
const logicalType = column.logicalType?.type || column.originalType;
|
|
1489
|
+
if (logicalType === 'FLOAT16') return false;
|
|
1490
|
+
if (logicalType === 'DECIMAL' || logicalType?.startsWith('DECIMAL_')) {
|
|
1491
|
+
return column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY';
|
|
1492
|
+
}
|
|
1493
|
+
if (logicalType === 'INTEGER' || logicalType?.startsWith('UINT_')) {
|
|
1494
|
+
return column.logicalType?.isSigned !== false && !logicalType.startsWith('UINT_');
|
|
1495
|
+
}
|
|
1496
|
+
return true;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
/** Counts the occurrences of each definition or repetition level. */
|
|
1500
|
+
function createLevelHistogram(
|
|
1501
|
+
levels: ParquetColumnChunk['rlevels'],
|
|
1502
|
+
maximumLevel: number
|
|
1503
|
+
): number[] {
|
|
1504
|
+
const histogram = new Array<number>(maximumLevel + 1).fill(0);
|
|
1505
|
+
for (const level of levels) {
|
|
1506
|
+
if (level >= 0 && level <= maximumLevel) histogram[level]++;
|
|
1507
|
+
}
|
|
1508
|
+
return histogram;
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
/** Converts writer-facing sort keys into Parquet leaf-column indexes. */
|
|
1512
|
+
function createSortingColumns(
|
|
1513
|
+
schema: ParquetSchema,
|
|
1514
|
+
sortingColumns: readonly ParquetSortingColumnOption[] | undefined
|
|
1515
|
+
): SortingColumn[] | undefined {
|
|
1516
|
+
if (!sortingColumns?.length) return undefined;
|
|
1517
|
+
const leafFields = schema.fieldList.filter(field => !field.isNested);
|
|
1518
|
+
const usedColumnIndexes = new Set<number>();
|
|
1519
|
+
return sortingColumns.map(sortKey => {
|
|
1520
|
+
const columnIndex = leafFields.findIndex(
|
|
1521
|
+
field =>
|
|
1522
|
+
(field.path.length === 1 && field.name === sortKey.column) ||
|
|
1523
|
+
(field.path.length > 1 && field.path.join('.') === sortKey.column)
|
|
1524
|
+
);
|
|
1525
|
+
if (columnIndex < 0) {
|
|
1526
|
+
throw new Error(`Unknown Parquet sorting column ${sortKey.column}`);
|
|
1527
|
+
}
|
|
1528
|
+
if (usedColumnIndexes.has(columnIndex)) {
|
|
1529
|
+
throw new Error(`Duplicate Parquet sorting column ${sortKey.column}`);
|
|
1530
|
+
}
|
|
1531
|
+
usedColumnIndexes.add(columnIndex);
|
|
1532
|
+
return new SortingColumn({
|
|
1533
|
+
column_idx: columnIndex,
|
|
1534
|
+
descending: Boolean(sortKey.descending),
|
|
1535
|
+
nulls_first: Boolean(sortKey.nullsFirst)
|
|
1536
|
+
});
|
|
1537
|
+
});
|
|
687
1538
|
}
|
|
688
1539
|
|
|
689
1540
|
/**
|
|
@@ -700,7 +1551,8 @@ async function encodeRowGroup(
|
|
|
700
1551
|
const metadata = new RowGroup({
|
|
701
1552
|
num_rows: int64(data.rowCount),
|
|
702
1553
|
columns: [],
|
|
703
|
-
total_byte_size: int64(0)
|
|
1554
|
+
total_byte_size: int64(0),
|
|
1555
|
+
sorting_columns: createSortingColumns(schema, opts.sortingColumns)
|
|
704
1556
|
});
|
|
705
1557
|
|
|
706
1558
|
let body: Uint8Array = new Uint8Array(0);
|
|
@@ -709,11 +1561,30 @@ async function encodeRowGroup(
|
|
|
709
1561
|
continue; // eslint-disable-line no-continue
|
|
710
1562
|
}
|
|
711
1563
|
|
|
712
|
-
const cchunkData = await encodeColumnChunk(
|
|
1564
|
+
const cchunkData = await encodeColumnChunk(
|
|
1565
|
+
field,
|
|
1566
|
+
data,
|
|
1567
|
+
body.length,
|
|
1568
|
+
opts,
|
|
1569
|
+
opts.rowGroupOrdinal ?? 0,
|
|
1570
|
+
metadata.columns.length
|
|
1571
|
+
);
|
|
713
1572
|
|
|
714
1573
|
const cchunk = new ColumnChunk({
|
|
715
1574
|
file_offset: int64(cchunkData.metadataOffset),
|
|
716
|
-
meta_data: cchunkData.metadata
|
|
1575
|
+
meta_data: cchunkData.encryptedColumnMetadata ? undefined : cchunkData.metadata,
|
|
1576
|
+
offset_index_offset:
|
|
1577
|
+
cchunkData.offsetIndexOffset === undefined
|
|
1578
|
+
? undefined
|
|
1579
|
+
: int64(cchunkData.offsetIndexOffset),
|
|
1580
|
+
offset_index_length: cchunkData.offsetIndexLength,
|
|
1581
|
+
column_index_offset:
|
|
1582
|
+
cchunkData.columnIndexOffset === undefined
|
|
1583
|
+
? undefined
|
|
1584
|
+
: int64(cchunkData.columnIndexOffset),
|
|
1585
|
+
column_index_length: cchunkData.columnIndexLength,
|
|
1586
|
+
crypto_metadata: cchunkData.cryptoMetadata,
|
|
1587
|
+
encrypted_column_metadata: cchunkData.encryptedColumnMetadata
|
|
717
1588
|
});
|
|
718
1589
|
|
|
719
1590
|
metadata.columns.push(cchunk);
|
|
@@ -728,19 +1599,44 @@ async function encodeRowGroup(
|
|
|
728
1599
|
/**
|
|
729
1600
|
* Encode a parquet file metadata footer
|
|
730
1601
|
*/
|
|
731
|
-
function
|
|
1602
|
+
function encodeFooterMetadata(
|
|
732
1603
|
schema: ParquetSchema,
|
|
733
1604
|
rowCount: number,
|
|
734
1605
|
rowGroups: RowGroup[],
|
|
735
|
-
userMetadata: Record<string, string
|
|
1606
|
+
userMetadata: Record<string, string>,
|
|
1607
|
+
footerSignature?: ParquetWriterFooterSignatureOptions
|
|
736
1608
|
): Uint8Array {
|
|
1609
|
+
const signatureAlgorithm = footerSignature?.algorithm ?? 'AES_GCM_V1';
|
|
1610
|
+
const signatureFileUnique = footerSignature?.fileUnique
|
|
1611
|
+
? new Uint8Array(footerSignature.fileUnique)
|
|
1612
|
+
: undefined;
|
|
737
1613
|
const metadata = new FileMetaData({
|
|
738
1614
|
version: PARQUET_VERSION,
|
|
739
1615
|
created_by: 'parquets',
|
|
740
1616
|
num_rows: int64(rowCount),
|
|
741
1617
|
row_groups: rowGroups,
|
|
742
1618
|
schema: [],
|
|
743
|
-
key_value_metadata: []
|
|
1619
|
+
key_value_metadata: [],
|
|
1620
|
+
encryption_algorithm: footerSignature
|
|
1621
|
+
? new EncryptionAlgorithm({
|
|
1622
|
+
...(signatureAlgorithm === 'AES_GCM_CTR_V1'
|
|
1623
|
+
? {
|
|
1624
|
+
AES_GCM_CTR_V1: {
|
|
1625
|
+
aad_prefix: footerSignature.aadPrefix,
|
|
1626
|
+
aad_file_unique: signatureFileUnique
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
: {
|
|
1630
|
+
AES_GCM_V1: {
|
|
1631
|
+
aad_prefix: footerSignature.aadPrefix,
|
|
1632
|
+
aad_file_unique: signatureFileUnique
|
|
1633
|
+
}
|
|
1634
|
+
})
|
|
1635
|
+
})
|
|
1636
|
+
: undefined,
|
|
1637
|
+
footer_signing_key_metadata: footerSignature
|
|
1638
|
+
? new Uint8Array(footerSignature.keyMetadata)
|
|
1639
|
+
: undefined
|
|
744
1640
|
});
|
|
745
1641
|
|
|
746
1642
|
for (const key in userMetadata) {
|
|
@@ -791,7 +1687,17 @@ function encodeFooter(
|
|
|
791
1687
|
metadata.schema.push(schemaElem);
|
|
792
1688
|
}
|
|
793
1689
|
|
|
794
|
-
|
|
1690
|
+
return serializeThrift(metadata);
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
/** Encodes an unsigned plaintext footer with its standard Parquet trailer. */
|
|
1694
|
+
function encodeFooter(
|
|
1695
|
+
schema: ParquetSchema,
|
|
1696
|
+
rowCount: number,
|
|
1697
|
+
rowGroups: RowGroup[],
|
|
1698
|
+
userMetadata: Record<string, string>
|
|
1699
|
+
): Uint8Array {
|
|
1700
|
+
const metadataEncoded = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata);
|
|
795
1701
|
const footerEncoded = new Uint8Array(metadataEncoded.length + 8);
|
|
796
1702
|
|
|
797
1703
|
footerEncoded.set(metadataEncoded);
|
|
@@ -800,6 +1706,78 @@ function encodeFooter(
|
|
|
800
1706
|
return footerEncoded;
|
|
801
1707
|
}
|
|
802
1708
|
|
|
1709
|
+
/** Encodes a plaintext footer with its trailing 28-byte authenticated signature. */
|
|
1710
|
+
async function encodeSignedFooter(
|
|
1711
|
+
schema: ParquetSchema,
|
|
1712
|
+
rowCount: number,
|
|
1713
|
+
rowGroups: RowGroup[],
|
|
1714
|
+
userMetadata: Record<string, string>,
|
|
1715
|
+
options: ParquetWriterFooterSignatureOptions
|
|
1716
|
+
): Promise<Uint8Array> {
|
|
1717
|
+
const metadata = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata, options);
|
|
1718
|
+
const signature = await createParquetFooterSignature(metadata, options);
|
|
1719
|
+
const footerEncoded = new Uint8Array(metadata.length + signature.length + 8);
|
|
1720
|
+
footerEncoded.set(metadata);
|
|
1721
|
+
footerEncoded.set(signature, metadata.length);
|
|
1722
|
+
writeUInt32LE(
|
|
1723
|
+
footerEncoded,
|
|
1724
|
+
metadata.length + signature.length,
|
|
1725
|
+
metadata.length + signature.length
|
|
1726
|
+
);
|
|
1727
|
+
footerEncoded.set(PARQUET_MAGIC_BYTES, metadata.length + signature.length + 4);
|
|
1728
|
+
return footerEncoded;
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
/** Wraps a serialized footer in Parquet modular-encryption metadata and AES-GCM. */
|
|
1732
|
+
async function encodeEncryptedFooter(
|
|
1733
|
+
footer: Uint8Array,
|
|
1734
|
+
options: ParquetWriterEncryptionOptions
|
|
1735
|
+
): Promise<Uint8Array> {
|
|
1736
|
+
const algorithm = options.algorithm ?? 'AES_GCM_V1';
|
|
1737
|
+
const fileUnique = options.fileUnique ? new Uint8Array(options.fileUnique) : createFileUnique();
|
|
1738
|
+
if (fileUnique.byteLength !== 8) {
|
|
1739
|
+
throw new Error(`Parquet encrypted footer file_unique must be 8 bytes`);
|
|
1740
|
+
}
|
|
1741
|
+
const algorithmParameters = {
|
|
1742
|
+
aad_prefix: options.aadPrefix && new Uint8Array(options.aadPrefix),
|
|
1743
|
+
aad_file_unique: fileUnique,
|
|
1744
|
+
supply_aad_prefix: undefined
|
|
1745
|
+
};
|
|
1746
|
+
const encryptionAlgorithm =
|
|
1747
|
+
algorithm === 'AES_GCM_CTR_V1'
|
|
1748
|
+
? new EncryptionAlgorithm({AES_GCM_CTR_V1: algorithmParameters})
|
|
1749
|
+
: new EncryptionAlgorithm({AES_GCM_V1: algorithmParameters});
|
|
1750
|
+
const cryptoMetadata = serializeThrift(
|
|
1751
|
+
new FileCryptoMetaData({
|
|
1752
|
+
encryption_algorithm: encryptionAlgorithm,
|
|
1753
|
+
key_metadata: options.keyMetadata && new Uint8Array(options.keyMetadata)
|
|
1754
|
+
})
|
|
1755
|
+
);
|
|
1756
|
+
const footerMetadata = footer.subarray(0, footer.byteLength - 8);
|
|
1757
|
+
const encryptedFooter = await encryptParquetModule(footerMetadata, {
|
|
1758
|
+
algorithm,
|
|
1759
|
+
aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'footer'),
|
|
1760
|
+
keyMetadata: options.keyMetadata,
|
|
1761
|
+
keyRetriever: options.keyRetriever
|
|
1762
|
+
});
|
|
1763
|
+
const metadataSize = cryptoMetadata.byteLength + encryptedFooter.byteLength;
|
|
1764
|
+
const trailer = new Uint8Array(8);
|
|
1765
|
+
writeUInt32LE(trailer, metadataSize, 0);
|
|
1766
|
+
trailer.set(encodeUtf8(PARQUET_MAGIC_ENCRYPTED), 4);
|
|
1767
|
+
return concatUint8Arrays([cryptoMetadata, encryptedFooter, trailer]);
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
/** Generates the eight-byte file identifier required by Parquet module AAD. */
|
|
1771
|
+
function createFileUnique(): Uint8Array {
|
|
1772
|
+
const fileUnique = new Uint8Array(8);
|
|
1773
|
+
const cryptoProvider = globalThis.crypto;
|
|
1774
|
+
if (!cryptoProvider?.getRandomValues) {
|
|
1775
|
+
throw new Error('Parquet encrypted footer requires Web Crypto random values');
|
|
1776
|
+
}
|
|
1777
|
+
cryptoProvider.getRandomValues(fileUnique);
|
|
1778
|
+
return fileUnique;
|
|
1779
|
+
}
|
|
1780
|
+
|
|
803
1781
|
/** Returns an explicit or inferred modern logical annotation for a writer field. */
|
|
804
1782
|
function getFieldLogicalType(field: ParquetField): ParquetLogicalType | undefined {
|
|
805
1783
|
if (field.logicalType) return field.logicalType;
|