@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15841 -3306
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +16528 -4928
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-encoded-page-types.d.ts +108 -0
- package/dist/parquet-encoded-page-types.d.ts.map +1 -0
- package/dist/parquet-encoded-page-types.js +5 -0
- package/dist/parquet-encoded-page-types.js.map +1 -0
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14121 -2796
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +51 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +651 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7378 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8501 -1045
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/encoded-pages.d.ts +24 -0
- package/dist/parquetjs/parser/encoded-pages.d.ts.map +1 -0
- package/dist/parquetjs/parser/encoded-pages.js +206 -0
- package/dist/parquetjs/parser/encoded-pages.js.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts +88 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +713 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -13
- package/src/bundled.ts +10 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +59 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-encoded-page-types.ts +122 -0
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +1024 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/encoded-pages.ts +278 -0
- package/src/parquetjs/parser/parquet-reader.ts +1112 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -143,6 +143,85 @@ export function decodeValues(
|
|
|
143
143
|
return output;
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
+
/** Encodes legacy Parquet BIT_PACKED values, including the optional page length envelope. */
|
|
147
|
+
export function encodeBitPackedValues(
|
|
148
|
+
type: PrimitiveType,
|
|
149
|
+
values: any[],
|
|
150
|
+
options: ParquetCodecOptions
|
|
151
|
+
): Uint8Array {
|
|
152
|
+
if (!('bitWidth' in options)) {
|
|
153
|
+
throw new Error('bitWidth is required');
|
|
154
|
+
}
|
|
155
|
+
const bitWidth = options.bitWidth!;
|
|
156
|
+
if (!Number.isInteger(bitWidth) || bitWidth < 0 || bitWidth > 32) {
|
|
157
|
+
throw new Error(`invalid bit width: ${bitWidth}`);
|
|
158
|
+
}
|
|
159
|
+
if (type !== 'BOOLEAN' && type !== 'INT32' && type !== 'INT64') {
|
|
160
|
+
throw new Error(`unsupported type: ${type}`);
|
|
161
|
+
}
|
|
162
|
+
const paddedValueCount = Math.ceil(values.length / 8) * 8;
|
|
163
|
+
const encoded = new Uint8Array(Math.ceil((paddedValueCount * bitWidth) / 8));
|
|
164
|
+
for (let valueIndex = 0; valueIndex < values.length; valueIndex++) {
|
|
165
|
+
let value = Number(values[valueIndex]);
|
|
166
|
+
for (let bitIndex = 0; bitIndex < bitWidth; bitIndex++) {
|
|
167
|
+
if (value & (1 << bitIndex)) {
|
|
168
|
+
const packedBit = valueIndex * bitWidth + bitIndex;
|
|
169
|
+
encoded[Math.floor(packedBit / 8)] |= 1 << (packedBit % 8);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (options.disableEnvelope) {
|
|
174
|
+
return encoded;
|
|
175
|
+
}
|
|
176
|
+
const envelope = new Uint8Array(encoded.length + 4);
|
|
177
|
+
writeUInt32LE(envelope, encoded.length, 0);
|
|
178
|
+
envelope.set(encoded, 4);
|
|
179
|
+
return envelope;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Decodes legacy Parquet BIT_PACKED values, including the optional page length envelope. */
|
|
183
|
+
export function decodeBitPackedValues(
|
|
184
|
+
type: PrimitiveType,
|
|
185
|
+
cursor: CursorBuffer,
|
|
186
|
+
count: number,
|
|
187
|
+
options: ParquetCodecOptions
|
|
188
|
+
): ParquetValueBuffer {
|
|
189
|
+
if (!('bitWidth' in options)) {
|
|
190
|
+
throw new Error('bitWidth is required');
|
|
191
|
+
}
|
|
192
|
+
const bitWidth = options.bitWidth!;
|
|
193
|
+
if (!Number.isInteger(bitWidth) || bitWidth < 0 || bitWidth > 64) {
|
|
194
|
+
throw new Error(`invalid bit width: ${bitWidth}`);
|
|
195
|
+
}
|
|
196
|
+
if (!Number.isInteger(count) || count < 0) {
|
|
197
|
+
throw new Error(`invalid value count: ${count}`);
|
|
198
|
+
}
|
|
199
|
+
const envelopeEnd = options.disableEnvelope
|
|
200
|
+
? undefined
|
|
201
|
+
: (() => {
|
|
202
|
+
assertReadable(cursor, 4);
|
|
203
|
+
const length =
|
|
204
|
+
cursor.buffer[cursor.offset] |
|
|
205
|
+
(cursor.buffer[cursor.offset + 1] << 8) |
|
|
206
|
+
(cursor.buffer[cursor.offset + 2] << 16) |
|
|
207
|
+
(cursor.buffer[cursor.offset + 3] << 24);
|
|
208
|
+
cursor.offset += 4;
|
|
209
|
+
if (length < 0) throw new Error('invalid BIT_PACKED length');
|
|
210
|
+
assertReadable(cursor, length);
|
|
211
|
+
return cursor.offset + length;
|
|
212
|
+
})();
|
|
213
|
+
const paddedValueCount = Math.ceil(count / 8) * 8;
|
|
214
|
+
const packedByteLength = Math.ceil((paddedValueCount * bitWidth) / 8);
|
|
215
|
+
assertReadable(cursor, packedByteLength);
|
|
216
|
+
const {output, outputOffset} = getParquetValueOutput(options, count);
|
|
217
|
+
decodeBitPackedRun(cursor, paddedValueCount, count, bitWidth, output, outputOffset);
|
|
218
|
+
if (envelopeEnd !== undefined) {
|
|
219
|
+
if (cursor.offset > envelopeEnd) throw new Error('invalid BIT_PACKED length');
|
|
220
|
+
cursor.offset = envelopeEnd;
|
|
221
|
+
}
|
|
222
|
+
return output;
|
|
223
|
+
}
|
|
224
|
+
|
|
146
225
|
/** Decodes one complete bit-packed run directly into the caller's output array. */
|
|
147
226
|
function decodeBitPackedRun(
|
|
148
227
|
cursor: CursorBuffer,
|
|
@@ -237,10 +316,43 @@ function decodeUint32BitPackedRun(
|
|
|
237
316
|
outputOffset: number,
|
|
238
317
|
dictionary?: readonly unknown[]
|
|
239
318
|
): void {
|
|
319
|
+
if (!dictionary && bitWidth <= 4) {
|
|
320
|
+
decodeNarrowUnsignedBitPackedRun(buffer, packedOffset, count, bitWidth, output, outputOffset);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
if (dictionary && bitWidth <= 4) {
|
|
324
|
+
decodeNarrowDictionaryBitPackedRun(
|
|
325
|
+
buffer,
|
|
326
|
+
packedOffset,
|
|
327
|
+
count,
|
|
328
|
+
bitWidth,
|
|
329
|
+
output,
|
|
330
|
+
outputOffset,
|
|
331
|
+
dictionary
|
|
332
|
+
);
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
if (dictionary && bitWidth === 10) {
|
|
336
|
+
decode10BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
if (dictionary && bitWidth === 12) {
|
|
340
|
+
decode12BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
240
343
|
if (bitWidth === 8) {
|
|
344
|
+
if (dictionary) {
|
|
345
|
+
for (let valueIndex = 0; valueIndex < count; valueIndex++) {
|
|
346
|
+
const value = buffer[packedOffset + valueIndex] ?? 0;
|
|
347
|
+
if (value >= dictionary.length) {
|
|
348
|
+
throw new Error(`Invalid Parquet dictionary index ${value}`);
|
|
349
|
+
}
|
|
350
|
+
output[outputOffset + valueIndex] = dictionary[value];
|
|
351
|
+
}
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
241
354
|
for (let valueIndex = 0; valueIndex < count; valueIndex++) {
|
|
242
|
-
|
|
243
|
-
output[outputOffset + valueIndex] = resolveDictionaryValue(value, dictionary);
|
|
355
|
+
output[outputOffset + valueIndex] = buffer[packedOffset + valueIndex] ?? 0;
|
|
244
356
|
}
|
|
245
357
|
return;
|
|
246
358
|
}
|
|
@@ -249,18 +361,476 @@ function decodeUint32BitPackedRun(
|
|
|
249
361
|
let packedBits = 0;
|
|
250
362
|
let packedBitCount = 0;
|
|
251
363
|
let byteOffset = packedOffset;
|
|
364
|
+
if (dictionary) {
|
|
365
|
+
for (let valueIndex = 0; valueIndex < count; valueIndex++) {
|
|
366
|
+
while (packedBitCount < bitWidth) {
|
|
367
|
+
packedBits |= (buffer[byteOffset++] ?? 0) << packedBitCount;
|
|
368
|
+
packedBitCount += 8;
|
|
369
|
+
}
|
|
370
|
+
const value = packedBits & mask;
|
|
371
|
+
if (value >= dictionary.length) {
|
|
372
|
+
throw new Error(`Invalid Parquet dictionary index ${value}`);
|
|
373
|
+
}
|
|
374
|
+
output[outputOffset + valueIndex] = dictionary[value];
|
|
375
|
+
packedBits >>>= bitWidth;
|
|
376
|
+
packedBitCount -= bitWidth;
|
|
377
|
+
}
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
252
380
|
for (let valueIndex = 0; valueIndex < count; valueIndex++) {
|
|
253
381
|
while (packedBitCount < bitWidth) {
|
|
254
382
|
packedBits |= (buffer[byteOffset++] ?? 0) << packedBitCount;
|
|
255
383
|
packedBitCount += 8;
|
|
256
384
|
}
|
|
257
385
|
const value = packedBits & mask;
|
|
258
|
-
output[outputOffset + valueIndex] =
|
|
386
|
+
output[outputOffset + valueIndex] = value;
|
|
387
|
+
packedBits >>>= bitWidth;
|
|
388
|
+
packedBitCount -= bitWidth;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** Decodes complete groups of eight narrow unsigned values without a per-value reservoir. */
|
|
393
|
+
function decodeNarrowUnsignedBitPackedRun(
|
|
394
|
+
buffer: Uint8Array,
|
|
395
|
+
packedOffset: number,
|
|
396
|
+
count: number,
|
|
397
|
+
bitWidth: number,
|
|
398
|
+
output: ParquetValueBuffer,
|
|
399
|
+
outputOffset: number
|
|
400
|
+
): void {
|
|
401
|
+
if (bitWidth === 1) {
|
|
402
|
+
decode1BitUnsignedRun(buffer, packedOffset, count, output, outputOffset);
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
if (bitWidth === 2) {
|
|
406
|
+
decode2BitUnsignedRun(buffer, packedOffset, count, output, outputOffset);
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
const mask = 2 ** bitWidth - 1;
|
|
410
|
+
let byteOffset = packedOffset;
|
|
411
|
+
let valueIndex = 0;
|
|
412
|
+
while (valueIndex < count) {
|
|
413
|
+
const packedBits =
|
|
414
|
+
(buffer[byteOffset] ?? 0) |
|
|
415
|
+
((buffer[byteOffset + 1] ?? 0) << 8) |
|
|
416
|
+
((buffer[byteOffset + 2] ?? 0) << 16) |
|
|
417
|
+
((buffer[byteOffset + 3] ?? 0) << 24);
|
|
418
|
+
const groupValueCount = Math.min(8, count - valueIndex);
|
|
419
|
+
for (let groupIndex = 0; groupIndex < groupValueCount; groupIndex++) {
|
|
420
|
+
output[outputOffset + valueIndex + groupIndex] =
|
|
421
|
+
(packedBits >>> (groupIndex * bitWidth)) & mask;
|
|
422
|
+
}
|
|
423
|
+
byteOffset += bitWidth;
|
|
424
|
+
valueIndex += groupValueCount;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/** Expands byte-aligned groups of eight one-bit level values. */
|
|
429
|
+
function decode1BitUnsignedRun(
|
|
430
|
+
buffer: Uint8Array,
|
|
431
|
+
packedOffset: number,
|
|
432
|
+
count: number,
|
|
433
|
+
output: ParquetValueBuffer,
|
|
434
|
+
outputOffset: number
|
|
435
|
+
): void {
|
|
436
|
+
let byteOffset = packedOffset;
|
|
437
|
+
let valueIndex = 0;
|
|
438
|
+
while (valueIndex + 8 <= count) {
|
|
439
|
+
const packedBits = buffer[byteOffset++] ?? 0;
|
|
440
|
+
output[outputOffset + valueIndex] = packedBits & 1;
|
|
441
|
+
output[outputOffset + valueIndex + 1] = (packedBits >>> 1) & 1;
|
|
442
|
+
output[outputOffset + valueIndex + 2] = (packedBits >>> 2) & 1;
|
|
443
|
+
output[outputOffset + valueIndex + 3] = (packedBits >>> 3) & 1;
|
|
444
|
+
output[outputOffset + valueIndex + 4] = (packedBits >>> 4) & 1;
|
|
445
|
+
output[outputOffset + valueIndex + 5] = (packedBits >>> 5) & 1;
|
|
446
|
+
output[outputOffset + valueIndex + 6] = (packedBits >>> 6) & 1;
|
|
447
|
+
output[outputOffset + valueIndex + 7] = packedBits >>> 7;
|
|
448
|
+
valueIndex += 8;
|
|
449
|
+
}
|
|
450
|
+
const packedBits = buffer[byteOffset] ?? 0;
|
|
451
|
+
for (let tailIndex = 0; valueIndex + tailIndex < count; tailIndex++) {
|
|
452
|
+
output[outputOffset + valueIndex + tailIndex] = (packedBits >>> tailIndex) & 1;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/** Expands two-byte groups of eight two-bit level values. */
|
|
457
|
+
function decode2BitUnsignedRun(
|
|
458
|
+
buffer: Uint8Array,
|
|
459
|
+
packedOffset: number,
|
|
460
|
+
count: number,
|
|
461
|
+
output: ParquetValueBuffer,
|
|
462
|
+
outputOffset: number
|
|
463
|
+
): void {
|
|
464
|
+
let byteOffset = packedOffset;
|
|
465
|
+
let valueIndex = 0;
|
|
466
|
+
while (valueIndex + 8 <= count) {
|
|
467
|
+
const packedBits = (buffer[byteOffset] ?? 0) | ((buffer[byteOffset + 1] ?? 0) << 8);
|
|
468
|
+
output[outputOffset + valueIndex] = packedBits & 3;
|
|
469
|
+
output[outputOffset + valueIndex + 1] = (packedBits >>> 2) & 3;
|
|
470
|
+
output[outputOffset + valueIndex + 2] = (packedBits >>> 4) & 3;
|
|
471
|
+
output[outputOffset + valueIndex + 3] = (packedBits >>> 6) & 3;
|
|
472
|
+
output[outputOffset + valueIndex + 4] = (packedBits >>> 8) & 3;
|
|
473
|
+
output[outputOffset + valueIndex + 5] = (packedBits >>> 10) & 3;
|
|
474
|
+
output[outputOffset + valueIndex + 6] = (packedBits >>> 12) & 3;
|
|
475
|
+
output[outputOffset + valueIndex + 7] = (packedBits >>> 14) & 3;
|
|
476
|
+
byteOffset += 2;
|
|
477
|
+
valueIndex += 8;
|
|
478
|
+
}
|
|
479
|
+
const packedBits = (buffer[byteOffset] ?? 0) | ((buffer[byteOffset + 1] ?? 0) << 8);
|
|
480
|
+
for (let tailIndex = 0; valueIndex + tailIndex < count; tailIndex++) {
|
|
481
|
+
output[outputOffset + valueIndex + tailIndex] = (packedBits >>> (tailIndex * 2)) & 3;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/** Resolves aligned groups of four 10-bit dictionary indices from five input bytes. */
|
|
486
|
+
function decode10BitDictionaryRun(
|
|
487
|
+
buffer: Uint8Array,
|
|
488
|
+
packedOffset: number,
|
|
489
|
+
count: number,
|
|
490
|
+
output: ParquetValueBuffer,
|
|
491
|
+
outputOffset: number,
|
|
492
|
+
dictionary: readonly unknown[]
|
|
493
|
+
): void {
|
|
494
|
+
let byteOffset = packedOffset;
|
|
495
|
+
let valueIndex = 0;
|
|
496
|
+
while (valueIndex + 4 <= count) {
|
|
497
|
+
const byte0 = buffer[byteOffset] ?? 0;
|
|
498
|
+
const byte1 = buffer[byteOffset + 1] ?? 0;
|
|
499
|
+
const byte2 = buffer[byteOffset + 2] ?? 0;
|
|
500
|
+
const byte3 = buffer[byteOffset + 3] ?? 0;
|
|
501
|
+
const byte4 = buffer[byteOffset + 4] ?? 0;
|
|
502
|
+
const index0 = byte0 | ((byte1 & 0x03) << 8);
|
|
503
|
+
const index1 = (byte1 >>> 2) | ((byte2 & 0x0f) << 6);
|
|
504
|
+
const index2 = (byte2 >>> 4) | ((byte3 & 0x3f) << 4);
|
|
505
|
+
const index3 = (byte3 >>> 6) | (byte4 << 2);
|
|
506
|
+
if (
|
|
507
|
+
index0 >= dictionary.length ||
|
|
508
|
+
index1 >= dictionary.length ||
|
|
509
|
+
index2 >= dictionary.length ||
|
|
510
|
+
index3 >= dictionary.length
|
|
511
|
+
) {
|
|
512
|
+
const invalidIndex = [index0, index1, index2, index3].find(
|
|
513
|
+
dictionaryIndex => dictionaryIndex >= dictionary.length
|
|
514
|
+
);
|
|
515
|
+
throw new Error(`Invalid Parquet dictionary index ${invalidIndex}`);
|
|
516
|
+
}
|
|
517
|
+
output[outputOffset + valueIndex] = dictionary[index0];
|
|
518
|
+
output[outputOffset + valueIndex + 1] = dictionary[index1];
|
|
519
|
+
output[outputOffset + valueIndex + 2] = dictionary[index2];
|
|
520
|
+
output[outputOffset + valueIndex + 3] = dictionary[index3];
|
|
521
|
+
byteOffset += 5;
|
|
522
|
+
valueIndex += 4;
|
|
523
|
+
}
|
|
524
|
+
decodeDictionaryBitPackedTail(
|
|
525
|
+
buffer,
|
|
526
|
+
byteOffset,
|
|
527
|
+
count - valueIndex,
|
|
528
|
+
10,
|
|
529
|
+
output,
|
|
530
|
+
outputOffset + valueIndex,
|
|
531
|
+
dictionary
|
|
532
|
+
);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/** Resolves aligned pairs of 12-bit dictionary indices from three input bytes. */
|
|
536
|
+
function decode12BitDictionaryRun(
|
|
537
|
+
buffer: Uint8Array,
|
|
538
|
+
packedOffset: number,
|
|
539
|
+
count: number,
|
|
540
|
+
output: ParquetValueBuffer,
|
|
541
|
+
outputOffset: number,
|
|
542
|
+
dictionary: readonly unknown[]
|
|
543
|
+
): void {
|
|
544
|
+
let byteOffset = packedOffset;
|
|
545
|
+
let valueIndex = 0;
|
|
546
|
+
while (valueIndex + 2 <= count) {
|
|
547
|
+
const byte0 = buffer[byteOffset] ?? 0;
|
|
548
|
+
const byte1 = buffer[byteOffset + 1] ?? 0;
|
|
549
|
+
const byte2 = buffer[byteOffset + 2] ?? 0;
|
|
550
|
+
const index0 = byte0 | ((byte1 & 0x0f) << 8);
|
|
551
|
+
const index1 = (byte1 >>> 4) | (byte2 << 4);
|
|
552
|
+
if (index0 >= dictionary.length || index1 >= dictionary.length) {
|
|
553
|
+
const invalidIndex = index0 >= dictionary.length ? index0 : index1;
|
|
554
|
+
throw new Error(`Invalid Parquet dictionary index ${invalidIndex}`);
|
|
555
|
+
}
|
|
556
|
+
output[outputOffset + valueIndex] = dictionary[index0];
|
|
557
|
+
output[outputOffset + valueIndex + 1] = dictionary[index1];
|
|
558
|
+
byteOffset += 3;
|
|
559
|
+
valueIndex += 2;
|
|
560
|
+
}
|
|
561
|
+
decodeDictionaryBitPackedTail(
|
|
562
|
+
buffer,
|
|
563
|
+
byteOffset,
|
|
564
|
+
count - valueIndex,
|
|
565
|
+
12,
|
|
566
|
+
output,
|
|
567
|
+
outputOffset + valueIndex,
|
|
568
|
+
dictionary
|
|
569
|
+
);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/** Resolves the byte-aligned remainder after a fixed-width dictionary group. */
|
|
573
|
+
function decodeDictionaryBitPackedTail(
|
|
574
|
+
buffer: Uint8Array,
|
|
575
|
+
packedOffset: number,
|
|
576
|
+
count: number,
|
|
577
|
+
bitWidth: number,
|
|
578
|
+
output: ParquetValueBuffer,
|
|
579
|
+
outputOffset: number,
|
|
580
|
+
dictionary: readonly unknown[]
|
|
581
|
+
): void {
|
|
582
|
+
const mask = 2 ** bitWidth - 1;
|
|
583
|
+
let packedBits = 0;
|
|
584
|
+
let packedBitCount = 0;
|
|
585
|
+
let byteOffset = packedOffset;
|
|
586
|
+
for (let valueIndex = 0; valueIndex < count; valueIndex++) {
|
|
587
|
+
while (packedBitCount < bitWidth) {
|
|
588
|
+
packedBits |= (buffer[byteOffset++] ?? 0) << packedBitCount;
|
|
589
|
+
packedBitCount += 8;
|
|
590
|
+
}
|
|
591
|
+
const dictionaryIndex = packedBits & mask;
|
|
592
|
+
if (dictionaryIndex >= dictionary.length) {
|
|
593
|
+
throw new Error(`Invalid Parquet dictionary index ${dictionaryIndex}`);
|
|
594
|
+
}
|
|
595
|
+
output[outputOffset + valueIndex] = dictionary[dictionaryIndex];
|
|
259
596
|
packedBits >>>= bitWidth;
|
|
260
597
|
packedBitCount -= bitWidth;
|
|
261
598
|
}
|
|
262
599
|
}
|
|
263
600
|
|
|
601
|
+
/** Resolves complete groups of eight narrow dictionary indices without a per-value reservoir. */
|
|
602
|
+
function decodeNarrowDictionaryBitPackedRun(
|
|
603
|
+
buffer: Uint8Array,
|
|
604
|
+
packedOffset: number,
|
|
605
|
+
count: number,
|
|
606
|
+
bitWidth: number,
|
|
607
|
+
output: ParquetValueBuffer,
|
|
608
|
+
outputOffset: number,
|
|
609
|
+
dictionary: readonly unknown[]
|
|
610
|
+
): void {
|
|
611
|
+
if (bitWidth === 1) {
|
|
612
|
+
decode1BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
if (bitWidth === 2) {
|
|
616
|
+
decode2BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
if (bitWidth === 3) {
|
|
620
|
+
decode3BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
if (bitWidth === 4) {
|
|
624
|
+
decode4BitDictionaryRun(buffer, packedOffset, count, output, outputOffset, dictionary);
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
const mask = 2 ** bitWidth - 1;
|
|
628
|
+
let byteOffset = packedOffset;
|
|
629
|
+
let valueIndex = 0;
|
|
630
|
+
while (valueIndex < count) {
|
|
631
|
+
const packedBits =
|
|
632
|
+
(buffer[byteOffset] ?? 0) |
|
|
633
|
+
((buffer[byteOffset + 1] ?? 0) << 8) |
|
|
634
|
+
((buffer[byteOffset + 2] ?? 0) << 16) |
|
|
635
|
+
((buffer[byteOffset + 3] ?? 0) << 24);
|
|
636
|
+
const groupValueCount = Math.min(8, count - valueIndex);
|
|
637
|
+
for (let groupIndex = 0; groupIndex < groupValueCount; groupIndex++) {
|
|
638
|
+
const dictionaryIndex = (packedBits >>> (groupIndex * bitWidth)) & mask;
|
|
639
|
+
if (dictionaryIndex >= dictionary.length) {
|
|
640
|
+
throw new Error(`Invalid Parquet dictionary index ${dictionaryIndex}`);
|
|
641
|
+
}
|
|
642
|
+
output[outputOffset + valueIndex + groupIndex] = dictionary[dictionaryIndex];
|
|
643
|
+
}
|
|
644
|
+
byteOffset += bitWidth;
|
|
645
|
+
valueIndex += groupValueCount;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/** Resolves byte-aligned groups of eight one-bit dictionary indices. */
|
|
650
|
+
function decode1BitDictionaryRun(
|
|
651
|
+
buffer: Uint8Array,
|
|
652
|
+
packedOffset: number,
|
|
653
|
+
count: number,
|
|
654
|
+
output: ParquetValueBuffer,
|
|
655
|
+
outputOffset: number,
|
|
656
|
+
dictionary: readonly unknown[]
|
|
657
|
+
): void {
|
|
658
|
+
let byteOffset = packedOffset;
|
|
659
|
+
let valueIndex = 0;
|
|
660
|
+
while (valueIndex + 8 <= count) {
|
|
661
|
+
const packedBits = buffer[byteOffset++] ?? 0;
|
|
662
|
+
if (dictionary.length < 2 && packedBits !== 0) {
|
|
663
|
+
throw new Error('Invalid Parquet dictionary index 1');
|
|
664
|
+
}
|
|
665
|
+
output[outputOffset + valueIndex] = dictionary[packedBits & 1];
|
|
666
|
+
output[outputOffset + valueIndex + 1] = dictionary[(packedBits >>> 1) & 1];
|
|
667
|
+
output[outputOffset + valueIndex + 2] = dictionary[(packedBits >>> 2) & 1];
|
|
668
|
+
output[outputOffset + valueIndex + 3] = dictionary[(packedBits >>> 3) & 1];
|
|
669
|
+
output[outputOffset + valueIndex + 4] = dictionary[(packedBits >>> 4) & 1];
|
|
670
|
+
output[outputOffset + valueIndex + 5] = dictionary[(packedBits >>> 5) & 1];
|
|
671
|
+
output[outputOffset + valueIndex + 6] = dictionary[(packedBits >>> 6) & 1];
|
|
672
|
+
output[outputOffset + valueIndex + 7] = dictionary[packedBits >>> 7];
|
|
673
|
+
valueIndex += 8;
|
|
674
|
+
}
|
|
675
|
+
decodeDictionaryBitPackedTail(
|
|
676
|
+
buffer,
|
|
677
|
+
byteOffset,
|
|
678
|
+
count - valueIndex,
|
|
679
|
+
1,
|
|
680
|
+
output,
|
|
681
|
+
outputOffset + valueIndex,
|
|
682
|
+
dictionary
|
|
683
|
+
);
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/** Resolves two-byte groups of eight two-bit dictionary indices. */
|
|
687
|
+
function decode2BitDictionaryRun(
|
|
688
|
+
buffer: Uint8Array,
|
|
689
|
+
packedOffset: number,
|
|
690
|
+
count: number,
|
|
691
|
+
output: ParquetValueBuffer,
|
|
692
|
+
outputOffset: number,
|
|
693
|
+
dictionary: readonly unknown[]
|
|
694
|
+
): void {
|
|
695
|
+
let byteOffset = packedOffset;
|
|
696
|
+
let valueIndex = 0;
|
|
697
|
+
while (valueIndex + 8 <= count) {
|
|
698
|
+
const packedBits = (buffer[byteOffset] ?? 0) | ((buffer[byteOffset + 1] ?? 0) << 8);
|
|
699
|
+
const index0 = packedBits & 3;
|
|
700
|
+
const index1 = (packedBits >>> 2) & 3;
|
|
701
|
+
const index2 = (packedBits >>> 4) & 3;
|
|
702
|
+
const index3 = (packedBits >>> 6) & 3;
|
|
703
|
+
const index4 = (packedBits >>> 8) & 3;
|
|
704
|
+
const index5 = (packedBits >>> 10) & 3;
|
|
705
|
+
const index6 = (packedBits >>> 12) & 3;
|
|
706
|
+
const index7 = packedBits >>> 14;
|
|
707
|
+
if (
|
|
708
|
+
index0 >= dictionary.length ||
|
|
709
|
+
index1 >= dictionary.length ||
|
|
710
|
+
index2 >= dictionary.length ||
|
|
711
|
+
index3 >= dictionary.length ||
|
|
712
|
+
index4 >= dictionary.length ||
|
|
713
|
+
index5 >= dictionary.length ||
|
|
714
|
+
index6 >= dictionary.length ||
|
|
715
|
+
index7 >= dictionary.length
|
|
716
|
+
) {
|
|
717
|
+
throw new Error('Invalid Parquet dictionary index');
|
|
718
|
+
}
|
|
719
|
+
output[outputOffset + valueIndex] = dictionary[index0];
|
|
720
|
+
output[outputOffset + valueIndex + 1] = dictionary[index1];
|
|
721
|
+
output[outputOffset + valueIndex + 2] = dictionary[index2];
|
|
722
|
+
output[outputOffset + valueIndex + 3] = dictionary[index3];
|
|
723
|
+
output[outputOffset + valueIndex + 4] = dictionary[index4];
|
|
724
|
+
output[outputOffset + valueIndex + 5] = dictionary[index5];
|
|
725
|
+
output[outputOffset + valueIndex + 6] = dictionary[index6];
|
|
726
|
+
output[outputOffset + valueIndex + 7] = dictionary[index7];
|
|
727
|
+
byteOffset += 2;
|
|
728
|
+
valueIndex += 8;
|
|
729
|
+
}
|
|
730
|
+
decodeDictionaryBitPackedTail(
|
|
731
|
+
buffer,
|
|
732
|
+
byteOffset,
|
|
733
|
+
count - valueIndex,
|
|
734
|
+
2,
|
|
735
|
+
output,
|
|
736
|
+
outputOffset + valueIndex,
|
|
737
|
+
dictionary
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
/** Resolves byte-aligned groups of eight three-bit dictionary indices. */
|
|
742
|
+
function decode3BitDictionaryRun(
|
|
743
|
+
buffer: Uint8Array,
|
|
744
|
+
packedOffset: number,
|
|
745
|
+
count: number,
|
|
746
|
+
output: ParquetValueBuffer,
|
|
747
|
+
outputOffset: number,
|
|
748
|
+
dictionary: readonly unknown[]
|
|
749
|
+
): void {
|
|
750
|
+
let byteOffset = packedOffset;
|
|
751
|
+
let valueIndex = 0;
|
|
752
|
+
while (valueIndex + 8 <= count) {
|
|
753
|
+
const packedBits =
|
|
754
|
+
(buffer[byteOffset] ?? 0) |
|
|
755
|
+
((buffer[byteOffset + 1] ?? 0) << 8) |
|
|
756
|
+
((buffer[byteOffset + 2] ?? 0) << 16);
|
|
757
|
+
const index0 = packedBits & 7;
|
|
758
|
+
const index1 = (packedBits >>> 3) & 7;
|
|
759
|
+
const index2 = (packedBits >>> 6) & 7;
|
|
760
|
+
const index3 = (packedBits >>> 9) & 7;
|
|
761
|
+
const index4 = (packedBits >>> 12) & 7;
|
|
762
|
+
const index5 = (packedBits >>> 15) & 7;
|
|
763
|
+
const index6 = (packedBits >>> 18) & 7;
|
|
764
|
+
const index7 = (packedBits >>> 21) & 7;
|
|
765
|
+
if (
|
|
766
|
+
index0 >= dictionary.length ||
|
|
767
|
+
index1 >= dictionary.length ||
|
|
768
|
+
index2 >= dictionary.length ||
|
|
769
|
+
index3 >= dictionary.length ||
|
|
770
|
+
index4 >= dictionary.length ||
|
|
771
|
+
index5 >= dictionary.length ||
|
|
772
|
+
index6 >= dictionary.length ||
|
|
773
|
+
index7 >= dictionary.length
|
|
774
|
+
) {
|
|
775
|
+
throw new Error('Invalid Parquet dictionary index');
|
|
776
|
+
}
|
|
777
|
+
output[outputOffset + valueIndex] = dictionary[index0];
|
|
778
|
+
output[outputOffset + valueIndex + 1] = dictionary[index1];
|
|
779
|
+
output[outputOffset + valueIndex + 2] = dictionary[index2];
|
|
780
|
+
output[outputOffset + valueIndex + 3] = dictionary[index3];
|
|
781
|
+
output[outputOffset + valueIndex + 4] = dictionary[index4];
|
|
782
|
+
output[outputOffset + valueIndex + 5] = dictionary[index5];
|
|
783
|
+
output[outputOffset + valueIndex + 6] = dictionary[index6];
|
|
784
|
+
output[outputOffset + valueIndex + 7] = dictionary[index7];
|
|
785
|
+
byteOffset += 3;
|
|
786
|
+
valueIndex += 8;
|
|
787
|
+
}
|
|
788
|
+
decodeDictionaryBitPackedTail(
|
|
789
|
+
buffer,
|
|
790
|
+
byteOffset,
|
|
791
|
+
count - valueIndex,
|
|
792
|
+
3,
|
|
793
|
+
output,
|
|
794
|
+
outputOffset + valueIndex,
|
|
795
|
+
dictionary
|
|
796
|
+
);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
/** Resolves pairs of four-bit dictionary indices from each input byte. */
|
|
800
|
+
function decode4BitDictionaryRun(
|
|
801
|
+
buffer: Uint8Array,
|
|
802
|
+
packedOffset: number,
|
|
803
|
+
count: number,
|
|
804
|
+
output: ParquetValueBuffer,
|
|
805
|
+
outputOffset: number,
|
|
806
|
+
dictionary: readonly unknown[]
|
|
807
|
+
): void {
|
|
808
|
+
let byteOffset = packedOffset;
|
|
809
|
+
let valueIndex = 0;
|
|
810
|
+
while (valueIndex + 2 <= count) {
|
|
811
|
+
const packedByte = buffer[byteOffset++] ?? 0;
|
|
812
|
+
const index0 = packedByte & 15;
|
|
813
|
+
const index1 = packedByte >>> 4;
|
|
814
|
+
if (index0 >= dictionary.length || index1 >= dictionary.length) {
|
|
815
|
+
throw new Error(
|
|
816
|
+
`Invalid Parquet dictionary index ${index0 >= dictionary.length ? index0 : index1}`
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
output[outputOffset + valueIndex] = dictionary[index0];
|
|
820
|
+
output[outputOffset + valueIndex + 1] = dictionary[index1];
|
|
821
|
+
valueIndex += 2;
|
|
822
|
+
}
|
|
823
|
+
decodeDictionaryBitPackedTail(
|
|
824
|
+
buffer,
|
|
825
|
+
byteOffset,
|
|
826
|
+
count - valueIndex,
|
|
827
|
+
4,
|
|
828
|
+
output,
|
|
829
|
+
outputOffset + valueIndex,
|
|
830
|
+
dictionary
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
|
|
264
834
|
/** Decodes a bit-packed run with an exact number-based bit reservoir. */
|
|
265
835
|
function decodeNumberBitPackedRun(
|
|
266
836
|
buffer: Uint8Array,
|