@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15005 -3143
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +15766 -4896
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14911 -4316
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +42 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +585 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7055 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8505 -1041
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +78 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +605 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -11
- package/src/bundled.ts +3 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +50 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +929 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/parquet-reader.ts +935 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
SchemaDefinition
|
|
18
18
|
} from '../schema/declare';
|
|
19
19
|
import {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs/index';
|
|
20
|
-
import type {ParquetValueBuffer} from '../codecs/declare';
|
|
20
|
+
import type {ParquetByteArrayOutput, ParquetValueBuffer} from '../codecs/declare';
|
|
21
21
|
import {
|
|
22
22
|
ConvertedType,
|
|
23
23
|
EdgeInterpolationAlgorithm,
|
|
@@ -32,11 +32,16 @@ import {decompress} from '../compression';
|
|
|
32
32
|
import type {TimeUnit as ParquetThriftTimeUnit} from '../parquet-thrift/TimeUnit';
|
|
33
33
|
import {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../lib/constants';
|
|
34
34
|
import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';
|
|
35
|
+
import {crc32} from '../utils/crc32';
|
|
36
|
+
|
|
37
|
+
/** Shared empty level storage for required flat columns, whose levels are implicit. */
|
|
38
|
+
const EMPTY_PARQUET_LEVEL_BUFFER = new Uint8Array(0);
|
|
35
39
|
|
|
36
40
|
/** Preallocated column destination used to bypass page-local value and level arrays. */
|
|
37
41
|
type ParquetPageDecodeTarget = {
|
|
38
42
|
values: ParquetValueBuffer;
|
|
39
43
|
valueOffset: number;
|
|
44
|
+
byteArrayOutput?: ParquetByteArrayOutput;
|
|
40
45
|
dictionary: readonly unknown[];
|
|
41
46
|
rlevels: ParquetLevelBuffer;
|
|
42
47
|
dlevels: ParquetLevelBuffer;
|
|
@@ -70,12 +75,14 @@ export async function decodeDataPages(
|
|
|
70
75
|
}
|
|
71
76
|
|
|
72
77
|
const outputCapacity = expectedLevelCount ?? 0;
|
|
78
|
+
const byteArrayOutput = createArrowByteArrayOutput(buffer, context, outputCapacity);
|
|
73
79
|
const data: ParquetColumnChunk = {
|
|
74
80
|
rlevels: createParquetLevelBuffer(context, outputCapacity, context.rLevelMax),
|
|
75
81
|
dlevels: createParquetLevelBuffer(context, outputCapacity, context.dLevelMax),
|
|
76
|
-
values: createParquetColumnValueBuffer(context, outputCapacity),
|
|
82
|
+
values: byteArrayOutput ? [] : createParquetColumnValueBuffer(context, outputCapacity),
|
|
77
83
|
pageHeaders: [],
|
|
78
|
-
count: 0
|
|
84
|
+
count: 0,
|
|
85
|
+
nullCount: 0
|
|
79
86
|
};
|
|
80
87
|
|
|
81
88
|
let dictionary = context.dictionary || [];
|
|
@@ -88,14 +95,19 @@ export async function decodeDataPages(
|
|
|
88
95
|
(expectedLevelCount === undefined || levelOffset < expectedLevelCount)
|
|
89
96
|
) {
|
|
90
97
|
// Looks like we have to decode these in sequence due to cursor updates?
|
|
91
|
-
const
|
|
98
|
+
const target = {
|
|
92
99
|
values: data.values,
|
|
93
100
|
valueOffset,
|
|
101
|
+
byteArrayOutput,
|
|
94
102
|
dictionary,
|
|
95
103
|
rlevels: data.rlevels,
|
|
96
104
|
dlevels: data.dlevels,
|
|
97
105
|
levelOffset
|
|
98
|
-
}
|
|
106
|
+
};
|
|
107
|
+
const page =
|
|
108
|
+
context.compression === 'UNCOMPRESSED'
|
|
109
|
+
? decodeUncompressedPage(cursor, context, target)
|
|
110
|
+
: await decodePage(cursor, context, target);
|
|
99
111
|
|
|
100
112
|
if (page.dictionary) {
|
|
101
113
|
dictionary = page.dictionary;
|
|
@@ -107,6 +119,7 @@ export async function decodeDataPages(
|
|
|
107
119
|
|
|
108
120
|
if (page.directValuesWritten !== undefined) {
|
|
109
121
|
valueOffset += page.directValuesWritten;
|
|
122
|
+
data.nullCount! += page.count - page.directValuesWritten;
|
|
110
123
|
} else {
|
|
111
124
|
const valueEncoding = getThriftEnum(
|
|
112
125
|
Encoding,
|
|
@@ -129,11 +142,114 @@ export async function decodeDataPages(
|
|
|
129
142
|
|
|
130
143
|
data.rlevels = trimParquetLevelBuffer(data.rlevels, levelOffset);
|
|
131
144
|
data.dlevels = trimParquetLevelBuffer(data.dlevels, levelOffset);
|
|
132
|
-
|
|
145
|
+
if (byteArrayOutput) {
|
|
146
|
+
data.values = [];
|
|
147
|
+
data.byteArrayData = {
|
|
148
|
+
data: byteArrayOutput.data.subarray(0, byteArrayOutput.byteLength),
|
|
149
|
+
valueOffsets: byteArrayOutput.valueOffsets.subarray(0, valueOffset + 1)
|
|
150
|
+
};
|
|
151
|
+
} else {
|
|
152
|
+
data.values = trimParquetValueBuffer(data.values, valueOffset);
|
|
153
|
+
}
|
|
133
154
|
|
|
134
155
|
return data;
|
|
135
156
|
}
|
|
136
157
|
|
|
158
|
+
/**
|
|
159
|
+
* Decodes an uncompressed column chunk synchronously.
|
|
160
|
+
*
|
|
161
|
+
* In-memory Parquet reads use this entry point to avoid one resolved Promise per selected column.
|
|
162
|
+
*/
|
|
163
|
+
export function decodeUncompressedDataPages(
|
|
164
|
+
buffer: Uint8Array,
|
|
165
|
+
context: ParquetReaderContext
|
|
166
|
+
): ParquetColumnChunk {
|
|
167
|
+
const cursor: CursorBuffer = {buffer, offset: 0, size: buffer.length};
|
|
168
|
+
const expectedLevelCount =
|
|
169
|
+
context.numValues === undefined ? undefined : Number(context.numValues);
|
|
170
|
+
if (
|
|
171
|
+
expectedLevelCount !== undefined &&
|
|
172
|
+
(!Number.isSafeInteger(expectedLevelCount) || expectedLevelCount < 0)
|
|
173
|
+
) {
|
|
174
|
+
throw new Error(`Invalid Parquet column value count ${expectedLevelCount}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const outputCapacity = expectedLevelCount ?? 0;
|
|
178
|
+
const byteArrayOutput = createArrowByteArrayOutput(buffer, context, outputCapacity);
|
|
179
|
+
const data: ParquetColumnChunk = {
|
|
180
|
+
rlevels: createParquetLevelBuffer(context, outputCapacity, context.rLevelMax),
|
|
181
|
+
dlevels: createParquetLevelBuffer(context, outputCapacity, context.dLevelMax),
|
|
182
|
+
values: byteArrayOutput ? [] : createParquetColumnValueBuffer(context, outputCapacity),
|
|
183
|
+
pageHeaders: [],
|
|
184
|
+
count: 0,
|
|
185
|
+
nullCount: 0
|
|
186
|
+
};
|
|
187
|
+
let dictionary = context.dictionary || [];
|
|
188
|
+
let levelOffset = 0;
|
|
189
|
+
let valueOffset = 0;
|
|
190
|
+
|
|
191
|
+
while (
|
|
192
|
+
cursor.offset < (cursor.size ?? cursor.buffer.length) &&
|
|
193
|
+
(expectedLevelCount === undefined || levelOffset < expectedLevelCount)
|
|
194
|
+
) {
|
|
195
|
+
const page = decodeUncompressedPage(cursor, context, {
|
|
196
|
+
values: data.values,
|
|
197
|
+
valueOffset,
|
|
198
|
+
byteArrayOutput,
|
|
199
|
+
dictionary,
|
|
200
|
+
rlevels: data.rlevels,
|
|
201
|
+
dlevels: data.dlevels,
|
|
202
|
+
levelOffset
|
|
203
|
+
});
|
|
204
|
+
if (page.dictionary) {
|
|
205
|
+
dictionary = page.dictionary;
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
levelOffset += page.count;
|
|
210
|
+
if (page.directValuesWritten !== undefined) {
|
|
211
|
+
valueOffset += page.directValuesWritten;
|
|
212
|
+
data.nullCount! += page.count - page.directValuesWritten;
|
|
213
|
+
} else {
|
|
214
|
+
const valueEncoding = getThriftEnum(
|
|
215
|
+
Encoding,
|
|
216
|
+
page.pageHeader.data_page_header?.encoding ?? page.pageHeader.data_page_header_v2?.encoding!
|
|
217
|
+
) as ParquetCodec;
|
|
218
|
+
const usesDictionary =
|
|
219
|
+
dictionary.length &&
|
|
220
|
+
(valueEncoding === 'PLAIN_DICTIONARY' || valueEncoding === 'RLE_DICTIONARY');
|
|
221
|
+
for (let index = 0; index < page.values.length; index++) {
|
|
222
|
+
const value = usesDictionary ? dictionary[Number(page.values[index])] : page.values[index];
|
|
223
|
+
if (value !== undefined) data.values[valueOffset++] = value;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
data.count += page.count;
|
|
227
|
+
data.pageHeaders.push(page.pageHeader);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
data.rlevels = trimParquetLevelBuffer(data.rlevels, levelOffset);
|
|
231
|
+
data.dlevels = trimParquetLevelBuffer(data.dlevels, levelOffset);
|
|
232
|
+
if (byteArrayOutput) {
|
|
233
|
+
data.values = [];
|
|
234
|
+
data.byteArrayData = {
|
|
235
|
+
data: byteArrayOutput.data.subarray(0, byteArrayOutput.byteLength),
|
|
236
|
+
valueOffsets: byteArrayOutput.valueOffsets.subarray(0, valueOffset + 1)
|
|
237
|
+
};
|
|
238
|
+
} else {
|
|
239
|
+
data.values = trimParquetValueBuffer(data.values, valueOffset);
|
|
240
|
+
}
|
|
241
|
+
return data;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Decodes a dictionary page from an uncompressed in-memory column chunk. */
|
|
245
|
+
export function decodeUncompressedDictionaryBuffer(
|
|
246
|
+
buffer: Uint8Array,
|
|
247
|
+
context: ParquetReaderContext
|
|
248
|
+
): unknown[] {
|
|
249
|
+
const cursor: CursorBuffer = {buffer, offset: 0, size: buffer.length};
|
|
250
|
+
return decodeUncompressedPage(cursor, context).dictionary!;
|
|
251
|
+
}
|
|
252
|
+
|
|
137
253
|
/**
|
|
138
254
|
* Decode parquet page based on page type
|
|
139
255
|
* @param cursor
|
|
@@ -150,6 +266,8 @@ export async function decodePage(
|
|
|
150
266
|
cursor.offset += length;
|
|
151
267
|
|
|
152
268
|
const pageType = getThriftEnum(PageType, pageHeader.type);
|
|
269
|
+
const pageEnd = cursor.offset + pageHeader.compressed_page_size;
|
|
270
|
+
verifyPageChecksum(cursor, pageHeader, pageEnd, context);
|
|
153
271
|
|
|
154
272
|
switch (pageType) {
|
|
155
273
|
case 'DATA_PAGE':
|
|
@@ -171,6 +289,56 @@ export async function decodePage(
|
|
|
171
289
|
return page;
|
|
172
290
|
}
|
|
173
291
|
|
|
292
|
+
/** Decodes one uncompressed page without introducing a Promise/microtask boundary. */
|
|
293
|
+
function decodeUncompressedPage(
|
|
294
|
+
cursor: CursorBuffer,
|
|
295
|
+
context: ParquetReaderContext,
|
|
296
|
+
target?: ParquetPageDecodeTarget
|
|
297
|
+
): ParquetPageData {
|
|
298
|
+
const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);
|
|
299
|
+
cursor.offset += length;
|
|
300
|
+
const pageEnd = cursor.offset + pageHeader.compressed_page_size;
|
|
301
|
+
verifyPageChecksum(cursor, pageHeader, pageEnd, context);
|
|
302
|
+
|
|
303
|
+
const pageType = getThriftEnum(PageType, pageHeader.type);
|
|
304
|
+
switch (pageType) {
|
|
305
|
+
case 'DATA_PAGE':
|
|
306
|
+
return decodeDataPageValues(cursor, pageHeader, context, target);
|
|
307
|
+
case 'DATA_PAGE_V2':
|
|
308
|
+
return decodeUncompressedDataPageV2(cursor, pageHeader, context, target);
|
|
309
|
+
case 'DICTIONARY_PAGE':
|
|
310
|
+
return {
|
|
311
|
+
dictionary: decodeUncompressedDictionaryPage(cursor, pageHeader, context),
|
|
312
|
+
dlevels: EMPTY_PARQUET_LEVEL_BUFFER,
|
|
313
|
+
rlevels: EMPTY_PARQUET_LEVEL_BUFFER,
|
|
314
|
+
values: [],
|
|
315
|
+
count: 0,
|
|
316
|
+
pageHeader
|
|
317
|
+
};
|
|
318
|
+
default:
|
|
319
|
+
throw new Error(`invalid page type: ${pageType}`);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** Verifies a page checksum before either the synchronous or asynchronous decode path consumes it. */
|
|
324
|
+
function verifyPageChecksum(
|
|
325
|
+
cursor: CursorBuffer,
|
|
326
|
+
pageHeader: PageHeader,
|
|
327
|
+
pageEnd: number,
|
|
328
|
+
context: ParquetReaderContext
|
|
329
|
+
): void {
|
|
330
|
+
if (!context.verifyPageChecksums) return;
|
|
331
|
+
if (pageEnd > (cursor.size ?? cursor.buffer.length)) {
|
|
332
|
+
throw new Error('Parquet page extends beyond the available buffer');
|
|
333
|
+
}
|
|
334
|
+
if (pageHeader.crc === undefined) return;
|
|
335
|
+
const expected = pageHeader.crc >>> 0;
|
|
336
|
+
const actual = crc32(cursor.buffer.subarray(cursor.offset, pageEnd));
|
|
337
|
+
if (actual !== expected) {
|
|
338
|
+
throw new Error(`Parquet page checksum mismatch: expected ${expected}, calculated ${actual}`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
174
342
|
/**
|
|
175
343
|
* Decode parquet schema
|
|
176
344
|
* @param schemaElements input schema elements data
|
|
@@ -437,17 +605,15 @@ async function decodeDataPage(
|
|
|
437
605
|
target?: ParquetPageDecodeTarget
|
|
438
606
|
): Promise<ParquetPageData> {
|
|
439
607
|
const cursorEnd = cursor.offset + header.compressed_page_size;
|
|
440
|
-
const valueCount = header.data_page_header?.num_values;
|
|
441
608
|
|
|
442
609
|
/* uncompress page */
|
|
443
610
|
let dataCursor = cursor;
|
|
444
611
|
|
|
445
612
|
if (context.compression !== 'UNCOMPRESSED') {
|
|
446
|
-
const
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
header.uncompressed_page_size
|
|
450
|
-
);
|
|
613
|
+
const compressedValues = cursor.buffer.slice(cursor.offset, cursorEnd);
|
|
614
|
+
const valuesBuf = context.decompressPage
|
|
615
|
+
? await context.decompressPage(compressedValues, header.uncompressed_page_size)
|
|
616
|
+
: await decompress(context.compression, compressedValues, header.uncompressed_page_size);
|
|
451
617
|
dataCursor = {
|
|
452
618
|
buffer: valuesBuf,
|
|
453
619
|
offset: 0,
|
|
@@ -456,6 +622,18 @@ async function decodeDataPage(
|
|
|
456
622
|
cursor.offset = cursorEnd;
|
|
457
623
|
}
|
|
458
624
|
|
|
625
|
+
return decodeDataPageValues(dataCursor, header, context, target);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
/** Decodes one uncompressed Data Page V1 body without introducing an asynchronous boundary. */
|
|
629
|
+
function decodeDataPageValues(
|
|
630
|
+
dataCursor: CursorBuffer,
|
|
631
|
+
header: PageHeader,
|
|
632
|
+
context: ParquetReaderContext,
|
|
633
|
+
target?: ParquetPageDecodeTarget
|
|
634
|
+
): ParquetPageData {
|
|
635
|
+
const valueCount = header.data_page_header?.num_values;
|
|
636
|
+
|
|
459
637
|
/* read repetition levels */
|
|
460
638
|
const rLevelEncoding = getThriftEnum(
|
|
461
639
|
Encoding,
|
|
@@ -506,7 +684,9 @@ async function decodeDataPage(
|
|
|
506
684
|
output: target?.values,
|
|
507
685
|
outputOffset: target?.valueOffset,
|
|
508
686
|
dictionary: isDictionaryEncoding(valueEncoding) ? target?.dictionary : undefined,
|
|
509
|
-
|
|
687
|
+
byteArrayOutput: target?.byteArrayOutput,
|
|
688
|
+
int64AsBigInt: shouldDecodeInt64AsBigInt(context),
|
|
689
|
+
int96AsTimestamp: context.int96AsTimestamp
|
|
510
690
|
};
|
|
511
691
|
|
|
512
692
|
const values = decodeValues(
|
|
@@ -615,14 +795,125 @@ async function decodeDataPageV2(
|
|
|
615
795
|
let valuesBuffer = cursor.buffer.subarray(valuesOffset, cursorEnd);
|
|
616
796
|
|
|
617
797
|
if (dataPageHeader.is_compressed !== false && context.compression !== 'UNCOMPRESSED') {
|
|
618
|
-
valuesBuffer =
|
|
619
|
-
context.
|
|
620
|
-
valuesBuffer,
|
|
621
|
-
|
|
798
|
+
valuesBuffer = context.decompressPage
|
|
799
|
+
? await context.decompressPage(valuesBuffer, valuesUncompressedByteLength)
|
|
800
|
+
: await decompress(context.compression, valuesBuffer, valuesUncompressedByteLength);
|
|
801
|
+
}
|
|
802
|
+
cursor.offset = cursorEnd;
|
|
803
|
+
return decodeDataPageV2Values(
|
|
804
|
+
valuesBuffer,
|
|
805
|
+
header,
|
|
806
|
+
context,
|
|
807
|
+
target,
|
|
808
|
+
rLevels,
|
|
809
|
+
dLevels,
|
|
810
|
+
valueCount,
|
|
811
|
+
valueCountNonNull,
|
|
812
|
+
valueEncoding
|
|
813
|
+
);
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/** Decodes one uncompressed Data Page V2 without awaiting an unused decompressor. */
|
|
817
|
+
function decodeUncompressedDataPageV2(
|
|
818
|
+
cursor: CursorBuffer,
|
|
819
|
+
header: PageHeader,
|
|
820
|
+
context: ParquetReaderContext,
|
|
821
|
+
target?: ParquetPageDecodeTarget
|
|
822
|
+
): ParquetPageData {
|
|
823
|
+
const dataPageHeader = header.data_page_header_v2;
|
|
824
|
+
if (!dataPageHeader) {
|
|
825
|
+
throw new Error('Missing Parquet data page v2 header');
|
|
826
|
+
}
|
|
827
|
+
const cursorEnd = cursor.offset + header.compressed_page_size;
|
|
828
|
+
const levelsOffset = cursor.offset;
|
|
829
|
+
const valueCount = dataPageHeader.num_values;
|
|
830
|
+
const valueCountNonNull = valueCount - dataPageHeader.num_nulls;
|
|
831
|
+
const valueEncoding = getThriftEnum(Encoding, dataPageHeader.encoding) as ParquetCodec;
|
|
832
|
+
if (
|
|
833
|
+
header.compressed_page_size < 0 ||
|
|
834
|
+
cursorEnd > (cursor.size ?? cursor.buffer.length) ||
|
|
835
|
+
valueCountNonNull < 0
|
|
836
|
+
) {
|
|
837
|
+
throw new Error('Invalid Parquet data page v2 header');
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
let rLevels: number[] = [];
|
|
841
|
+
if (context.column.rLevelMax > 0) {
|
|
842
|
+
const repetitionLevelCursor = createPageSliceCursor(
|
|
843
|
+
cursor,
|
|
844
|
+
levelsOffset,
|
|
845
|
+
dataPageHeader.repetition_levels_byte_length
|
|
846
|
+
);
|
|
847
|
+
rLevels = decodeLevels(
|
|
848
|
+
repetitionLevelCursor,
|
|
849
|
+
valueCount,
|
|
850
|
+
context.column.rLevelMax,
|
|
851
|
+
PARQUET_RDLVL_ENCODING,
|
|
852
|
+
true,
|
|
853
|
+
target?.rlevels,
|
|
854
|
+
target?.levelOffset
|
|
622
855
|
);
|
|
856
|
+
} else if (!target) {
|
|
857
|
+
rLevels = new Array(valueCount).fill(0);
|
|
623
858
|
}
|
|
624
|
-
|
|
859
|
+
|
|
860
|
+
const definitionLevelsOffset = levelsOffset + dataPageHeader.repetition_levels_byte_length;
|
|
861
|
+
let dLevels: number[] = [];
|
|
862
|
+
if (context.column.dLevelMax > 0) {
|
|
863
|
+
const definitionLevelCursor = createPageSliceCursor(
|
|
864
|
+
cursor,
|
|
865
|
+
definitionLevelsOffset,
|
|
866
|
+
dataPageHeader.definition_levels_byte_length
|
|
867
|
+
);
|
|
868
|
+
dLevels = decodeLevels(
|
|
869
|
+
definitionLevelCursor,
|
|
870
|
+
valueCount,
|
|
871
|
+
context.column.dLevelMax,
|
|
872
|
+
PARQUET_RDLVL_ENCODING,
|
|
873
|
+
true,
|
|
874
|
+
target?.dlevels,
|
|
875
|
+
target?.levelOffset
|
|
876
|
+
);
|
|
877
|
+
} else if (!target) {
|
|
878
|
+
dLevels = new Array(valueCount).fill(0);
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
const valuesOffset = definitionLevelsOffset + dataPageHeader.definition_levels_byte_length;
|
|
882
|
+
const valuesUncompressedByteLength =
|
|
883
|
+
header.uncompressed_page_size -
|
|
884
|
+
dataPageHeader.repetition_levels_byte_length -
|
|
885
|
+
dataPageHeader.definition_levels_byte_length;
|
|
886
|
+
if (valuesOffset > cursorEnd || valuesUncompressedByteLength < 0) {
|
|
887
|
+
throw new Error('Invalid Parquet data page v2 level lengths');
|
|
888
|
+
}
|
|
889
|
+
const valuesBuffer = cursor.buffer.subarray(valuesOffset, cursorEnd);
|
|
625
890
|
cursor.offset = cursorEnd;
|
|
891
|
+
return decodeDataPageV2Values(
|
|
892
|
+
valuesBuffer,
|
|
893
|
+
header,
|
|
894
|
+
context,
|
|
895
|
+
target,
|
|
896
|
+
rLevels,
|
|
897
|
+
dLevels,
|
|
898
|
+
valueCount,
|
|
899
|
+
valueCountNonNull,
|
|
900
|
+
valueEncoding
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
/** Decodes a prepared Data Page V2 value section without an asynchronous boundary. */
|
|
905
|
+
function decodeDataPageV2Values(
|
|
906
|
+
valuesBuffer: Uint8Array,
|
|
907
|
+
header: PageHeader,
|
|
908
|
+
context: ParquetReaderContext,
|
|
909
|
+
target: ParquetPageDecodeTarget | undefined,
|
|
910
|
+
rLevels: number[],
|
|
911
|
+
dLevels: number[],
|
|
912
|
+
valueCount: number,
|
|
913
|
+
valueCountNonNull: number,
|
|
914
|
+
valueEncoding: ParquetCodec
|
|
915
|
+
): ParquetPageData {
|
|
916
|
+
const valuesBufCursor = {buffer: valuesBuffer, offset: 0, size: valuesBuffer.length};
|
|
626
917
|
|
|
627
918
|
const decodeOptions = {
|
|
628
919
|
typeLength: context.column.typeLength,
|
|
@@ -631,7 +922,9 @@ async function decodeDataPageV2(
|
|
|
631
922
|
output: target?.values,
|
|
632
923
|
outputOffset: target?.valueOffset,
|
|
633
924
|
dictionary: isDictionaryEncoding(valueEncoding) ? target?.dictionary : undefined,
|
|
634
|
-
|
|
925
|
+
byteArrayOutput: target?.byteArrayOutput,
|
|
926
|
+
int64AsBigInt: shouldDecodeInt64AsBigInt(context),
|
|
927
|
+
int96AsTimestamp: context.int96AsTimestamp
|
|
635
928
|
};
|
|
636
929
|
|
|
637
930
|
const values = decodeValues(
|
|
@@ -674,12 +967,114 @@ function decodeLevels(
|
|
|
674
967
|
return output ? [] : levels;
|
|
675
968
|
}
|
|
676
969
|
|
|
970
|
+
/**
|
|
971
|
+
* Allocates a compact Arrow-compatible destination for supported uncompressed byte pages.
|
|
972
|
+
*
|
|
973
|
+
* Scanning only the small Thrift page headers avoids committing to the fast path when a column
|
|
974
|
+
* mixes in an encoding that still requires materialized values. The page buffer is a useful
|
|
975
|
+
* initial capacity; the destination grows for prefix-reconstructed delta values when necessary.
|
|
976
|
+
*/
|
|
977
|
+
function createArrowByteArrayOutput(
|
|
978
|
+
buffer: Uint8Array,
|
|
979
|
+
context: ParquetReaderContext,
|
|
980
|
+
valueCapacity: number
|
|
981
|
+
): ParquetByteArrayOutput | undefined {
|
|
982
|
+
if (
|
|
983
|
+
!context.useArrowByteArrayBuffers ||
|
|
984
|
+
context.compression !== 'UNCOMPRESSED' ||
|
|
985
|
+
!supportsArrowByteArrayOutput(context.column) ||
|
|
986
|
+
valueCapacity === 0
|
|
987
|
+
) {
|
|
988
|
+
return undefined;
|
|
989
|
+
}
|
|
990
|
+
if (
|
|
991
|
+
!context.hasOnlyArrowByteArrayDataPages &&
|
|
992
|
+
!hasOnlyArrowByteArrayDataPages(buffer, valueCapacity)
|
|
993
|
+
) {
|
|
994
|
+
return undefined;
|
|
995
|
+
}
|
|
996
|
+
return {
|
|
997
|
+
data:
|
|
998
|
+
context.column.primitiveType === 'FIXED_LEN_BYTE_ARRAY'
|
|
999
|
+
? new Uint8Array(0)
|
|
1000
|
+
: new Uint8Array(Math.max(1, buffer.byteLength)),
|
|
1001
|
+
valueOffsets: new Int32Array(valueCapacity + 1),
|
|
1002
|
+
byteLength: 0
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
/** Returns whether a BYTE_ARRAY field maps directly to Arrow Utf8 or Binary without conversion. */
|
|
1007
|
+
function supportsArrowByteArrayOutput(column: ParquetReaderContext['column']): boolean {
|
|
1008
|
+
if (column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY') {
|
|
1009
|
+
return false;
|
|
1010
|
+
}
|
|
1011
|
+
if (column.logicalType) {
|
|
1012
|
+
return (
|
|
1013
|
+
column.logicalType.type === 'STRING' ||
|
|
1014
|
+
column.logicalType.type === 'ENUM' ||
|
|
1015
|
+
column.logicalType.type === 'JSON' ||
|
|
1016
|
+
column.logicalType.type === 'BSON' ||
|
|
1017
|
+
column.logicalType.type === 'VARIANT' ||
|
|
1018
|
+
column.logicalType.type === 'GEOMETRY' ||
|
|
1019
|
+
column.logicalType.type === 'GEOGRAPHY'
|
|
1020
|
+
);
|
|
1021
|
+
}
|
|
1022
|
+
return (
|
|
1023
|
+
!column.originalType ||
|
|
1024
|
+
column.originalType === 'UTF8' ||
|
|
1025
|
+
column.originalType === 'ENUM' ||
|
|
1026
|
+
column.originalType === 'JSON' ||
|
|
1027
|
+
column.originalType === 'BSON' ||
|
|
1028
|
+
column.originalType === 'INTERVAL'
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/** Returns whether a complete column buffer contains only direct Arrow byte-array data pages. */
|
|
1033
|
+
function hasOnlyArrowByteArrayDataPages(buffer: Uint8Array, expectedLevelCount: number): boolean {
|
|
1034
|
+
let offset = 0;
|
|
1035
|
+
let levelCount = 0;
|
|
1036
|
+
while (offset < buffer.byteLength && levelCount < expectedLevelCount) {
|
|
1037
|
+
const {pageHeader, length} = decodePageHeader(buffer, offset);
|
|
1038
|
+
const pageType = getThriftEnum(PageType, pageHeader.type);
|
|
1039
|
+
if (pageType !== 'DATA_PAGE' && pageType !== 'DATA_PAGE_V2') {
|
|
1040
|
+
return false;
|
|
1041
|
+
}
|
|
1042
|
+
const encoding = getThriftEnum(
|
|
1043
|
+
Encoding,
|
|
1044
|
+
pageHeader.data_page_header?.encoding ?? pageHeader.data_page_header_v2?.encoding!
|
|
1045
|
+
);
|
|
1046
|
+
if (
|
|
1047
|
+
encoding !== 'PLAIN' &&
|
|
1048
|
+
encoding !== 'DELTA_LENGTH_BYTE_ARRAY' &&
|
|
1049
|
+
encoding !== 'DELTA_BYTE_ARRAY'
|
|
1050
|
+
) {
|
|
1051
|
+
return false;
|
|
1052
|
+
}
|
|
1053
|
+
const valueCount =
|
|
1054
|
+
pageHeader.data_page_header?.num_values ?? pageHeader.data_page_header_v2?.num_values;
|
|
1055
|
+
if (valueCount === undefined || valueCount < 0) {
|
|
1056
|
+
return false;
|
|
1057
|
+
}
|
|
1058
|
+
levelCount += valueCount;
|
|
1059
|
+
offset += length + pageHeader.compressed_page_size;
|
|
1060
|
+
if (offset > buffer.byteLength) {
|
|
1061
|
+
return false;
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
return levelCount === expectedLevelCount;
|
|
1065
|
+
}
|
|
1066
|
+
|
|
677
1067
|
/** Allocates compact unsigned storage for repetition and definition levels. */
|
|
678
1068
|
function createParquetLevelBuffer(
|
|
679
1069
|
context: ParquetReaderContext,
|
|
680
1070
|
capacity: number,
|
|
681
1071
|
levelMax: number
|
|
682
1072
|
): ParquetLevelBuffer {
|
|
1073
|
+
if (context.useTypedLevelBuffers && levelMax === 0) {
|
|
1074
|
+
// Required flat columns have no encoded levels. Avoid allocating two row-sized zero buffers
|
|
1075
|
+
// that neither direct Arrow construction nor level decoding will inspect.
|
|
1076
|
+
return EMPTY_PARQUET_LEVEL_BUFFER;
|
|
1077
|
+
}
|
|
683
1078
|
if (!context.useTypedLevelBuffers || capacity === 0) {
|
|
684
1079
|
return levelMax > 0 ? new Array<number>(capacity) : new Array<number>(capacity).fill(0);
|
|
685
1080
|
}
|
|
@@ -698,6 +1093,7 @@ function trimParquetLevelBuffer(levels: ParquetLevelBuffer, length: number): Par
|
|
|
698
1093
|
levels.length = length;
|
|
699
1094
|
return levels;
|
|
700
1095
|
}
|
|
1096
|
+
if (levels.length === 0 || levels.length === length) return levels;
|
|
701
1097
|
return levels.subarray(0, length) as ParquetLevelBuffer;
|
|
702
1098
|
}
|
|
703
1099
|
|
|
@@ -727,6 +1123,7 @@ function createParquetColumnValueBuffer(
|
|
|
727
1123
|
case 'INT64':
|
|
728
1124
|
return new BigInt64Array(capacity);
|
|
729
1125
|
case 'INT96':
|
|
1126
|
+
return context.int96AsTimestamp ? new BigInt64Array(capacity) : new Float64Array(capacity);
|
|
730
1127
|
case 'DOUBLE':
|
|
731
1128
|
return new Float64Array(capacity);
|
|
732
1129
|
case 'FLOAT':
|
|
@@ -792,11 +1189,10 @@ async function decodeDictionaryPage(
|
|
|
792
1189
|
cursor.offset = cursorEnd;
|
|
793
1190
|
|
|
794
1191
|
if (context.compression !== 'UNCOMPRESSED') {
|
|
795
|
-
const
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
pageHeader.uncompressed_page_size
|
|
799
|
-
);
|
|
1192
|
+
const compressedValues = dictCursor.buffer.subarray(dictCursor.offset);
|
|
1193
|
+
const valuesBuf = context.decompressPage
|
|
1194
|
+
? await context.decompressPage(compressedValues, pageHeader.uncompressed_page_size)
|
|
1195
|
+
: await decompress(context.compression, compressedValues, pageHeader.uncompressed_page_size);
|
|
800
1196
|
|
|
801
1197
|
dictCursor = {
|
|
802
1198
|
buffer: valuesBuf,
|
|
@@ -807,6 +1203,34 @@ async function decodeDictionaryPage(
|
|
|
807
1203
|
cursor.offset = cursorEnd;
|
|
808
1204
|
}
|
|
809
1205
|
|
|
1206
|
+
return decodeDictionaryValues(dictCursor, pageHeader, context);
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/** Decodes an uncompressed dictionary page directly from a view of the column chunk. */
|
|
1210
|
+
function decodeUncompressedDictionaryPage(
|
|
1211
|
+
cursor: CursorBuffer,
|
|
1212
|
+
pageHeader: PageHeader,
|
|
1213
|
+
context: ParquetReaderContext
|
|
1214
|
+
): (string | ArrayBuffer)[] {
|
|
1215
|
+
const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
|
|
1216
|
+
if (cursorEnd > (cursor.size ?? cursor.buffer.length)) {
|
|
1217
|
+
throw new Error('Parquet dictionary page exceeds the available buffer');
|
|
1218
|
+
}
|
|
1219
|
+
const dictionaryCursor: CursorBuffer = {
|
|
1220
|
+
buffer: cursor.buffer.subarray(cursor.offset, cursorEnd),
|
|
1221
|
+
offset: 0,
|
|
1222
|
+
size: pageHeader.compressed_page_size
|
|
1223
|
+
};
|
|
1224
|
+
cursor.offset = cursorEnd;
|
|
1225
|
+
return decodeDictionaryValues(dictionaryCursor, pageHeader, context);
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
/** Decodes one dictionary body from its current cursor without copying uncompressed page bytes. */
|
|
1229
|
+
function decodeDictionaryValues(
|
|
1230
|
+
dictionaryCursor: CursorBuffer,
|
|
1231
|
+
pageHeader: PageHeader,
|
|
1232
|
+
context: ParquetReaderContext
|
|
1233
|
+
): (string | ArrayBuffer)[] {
|
|
810
1234
|
const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
|
|
811
1235
|
const declaredEncoding = getThriftEnum(
|
|
812
1236
|
Encoding,
|
|
@@ -822,12 +1246,13 @@ async function decodeDictionaryPage(
|
|
|
822
1246
|
const decodedDictionaryValues = decodeValues(
|
|
823
1247
|
context.column.primitiveType!,
|
|
824
1248
|
dictionaryValueEncoding,
|
|
825
|
-
|
|
1249
|
+
dictionaryCursor,
|
|
826
1250
|
numValues,
|
|
827
1251
|
{
|
|
828
1252
|
...context,
|
|
829
1253
|
typeLength: context.column.typeLength,
|
|
830
|
-
int64AsBigInt: shouldDecodeInt64AsBigInt(context)
|
|
1254
|
+
int64AsBigInt: shouldDecodeInt64AsBigInt(context),
|
|
1255
|
+
int96AsTimestamp: context.int96AsTimestamp
|
|
831
1256
|
} as ParquetCodecOptions
|
|
832
1257
|
);
|
|
833
1258
|
|