@loaders.gl/parquet 3.3.0-alpha.4 → 3.3.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +14 -14
- package/dist/dist.min.js.map +2 -2
- package/dist/es5/bundle.js +0 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/constants.js +3 -1
- package/dist/es5/constants.js.map +1 -1
- package/dist/es5/index.js +23 -39
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/convert-schema.js +2 -11
- package/dist/es5/lib/convert-schema.js.map +1 -1
- package/dist/es5/lib/parse-parquet.js +29 -72
- package/dist/es5/lib/parse-parquet.js.map +1 -1
- package/dist/es5/lib/read-array-buffer.js +0 -10
- package/dist/es5/lib/read-array-buffer.js.map +1 -1
- package/dist/es5/lib/wasm/encode-parquet-wasm.js +0 -11
- package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -1
- package/dist/es5/lib/wasm/load-wasm/index.js +0 -1
- package/dist/es5/lib/wasm/load-wasm/index.js.map +1 -1
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +0 -14
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +0 -10
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
- package/dist/es5/lib/wasm/parse-parquet-wasm.js +1 -19
- package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -1
- package/dist/es5/parquet-loader.js +2 -1
- package/dist/es5/parquet-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-loader.js +2 -1
- package/dist/es5/parquet-wasm-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-writer.js +1 -3
- package/dist/es5/parquet-wasm-writer.js.map +1 -1
- package/dist/es5/parquet-writer.js +1 -2
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/es5/parquetjs/codecs/declare.js.map +1 -1
- package/dist/es5/parquetjs/codecs/dictionary.js +2 -9
- package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/es5/parquetjs/codecs/index.js +0 -8
- package/dist/es5/parquetjs/codecs/index.js.map +1 -1
- package/dist/es5/parquetjs/codecs/plain.js +1 -77
- package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
- package/dist/es5/parquetjs/codecs/rle.js +1 -39
- package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
- package/dist/es5/parquetjs/compression.js +5 -30
- package/dist/es5/parquetjs/compression.js.map +1 -1
- package/dist/es5/parquetjs/encoder/writer.js +31 -149
- package/dist/es5/parquetjs/encoder/writer.js.map +1 -1
- package/dist/es5/parquetjs/file.js +3 -12
- package/dist/es5/parquetjs/file.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +0 -48
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +0 -47
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +0 -82
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +0 -25
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +0 -39
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -51
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DateType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +0 -26
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +0 -59
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IntType.js +0 -26
- package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +0 -26
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ListType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +0 -85
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MapType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/NullType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +0 -25
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +0 -54
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +0 -31
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +0 -41
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +0 -59
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js +0 -42
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/StringType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js +0 -27
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +0 -27
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/index.js +0 -86
- package/dist/es5/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/es5/parquetjs/parser/decoders.js +3 -82
- package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-cursor.js +5 -37
- package/dist/es5/parquetjs/parser/parquet-cursor.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +2 -88
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-reader.js +14 -67
- package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/es5/parquetjs/schema/declare.js +3 -7
- package/dist/es5/parquetjs/schema/declare.js.map +1 -1
- package/dist/es5/parquetjs/schema/schema.js +6 -34
- package/dist/es5/parquetjs/schema/schema.js.map +1 -1
- package/dist/es5/parquetjs/schema/shred.js +11 -41
- package/dist/es5/parquetjs/schema/shred.js.map +1 -1
- package/dist/es5/parquetjs/schema/types.js +3 -84
- package/dist/es5/parquetjs/schema/types.js.map +1 -1
- package/dist/es5/parquetjs/utils/buffer-utils.js +0 -2
- package/dist/es5/parquetjs/utils/buffer-utils.js.map +1 -1
- package/dist/es5/parquetjs/utils/file-utils.js +1 -7
- package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/es5/parquetjs/utils/read-utils.js +6 -38
- package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/es5/workers/parquet-worker.js +0 -2
- package/dist/es5/workers/parquet-worker.js.map +1 -1
- package/dist/esm/bundle.js +1 -1
- package/dist/esm/bundle.js.map +1 -1
- package/dist/esm/constants.js +3 -0
- package/dist/esm/constants.js.map +1 -1
- package/dist/esm/index.js +10 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/convert-schema.js +1 -7
- package/dist/esm/lib/convert-schema.js.map +1 -1
- package/dist/esm/lib/parse-parquet.js +2 -5
- package/dist/esm/lib/parse-parquet.js.map +1 -1
- package/dist/esm/lib/read-array-buffer.js +2 -1
- package/dist/esm/lib/read-array-buffer.js.map +1 -1
- package/dist/esm/lib/wasm/encode-parquet-wasm.js +1 -1
- package/dist/esm/lib/wasm/encode-parquet-wasm.js.map +1 -1
- package/dist/esm/lib/wasm/load-wasm/index.js.map +1 -1
- package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
- package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
- package/dist/esm/lib/wasm/parse-parquet-wasm.js +2 -3
- package/dist/esm/lib/wasm/parse-parquet-wasm.js.map +1 -1
- package/dist/esm/parquet-loader.js +3 -1
- package/dist/esm/parquet-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-loader.js +3 -1
- package/dist/esm/parquet-wasm-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-writer.js +2 -1
- package/dist/esm/parquet-wasm-writer.js.map +1 -1
- package/dist/esm/parquet-writer.js +2 -2
- package/dist/esm/parquet-writer.js.map +1 -1
- package/dist/esm/parquetjs/codecs/declare.js.map +1 -1
- package/dist/esm/parquetjs/codecs/dictionary.js +2 -1
- package/dist/esm/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/esm/parquetjs/codecs/index.js +2 -0
- package/dist/esm/parquetjs/codecs/index.js.map +1 -1
- package/dist/esm/parquetjs/codecs/plain.js +2 -68
- package/dist/esm/parquetjs/codecs/plain.js.map +1 -1
- package/dist/esm/parquetjs/codecs/rle.js +3 -29
- package/dist/esm/parquetjs/codecs/rle.js.map +1 -1
- package/dist/esm/parquetjs/compression.js +9 -5
- package/dist/esm/parquetjs/compression.js.map +1 -1
- package/dist/esm/parquetjs/encoder/writer.js +21 -51
- package/dist/esm/parquetjs/encoder/writer.js.map +1 -1
- package/dist/esm/parquetjs/file.js +1 -0
- package/dist/esm/parquetjs/file.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +0 -44
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +0 -42
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +0 -82
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +0 -18
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +0 -34
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -49
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DateType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +2 -53
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/IntType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ListType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +0 -90
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/MapType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/NullType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +0 -16
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +0 -49
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageType.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +0 -33
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +0 -59
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js +0 -38
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/StringType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Type.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Type.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/index.js +1 -0
- package/dist/esm/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/esm/parquetjs/parser/decoders.js +9 -39
- package/dist/esm/parquetjs/parser/decoders.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-cursor.js +1 -13
- package/dist/esm/parquetjs/parser/parquet-cursor.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +6 -32
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-reader.js +1 -18
- package/dist/esm/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/esm/parquetjs/schema/declare.js +4 -4
- package/dist/esm/parquetjs/schema/declare.js.map +1 -1
- package/dist/esm/parquetjs/schema/schema.js +3 -29
- package/dist/esm/parquetjs/schema/schema.js.map +1 -1
- package/dist/esm/parquetjs/schema/shred.js +7 -22
- package/dist/esm/parquetjs/schema/shred.js.map +1 -1
- package/dist/esm/parquetjs/schema/types.js +3 -78
- package/dist/esm/parquetjs/schema/types.js.map +1 -1
- package/dist/esm/parquetjs/utils/buffer-utils.js +2 -1
- package/dist/esm/parquetjs/utils/buffer-utils.js.map +1 -1
- package/dist/esm/parquetjs/utils/file-utils.js +1 -0
- package/dist/esm/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/esm/parquetjs/utils/read-utils.js +5 -12
- package/dist/esm/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/esm/workers/parquet-worker.js.map +1 -1
- package/dist/lib/wasm/load-wasm/load-wasm-browser.js +5 -1
- package/dist/lib/wasm/load-wasm/load-wasm-node.js +5 -1
- package/dist/parquet-worker.js +14 -14
- package/dist/parquet-worker.js.map +2 -2
- package/dist/parquetjs/codecs/index.js +5 -1
- package/dist/parquetjs/encoder/writer.d.ts +1 -0
- package/dist/parquetjs/encoder/writer.d.ts.map +1 -1
- package/dist/parquetjs/encoder/writer.js +5 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +5 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +5 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +5 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +5 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +5 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +5 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +5 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +5 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +5 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +5 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +5 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +5 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +5 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +5 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +5 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +5 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +5 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +5 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +5 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +5 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +5 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +5 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +5 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +5 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +5 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +5 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +5 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +5 -1
- package/dist/parquetjs/parquet-thrift/index.js +5 -1
- package/dist/parquetjs/schema/shred.js +5 -1
- package/dist/parquetjs/utils/file-utils.d.ts +2 -0
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/package.json +6 -6
|
@@ -1,170 +1,125 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
|
-
exports.encodeValues = encodeValues;
|
|
9
7
|
exports.decodeValues = decodeValues;
|
|
10
|
-
|
|
8
|
+
exports.encodeValues = encodeValues;
|
|
11
9
|
var _int = _interopRequireDefault(require("int53"));
|
|
12
10
|
|
|
13
11
|
function encodeValues(type, values, opts) {
|
|
14
12
|
switch (type) {
|
|
15
13
|
case 'BOOLEAN':
|
|
16
14
|
return encodeValues_BOOLEAN(values);
|
|
17
|
-
|
|
18
15
|
case 'INT32':
|
|
19
16
|
return encodeValues_INT32(values);
|
|
20
|
-
|
|
21
17
|
case 'INT64':
|
|
22
18
|
return encodeValues_INT64(values);
|
|
23
|
-
|
|
24
19
|
case 'INT96':
|
|
25
20
|
return encodeValues_INT96(values);
|
|
26
|
-
|
|
27
21
|
case 'FLOAT':
|
|
28
22
|
return encodeValues_FLOAT(values);
|
|
29
|
-
|
|
30
23
|
case 'DOUBLE':
|
|
31
24
|
return encodeValues_DOUBLE(values);
|
|
32
|
-
|
|
33
25
|
case 'BYTE_ARRAY':
|
|
34
26
|
return encodeValues_BYTE_ARRAY(values);
|
|
35
|
-
|
|
36
27
|
case 'FIXED_LEN_BYTE_ARRAY':
|
|
37
28
|
return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
|
|
38
|
-
|
|
39
29
|
default:
|
|
40
30
|
throw new Error("unsupported type: ".concat(type));
|
|
41
31
|
}
|
|
42
32
|
}
|
|
43
|
-
|
|
44
33
|
function decodeValues(type, cursor, count, opts) {
|
|
45
34
|
switch (type) {
|
|
46
35
|
case 'BOOLEAN':
|
|
47
36
|
return decodeValues_BOOLEAN(cursor, count);
|
|
48
|
-
|
|
49
37
|
case 'INT32':
|
|
50
38
|
return decodeValues_INT32(cursor, count);
|
|
51
|
-
|
|
52
39
|
case 'INT64':
|
|
53
40
|
return decodeValues_INT64(cursor, count);
|
|
54
|
-
|
|
55
41
|
case 'INT96':
|
|
56
42
|
return decodeValues_INT96(cursor, count);
|
|
57
|
-
|
|
58
43
|
case 'FLOAT':
|
|
59
44
|
return decodeValues_FLOAT(cursor, count);
|
|
60
|
-
|
|
61
45
|
case 'DOUBLE':
|
|
62
46
|
return decodeValues_DOUBLE(cursor, count);
|
|
63
|
-
|
|
64
47
|
case 'BYTE_ARRAY':
|
|
65
48
|
return decodeValues_BYTE_ARRAY(cursor, count);
|
|
66
|
-
|
|
67
49
|
case 'FIXED_LEN_BYTE_ARRAY':
|
|
68
50
|
return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
|
|
69
|
-
|
|
70
51
|
default:
|
|
71
52
|
throw new Error("unsupported type: ".concat(type));
|
|
72
53
|
}
|
|
73
54
|
}
|
|
74
|
-
|
|
75
55
|
function encodeValues_BOOLEAN(values) {
|
|
76
56
|
var buf = Buffer.alloc(Math.ceil(values.length / 8));
|
|
77
57
|
buf.fill(0);
|
|
78
|
-
|
|
79
58
|
for (var i = 0; i < values.length; i++) {
|
|
80
59
|
if (values[i]) {
|
|
81
60
|
buf[Math.floor(i / 8)] |= 1 << i % 8;
|
|
82
61
|
}
|
|
83
62
|
}
|
|
84
|
-
|
|
85
63
|
return buf;
|
|
86
64
|
}
|
|
87
|
-
|
|
88
65
|
function decodeValues_BOOLEAN(cursor, count) {
|
|
89
66
|
var values = [];
|
|
90
|
-
|
|
91
67
|
for (var i = 0; i < count; i++) {
|
|
92
68
|
var b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
|
|
93
69
|
values.push((b & 1 << i % 8) > 0);
|
|
94
70
|
}
|
|
95
|
-
|
|
96
71
|
cursor.offset += Math.ceil(count / 8);
|
|
97
72
|
return values;
|
|
98
73
|
}
|
|
99
|
-
|
|
100
74
|
function encodeValues_INT32(values) {
|
|
101
75
|
var buf = Buffer.alloc(4 * values.length);
|
|
102
|
-
|
|
103
76
|
for (var i = 0; i < values.length; i++) {
|
|
104
77
|
buf.writeInt32LE(values[i], i * 4);
|
|
105
78
|
}
|
|
106
|
-
|
|
107
79
|
return buf;
|
|
108
80
|
}
|
|
109
|
-
|
|
110
81
|
function decodeValues_INT32(cursor, count) {
|
|
111
82
|
var values = [];
|
|
112
|
-
|
|
113
83
|
for (var i = 0; i < count; i++) {
|
|
114
84
|
values.push(cursor.buffer.readInt32LE(cursor.offset));
|
|
115
85
|
cursor.offset += 4;
|
|
116
86
|
}
|
|
117
|
-
|
|
118
87
|
return values;
|
|
119
88
|
}
|
|
120
|
-
|
|
121
89
|
function encodeValues_INT64(values) {
|
|
122
90
|
var buf = Buffer.alloc(8 * values.length);
|
|
123
|
-
|
|
124
91
|
for (var i = 0; i < values.length; i++) {
|
|
125
92
|
_int.default.writeInt64LE(values[i], buf, i * 8);
|
|
126
93
|
}
|
|
127
|
-
|
|
128
94
|
return buf;
|
|
129
95
|
}
|
|
130
|
-
|
|
131
96
|
function decodeValues_INT64(cursor, count) {
|
|
132
97
|
var values = [];
|
|
133
|
-
|
|
134
98
|
for (var i = 0; i < count; i++) {
|
|
135
99
|
values.push(_int.default.readInt64LE(cursor.buffer, cursor.offset));
|
|
136
100
|
cursor.offset += 8;
|
|
137
101
|
}
|
|
138
|
-
|
|
139
102
|
return values;
|
|
140
103
|
}
|
|
141
|
-
|
|
142
104
|
function encodeValues_INT96(values) {
|
|
143
105
|
var buf = Buffer.alloc(12 * values.length);
|
|
144
|
-
|
|
145
106
|
for (var i = 0; i < values.length; i++) {
|
|
146
107
|
if (values[i] >= 0) {
|
|
147
108
|
_int.default.writeInt64LE(values[i], buf, i * 12);
|
|
148
|
-
|
|
149
109
|
buf.writeUInt32LE(0, i * 12 + 8);
|
|
150
110
|
} else {
|
|
151
111
|
_int.default.writeInt64LE(~-values[i] + 1, buf, i * 12);
|
|
152
|
-
|
|
153
112
|
buf.writeUInt32LE(0xffffffff, i * 12 + 8);
|
|
154
113
|
}
|
|
155
114
|
}
|
|
156
115
|
|
|
157
116
|
return buf;
|
|
158
117
|
}
|
|
159
|
-
|
|
160
118
|
function decodeValues_INT96(cursor, count) {
|
|
161
119
|
var values = [];
|
|
162
|
-
|
|
163
120
|
for (var i = 0; i < count; i++) {
|
|
164
121
|
var low = _int.default.readInt64LE(cursor.buffer, cursor.offset);
|
|
165
|
-
|
|
166
122
|
var high = cursor.buffer.readUInt32LE(cursor.offset + 8);
|
|
167
|
-
|
|
168
123
|
if (high === 0xffffffff) {
|
|
169
124
|
values.push(~-low + 1);
|
|
170
125
|
} else {
|
|
@@ -173,115 +128,84 @@ function decodeValues_INT96(cursor, count) {
|
|
|
173
128
|
|
|
174
129
|
cursor.offset += 12;
|
|
175
130
|
}
|
|
176
|
-
|
|
177
131
|
return values;
|
|
178
132
|
}
|
|
179
|
-
|
|
180
133
|
function encodeValues_FLOAT(values) {
|
|
181
134
|
var buf = Buffer.alloc(4 * values.length);
|
|
182
|
-
|
|
183
135
|
for (var i = 0; i < values.length; i++) {
|
|
184
136
|
buf.writeFloatLE(values[i], i * 4);
|
|
185
137
|
}
|
|
186
|
-
|
|
187
138
|
return buf;
|
|
188
139
|
}
|
|
189
|
-
|
|
190
140
|
function decodeValues_FLOAT(cursor, count) {
|
|
191
141
|
var values = [];
|
|
192
|
-
|
|
193
142
|
for (var i = 0; i < count; i++) {
|
|
194
143
|
values.push(cursor.buffer.readFloatLE(cursor.offset));
|
|
195
144
|
cursor.offset += 4;
|
|
196
145
|
}
|
|
197
|
-
|
|
198
146
|
return values;
|
|
199
147
|
}
|
|
200
|
-
|
|
201
148
|
function encodeValues_DOUBLE(values) {
|
|
202
149
|
var buf = Buffer.alloc(8 * values.length);
|
|
203
|
-
|
|
204
150
|
for (var i = 0; i < values.length; i++) {
|
|
205
151
|
buf.writeDoubleLE(values[i], i * 8);
|
|
206
152
|
}
|
|
207
|
-
|
|
208
153
|
return buf;
|
|
209
154
|
}
|
|
210
|
-
|
|
211
155
|
function decodeValues_DOUBLE(cursor, count) {
|
|
212
156
|
var values = [];
|
|
213
|
-
|
|
214
157
|
for (var i = 0; i < count; i++) {
|
|
215
158
|
values.push(cursor.buffer.readDoubleLE(cursor.offset));
|
|
216
159
|
cursor.offset += 8;
|
|
217
160
|
}
|
|
218
|
-
|
|
219
161
|
return values;
|
|
220
162
|
}
|
|
221
|
-
|
|
222
163
|
function encodeValues_BYTE_ARRAY(values) {
|
|
223
164
|
var buf_len = 0;
|
|
224
|
-
|
|
225
165
|
for (var i = 0; i < values.length; i++) {
|
|
226
166
|
values[i] = Buffer.from(values[i]);
|
|
227
167
|
buf_len += 4 + values[i].length;
|
|
228
168
|
}
|
|
229
|
-
|
|
230
169
|
var buf = Buffer.alloc(buf_len);
|
|
231
170
|
var buf_pos = 0;
|
|
232
|
-
|
|
233
171
|
for (var _i = 0; _i < values.length; _i++) {
|
|
234
172
|
buf.writeUInt32LE(values[_i].length, buf_pos);
|
|
235
|
-
|
|
236
173
|
values[_i].copy(buf, buf_pos + 4);
|
|
237
|
-
|
|
238
174
|
buf_pos += 4 + values[_i].length;
|
|
239
175
|
}
|
|
240
|
-
|
|
241
176
|
return buf;
|
|
242
177
|
}
|
|
243
|
-
|
|
244
178
|
function decodeValues_BYTE_ARRAY(cursor, count) {
|
|
245
179
|
var values = [];
|
|
246
|
-
|
|
247
180
|
for (var i = 0; i < count; i++) {
|
|
248
181
|
var len = cursor.buffer.readUInt32LE(cursor.offset);
|
|
249
182
|
cursor.offset += 4;
|
|
250
183
|
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
|
|
251
184
|
cursor.offset += len;
|
|
252
185
|
}
|
|
253
|
-
|
|
254
186
|
return values;
|
|
255
187
|
}
|
|
256
|
-
|
|
257
188
|
function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
|
|
258
189
|
if (!opts.typeLength) {
|
|
259
190
|
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
|
|
260
191
|
}
|
|
261
|
-
|
|
262
192
|
for (var i = 0; i < values.length; i++) {
|
|
263
193
|
values[i] = Buffer.from(values[i]);
|
|
264
|
-
|
|
265
194
|
if (values[i].length !== opts.typeLength) {
|
|
266
195
|
throw new Error("invalid value for FIXED_LEN_BYTE_ARRAY: ".concat(values[i]));
|
|
267
196
|
}
|
|
268
197
|
}
|
|
269
|
-
|
|
270
198
|
return Buffer.concat(values);
|
|
271
199
|
}
|
|
272
|
-
|
|
273
200
|
function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
|
|
274
201
|
var values = [];
|
|
275
|
-
|
|
276
202
|
if (!opts.typeLength) {
|
|
277
203
|
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
|
|
278
204
|
}
|
|
279
|
-
|
|
280
205
|
for (var i = 0; i < count; i++) {
|
|
281
206
|
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
|
|
282
207
|
cursor.offset += opts.typeLength;
|
|
283
208
|
}
|
|
284
|
-
|
|
285
209
|
return values;
|
|
286
210
|
}
|
|
287
211
|
//# sourceMappingURL=plain.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/plain.ts"],"names":["encodeValues","type","values","opts","encodeValues_BOOLEAN","encodeValues_INT32","encodeValues_INT64","encodeValues_INT96","encodeValues_FLOAT","encodeValues_DOUBLE","encodeValues_BYTE_ARRAY","encodeValues_FIXED_LEN_BYTE_ARRAY","Error","decodeValues","cursor","count","decodeValues_BOOLEAN","decodeValues_INT32","decodeValues_INT64","decodeValues_INT96","decodeValues_FLOAT","decodeValues_DOUBLE","decodeValues_BYTE_ARRAY","decodeValues_FIXED_LEN_BYTE_ARRAY","buf","Buffer","alloc","Math","ceil","length","fill","i","floor","b","buffer","offset","push","writeInt32LE","readInt32LE","INT53","writeInt64LE","readInt64LE","writeUInt32LE","low","high","readUInt32LE","writeFloatLE","readFloatLE","writeDoubleLE","readDoubleLE","buf_len","from","buf_pos","copy","len","slice","typeLength","concat"],"mappings":";;;;;;;;;;AAIA;;AAEO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOG,oBAAoB,CAACF,MAAD,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACH,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACJ,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACL,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOM,kBAAkB,CAACN,MAAD,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOO,mBAAmB,CAACP,MAAD,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOQ,uBAAuB,CAACR,MAAD,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOS,iCAAiC,CAACT,MAAD,EAASC,IAAT,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAEM,SAASY,YAAT,CACLZ,IADK,EAELa,MAFK,EAGLC,KAHK,EAILZ,IAJK,EAKE;AACP,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOe,oBAAoB,CAACF,MAAD,EAASC,KAAT,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOE,kBAAkB,CAACH,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACJ,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACL,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACN,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOM,mBAAmB,CAACP,MAAD,EAASC,KAAT,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOO,uBAAuB,CAACR,MAAD,EAASC,KAAT,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOQ,iCAAiC,CAACT,MAAD,EAASC,KAAT,EAAgBZ,IAAhB,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAED,SAASG,oBAAT,CAA8BF,MAA9B,EAAyD;AACvD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAaC,IAAI,CAACC,IAAL,CAAU1B,MAAM,CAAC2B,MAAP,GAAgB,CAA1B,CAAb,CAAZ;AACAL,EAAAA,GAAG,CAACM,IAAJ,CAAS,CAAT;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAV,EAAe;AACbP,MAAAA,GAAG,CAACG,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASR,oBAAT,CAA8BF,MAA9B,EAAoDC,KAApD,EAA8E;AAC5E,MAAMb,MAAiB,GAAG,EAA1B;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAME,CAAC,GAAGnB,MAAM,CAACoB,MAAP,CAAcpB,MAAM,CAACqB,MAAP,GAAgBR,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAA9B,CAAV;AACA7B,IAAAA,MAAM,CAACkC,IAAP,CAAY,CAACH,CAAC,GAAI,KAAKF,CAAC,GAAG,CAAf,IAAqB,CAAjC;AACD;;AACDjB,EAAAA,MAAM,CAACqB,MAAP,IAAiBR,IAAI,CAACC,IAAL,CAAUb,KAAK,GAAG,CAAlB,CAAjB;AACA,SAAOb,MAAP;AACD;;AAED,SAASG,kBAAT,CAA4BH,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACa,YAAJ,CAAiBnC,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASP,kBAAT,CAA4BH,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcI,WAAd,CAA0BxB,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASI,kBAAT,CAA4BJ,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCQ,iBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,CAAvC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASN,kBAAT,CAA4BJ,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYG,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASK,kBAAT,CAA4BL,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,KAAKxB,MAAM,CAAC2B,MAAzB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,IAAa,CAAjB,EAAoB;AAClBQ,mBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,EAAvC;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,CAAlB,EAAqBX,CAAC,GAAG,EAAJ,GAAS,CAA9B;AACD,KAHD,MAGO;AACLQ,mBAAMC,YAAN,CAAmB,CAAC,CAACtC,MAAM,CAAC6B,CAAD,CAAR,GAAc,CAAjC,EAAoCP,GAApC,EAAyCO,CAAC,GAAG,EAA7C;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,UAAlB,EAA8BX,CAAC,GAAG,EAAJ,GAAS,CAAvC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASL,kBAAT,CAA4BL,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAMY,GAAG,GAAGJ,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;;AACA,QAAMS,IAAI,GAAG9B,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAP,GAAgB,CAA3C,CAAb;;AACA,QAAIS,IAAI,KAAK,UAAb,EAAyB;AACvB1C,MAAAA,MAAM,CAACkC,IAAP,CAAY,CAAC,CAACO,GAAF,GAAQ,CAApB;AACD,KAFD,MAEO;AACLzC,MAAAA,MAAM,CAACkC,IAAP,CAAYO,GAAZ;AACD;;AACD7B,IAAAA,MAAM,CAACqB,MAAP,IAAiB,EAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASM,kBAAT,CAA4BN,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACsB,YAAJ,CAAiB5C,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASJ,kBAAT,CAA4BN,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAca,WAAd,CAA0BjC,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASO,mBAAT,CAA6BP,MAA7B,EAAuD;AACrD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACwB,aAAJ,CAAkB9C,MAAM,CAAC6B,CAAD,CAAxB,EAA6BA,CAAC,GAAG,CAAjC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASH,mBAAT,CAA6BP,MAA7B,EAAmDC,KAAnD,EAA4E;AAC1E,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAce,YAAd,CAA2BnC,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASQ,uBAAT,CAAiCR,MAAjC,EAA2D;AAEzD,MAAIgD,OAAO,GAAG,CAAd;;AACA,OAAK,IAAInB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;AACAmB,IAAAA,OAAO,IAAI,IAAIhD,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,MAAML,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAawB,OAAb,CAAZ;AAEA,MAAIE,OAAO,GAAG,CAAd;;AACA,OAAK,IAAIrB,EAAC,GAAG,CAAb,EAAgBA,EAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,EAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACkB,aAAJ,CAAkBxC,MAAM,CAAC6B,EAAD,CAAN,CAAUF,MAA5B,EAAoCuB,OAApC;;AACAlD,IAAAA,MAAM,CAAC6B,EAAD,CAAN,CAAUsB,IAAV,CAAe7B,GAAf,EAAoB4B,OAAO,GAAG,CAA9B;;AACAA,IAAAA,OAAO,IAAI,IAAIlD,MAAM,CAAC6B,EAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,SAAOL,GAAP;AACD;;AAED,SAASF,uBAAT,CAAiCR,MAAjC,EAAuDC,KAAvD,EAAgF;AAC9E,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAMuB,GAAG,GAAGxC,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACAjC,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBmB,GAAnD,CAAZ;AACAxC,IAAAA,MAAM,CAACqB,MAAP,IAAiBmB,GAAjB;AACD;;AACD,SAAOpD,MAAP;AACD;;AAED,SAASS,iCAAT,CAA2CT,MAA3C,EAA6DC,IAA7D,EAAgG;AAC9F,MAAI,CAACA,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;;AACA,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAV,KAAqB1B,IAAI,CAACqD,UAA9B,EAA0C;AACxC,YAAM,IAAI5C,KAAJ,mDAAqDV,MAAM,CAAC6B,CAAD,CAA3D,EAAN;AACD;AACF;;AACD,SAAON,MAAM,CAACgC,MAAP,CAAcvD,MAAd,CAAP;AACD;;AAED,SAASqB,iCAAT,CACET,MADF,EAEEC,KAFF,EAGEZ,IAHF,EAIY;AACV,MAAMD,MAAgB,GAAG,EAAzB;;AACA,MAAI,CAACC,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBhC,IAAI,CAACqD,UAAxD,CAAZ;AACA1C,IAAAA,MAAM,CAACqB,MAAP,IAAiBhC,IAAI,CAACqD,UAAtB;AACD;;AACD,SAAOtD,MAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport INT53 from 'int53';\n\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n switch (type) {\n case 'BOOLEAN':\n return encodeValues_BOOLEAN(values);\n case 'INT32':\n return encodeValues_INT32(values);\n case 'INT64':\n return encodeValues_INT64(values);\n case 'INT96':\n return encodeValues_INT96(values);\n case 'FLOAT':\n return encodeValues_FLOAT(values);\n case 'DOUBLE':\n return encodeValues_DOUBLE(values);\n case 'BYTE_ARRAY':\n return encodeValues_BYTE_ARRAY(values);\n case 'FIXED_LEN_BYTE_ARRAY':\n return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n switch (type) {\n case 'BOOLEAN':\n return decodeValues_BOOLEAN(cursor, count);\n case 'INT32':\n return decodeValues_INT32(cursor, count);\n case 'INT64':\n return decodeValues_INT64(cursor, count);\n case 'INT96':\n return decodeValues_INT96(cursor, count);\n case 'FLOAT':\n return decodeValues_FLOAT(cursor, count);\n case 'DOUBLE':\n return decodeValues_DOUBLE(cursor, count);\n case 'BYTE_ARRAY':\n return decodeValues_BYTE_ARRAY(cursor, count);\n case 'FIXED_LEN_BYTE_ARRAY':\n return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nfunction encodeValues_BOOLEAN(values: boolean[]): Buffer {\n const buf = Buffer.alloc(Math.ceil(values.length / 8));\n buf.fill(0);\n for (let i = 0; i < values.length; i++) {\n if (values[i]) {\n buf[Math.floor(i / 8)] |= 1 << i % 8;\n }\n }\n return buf;\n}\n\nfunction decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {\n const values: boolean[] = [];\n for (let i = 0; i < count; i++) {\n const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];\n values.push((b & (1 << i % 8)) > 0);\n }\n cursor.offset += Math.ceil(count / 8);\n return values;\n}\n\nfunction encodeValues_INT32(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeInt32LE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_INT32(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readInt32LE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_INT64(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n INT53.writeInt64LE(values[i], buf, i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_INT64(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_INT96(values: number[]): Buffer {\n const buf = Buffer.alloc(12 * values.length);\n for (let i = 0; i < values.length; i++) {\n if (values[i] >= 0) {\n INT53.writeInt64LE(values[i], buf, i * 12);\n buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision\n } else {\n INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);\n buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision\n }\n }\n return buf;\n}\n\nfunction decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n const low = INT53.readInt64LE(cursor.buffer, cursor.offset);\n const high = cursor.buffer.readUInt32LE(cursor.offset + 8);\n if (high === 0xffffffff) {\n values.push(~-low + 1); // truncate to 64 actual precision\n } else {\n values.push(low); // truncate to 64 actual precision\n }\n cursor.offset += 12;\n }\n return values;\n}\n\nfunction encodeValues_FLOAT(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeFloatLE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_FLOAT(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readFloatLE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_DOUBLE(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeDoubleLE(values[i], i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_DOUBLE(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readDoubleLE(cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {\n // tslint:disable-next-line:variable-name\n let buf_len = 0;\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n buf_len += 4 + values[i].length;\n }\n const buf = Buffer.alloc(buf_len);\n // tslint:disable-next-line:variable-name\n let buf_pos = 0;\n for (let i = 0; i < values.length; i++) {\n buf.writeUInt32LE(values[i].length, buf_pos);\n values[i].copy(buf, buf_pos + 4);\n buf_pos += 4 + values[i].length;\n }\n return buf;\n}\n\nfunction decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[] {\n const values: Buffer[] = [];\n for (let i = 0; i < count; i++) {\n const len = cursor.buffer.readUInt32LE(cursor.offset);\n cursor.offset += 4;\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));\n cursor.offset += len;\n }\n return values;\n}\n\nfunction encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecOptions): Buffer {\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n if (values[i].length !== opts.typeLength) {\n throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);\n }\n }\n return Buffer.concat(values);\n}\n\nfunction decodeValues_FIXED_LEN_BYTE_ARRAY(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): Buffer[] {\n const values: Buffer[] = [];\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));\n cursor.offset += opts.typeLength;\n }\n return values;\n}\n"],"file":"plain.js"}
|
|
1
|
+
{"version":3,"file":"plain.js","names":["encodeValues","type","values","opts","encodeValues_BOOLEAN","encodeValues_INT32","encodeValues_INT64","encodeValues_INT96","encodeValues_FLOAT","encodeValues_DOUBLE","encodeValues_BYTE_ARRAY","encodeValues_FIXED_LEN_BYTE_ARRAY","Error","decodeValues","cursor","count","decodeValues_BOOLEAN","decodeValues_INT32","decodeValues_INT64","decodeValues_INT96","decodeValues_FLOAT","decodeValues_DOUBLE","decodeValues_BYTE_ARRAY","decodeValues_FIXED_LEN_BYTE_ARRAY","buf","Buffer","alloc","Math","ceil","length","fill","i","floor","b","buffer","offset","push","writeInt32LE","readInt32LE","INT53","writeInt64LE","readInt64LE","writeUInt32LE","low","high","readUInt32LE","writeFloatLE","readFloatLE","writeDoubleLE","readDoubleLE","buf_len","from","buf_pos","copy","len","slice","typeLength","concat"],"sources":["../../../../src/parquetjs/codecs/plain.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport INT53 from 'int53';\n\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n switch (type) {\n case 'BOOLEAN':\n return encodeValues_BOOLEAN(values);\n case 'INT32':\n return encodeValues_INT32(values);\n case 'INT64':\n return encodeValues_INT64(values);\n case 'INT96':\n return encodeValues_INT96(values);\n case 'FLOAT':\n return encodeValues_FLOAT(values);\n case 'DOUBLE':\n return encodeValues_DOUBLE(values);\n case 'BYTE_ARRAY':\n return encodeValues_BYTE_ARRAY(values);\n case 'FIXED_LEN_BYTE_ARRAY':\n return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n switch (type) {\n case 'BOOLEAN':\n return decodeValues_BOOLEAN(cursor, count);\n case 'INT32':\n return decodeValues_INT32(cursor, count);\n case 'INT64':\n return decodeValues_INT64(cursor, count);\n case 'INT96':\n return decodeValues_INT96(cursor, count);\n case 'FLOAT':\n return decodeValues_FLOAT(cursor, count);\n case 'DOUBLE':\n return decodeValues_DOUBLE(cursor, count);\n case 'BYTE_ARRAY':\n return decodeValues_BYTE_ARRAY(cursor, count);\n case 'FIXED_LEN_BYTE_ARRAY':\n return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nfunction encodeValues_BOOLEAN(values: boolean[]): Buffer {\n const buf = Buffer.alloc(Math.ceil(values.length / 8));\n buf.fill(0);\n for (let i = 0; i < values.length; i++) {\n if (values[i]) {\n buf[Math.floor(i / 8)] |= 1 << i % 8;\n }\n }\n return buf;\n}\n\nfunction decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {\n const values: boolean[] = [];\n for (let i = 0; i < count; i++) {\n const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];\n values.push((b & (1 << i % 8)) > 0);\n }\n cursor.offset += Math.ceil(count / 8);\n return values;\n}\n\nfunction encodeValues_INT32(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeInt32LE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_INT32(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readInt32LE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_INT64(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n INT53.writeInt64LE(values[i], buf, i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_INT64(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_INT96(values: number[]): Buffer {\n const buf = Buffer.alloc(12 * values.length);\n for (let i = 0; i < values.length; i++) {\n if (values[i] >= 0) {\n INT53.writeInt64LE(values[i], buf, i * 12);\n buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision\n } else {\n INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);\n buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision\n }\n }\n return buf;\n}\n\nfunction decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n const low = INT53.readInt64LE(cursor.buffer, cursor.offset);\n const high = cursor.buffer.readUInt32LE(cursor.offset + 8);\n if (high === 0xffffffff) {\n values.push(~-low + 1); // truncate to 64 actual precision\n } else {\n values.push(low); // truncate to 64 actual precision\n }\n cursor.offset += 12;\n }\n return values;\n}\n\nfunction encodeValues_FLOAT(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeFloatLE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_FLOAT(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readFloatLE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_DOUBLE(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeDoubleLE(values[i], i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_DOUBLE(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readDoubleLE(cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {\n // tslint:disable-next-line:variable-name\n let buf_len = 0;\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n buf_len += 4 + values[i].length;\n }\n const buf = Buffer.alloc(buf_len);\n // tslint:disable-next-line:variable-name\n let buf_pos = 0;\n for (let i = 0; i < values.length; i++) {\n buf.writeUInt32LE(values[i].length, buf_pos);\n values[i].copy(buf, buf_pos + 4);\n buf_pos += 4 + values[i].length;\n }\n return buf;\n}\n\nfunction decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[] {\n const values: Buffer[] = [];\n for (let i = 0; i < count; i++) {\n const len = cursor.buffer.readUInt32LE(cursor.offset);\n cursor.offset += 4;\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));\n cursor.offset += len;\n }\n return values;\n}\n\nfunction encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecOptions): Buffer {\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n if (values[i].length !== opts.typeLength) {\n throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);\n }\n }\n return Buffer.concat(values);\n}\n\nfunction decodeValues_FIXED_LEN_BYTE_ARRAY(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): Buffer[] {\n const values: Buffer[] = [];\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));\n cursor.offset += opts.typeLength;\n }\n return values;\n}\n"],"mappings":";;;;;;;;AAIA;;AAEO,SAASA,YAAY,CAC1BC,IAAmB,EACnBC,MAAa,EACbC,IAAyB,EACjB;EACR,QAAQF,IAAI;IACV,KAAK,SAAS;MACZ,OAAOG,oBAAoB,CAACF,MAAM,CAAC;IACrC,KAAK,OAAO;MACV,OAAOG,kBAAkB,CAACH,MAAM,CAAC;IACnC,KAAK,OAAO;MACV,OAAOI,kBAAkB,CAACJ,MAAM,CAAC;IACnC,KAAK,OAAO;MACV,OAAOK,kBAAkB,CAACL,MAAM,CAAC;IACnC,KAAK,OAAO;MACV,OAAOM,kBAAkB,CAACN,MAAM,CAAC;IACnC,KAAK,QAAQ;MACX,OAAOO,mBAAmB,CAACP,MAAM,CAAC;IACpC,KAAK,YAAY;MACf,OAAOQ,uBAAuB,CAACR,MAAM,CAAC;IACxC,KAAK,sBAAsB;MACzB,OAAOS,iCAAiC,CAACT,MAAM,EAAEC,IAAI,CAAC;IACxD;MACE,MAAM,IAAIS,KAAK,6BAAsBX,IAAI,EAAG;EAAC;AAEnD;AAEO,SAASY,YAAY,CAC1BZ,IAAmB,EACnBa,MAAoB,EACpBC,KAAa,EACbZ,IAAyB,EAClB;EACP,QAAQF,IAAI;IACV,KAAK,SAAS;MACZ,OAAOe,oBAAoB,CAACF,MAAM,EAAEC,KAAK,CAAC;IAC5C,KAAK,OAAO;MACV,OAAOE,kBAAkB,CAACH,MAAM,EAAEC,KAAK,CAAC;IAC1C,KAAK,OAAO;MACV,OAAOG,kBAAkB,CAACJ,MAAM,EAAEC,KAAK,CAAC;IAC1C,KAAK,OAAO;MACV,OAAOI,kBAAkB,CAACL,MAAM,EAAEC,KAAK,CAAC;IAC1C,KAAK,OAAO;MACV,OAAOK,kBAAkB,CAACN,MAAM,EAAEC,KAAK,CAAC;IAC1C,KAAK,QAAQ;MACX,OAAOM,mBAAmB,CAACP,MAAM,EAAEC,KAAK,CAAC;IAC3C,KAAK,YAAY;MACf,OAAOO,uBAAuB,CAACR,MAAM,EAAEC,KAAK,CAAC;IAC/C,KAAK,sBAAsB;MACzB,OAAOQ,iCAAiC,CAACT,MAAM,EAAEC,KAAK,EAAEZ,IAAI,CAAC;IAC/D;MACE,MAAM,IAAIS,KAAK,6BAAsBX,IAAI,EAAG;EAAC;AAEnD;AAEA,SAASG,oBAAoB,CAACF,MAAiB,EAAU;EACvD,IAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAK,CAACC,IAAI,CAACC,IAAI,CAAC1B,MAAM,CAAC2B,MAAM,GAAG,CAAC,CAAC,CAAC;EACtDL,GAAG,CAACM,IAAI,CAAC,CAAC,CAAC;EACX,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtC,IAAI7B,MAAM,CAAC6B,CAAC,CAAC,EAAE;MACbP,GAAG,CAACG,IAAI,CAACK,KAAK,CAACD,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAIA,CAAC,GAAG,CAAC;IACtC;EACF;EACA,OAAOP,GAAG;AACZ;AAEA,SAASR,oBAAoB,CAACF,MAAoB,EAAEC,KAAa,EAAa;EAC5E,IAAMb,MAAiB,GAAG,EAAE;EAC5B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B,IAAME,CAAC,GAAGnB,MAAM,CAACoB,MAAM,CAACpB,MAAM,CAACqB,MAAM,GAAGR,IAAI,CAACK,KAAK,CAACD,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D7B,MAAM,CAACkC,IAAI,CAAC,CAACH,CAAC,GAAI,CAAC,IAAIF,CAAC,GAAG,CAAE,IAAI,CAAC,CAAC;EACrC;EACAjB,MAAM,CAACqB,MAAM,IAAIR,IAAI,CAACC,IAAI,CAACb,KAAK,GAAG,CAAC,CAAC;EACrC,OAAOb,MAAM;AACf;AAEA,SAASG,kBAAkB,CAACH,MAAgB,EAAU;EACpD,IAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,GAAGxB,MAAM,CAAC2B,MAAM,CAAC;EAC3C,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtCP,GAAG,CAACa,YAAY,CAACnC,MAAM,CAAC6B,CAAC,CAAC,EAAEA,CAAC,GAAG,CAAC,CAAC;EACpC;EACA,OAAOP,GAAG;AACZ;AAEA,SAASP,kBAAkB,CAACH,MAAoB,EAAEC,KAAa,EAAY;EACzE,IAAMb,MAAgB,GAAG,EAAE;EAC3B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B7B,MAAM,CAACkC,IAAI,CAACtB,MAAM,CAACoB,MAAM,CAACI,WAAW,CAACxB,MAAM,CAACqB,MAAM,CAAC,CAAC;IACrDrB,MAAM,CAACqB,MAAM,IAAI,CAAC;EACpB;EACA,OAAOjC,MAAM;AACf;AAEA,SAASI,kBAAkB,CAACJ,MAAgB,EAAU;EACpD,IAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,GAAGxB,MAAM,CAAC2B,MAAM,CAAC;EAC3C,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtCQ,YAAK,CAACC,YAAY,CAACtC,MAAM,CAAC6B,CAAC,CAAC,EAAEP,GAAG,EAAEO,CAAC,GAAG,CAAC,CAAC;EAC3C;EACA,OAAOP,GAAG;AACZ;AAEA,SAASN,kBAAkB,CAACJ,MAAoB,EAAEC,KAAa,EAAY;EACzE,IAAMb,MAAgB,GAAG,EAAE;EAC3B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B7B,MAAM,CAACkC,IAAI,CAACG,YAAK,CAACE,WAAW,CAAC3B,MAAM,CAACoB,MAAM,EAAEpB,MAAM,CAACqB,MAAM,CAAC,CAAC;IAC5DrB,MAAM,CAACqB,MAAM,IAAI,CAAC;EACpB;EACA,OAAOjC,MAAM;AACf;AAEA,SAASK,kBAAkB,CAACL,MAAgB,EAAU;EACpD,IAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,EAAE,GAAGxB,MAAM,CAAC2B,MAAM,CAAC;EAC5C,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtC,IAAI7B,MAAM,CAAC6B,CAAC,CAAC,IAAI,CAAC,EAAE;MAClBQ,YAAK,CAACC,YAAY,CAACtC,MAAM,CAAC6B,CAAC,CAAC,EAAEP,GAAG,EAAEO,CAAC,GAAG,EAAE,CAAC;MAC1CP,GAAG,CAACkB,aAAa,CAAC,CAAC,EAAEX,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAClC,CAAC,MAAM;MACLQ,YAAK,CAACC,YAAY,CAAC,CAAC,CAACtC,MAAM,CAAC6B,CAAC,CAAC,GAAG,CAAC,EAAEP,GAAG,EAAEO,CAAC,GAAG,EAAE,CAAC;MAChDP,GAAG,CAACkB,aAAa,CAAC,UAAU,EAAEX,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC3C;EACF;;EACA,OAAOP,GAAG;AACZ;AAEA,SAASL,kBAAkB,CAACL,MAAoB,EAAEC,KAAa,EAAY;EACzE,IAAMb,MAAgB,GAAG,EAAE;EAC3B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B,IAAMY,GAAG,GAAGJ,YAAK,CAACE,WAAW,CAAC3B,MAAM,CAACoB,MAAM,EAAEpB,MAAM,CAACqB,MAAM,CAAC;IAC3D,IAAMS,IAAI,GAAG9B,MAAM,CAACoB,MAAM,CAACW,YAAY,CAAC/B,MAAM,CAACqB,MAAM,GAAG,CAAC,CAAC;IAC1D,IAAIS,IAAI,KAAK,UAAU,EAAE;MACvB1C,MAAM,CAACkC,IAAI,CAAC,CAAC,CAACO,GAAG,GAAG,CAAC,CAAC;IACxB,CAAC,MAAM;MACLzC,MAAM,CAACkC,IAAI,CAACO,GAAG,CAAC;IAClB;;IACA7B,MAAM,CAACqB,MAAM,IAAI,EAAE;EACrB;EACA,OAAOjC,MAAM;AACf;AAEA,SAASM,kBAAkB,CAACN,MAAgB,EAAU;EACpD,IAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,GAAGxB,MAAM,CAAC2B,MAAM,CAAC;EAC3C,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtCP,GAAG,CAACsB,YAAY,CAAC5C,MAAM,CAAC6B,CAAC,CAAC,EAAEA,CAAC,GAAG,CAAC,CAAC;EACpC;EACA,OAAOP,GAAG;AACZ;AAEA,SAASJ,kBAAkB,CAACN,MAAoB,EAAEC,KAAa,EAAY;EACzE,IAAMb,MAAgB,GAAG,EAAE;EAC3B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B7B,MAAM,CAACkC,IAAI,CAACtB,MAAM,CAACoB,MAAM,CAACa,WAAW,CAACjC,MAAM,CAACqB,MAAM,CAAC,CAAC;IACrDrB,MAAM,CAACqB,MAAM,IAAI,CAAC;EACpB;EACA,OAAOjC,MAAM;AACf;AAEA,SAASO,mBAAmB,CAACP,MAAgB,EAAU;EACrD,IAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,GAAGxB,MAAM,CAAC2B,MAAM,CAAC;EAC3C,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtCP,GAAG,CAACwB,aAAa,CAAC9C,MAAM,CAAC6B,CAAC,CAAC,EAAEA,CAAC,GAAG,CAAC,CAAC;EACrC;EACA,OAAOP,GAAG;AACZ;AAEA,SAASH,mBAAmB,CAACP,MAAoB,EAAEC,KAAa,EAAY;EAC1E,IAAMb,MAAgB,GAAG,EAAE;EAC3B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B7B,MAAM,CAACkC,IAAI,CAACtB,MAAM,CAACoB,MAAM,CAACe,YAAY,CAACnC,MAAM,CAACqB,MAAM,CAAC,CAAC;IACtDrB,MAAM,CAACqB,MAAM,IAAI,CAAC;EACpB;EACA,OAAOjC,MAAM;AACf;AAEA,SAASQ,uBAAuB,CAACR,MAAgB,EAAU;EAEzD,IAAIgD,OAAO,GAAG,CAAC;EACf,KAAK,IAAInB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtC7B,MAAM,CAAC6B,CAAC,CAAC,GAAGN,MAAM,CAAC0B,IAAI,CAACjD,MAAM,CAAC6B,CAAC,CAAC,CAAC;IAClCmB,OAAO,IAAI,CAAC,GAAGhD,MAAM,CAAC6B,CAAC,CAAC,CAACF,MAAM;EACjC;EACA,IAAML,GAAG,GAAGC,MAAM,CAACC,KAAK,CAACwB,OAAO,CAAC;EAEjC,IAAIE,OAAO,GAAG,CAAC;EACf,KAAK,IAAIrB,EAAC,GAAG,CAAC,EAAEA,EAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,EAAC,EAAE,EAAE;IACtCP,GAAG,CAACkB,aAAa,CAACxC,MAAM,CAAC6B,EAAC,CAAC,CAACF,MAAM,EAAEuB,OAAO,CAAC;IAC5ClD,MAAM,CAAC6B,EAAC,CAAC,CAACsB,IAAI,CAAC7B,GAAG,EAAE4B,OAAO,GAAG,CAAC,CAAC;IAChCA,OAAO,IAAI,CAAC,GAAGlD,MAAM,CAAC6B,EAAC,CAAC,CAACF,MAAM;EACjC;EACA,OAAOL,GAAG;AACZ;AAEA,SAASF,uBAAuB,CAACR,MAAoB,EAAEC,KAAa,EAAY;EAC9E,IAAMb,MAAgB,GAAG,EAAE;EAC3B,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B,IAAMuB,GAAG,GAAGxC,MAAM,CAACoB,MAAM,CAACW,YAAY,CAAC/B,MAAM,CAACqB,MAAM,CAAC;IACrDrB,MAAM,CAACqB,MAAM,IAAI,CAAC;IAClBjC,MAAM,CAACkC,IAAI,CAACtB,MAAM,CAACoB,MAAM,CAACqB,KAAK,CAACzC,MAAM,CAACqB,MAAM,EAAErB,MAAM,CAACqB,MAAM,GAAGmB,GAAG,CAAC,CAAC;IACpExC,MAAM,CAACqB,MAAM,IAAImB,GAAG;EACtB;EACA,OAAOpD,MAAM;AACf;AAEA,SAASS,iCAAiC,CAACT,MAAgB,EAAEC,IAAyB,EAAU;EAC9F,IAAI,CAACA,IAAI,CAACqD,UAAU,EAAE;IACpB,MAAM,IAAI5C,KAAK,CAAC,gEAAgE,CAAC;EACnF;EACA,KAAK,IAAImB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG7B,MAAM,CAAC2B,MAAM,EAAEE,CAAC,EAAE,EAAE;IACtC7B,MAAM,CAAC6B,CAAC,CAAC,GAAGN,MAAM,CAAC0B,IAAI,CAACjD,MAAM,CAAC6B,CAAC,CAAC,CAAC;IAClC,IAAI7B,MAAM,CAAC6B,CAAC,CAAC,CAACF,MAAM,KAAK1B,IAAI,CAACqD,UAAU,EAAE;MACxC,MAAM,IAAI5C,KAAK,mDAA4CV,MAAM,CAAC6B,CAAC,CAAC,EAAG;IACzE;EACF;EACA,OAAON,MAAM,CAACgC,MAAM,CAACvD,MAAM,CAAC;AAC9B;AAEA,SAASqB,iCAAiC,CACxCT,MAAoB,EACpBC,KAAa,EACbZ,IAAyB,EACf;EACV,IAAMD,MAAgB,GAAG,EAAE;EAC3B,IAAI,CAACC,IAAI,CAACqD,UAAU,EAAE;IACpB,MAAM,IAAI5C,KAAK,CAAC,gEAAgE,CAAC;EACnF;EACA,KAAK,IAAImB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGhB,KAAK,EAAEgB,CAAC,EAAE,EAAE;IAC9B7B,MAAM,CAACkC,IAAI,CAACtB,MAAM,CAACoB,MAAM,CAACqB,KAAK,CAACzC,MAAM,CAACqB,MAAM,EAAErB,MAAM,CAACqB,MAAM,GAAGhC,IAAI,CAACqD,UAAU,CAAC,CAAC;IAChF1C,MAAM,CAACqB,MAAM,IAAIhC,IAAI,CAACqD,UAAU;EAClC;EACA,OAAOtD,MAAM;AACf"}
|
|
@@ -1,22 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
|
-
exports.encodeValues = encodeValues;
|
|
9
7
|
exports.decodeValues = decodeValues;
|
|
10
|
-
|
|
8
|
+
exports.encodeValues = encodeValues;
|
|
11
9
|
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
|
|
12
|
-
|
|
13
10
|
var _varint = _interopRequireDefault(require("varint"));
|
|
14
11
|
|
|
15
12
|
function encodeValues(type, values, opts) {
|
|
16
13
|
if (!('bitWidth' in opts)) {
|
|
17
14
|
throw new Error('bitWidth is required');
|
|
18
15
|
}
|
|
19
|
-
|
|
20
16
|
switch (type) {
|
|
21
17
|
case 'BOOLEAN':
|
|
22
18
|
case 'INT32':
|
|
@@ -25,22 +21,18 @@ function encodeValues(type, values, opts) {
|
|
|
25
21
|
return parseInt(x, 10);
|
|
26
22
|
});
|
|
27
23
|
break;
|
|
28
|
-
|
|
29
24
|
default:
|
|
30
25
|
throw new Error("unsupported type: ".concat(type));
|
|
31
26
|
}
|
|
32
|
-
|
|
33
27
|
var buf = Buffer.alloc(0);
|
|
34
28
|
var run = [];
|
|
35
29
|
var repeats = 0;
|
|
36
|
-
|
|
37
30
|
for (var i = 0; i < values.length; i++) {
|
|
38
31
|
if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
|
|
39
32
|
if (run.length) {
|
|
40
33
|
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
41
34
|
run = [];
|
|
42
35
|
}
|
|
43
|
-
|
|
44
36
|
repeats = 1;
|
|
45
37
|
} else if (repeats > 0 && values[i] === values[i - 1]) {
|
|
46
38
|
repeats += 1;
|
|
@@ -49,90 +41,67 @@ function encodeValues(type, values, opts) {
|
|
|
49
41
|
buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
|
|
50
42
|
repeats = 0;
|
|
51
43
|
}
|
|
52
|
-
|
|
53
44
|
run.push(values[i]);
|
|
54
45
|
}
|
|
55
46
|
}
|
|
56
|
-
|
|
57
47
|
if (repeats) {
|
|
58
48
|
buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
|
|
59
49
|
} else if (run.length) {
|
|
60
50
|
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
61
51
|
}
|
|
62
|
-
|
|
63
52
|
if (opts.disableEnvelope) {
|
|
64
53
|
return buf;
|
|
65
54
|
}
|
|
66
|
-
|
|
67
55
|
var envelope = Buffer.alloc(buf.length + 4);
|
|
68
56
|
envelope.writeUInt32LE(buf.length, undefined);
|
|
69
57
|
buf.copy(envelope, 4);
|
|
70
58
|
return envelope;
|
|
71
59
|
}
|
|
72
|
-
|
|
73
60
|
function decodeValues(type, cursor, count, opts) {
|
|
74
61
|
if (!('bitWidth' in opts)) {
|
|
75
62
|
throw new Error('bitWidth is required');
|
|
76
63
|
}
|
|
77
|
-
|
|
78
64
|
if (!opts.disableEnvelope) {
|
|
79
65
|
cursor.offset += 4;
|
|
80
66
|
}
|
|
81
|
-
|
|
82
67
|
var values = [];
|
|
83
|
-
|
|
84
68
|
while (values.length < count) {
|
|
85
69
|
var header = _varint.default.decode(cursor.buffer, cursor.offset);
|
|
86
|
-
|
|
87
70
|
cursor.offset += _varint.default.encodingLength(header);
|
|
88
|
-
|
|
89
71
|
if (header & 1) {
|
|
90
72
|
var _values;
|
|
91
|
-
|
|
92
73
|
var _count = (header >> 1) * 8;
|
|
93
|
-
|
|
94
74
|
(_values = values).push.apply(_values, (0, _toConsumableArray2.default)(decodeRunBitpacked(cursor, _count, opts)));
|
|
95
75
|
} else {
|
|
96
76
|
var _values2;
|
|
97
|
-
|
|
98
77
|
var _count2 = header >> 1;
|
|
99
|
-
|
|
100
78
|
(_values2 = values).push.apply(_values2, (0, _toConsumableArray2.default)(decodeRunRepeated(cursor, _count2, opts)));
|
|
101
79
|
}
|
|
102
80
|
}
|
|
103
|
-
|
|
104
81
|
values = values.slice(0, count);
|
|
105
|
-
|
|
106
82
|
if (values.length !== count) {
|
|
107
83
|
throw new Error('invalid RLE encoding');
|
|
108
84
|
}
|
|
109
|
-
|
|
110
85
|
return values;
|
|
111
86
|
}
|
|
112
|
-
|
|
113
87
|
function decodeRunBitpacked(cursor, count, opts) {
|
|
114
88
|
var bitWidth = opts.bitWidth;
|
|
115
|
-
|
|
116
89
|
if (count % 8 !== 0) {
|
|
117
90
|
throw new Error('must be a multiple of 8');
|
|
118
91
|
}
|
|
119
92
|
|
|
120
93
|
var values = new Array(count).fill(0);
|
|
121
|
-
|
|
122
94
|
for (var b = 0; b < bitWidth * count; b++) {
|
|
123
95
|
if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
|
|
124
96
|
values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
|
|
125
97
|
}
|
|
126
98
|
}
|
|
127
|
-
|
|
128
99
|
cursor.offset += bitWidth * (count / 8);
|
|
129
100
|
return values;
|
|
130
101
|
}
|
|
131
|
-
|
|
132
102
|
function decodeRunRepeated(cursor, count, opts) {
|
|
133
103
|
var bitWidth = opts.bitWidth;
|
|
134
104
|
var value = 0;
|
|
135
|
-
|
|
136
105
|
for (var i = 0; i < Math.ceil(bitWidth / 8); i++) {
|
|
137
106
|
value << 8;
|
|
138
107
|
value += cursor.buffer[cursor.offset];
|
|
@@ -141,29 +110,22 @@ function decodeRunRepeated(cursor, count, opts) {
|
|
|
141
110
|
|
|
142
111
|
return new Array(count).fill(value);
|
|
143
112
|
}
|
|
144
|
-
|
|
145
113
|
function encodeRunBitpacked(values, opts) {
|
|
146
114
|
var bitWidth = opts.bitWidth;
|
|
147
|
-
|
|
148
115
|
for (var i = 0; i < values.length % 8; i++) {
|
|
149
116
|
values.push(0);
|
|
150
117
|
}
|
|
151
|
-
|
|
152
118
|
var buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
|
|
153
|
-
|
|
154
119
|
for (var b = 0; b < bitWidth * values.length; b++) {
|
|
155
120
|
if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
|
|
156
121
|
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
157
122
|
}
|
|
158
123
|
}
|
|
159
|
-
|
|
160
124
|
return Buffer.concat([Buffer.from(_varint.default.encode(values.length / 8 << 1 | 1)), buf]);
|
|
161
125
|
}
|
|
162
|
-
|
|
163
126
|
function encodeRunRepeated(value, count, opts) {
|
|
164
127
|
var bitWidth = opts.bitWidth;
|
|
165
128
|
var buf = Buffer.alloc(Math.ceil(bitWidth / 8));
|
|
166
|
-
|
|
167
129
|
for (var i = 0; i < buf.length; i++) {
|
|
168
130
|
buf.writeUInt8(value & 0xff, i);
|
|
169
131
|
value >> 8;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAEhD6B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AAEA6B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n // eslint-disable-next-line\n value << 8; // TODO - this looks wrong\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n // eslint-disable-next-line\n value >> 8; // TODO - this looks wrong\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
|
|
1
|
+
{"version":3,"file":"rle.js","names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"sources":["../../../../src/parquetjs/codecs/rle.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n // eslint-disable-next-line\n value << 8; // TODO - this looks wrong\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n // eslint-disable-next-line\n value >> 8; // TODO - this looks wrong\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"mappings":";;;;;;;;;AAIA;;AAGO,SAASA,YAAY,CAC1BC,IAAmB,EACnBC,MAAa,EACbC,IAAyB,EACjB;EACR,IAAI,EAAE,UAAU,IAAIA,IAAI,CAAC,EAAE;IACzB,MAAM,IAAIC,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,QAAQH,IAAI;IACV,KAAK,SAAS;IACd,KAAK,OAAO;IACZ,KAAK,OAAO;MAEVC,MAAM,GAAGA,MAAM,CAACG,GAAG,CAAC,UAACC,CAAC;QAAA,OAAKC,QAAQ,CAACD,CAAC,EAAE,EAAE,CAAC;MAAA,EAAC;MAC3C;IAEF;MACE,MAAM,IAAIF,KAAK,6BAAsBH,IAAI,EAAG;EAAC;EAGjD,IAAIO,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,CAAC;EACzB,IAAIC,GAAU,GAAG,EAAE;EACnB,IAAIC,OAAO,GAAG,CAAC;EAEf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGX,MAAM,CAACY,MAAM,EAAED,CAAC,EAAE,EAAE;IAGtC,IAAID,OAAO,KAAK,CAAC,IAAID,GAAG,CAACG,MAAM,GAAG,CAAC,KAAK,CAAC,IAAIZ,MAAM,CAACW,CAAC,CAAC,KAAKX,MAAM,CAACW,CAAC,GAAG,CAAC,CAAC,EAAE;MAExE,IAAIF,GAAG,CAACG,MAAM,EAAE;QACdN,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAEQ,kBAAkB,CAACL,GAAG,EAAER,IAAI,CAAC,CAAC,CAAC;QACzDQ,GAAG,GAAG,EAAE;MACV;MACAC,OAAO,GAAG,CAAC;IACb,CAAC,MAAM,IAAIA,OAAO,GAAG,CAAC,IAAIV,MAAM,CAACW,CAAC,CAAC,KAAKX,MAAM,CAACW,CAAC,GAAG,CAAC,CAAC,EAAE;MACrDD,OAAO,IAAI,CAAC;IACd,CAAC,MAAM;MAEL,IAAIA,OAAO,EAAE;QACXJ,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAES,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAC,CAAC,EAAED,OAAO,EAAET,IAAI,CAAC,CAAC,CAAC;QAC3ES,OAAO,GAAG,CAAC;MACb;MACAD,GAAG,CAACO,IAAI,CAAChB,MAAM,CAACW,CAAC,CAAC,CAAC;IACrB;EACF;EAEA,IAAID,OAAO,EAAE;IACXJ,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAES,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAM,GAAG,CAAC,CAAC,EAAEF,OAAO,EAAET,IAAI,CAAC,CAAC,CAAC;EACzF,CAAC,MAAM,IAAIQ,GAAG,CAACG,MAAM,EAAE;IACrBN,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAEQ,kBAAkB,CAACL,GAAG,EAAER,IAAI,CAAC,CAAC,CAAC;EAC3D;EAEA,IAAIA,IAAI,CAACgB,eAAe,EAAE;IACxB,OAAOX,GAAG;EACZ;EAEA,IAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAK,CAACF,GAAG,CAACM,MAAM,GAAG,CAAC,CAAC;EAC7CM,QAAQ,CAACC,aAAa,CAACb,GAAG,CAACM,MAAM,EAAEQ,SAAS,CAAC;EAC7Cd,GAAG,CAACe,IAAI,CAACH,QAAQ,EAAE,CAAC,CAAC;EAErB,OAAOA,QAAQ;AACjB;AAEO,SAASI,YAAY,CAC1BvB,IAAmB,EACnBwB,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EACV,IAAI,EAAE,UAAU,IAAIA,IAAI,CAAC,EAAE;IACzB,MAAM,IAAIC,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,IAAI,CAACD,IAAI,CAACgB,eAAe,EAAE;IACzBM,MAAM,CAACE,MAAM,IAAI,CAAC;EACpB;EAEA,IAAIzB,MAAgB,GAAG,EAAE;EACzB,OAAOA,MAAM,CAACY,MAAM,GAAGY,KAAK,EAAE;IAC5B,IAAME,MAAM,GAAGC,eAAM,CAACC,MAAM,CAACL,MAAM,CAACM,MAAM,EAAEN,MAAM,CAACE,MAAM,CAAC;IAC1DF,MAAM,CAACE,MAAM,IAAIE,eAAM,CAACG,cAAc,CAACJ,MAAM,CAAC;IAC9C,IAAIA,MAAM,GAAG,CAAC,EAAE;MAAA;MACd,IAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAC,IAAI,CAAC;MAC/B,WAAA1B,MAAM,EAACgB,IAAI,iDAAIe,kBAAkB,CAACR,MAAM,EAAEC,MAAK,EAAEvB,IAAI,CAAC,EAAC;IACzD,CAAC,MAAM;MAAA;MACL,IAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAC;MACzB,YAAA1B,MAAM,EAACgB,IAAI,kDAAIgB,iBAAiB,CAACT,MAAM,EAAEC,OAAK,EAAEvB,IAAI,CAAC,EAAC;IACxD;EACF;EACAD,MAAM,GAAGA,MAAM,CAACiC,KAAK,CAAC,CAAC,EAAET,KAAK,CAAC;EAE/B,IAAIxB,MAAM,CAACY,MAAM,KAAKY,KAAK,EAAE;IAC3B,MAAM,IAAItB,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,OAAOF,MAAM;AACf;AAEA,SAAS+B,kBAAkB,CACzBR,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EAEV,IAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAAQ;EAEtC,IAAIV,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE;IACnB,MAAM,IAAItB,KAAK,CAAC,yBAAyB,CAAC;EAC5C;;EAGA,IAAMF,MAAM,GAAG,IAAImC,KAAK,CAACX,KAAK,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EACvC,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,QAAQ,GAAGV,KAAK,EAAEa,CAAC,EAAE,EAAE;IACzC,IAAId,MAAM,CAACM,MAAM,CAACN,MAAM,CAACE,MAAM,GAAGa,IAAI,CAACC,KAAK,CAACF,CAAC,GAAG,CAAC,CAAC,CAAC,GAAI,CAAC,IAAIA,CAAC,GAAG,CAAE,EAAE;MACnErC,MAAM,CAACsC,IAAI,CAACC,KAAK,CAACF,CAAC,GAAGH,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAIG,CAAC,GAAGH,QAAQ;IACvD;EACF;EAEAX,MAAM,CAACE,MAAM,IAAIS,QAAQ,IAAIV,KAAK,GAAG,CAAC,CAAC;EACvC,OAAOxB,MAAM;AACf;AAEA,SAASgC,iBAAiB,CACxBT,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EAEV,IAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAAQ;EAEtC,IAAIM,KAAK,GAAG,CAAC;EACb,KAAK,IAAI7B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2B,IAAI,CAACG,IAAI,CAACP,QAAQ,GAAG,CAAC,CAAC,EAAEvB,CAAC,EAAE,EAAE;IAEhD6B,KAAK,IAAI,CAAC;IACVA,KAAK,IAAIjB,MAAM,CAACM,MAAM,CAACN,MAAM,CAACE,MAAM,CAAC;IACrCF,MAAM,CAACE,MAAM,IAAI,CAAC;EACpB;;EAGA,OAAO,IAAIU,KAAK,CAACX,KAAK,CAAC,CAACY,IAAI,CAACI,KAAK,CAAC;AACrC;AAEA,SAAS1B,kBAAkB,CAACd,MAAgB,EAAEC,IAAyB,EAAU;EAE/E,IAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAAQ;EAEtC,KAAK,IAAIvB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGX,MAAM,CAACY,MAAM,GAAG,CAAC,EAAED,CAAC,EAAE,EAAE;IAC1CX,MAAM,CAACgB,IAAI,CAAC,CAAC,CAAC;EAChB;EAEA,IAAMV,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC8B,IAAI,CAACG,IAAI,CAACP,QAAQ,IAAIlC,MAAM,CAACY,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;EACnE,KAAK,IAAIyB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAM,EAAEyB,CAAC,EAAE,EAAE;IACjD,IAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAK,CAACF,CAAC,GAAGH,QAAQ,CAAC,CAAC,GAAI,CAAC,IAAIG,CAAC,GAAGH,QAAS,IAAI,CAAC,EAAE;MAChE5B,GAAG,CAACgC,IAAI,CAACC,KAAK,CAACF,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAIA,CAAC,GAAG,CAAC;IACtC;EACF;EAEA,OAAO9B,MAAM,CAACM,MAAM,CAAC,CAACN,MAAM,CAACmC,IAAI,CAACf,eAAM,CAACgB,MAAM,CAAG3C,MAAM,CAACY,MAAM,GAAG,CAAC,IAAK,CAAC,GAAI,CAAC,CAAC,CAAC,EAAEN,GAAG,CAAC,CAAC;AACzF;AAEA,SAASS,iBAAiB,CAACyB,KAAa,EAAEhB,KAAa,EAAEvB,IAAyB,EAAU;EAE1F,IAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAAQ;EAEtC,IAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC8B,IAAI,CAACG,IAAI,CAACP,QAAQ,GAAG,CAAC,CAAC,CAAC;EAEjD,KAAK,IAAIvB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGL,GAAG,CAACM,MAAM,EAAED,CAAC,EAAE,EAAE;IACnCL,GAAG,CAACsC,UAAU,CAACJ,KAAK,GAAG,IAAI,EAAE7B,CAAC,CAAC;IAE/B6B,KAAK,IAAI,CAAC;EACZ;;EAEA,OAAOjC,MAAM,CAACM,MAAM,CAAC,CAACN,MAAM,CAACmC,IAAI,CAACf,eAAM,CAACgB,MAAM,CAACnB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAElB,GAAG,CAAC,CAAC;AACrE"}
|