@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.d.ts +2 -0
- package/dist/bundle.d.ts.map +1 -0
- package/dist/bundle.js +5 -0
- package/dist/constants.d.ts +15 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +18 -0
- package/dist/dist.min.js +27 -13
- package/dist/dist.min.js.map +7 -1
- package/dist/es5/bundle.js +1 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/constants.js +5 -5
- package/dist/es5/constants.js.map +1 -1
- package/dist/es5/index.js +16 -45
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/convert-schema.js +13 -13
- package/dist/es5/lib/convert-schema.js.map +1 -1
- package/dist/es5/lib/parse-parquet.js +19 -154
- package/dist/es5/lib/parse-parquet.js.map +1 -1
- package/dist/es5/lib/read-array-buffer.js +6 -43
- package/dist/es5/lib/read-array-buffer.js.map +1 -1
- package/dist/es5/parquet-loader.js +4 -4
- package/dist/es5/parquet-loader.js.map +1 -1
- package/dist/es5/parquet-writer.js +4 -4
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/es5/parquetjs/codecs/dictionary.js +2 -10
- package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/es5/parquetjs/codecs/index.js +4 -6
- package/dist/es5/parquetjs/codecs/index.js.map +1 -1
- package/dist/es5/parquetjs/codecs/plain.js +41 -43
- package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
- package/dist/es5/parquetjs/codecs/rle.js +25 -35
- package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
- package/dist/es5/parquetjs/compression.js +28 -122
- package/dist/es5/parquetjs/compression.js.map +1 -1
- package/dist/es5/parquetjs/encoder/writer.js +301 -737
- package/dist/es5/parquetjs/encoder/writer.js.map +1 -1
- package/dist/es5/parquetjs/file.js +15 -15
- package/dist/es5/parquetjs/file.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +141 -152
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +147 -160
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +248 -259
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +67 -79
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +113 -124
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +158 -169
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DateType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +68 -79
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +83 -94
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +170 -182
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IntType.js +68 -79
- package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +68 -79
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ListType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +319 -343
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MapType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/NullType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +64 -75
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +83 -94
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +158 -169
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +83 -94
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +113 -124
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +188 -199
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +83 -94
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js +124 -135
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/StringType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js +68 -79
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +88 -101
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +68 -79
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +31 -45
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
- package/dist/es5/parquetjs/parser/decoders.js +218 -397
- package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-cursor.js +62 -180
- package/dist/es5/parquetjs/parser/parquet-cursor.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +124 -408
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-reader.js +91 -369
- package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/es5/parquetjs/schema/declare.js +9 -11
- package/dist/es5/parquetjs/schema/declare.js.map +1 -1
- package/dist/es5/parquetjs/schema/schema.js +73 -87
- package/dist/es5/parquetjs/schema/schema.js.map +1 -1
- package/dist/es5/parquetjs/schema/shred.js +56 -96
- package/dist/es5/parquetjs/schema/shred.js.map +1 -1
- package/dist/es5/parquetjs/schema/types.js +39 -40
- package/dist/es5/parquetjs/schema/types.js.map +1 -1
- package/dist/es5/parquetjs/utils/buffer-utils.js +1 -1
- package/dist/es5/parquetjs/utils/buffer-utils.js.map +1 -1
- package/dist/es5/parquetjs/utils/file-utils.js +8 -65
- package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/es5/parquetjs/utils/read-utils.js +22 -50
- package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/esm/index.js +2 -3
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/parquet-loader.js +1 -1
- package/dist/esm/parquet-loader.js.map +1 -1
- package/dist/esm/parquet-writer.js +1 -1
- package/dist/esm/parquet-writer.js.map +1 -1
- package/dist/esm/parquetjs/codecs/plain.js +3 -3
- package/dist/esm/parquetjs/codecs/plain.js.map +1 -1
- package/dist/esm/parquetjs/codecs/rle.js +1 -1
- package/dist/esm/parquetjs/codecs/rle.js.map +1 -1
- package/dist/esm/parquetjs/compression.js +4 -13
- package/dist/esm/parquetjs/compression.js.map +1 -1
- package/dist/esm/parquetjs/encoder/writer.js +1 -1
- package/dist/esm/parquetjs/encoder/writer.js.map +1 -1
- package/dist/esm/parquetjs/parser/decoders.js +4 -4
- package/dist/esm/parquetjs/parser/decoders.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +4 -13
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-reader.js +0 -13
- package/dist/esm/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/esm/parquetjs/schema/schema.js +3 -3
- package/dist/esm/parquetjs/schema/schema.js.map +1 -1
- package/dist/esm/parquetjs/schema/shred.js +2 -2
- package/dist/esm/parquetjs/schema/shred.js.map +1 -1
- package/dist/esm/parquetjs/schema/types.js +20 -20
- package/dist/esm/parquetjs/schema/types.js.map +1 -1
- package/dist/esm/parquetjs/utils/file-utils.js +0 -45
- package/dist/esm/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/index.d.ts +28 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/lib/convert-schema.d.ts +8 -0
- package/dist/lib/convert-schema.d.ts.map +1 -0
- package/dist/lib/convert-schema.js +70 -0
- package/dist/lib/parse-parquet.d.ts +4 -0
- package/dist/lib/parse-parquet.d.ts.map +1 -0
- package/dist/lib/parse-parquet.js +28 -0
- package/dist/lib/read-array-buffer.d.ts +19 -0
- package/dist/lib/read-array-buffer.d.ts.map +1 -0
- package/dist/lib/read-array-buffer.js +29 -0
- package/dist/parquet-loader.d.ts +23 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +27 -0
- package/dist/parquet-worker.js +27 -13
- package/dist/parquet-worker.js.map +7 -1
- package/dist/parquet-writer.d.ts +4 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +21 -0
- package/dist/parquetjs/codecs/declare.d.ts +17 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -0
- package/dist/parquetjs/codecs/declare.js +2 -0
- package/dist/parquetjs/codecs/dictionary.d.ts +3 -0
- package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -0
- package/dist/parquetjs/codecs/dictionary.js +14 -0
- package/dist/parquetjs/codecs/index.d.ts +5 -0
- package/dist/parquetjs/codecs/index.d.ts.map +1 -0
- package/dist/parquetjs/codecs/index.js +51 -0
- package/dist/parquetjs/codecs/plain.d.ts +6 -0
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -0
- package/dist/parquetjs/codecs/plain.js +211 -0
- package/dist/parquetjs/codecs/rle.d.ts +6 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -0
- package/dist/parquetjs/codecs/rle.js +145 -0
- package/dist/parquetjs/compression.d.ts +23 -0
- package/dist/parquetjs/compression.d.ts.map +1 -0
- package/dist/parquetjs/compression.js +168 -0
- package/dist/parquetjs/encoder/writer.d.ts +123 -0
- package/dist/parquetjs/encoder/writer.d.ts.map +1 -0
- package/dist/parquetjs/encoder/writer.js +478 -0
- package/dist/parquetjs/file.d.ts +10 -0
- package/dist/parquetjs/file.d.ts.map +1 -0
- package/dist/parquetjs/file.js +99 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +15 -0
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BsonType.js +58 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +25 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +207 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +213 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +42 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +398 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +13 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +104 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +20 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.d.ts +25 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +34 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +21 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +166 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +27 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +226 -0
- package/dist/parquetjs/parquet-thrift/DateType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/DateType.js +58 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +13 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.js +105 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +122 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js +20 -0
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EnumType.js +58 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +15 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +28 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +256 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +58 -0
- package/dist/parquetjs/parquet-thrift/IntType.d.ts +13 -0
- package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/IntType.js +105 -0
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/JsonType.js +58 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +13 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.js +102 -0
- package/dist/parquetjs/parquet-thrift/ListType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ListType.js +58 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +61 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.js +380 -0
- package/dist/parquetjs/parquet-thrift/MapType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/MapType.js +58 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +58 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +58 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.js +58 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +12 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +97 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +17 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +127 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +30 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.js +216 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.js +141 -0
- package/dist/parquetjs/parquet-thrift/PageType.d.ts +7 -0
- package/dist/parquetjs/parquet-thrift/PageType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageType.js +16 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +20 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.js +182 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +33 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +239 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +15 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +127 -0
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts +23 -0
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Statistics.js +176 -0
- package/dist/parquetjs/parquet-thrift/StringType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/StringType.js +58 -0
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts +14 -0
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimeType.js +106 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +17 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +127 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +14 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.js +106 -0
- package/dist/parquetjs/parquet-thrift/Type.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/Type.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Type.js +20 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +58 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.js +58 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +44 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.js +61 -0
- package/dist/parquetjs/parser/decoders.d.ts +34 -0
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -0
- package/dist/parquetjs/parser/decoders.js +318 -0
- package/dist/parquetjs/parser/parquet-cursor.d.ts +36 -0
- package/dist/parquetjs/parser/parquet-cursor.d.ts.map +1 -0
- package/dist/parquetjs/parser/parquet-cursor.js +74 -0
- package/dist/parquetjs/parser/parquet-envelope-reader.d.ts +40 -0
- package/dist/parquetjs/parser/parquet-envelope-reader.d.ts.map +1 -0
- package/dist/parquetjs/parser/parquet-envelope-reader.js +136 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts +68 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.js +134 -0
- package/dist/parquetjs/schema/declare.d.ts +80 -0
- package/dist/parquetjs/schema/declare.d.ts.map +1 -0
- package/dist/parquetjs/schema/declare.js +10 -0
- package/dist/parquetjs/schema/schema.d.ts +26 -0
- package/dist/parquetjs/schema/schema.d.ts.map +1 -0
- package/dist/parquetjs/schema/schema.js +162 -0
- package/dist/parquetjs/schema/shred.d.ts +48 -0
- package/dist/parquetjs/schema/shred.d.ts.map +1 -0
- package/dist/parquetjs/schema/shred.js +225 -0
- package/dist/parquetjs/schema/types.d.ts +20 -0
- package/dist/parquetjs/schema/types.d.ts.map +1 -0
- package/dist/parquetjs/schema/types.js +418 -0
- package/dist/parquetjs/utils/buffer-utils.d.ts +10 -0
- package/dist/parquetjs/utils/buffer-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/buffer-utils.js +22 -0
- package/dist/parquetjs/utils/file-utils.d.ts +16 -0
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/file-utils.js +46 -0
- package/dist/parquetjs/utils/read-utils.d.ts +25 -0
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/read-utils.js +109 -0
- package/dist/workers/parquet-worker.d.ts +2 -0
- package/dist/workers/parquet-worker.d.ts.map +1 -0
- package/dist/workers/parquet-worker.js +5 -0
- package/package.json +8 -8
- package/src/index.ts +3 -3
- package/src/parquetjs/compression.ts +10 -10
- package/src/parquetjs/parser/decoders.ts +1 -1
- package/src/parquetjs/parser/parquet-envelope-reader.ts +0 -11
- package/src/parquetjs/parser/parquet-reader.ts +0 -16
- package/src/parquetjs/utils/file-utils.ts +0 -49
|
@@ -1,27 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
3
|
Object.defineProperty(exports, "__esModule", {
|
|
6
4
|
value: true
|
|
7
5
|
});
|
|
8
6
|
exports.decodeValues = decodeValues;
|
|
9
7
|
exports.encodeValues = encodeValues;
|
|
10
8
|
|
|
11
|
-
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
12
|
-
|
|
13
9
|
var _rle = require("./rle");
|
|
14
10
|
|
|
15
|
-
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
16
|
-
|
|
17
|
-
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
18
|
-
|
|
19
11
|
function decodeValues(type, cursor, count, opts) {
|
|
20
12
|
opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
|
|
21
13
|
cursor.offset += 1;
|
|
22
|
-
return (0, _rle.decodeValues)(type, cursor, count,
|
|
14
|
+
return (0, _rle.decodeValues)(type, cursor, count, { ...opts,
|
|
23
15
|
disableEnvelope: true
|
|
24
|
-
})
|
|
16
|
+
});
|
|
25
17
|
}
|
|
26
18
|
|
|
27
19
|
function encodeValues(type, cursor, count, opts) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/dictionary.ts"],"names":["decodeValues","type","cursor","count","opts","bitWidth","buffer","slice","offset","readInt8","disableEnvelope","encodeValues","Error"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../../../src/parquetjs/codecs/dictionary.ts"],"names":["decodeValues","type","cursor","count","opts","bitWidth","buffer","slice","offset","readInt8","disableEnvelope","encodeValues","Error"],"mappings":";;;;;;;;AAAA;;AAEO,SAASA,YAAT,CAAsBC,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtDA,EAAAA,IAAI,CAACC,QAAL,GAAgBH,MAAM,CAACI,MAAP,CAAcC,KAAd,CAAoBL,MAAM,CAACM,MAA3B,EAAmCN,MAAM,CAACM,MAAP,GAAgB,CAAnD,EAAsDC,QAAtD,CAA+D,CAA/D,CAAhB;AACAP,EAAAA,MAAM,CAACM,MAAP,IAAiB,CAAjB;AACA,SAAO,uBAAgBP,IAAhB,EAAsBC,MAAtB,EAA8BC,KAA9B,EAAqC,EAAC,GAAGC,IAAJ;AAAUM,IAAAA,eAAe,EAAE;AAA3B,GAArC,CAAP;AACD;;AAEM,SAASC,YAAT,CAAsBV,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtD,QAAM,IAAIQ,KAAJ,CAAU,kDAAV,CAAN;AACD","sourcesContent":["import {decodeValues as decodeRleValues} from './rle';\n\nexport function decodeValues(type, cursor, count, opts) {\n opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);\n cursor.offset += 1;\n return decodeRleValues(type, cursor, count, {...opts, disableEnvelope: true});\n}\n\nexport function encodeValues(type, cursor, count, opts) {\n throw new Error('Encode dictionary functionality is not supported');\n}\n"],"file":"dictionary.js"}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
var _typeof = require("@babel/runtime/helpers/typeof");
|
|
4
|
-
|
|
5
3
|
Object.defineProperty(exports, "__esModule", {
|
|
6
4
|
value: true
|
|
7
5
|
});
|
|
@@ -24,17 +22,17 @@ Object.keys(_declare).forEach(function (key) {
|
|
|
24
22
|
if (key in exports && exports[key] === _declare[key]) return;
|
|
25
23
|
Object.defineProperty(exports, key, {
|
|
26
24
|
enumerable: true,
|
|
27
|
-
get: function
|
|
25
|
+
get: function () {
|
|
28
26
|
return _declare[key];
|
|
29
27
|
}
|
|
30
28
|
});
|
|
31
29
|
});
|
|
32
30
|
|
|
33
|
-
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function
|
|
31
|
+
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
|
34
32
|
|
|
35
|
-
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null ||
|
|
33
|
+
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
|
|
36
34
|
|
|
37
|
-
|
|
35
|
+
const PARQUET_CODECS = {
|
|
38
36
|
PLAIN: {
|
|
39
37
|
encodeValues: PLAIN.encodeValues,
|
|
40
38
|
decodeValues: PLAIN.decodeValues
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PARQUET_CODECS","PLAIN","encodeValues","decodeValues","RLE","PLAIN_DICTIONARY","DICTIONARY","RLE_DICTIONARY"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PARQUET_CODECS","PLAIN","encodeValues","decodeValues","RLE","PLAIN_DICTIONARY","DICTIONARY","RLE_DICTIONARY"],"mappings":";;;;;;;;;;AAGA;;AACA;;AACA;;AAEA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;;;;AAEO,MAAMA,cAAqD,GAAG;AACnEC,EAAAA,KAAK,EAAE;AACLC,IAAAA,YAAY,EAAED,KAAK,CAACC,YADf;AAELC,IAAAA,YAAY,EAAEF,KAAK,CAACE;AAFf,GAD4D;AAKnEC,EAAAA,GAAG,EAAE;AACHF,IAAAA,YAAY,EAAEE,GAAG,CAACF,YADf;AAEHC,IAAAA,YAAY,EAAEC,GAAG,CAACD;AAFf,GAL8D;AAUnEE,EAAAA,gBAAgB,EAAE;AAEhBH,IAAAA,YAAY,EAAEI,UAAU,CAACJ,YAFT;AAGhBC,IAAAA,YAAY,EAAEG,UAAU,CAACH;AAHT,GAViD;AAgBnEI,EAAAA,cAAc,EAAE;AAEdL,IAAAA,YAAY,EAAEI,UAAU,CAACJ,YAFX;AAGdC,IAAAA,YAAY,EAAEG,UAAU,CAACH;AAHX;AAhBmD,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\nimport * as DICTIONARY from './dictionary';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n PLAIN: {\n encodeValues: PLAIN.encodeValues,\n decodeValues: PLAIN.decodeValues\n },\n RLE: {\n encodeValues: RLE.encodeValues,\n decodeValues: RLE.decodeValues\n },\n // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.\n PLAIN_DICTIONARY: {\n // @ts-ignore\n encodeValues: DICTIONARY.encodeValues,\n decodeValues: DICTIONARY.decodeValues\n },\n // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.\n RLE_DICTIONARY: {\n // @ts-ignore\n encodeValues: DICTIONARY.encodeValues,\n decodeValues: DICTIONARY.decodeValues\n }\n};\n"],"file":"index.js"}
|
|
@@ -73,10 +73,10 @@ function decodeValues(type, cursor, count, opts) {
|
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
function encodeValues_BOOLEAN(values) {
|
|
76
|
-
|
|
76
|
+
const buf = Buffer.alloc(Math.ceil(values.length / 8));
|
|
77
77
|
buf.fill(0);
|
|
78
78
|
|
|
79
|
-
for (
|
|
79
|
+
for (let i = 0; i < values.length; i++) {
|
|
80
80
|
if (values[i]) {
|
|
81
81
|
buf[Math.floor(i / 8)] |= 1 << i % 8;
|
|
82
82
|
}
|
|
@@ -86,10 +86,10 @@ function encodeValues_BOOLEAN(values) {
|
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
function decodeValues_BOOLEAN(cursor, count) {
|
|
89
|
-
|
|
89
|
+
const values = [];
|
|
90
90
|
|
|
91
|
-
for (
|
|
92
|
-
|
|
91
|
+
for (let i = 0; i < count; i++) {
|
|
92
|
+
const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
|
|
93
93
|
values.push((b & 1 << i % 8) > 0);
|
|
94
94
|
}
|
|
95
95
|
|
|
@@ -98,9 +98,9 @@ function decodeValues_BOOLEAN(cursor, count) {
|
|
|
98
98
|
}
|
|
99
99
|
|
|
100
100
|
function encodeValues_INT32(values) {
|
|
101
|
-
|
|
101
|
+
const buf = Buffer.alloc(4 * values.length);
|
|
102
102
|
|
|
103
|
-
for (
|
|
103
|
+
for (let i = 0; i < values.length; i++) {
|
|
104
104
|
buf.writeInt32LE(values[i], i * 4);
|
|
105
105
|
}
|
|
106
106
|
|
|
@@ -108,9 +108,9 @@ function encodeValues_INT32(values) {
|
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
function decodeValues_INT32(cursor, count) {
|
|
111
|
-
|
|
111
|
+
const values = [];
|
|
112
112
|
|
|
113
|
-
for (
|
|
113
|
+
for (let i = 0; i < count; i++) {
|
|
114
114
|
values.push(cursor.buffer.readInt32LE(cursor.offset));
|
|
115
115
|
cursor.offset += 4;
|
|
116
116
|
}
|
|
@@ -119,9 +119,9 @@ function decodeValues_INT32(cursor, count) {
|
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
function encodeValues_INT64(values) {
|
|
122
|
-
|
|
122
|
+
const buf = Buffer.alloc(8 * values.length);
|
|
123
123
|
|
|
124
|
-
for (
|
|
124
|
+
for (let i = 0; i < values.length; i++) {
|
|
125
125
|
_int.default.writeInt64LE(values[i], buf, i * 8);
|
|
126
126
|
}
|
|
127
127
|
|
|
@@ -129,9 +129,9 @@ function encodeValues_INT64(values) {
|
|
|
129
129
|
}
|
|
130
130
|
|
|
131
131
|
function decodeValues_INT64(cursor, count) {
|
|
132
|
-
|
|
132
|
+
const values = [];
|
|
133
133
|
|
|
134
|
-
for (
|
|
134
|
+
for (let i = 0; i < count; i++) {
|
|
135
135
|
values.push(_int.default.readInt64LE(cursor.buffer, cursor.offset));
|
|
136
136
|
cursor.offset += 8;
|
|
137
137
|
}
|
|
@@ -140,9 +140,9 @@ function decodeValues_INT64(cursor, count) {
|
|
|
140
140
|
}
|
|
141
141
|
|
|
142
142
|
function encodeValues_INT96(values) {
|
|
143
|
-
|
|
143
|
+
const buf = Buffer.alloc(12 * values.length);
|
|
144
144
|
|
|
145
|
-
for (
|
|
145
|
+
for (let i = 0; i < values.length; i++) {
|
|
146
146
|
if (values[i] >= 0) {
|
|
147
147
|
_int.default.writeInt64LE(values[i], buf, i * 12);
|
|
148
148
|
|
|
@@ -158,12 +158,12 @@ function encodeValues_INT96(values) {
|
|
|
158
158
|
}
|
|
159
159
|
|
|
160
160
|
function decodeValues_INT96(cursor, count) {
|
|
161
|
-
|
|
161
|
+
const values = [];
|
|
162
162
|
|
|
163
|
-
for (
|
|
164
|
-
|
|
163
|
+
for (let i = 0; i < count; i++) {
|
|
164
|
+
const low = _int.default.readInt64LE(cursor.buffer, cursor.offset);
|
|
165
165
|
|
|
166
|
-
|
|
166
|
+
const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
|
|
167
167
|
|
|
168
168
|
if (high === 0xffffffff) {
|
|
169
169
|
values.push(~-low + 1);
|
|
@@ -178,9 +178,9 @@ function decodeValues_INT96(cursor, count) {
|
|
|
178
178
|
}
|
|
179
179
|
|
|
180
180
|
function encodeValues_FLOAT(values) {
|
|
181
|
-
|
|
181
|
+
const buf = Buffer.alloc(4 * values.length);
|
|
182
182
|
|
|
183
|
-
for (
|
|
183
|
+
for (let i = 0; i < values.length; i++) {
|
|
184
184
|
buf.writeFloatLE(values[i], i * 4);
|
|
185
185
|
}
|
|
186
186
|
|
|
@@ -188,9 +188,9 @@ function encodeValues_FLOAT(values) {
|
|
|
188
188
|
}
|
|
189
189
|
|
|
190
190
|
function decodeValues_FLOAT(cursor, count) {
|
|
191
|
-
|
|
191
|
+
const values = [];
|
|
192
192
|
|
|
193
|
-
for (
|
|
193
|
+
for (let i = 0; i < count; i++) {
|
|
194
194
|
values.push(cursor.buffer.readFloatLE(cursor.offset));
|
|
195
195
|
cursor.offset += 4;
|
|
196
196
|
}
|
|
@@ -199,9 +199,9 @@ function decodeValues_FLOAT(cursor, count) {
|
|
|
199
199
|
}
|
|
200
200
|
|
|
201
201
|
function encodeValues_DOUBLE(values) {
|
|
202
|
-
|
|
202
|
+
const buf = Buffer.alloc(8 * values.length);
|
|
203
203
|
|
|
204
|
-
for (
|
|
204
|
+
for (let i = 0; i < values.length; i++) {
|
|
205
205
|
buf.writeDoubleLE(values[i], i * 8);
|
|
206
206
|
}
|
|
207
207
|
|
|
@@ -209,9 +209,9 @@ function encodeValues_DOUBLE(values) {
|
|
|
209
209
|
}
|
|
210
210
|
|
|
211
211
|
function decodeValues_DOUBLE(cursor, count) {
|
|
212
|
-
|
|
212
|
+
const values = [];
|
|
213
213
|
|
|
214
|
-
for (
|
|
214
|
+
for (let i = 0; i < count; i++) {
|
|
215
215
|
values.push(cursor.buffer.readDoubleLE(cursor.offset));
|
|
216
216
|
cursor.offset += 8;
|
|
217
217
|
}
|
|
@@ -220,32 +220,30 @@ function decodeValues_DOUBLE(cursor, count) {
|
|
|
220
220
|
}
|
|
221
221
|
|
|
222
222
|
function encodeValues_BYTE_ARRAY(values) {
|
|
223
|
-
|
|
223
|
+
let buf_len = 0;
|
|
224
224
|
|
|
225
|
-
for (
|
|
225
|
+
for (let i = 0; i < values.length; i++) {
|
|
226
226
|
values[i] = Buffer.from(values[i]);
|
|
227
227
|
buf_len += 4 + values[i].length;
|
|
228
228
|
}
|
|
229
229
|
|
|
230
|
-
|
|
231
|
-
|
|
230
|
+
const buf = Buffer.alloc(buf_len);
|
|
231
|
+
let buf_pos = 0;
|
|
232
232
|
|
|
233
|
-
for (
|
|
234
|
-
buf.writeUInt32LE(values[
|
|
235
|
-
|
|
236
|
-
values[
|
|
237
|
-
|
|
238
|
-
buf_pos += 4 + values[_i].length;
|
|
233
|
+
for (let i = 0; i < values.length; i++) {
|
|
234
|
+
buf.writeUInt32LE(values[i].length, buf_pos);
|
|
235
|
+
values[i].copy(buf, buf_pos + 4);
|
|
236
|
+
buf_pos += 4 + values[i].length;
|
|
239
237
|
}
|
|
240
238
|
|
|
241
239
|
return buf;
|
|
242
240
|
}
|
|
243
241
|
|
|
244
242
|
function decodeValues_BYTE_ARRAY(cursor, count) {
|
|
245
|
-
|
|
243
|
+
const values = [];
|
|
246
244
|
|
|
247
|
-
for (
|
|
248
|
-
|
|
245
|
+
for (let i = 0; i < count; i++) {
|
|
246
|
+
const len = cursor.buffer.readUInt32LE(cursor.offset);
|
|
249
247
|
cursor.offset += 4;
|
|
250
248
|
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
|
|
251
249
|
cursor.offset += len;
|
|
@@ -259,7 +257,7 @@ function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
|
|
|
259
257
|
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
|
|
260
258
|
}
|
|
261
259
|
|
|
262
|
-
for (
|
|
260
|
+
for (let i = 0; i < values.length; i++) {
|
|
263
261
|
values[i] = Buffer.from(values[i]);
|
|
264
262
|
|
|
265
263
|
if (values[i].length !== opts.typeLength) {
|
|
@@ -271,13 +269,13 @@ function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
|
|
|
271
269
|
}
|
|
272
270
|
|
|
273
271
|
function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
|
|
274
|
-
|
|
272
|
+
const values = [];
|
|
275
273
|
|
|
276
274
|
if (!opts.typeLength) {
|
|
277
275
|
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
|
|
278
276
|
}
|
|
279
277
|
|
|
280
|
-
for (
|
|
278
|
+
for (let i = 0; i < count; i++) {
|
|
281
279
|
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
|
|
282
280
|
cursor.offset += opts.typeLength;
|
|
283
281
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/plain.ts"],"names":["encodeValues","type","values","opts","encodeValues_BOOLEAN","encodeValues_INT32","encodeValues_INT64","encodeValues_INT96","encodeValues_FLOAT","encodeValues_DOUBLE","encodeValues_BYTE_ARRAY","encodeValues_FIXED_LEN_BYTE_ARRAY","Error","decodeValues","cursor","count","decodeValues_BOOLEAN","decodeValues_INT32","decodeValues_INT64","decodeValues_INT96","decodeValues_FLOAT","decodeValues_DOUBLE","decodeValues_BYTE_ARRAY","decodeValues_FIXED_LEN_BYTE_ARRAY","buf","Buffer","alloc","Math","ceil","length","fill","i","floor","b","buffer","offset","push","writeInt32LE","readInt32LE","INT53","writeInt64LE","readInt64LE","writeUInt32LE","low","high","readUInt32LE","writeFloatLE","readFloatLE","writeDoubleLE","readDoubleLE","buf_len","from","buf_pos","copy","len","slice","typeLength","concat"],"mappings":";;;;;;;;;;AAIA;;AAEO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOG,oBAAoB,CAACF,MAAD,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACH,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACJ,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACL,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOM,kBAAkB,CAACN,MAAD,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOO,mBAAmB,CAACP,MAAD,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOQ,uBAAuB,CAACR,MAAD,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOS,iCAAiC,CAACT,MAAD,EAASC,IAAT,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAEM,SAASY,YAAT,CACLZ,IADK,EAELa,MAFK,EAGLC,KAHK,EAILZ,IAJK,EAKE;AACP,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOe,oBAAoB,CAACF,MAAD,EAASC,KAAT,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOE,kBAAkB,CAACH,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACJ,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACL,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACN,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOM,mBAAmB,CAACP,MAAD,EAASC,KAAT,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOO,uBAAuB,CAACR,MAAD,EAASC,KAAT,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOQ,iCAAiC,CAACT,MAAD,EAASC,KAAT,EAAgBZ,IAAhB,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAED,SAASG,oBAAT,CAA8BF,MAA9B,EAAyD;AACvD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAaC,IAAI,CAACC,IAAL,CAAU1B,MAAM,CAAC2B,MAAP,GAAgB,CAA1B,CAAb,CAAZ;AACAL,EAAAA,GAAG,CAACM,IAAJ,CAAS,CAAT;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAV,EAAe;AACbP,MAAAA,GAAG,CAACG,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASR,oBAAT,CAA8BF,MAA9B,EAAoDC,KAApD,EAA8E;AAC5E,MAAMb,MAAiB,GAAG,EAA1B;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAME,CAAC,GAAGnB,MAAM,CAACoB,MAAP,CAAcpB,MAAM,CAACqB,MAAP,GAAgBR,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAA9B,CAAV;AACA7B,IAAAA,MAAM,CAACkC,IAAP,CAAY,CAACH,CAAC,GAAI,KAAKF,CAAC,GAAG,CAAf,IAAqB,CAAjC;AACD;;AACDjB,EAAAA,MAAM,CAACqB,MAAP,IAAiBR,IAAI,CAACC,IAAL,CAAUb,KAAK,GAAG,CAAlB,CAAjB;AACA,SAAOb,MAAP;AACD;;AAED,SAASG,kBAAT,CAA4BH,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACa,YAAJ,CAAiBnC,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASP,kBAAT,CAA4BH,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcI,WAAd,CAA0BxB,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASI,kBAAT,CAA4BJ,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCQ,iBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,CAAvC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASN,kBAAT,CAA4BJ,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYG,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASK,kBAAT,CAA4BL,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,KAAKxB,MAAM,CAAC2B,MAAzB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,IAAa,CAAjB,EAAoB;AAClBQ,mBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,EAAvC;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,CAAlB,EAAqBX,CAAC,GAAG,EAAJ,GAAS,CAA9B;AACD,KAHD,MAGO;AACLQ,mBAAMC,YAAN,CAAmB,CAAC,CAACtC,MAAM,CAAC6B,CAAD,CAAR,GAAc,CAAjC,EAAoCP,GAApC,EAAyCO,CAAC,GAAG,EAA7C;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,UAAlB,EAA8BX,CAAC,GAAG,EAAJ,GAAS,CAAvC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASL,kBAAT,CAA4BL,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAMY,GAAG,GAAGJ,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;;AACA,QAAMS,IAAI,GAAG9B,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAP,GAAgB,CAA3C,CAAb;;AACA,QAAIS,IAAI,KAAK,UAAb,EAAyB;AACvB1C,MAAAA,MAAM,CAACkC,IAAP,CAAY,CAAC,CAACO,GAAF,GAAQ,CAApB;AACD,KAFD,MAEO;AACLzC,MAAAA,MAAM,CAACkC,IAAP,CAAYO,GAAZ;AACD;;AACD7B,IAAAA,MAAM,CAACqB,MAAP,IAAiB,EAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASM,kBAAT,CAA4BN,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACsB,YAAJ,CAAiB5C,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASJ,kBAAT,CAA4BN,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAca,WAAd,CAA0BjC,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASO,mBAAT,CAA6BP,MAA7B,EAAuD;AACrD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACwB,aAAJ,CAAkB9C,MAAM,CAAC6B,CAAD,CAAxB,EAA6BA,CAAC,GAAG,CAAjC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASH,mBAAT,CAA6BP,MAA7B,EAAmDC,KAAnD,EAA4E;AAC1E,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAce,YAAd,CAA2BnC,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASQ,uBAAT,CAAiCR,MAAjC,EAA2D;AAEzD,MAAIgD,OAAO,GAAG,CAAd;;AACA,OAAK,IAAInB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;AACAmB,IAAAA,OAAO,IAAI,IAAIhD,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,MAAML,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAawB,OAAb,CAAZ;AAEA,MAAIE,OAAO,GAAG,CAAd;;AACA,OAAK,IAAIrB,EAAC,GAAG,CAAb,EAAgBA,EAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,EAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACkB,aAAJ,CAAkBxC,MAAM,CAAC6B,EAAD,CAAN,CAAUF,MAA5B,EAAoCuB,OAApC;;AACAlD,IAAAA,MAAM,CAAC6B,EAAD,CAAN,CAAUsB,IAAV,CAAe7B,GAAf,EAAoB4B,OAAO,GAAG,CAA9B;;AACAA,IAAAA,OAAO,IAAI,IAAIlD,MAAM,CAAC6B,EAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,SAAOL,GAAP;AACD;;AAED,SAASF,uBAAT,CAAiCR,MAAjC,EAAuDC,KAAvD,EAAgF;AAC9E,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAMuB,GAAG,GAAGxC,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACAjC,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBmB,GAAnD,CAAZ;AACAxC,IAAAA,MAAM,CAACqB,MAAP,IAAiBmB,GAAjB;AACD;;AACD,SAAOpD,MAAP;AACD;;AAED,SAASS,iCAAT,CAA2CT,MAA3C,EAA6DC,IAA7D,EAAgG;AAC9F,MAAI,CAACA,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;;AACA,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAV,KAAqB1B,IAAI,CAACqD,UAA9B,EAA0C;AACxC,YAAM,IAAI5C,KAAJ,mDAAqDV,MAAM,CAAC6B,CAAD,CAA3D,EAAN;AACD;AACF;;AACD,SAAON,MAAM,CAACgC,MAAP,CAAcvD,MAAd,CAAP;AACD;;AAED,SAASqB,iCAAT,CACET,MADF,EAEEC,KAFF,EAGEZ,IAHF,EAIY;AACV,MAAMD,MAAgB,GAAG,EAAzB;;AACA,MAAI,CAACC,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBhC,IAAI,CAACqD,UAAxD,CAAZ;AACA1C,IAAAA,MAAM,CAACqB,MAAP,IAAiBhC,IAAI,CAACqD,UAAtB;AACD;;AACD,SAAOtD,MAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport INT53 from 'int53';\n\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n switch (type) {\n case 'BOOLEAN':\n return encodeValues_BOOLEAN(values);\n case 'INT32':\n return encodeValues_INT32(values);\n case 'INT64':\n return encodeValues_INT64(values);\n case 'INT96':\n return encodeValues_INT96(values);\n case 'FLOAT':\n return encodeValues_FLOAT(values);\n case 'DOUBLE':\n return encodeValues_DOUBLE(values);\n case 'BYTE_ARRAY':\n return encodeValues_BYTE_ARRAY(values);\n case 'FIXED_LEN_BYTE_ARRAY':\n return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n switch (type) {\n case 'BOOLEAN':\n return decodeValues_BOOLEAN(cursor, count);\n case 'INT32':\n return decodeValues_INT32(cursor, count);\n case 'INT64':\n return decodeValues_INT64(cursor, count);\n case 'INT96':\n return decodeValues_INT96(cursor, count);\n case 'FLOAT':\n return decodeValues_FLOAT(cursor, count);\n case 'DOUBLE':\n return decodeValues_DOUBLE(cursor, count);\n case 'BYTE_ARRAY':\n return decodeValues_BYTE_ARRAY(cursor, count);\n case 'FIXED_LEN_BYTE_ARRAY':\n return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nfunction encodeValues_BOOLEAN(values: boolean[]): Buffer {\n const buf = Buffer.alloc(Math.ceil(values.length / 8));\n buf.fill(0);\n for (let i = 0; i < values.length; i++) {\n if (values[i]) {\n buf[Math.floor(i / 8)] |= 1 << i % 8;\n }\n }\n return buf;\n}\n\nfunction decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {\n const values: boolean[] = [];\n for (let i = 0; i < count; i++) {\n const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];\n values.push((b & (1 << i % 8)) > 0);\n }\n cursor.offset += Math.ceil(count / 8);\n return values;\n}\n\nfunction encodeValues_INT32(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeInt32LE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_INT32(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readInt32LE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_INT64(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n INT53.writeInt64LE(values[i], buf, i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_INT64(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_INT96(values: number[]): Buffer {\n const buf = Buffer.alloc(12 * values.length);\n for (let i = 0; i < values.length; i++) {\n if (values[i] >= 0) {\n INT53.writeInt64LE(values[i], buf, i * 12);\n buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision\n } else {\n INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);\n buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision\n }\n }\n return buf;\n}\n\nfunction decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n const low = INT53.readInt64LE(cursor.buffer, cursor.offset);\n const high = cursor.buffer.readUInt32LE(cursor.offset + 8);\n if (high === 0xffffffff) {\n values.push(~-low + 1); // truncate to 64 actual precision\n } else {\n values.push(low); // truncate to 64 actual precision\n }\n cursor.offset += 12;\n }\n return values;\n}\n\nfunction encodeValues_FLOAT(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeFloatLE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_FLOAT(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readFloatLE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_DOUBLE(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeDoubleLE(values[i], i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_DOUBLE(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readDoubleLE(cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {\n // tslint:disable-next-line:variable-name\n let buf_len = 0;\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n buf_len += 4 + values[i].length;\n }\n const buf = Buffer.alloc(buf_len);\n // tslint:disable-next-line:variable-name\n let buf_pos = 0;\n for (let i = 0; i < values.length; i++) {\n buf.writeUInt32LE(values[i].length, buf_pos);\n values[i].copy(buf, buf_pos + 4);\n buf_pos += 4 + values[i].length;\n }\n return buf;\n}\n\nfunction decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[] {\n const values: Buffer[] = [];\n for (let i = 0; i < count; i++) {\n const len = cursor.buffer.readUInt32LE(cursor.offset);\n cursor.offset += 4;\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));\n cursor.offset += len;\n }\n return values;\n}\n\nfunction encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecOptions): Buffer {\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n if (values[i].length !== opts.typeLength) {\n throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);\n }\n }\n return Buffer.concat(values);\n}\n\nfunction decodeValues_FIXED_LEN_BYTE_ARRAY(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): Buffer[] {\n const values: Buffer[] = [];\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));\n cursor.offset += opts.typeLength;\n }\n return values;\n}\n"],"file":"plain.js"}
|
|
1
|
+
{"version":3,"sources":["../../../../src/parquetjs/codecs/plain.ts"],"names":["encodeValues","type","values","opts","encodeValues_BOOLEAN","encodeValues_INT32","encodeValues_INT64","encodeValues_INT96","encodeValues_FLOAT","encodeValues_DOUBLE","encodeValues_BYTE_ARRAY","encodeValues_FIXED_LEN_BYTE_ARRAY","Error","decodeValues","cursor","count","decodeValues_BOOLEAN","decodeValues_INT32","decodeValues_INT64","decodeValues_INT96","decodeValues_FLOAT","decodeValues_DOUBLE","decodeValues_BYTE_ARRAY","decodeValues_FIXED_LEN_BYTE_ARRAY","buf","Buffer","alloc","Math","ceil","length","fill","i","floor","b","buffer","offset","push","writeInt32LE","readInt32LE","INT53","writeInt64LE","readInt64LE","writeUInt32LE","low","high","readUInt32LE","writeFloatLE","readFloatLE","writeDoubleLE","readDoubleLE","buf_len","from","buf_pos","copy","len","slice","typeLength","concat"],"mappings":";;;;;;;;;;AAIA;;AAEO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOG,oBAAoB,CAACF,MAAD,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACH,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACJ,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACL,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOM,kBAAkB,CAACN,MAAD,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOO,mBAAmB,CAACP,MAAD,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOQ,uBAAuB,CAACR,MAAD,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOS,iCAAiC,CAACT,MAAD,EAASC,IAAT,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAEM,SAASY,YAAT,CACLZ,IADK,EAELa,MAFK,EAGLC,KAHK,EAILZ,IAJK,EAKE;AACP,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOe,oBAAoB,CAACF,MAAD,EAASC,KAAT,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOE,kBAAkB,CAACH,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACJ,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACL,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACN,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOM,mBAAmB,CAACP,MAAD,EAASC,KAAT,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOO,uBAAuB,CAACR,MAAD,EAASC,KAAT,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOQ,iCAAiC,CAACT,MAAD,EAASC,KAAT,EAAgBZ,IAAhB,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAED,SAASG,oBAAT,CAA8BF,MAA9B,EAAyD;AACvD,QAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAaC,IAAI,CAACC,IAAL,CAAU1B,MAAM,CAAC2B,MAAP,GAAgB,CAA1B,CAAb,CAAZ;AACAL,EAAAA,GAAG,CAACM,IAAJ,CAAS,CAAT;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAV,EAAe;AACbP,MAAAA,GAAG,CAACG,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASR,oBAAT,CAA8BF,MAA9B,EAAoDC,KAApD,EAA8E;AAC5E,QAAMb,MAAiB,GAAG,EAA1B;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,UAAME,CAAC,GAAGnB,MAAM,CAACoB,MAAP,CAAcpB,MAAM,CAACqB,MAAP,GAAgBR,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAA9B,CAAV;AACA7B,IAAAA,MAAM,CAACkC,IAAP,CAAY,CAACH,CAAC,GAAI,KAAKF,CAAC,GAAG,CAAf,IAAqB,CAAjC;AACD;;AACDjB,EAAAA,MAAM,CAACqB,MAAP,IAAiBR,IAAI,CAACC,IAAL,CAAUb,KAAK,GAAG,CAAlB,CAAjB;AACA,SAAOb,MAAP;AACD;;AAED,SAASG,kBAAT,CAA4BH,MAA5B,EAAsD;AACpD,QAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACa,YAAJ,CAAiBnC,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASP,kBAAT,CAA4BH,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,QAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcI,WAAd,CAA0BxB,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASI,kBAAT,CAA4BJ,MAA5B,EAAsD;AACpD,QAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCQ,iBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,CAAvC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASN,kBAAT,CAA4BJ,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,QAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYG,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASK,kBAAT,CAA4BL,MAA5B,EAAsD;AACpD,QAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,KAAKxB,MAAM,CAAC2B,MAAzB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,IAAa,CAAjB,EAAoB;AAClBQ,mBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,EAAvC;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,CAAlB,EAAqBX,CAAC,GAAG,EAAJ,GAAS,CAA9B;AACD,KAHD,MAGO;AACLQ,mBAAMC,YAAN,CAAmB,CAAC,CAACtC,MAAM,CAAC6B,CAAD,CAAR,GAAc,CAAjC,EAAoCP,GAApC,EAAyCO,CAAC,GAAG,EAA7C;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,UAAlB,EAA8BX,CAAC,GAAG,EAAJ,GAAS,CAAvC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASL,kBAAT,CAA4BL,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,QAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,UAAMY,GAAG,GAAGJ,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;;AACA,UAAMS,IAAI,GAAG9B,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAP,GAAgB,CAA3C,CAAb;;AACA,QAAIS,IAAI,KAAK,UAAb,EAAyB;AACvB1C,MAAAA,MAAM,CAACkC,IAAP,CAAY,CAAC,CAACO,GAAF,GAAQ,CAApB;AACD,KAFD,MAEO;AACLzC,MAAAA,MAAM,CAACkC,IAAP,CAAYO,GAAZ;AACD;;AACD7B,IAAAA,MAAM,CAACqB,MAAP,IAAiB,EAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASM,kBAAT,CAA4BN,MAA5B,EAAsD;AACpD,QAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACsB,YAAJ,CAAiB5C,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASJ,kBAAT,CAA4BN,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,QAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAca,WAAd,CAA0BjC,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASO,mBAAT,CAA6BP,MAA7B,EAAuD;AACrD,QAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACwB,aAAJ,CAAkB9C,MAAM,CAAC6B,CAAD,CAAxB,EAA6BA,CAAC,GAAG,CAAjC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASH,mBAAT,CAA6BP,MAA7B,EAAmDC,KAAnD,EAA4E;AAC1E,QAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAce,YAAd,CAA2BnC,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASQ,uBAAT,CAAiCR,MAAjC,EAA2D;AAEzD,MAAIgD,OAAO,GAAG,CAAd;;AACA,OAAK,IAAInB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;AACAmB,IAAAA,OAAO,IAAI,IAAIhD,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,QAAML,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAawB,OAAb,CAAZ;AAEA,MAAIE,OAAO,GAAG,CAAd;;AACA,OAAK,IAAIrB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACkB,aAAJ,CAAkBxC,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAA5B,EAAoCuB,OAApC;AACAlD,IAAAA,MAAM,CAAC6B,CAAD,CAAN,CAAUsB,IAAV,CAAe7B,GAAf,EAAoB4B,OAAO,GAAG,CAA9B;AACAA,IAAAA,OAAO,IAAI,IAAIlD,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,SAAOL,GAAP;AACD;;AAED,SAASF,uBAAT,CAAiCR,MAAjC,EAAuDC,KAAvD,EAAgF;AAC9E,QAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,UAAMuB,GAAG,GAAGxC,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACAjC,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBmB,GAAnD,CAAZ;AACAxC,IAAAA,MAAM,CAACqB,MAAP,IAAiBmB,GAAjB;AACD;;AACD,SAAOpD,MAAP;AACD;;AAED,SAASS,iCAAT,CAA2CT,MAA3C,EAA6DC,IAA7D,EAAgG;AAC9F,MAAI,CAACA,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;;AACA,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAV,KAAqB1B,IAAI,CAACqD,UAA9B,EAA0C;AACxC,YAAM,IAAI5C,KAAJ,mDAAqDV,MAAM,CAAC6B,CAAD,CAA3D,EAAN;AACD;AACF;;AACD,SAAON,MAAM,CAACgC,MAAP,CAAcvD,MAAd,CAAP;AACD;;AAED,SAASqB,iCAAT,CACET,MADF,EAEEC,KAFF,EAGEZ,IAHF,EAIY;AACV,QAAMD,MAAgB,GAAG,EAAzB;;AACA,MAAI,CAACC,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBhC,IAAI,CAACqD,UAAxD,CAAZ;AACA1C,IAAAA,MAAM,CAACqB,MAAP,IAAiBhC,IAAI,CAACqD,UAAtB;AACD;;AACD,SAAOtD,MAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport INT53 from 'int53';\n\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n switch (type) {\n case 'BOOLEAN':\n return encodeValues_BOOLEAN(values);\n case 'INT32':\n return encodeValues_INT32(values);\n case 'INT64':\n return encodeValues_INT64(values);\n case 'INT96':\n return encodeValues_INT96(values);\n case 'FLOAT':\n return encodeValues_FLOAT(values);\n case 'DOUBLE':\n return encodeValues_DOUBLE(values);\n case 'BYTE_ARRAY':\n return encodeValues_BYTE_ARRAY(values);\n case 'FIXED_LEN_BYTE_ARRAY':\n return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n switch (type) {\n case 'BOOLEAN':\n return decodeValues_BOOLEAN(cursor, count);\n case 'INT32':\n return decodeValues_INT32(cursor, count);\n case 'INT64':\n return decodeValues_INT64(cursor, count);\n case 'INT96':\n return decodeValues_INT96(cursor, count);\n case 'FLOAT':\n return decodeValues_FLOAT(cursor, count);\n case 'DOUBLE':\n return decodeValues_DOUBLE(cursor, count);\n case 'BYTE_ARRAY':\n return decodeValues_BYTE_ARRAY(cursor, count);\n case 'FIXED_LEN_BYTE_ARRAY':\n return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n}\n\nfunction encodeValues_BOOLEAN(values: boolean[]): Buffer {\n const buf = Buffer.alloc(Math.ceil(values.length / 8));\n buf.fill(0);\n for (let i = 0; i < values.length; i++) {\n if (values[i]) {\n buf[Math.floor(i / 8)] |= 1 << i % 8;\n }\n }\n return buf;\n}\n\nfunction decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {\n const values: boolean[] = [];\n for (let i = 0; i < count; i++) {\n const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];\n values.push((b & (1 << i % 8)) > 0);\n }\n cursor.offset += Math.ceil(count / 8);\n return values;\n}\n\nfunction encodeValues_INT32(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeInt32LE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_INT32(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readInt32LE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_INT64(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n INT53.writeInt64LE(values[i], buf, i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_INT64(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_INT96(values: number[]): Buffer {\n const buf = Buffer.alloc(12 * values.length);\n for (let i = 0; i < values.length; i++) {\n if (values[i] >= 0) {\n INT53.writeInt64LE(values[i], buf, i * 12);\n buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision\n } else {\n INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);\n buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision\n }\n }\n return buf;\n}\n\nfunction decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n const low = INT53.readInt64LE(cursor.buffer, cursor.offset);\n const high = cursor.buffer.readUInt32LE(cursor.offset + 8);\n if (high === 0xffffffff) {\n values.push(~-low + 1); // truncate to 64 actual precision\n } else {\n values.push(low); // truncate to 64 actual precision\n }\n cursor.offset += 12;\n }\n return values;\n}\n\nfunction encodeValues_FLOAT(values: number[]): Buffer {\n const buf = Buffer.alloc(4 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeFloatLE(values[i], i * 4);\n }\n return buf;\n}\n\nfunction decodeValues_FLOAT(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readFloatLE(cursor.offset));\n cursor.offset += 4;\n }\n return values;\n}\n\nfunction encodeValues_DOUBLE(values: number[]): Buffer {\n const buf = Buffer.alloc(8 * values.length);\n for (let i = 0; i < values.length; i++) {\n buf.writeDoubleLE(values[i], i * 8);\n }\n return buf;\n}\n\nfunction decodeValues_DOUBLE(cursor: CursorBuffer, count: number): number[] {\n const values: number[] = [];\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.readDoubleLE(cursor.offset));\n cursor.offset += 8;\n }\n return values;\n}\n\nfunction encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {\n // tslint:disable-next-line:variable-name\n let buf_len = 0;\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n buf_len += 4 + values[i].length;\n }\n const buf = Buffer.alloc(buf_len);\n // tslint:disable-next-line:variable-name\n let buf_pos = 0;\n for (let i = 0; i < values.length; i++) {\n buf.writeUInt32LE(values[i].length, buf_pos);\n values[i].copy(buf, buf_pos + 4);\n buf_pos += 4 + values[i].length;\n }\n return buf;\n}\n\nfunction decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[] {\n const values: Buffer[] = [];\n for (let i = 0; i < count; i++) {\n const len = cursor.buffer.readUInt32LE(cursor.offset);\n cursor.offset += 4;\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));\n cursor.offset += len;\n }\n return values;\n}\n\nfunction encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecOptions): Buffer {\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < values.length; i++) {\n values[i] = Buffer.from(values[i]);\n if (values[i].length !== opts.typeLength) {\n throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);\n }\n }\n return Buffer.concat(values);\n}\n\nfunction decodeValues_FIXED_LEN_BYTE_ARRAY(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): Buffer[] {\n const values: Buffer[] = [];\n if (!opts.typeLength) {\n throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n }\n for (let i = 0; i < count; i++) {\n values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));\n cursor.offset += opts.typeLength;\n }\n return values;\n}\n"],"file":"plain.js"}
|
|
@@ -8,8 +8,6 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
8
8
|
exports.encodeValues = encodeValues;
|
|
9
9
|
exports.decodeValues = decodeValues;
|
|
10
10
|
|
|
11
|
-
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
|
|
12
|
-
|
|
13
11
|
var _varint = _interopRequireDefault(require("varint"));
|
|
14
12
|
|
|
15
13
|
function encodeValues(type, values, opts) {
|
|
@@ -21,20 +19,18 @@ function encodeValues(type, values, opts) {
|
|
|
21
19
|
case 'BOOLEAN':
|
|
22
20
|
case 'INT32':
|
|
23
21
|
case 'INT64':
|
|
24
|
-
values = values.map(
|
|
25
|
-
return parseInt(x, 10);
|
|
26
|
-
});
|
|
22
|
+
values = values.map(x => parseInt(x, 10));
|
|
27
23
|
break;
|
|
28
24
|
|
|
29
25
|
default:
|
|
30
26
|
throw new Error("unsupported type: ".concat(type));
|
|
31
27
|
}
|
|
32
28
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
29
|
+
let buf = Buffer.alloc(0);
|
|
30
|
+
let run = [];
|
|
31
|
+
let repeats = 0;
|
|
36
32
|
|
|
37
|
-
for (
|
|
33
|
+
for (let i = 0; i < values.length; i++) {
|
|
38
34
|
if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
|
|
39
35
|
if (run.length) {
|
|
40
36
|
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
@@ -64,7 +60,7 @@ function encodeValues(type, values, opts) {
|
|
|
64
60
|
return buf;
|
|
65
61
|
}
|
|
66
62
|
|
|
67
|
-
|
|
63
|
+
const envelope = Buffer.alloc(buf.length + 4);
|
|
68
64
|
envelope.writeUInt32LE(buf.length, undefined);
|
|
69
65
|
buf.copy(envelope, 4);
|
|
70
66
|
return envelope;
|
|
@@ -79,25 +75,19 @@ function decodeValues(type, cursor, count, opts) {
|
|
|
79
75
|
cursor.offset += 4;
|
|
80
76
|
}
|
|
81
77
|
|
|
82
|
-
|
|
78
|
+
let values = [];
|
|
83
79
|
|
|
84
80
|
while (values.length < count) {
|
|
85
|
-
|
|
81
|
+
const header = _varint.default.decode(cursor.buffer, cursor.offset);
|
|
86
82
|
|
|
87
83
|
cursor.offset += _varint.default.encodingLength(header);
|
|
88
84
|
|
|
89
85
|
if (header & 1) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
var _count = (header >> 1) * 8;
|
|
93
|
-
|
|
94
|
-
(_values = values).push.apply(_values, (0, _toConsumableArray2.default)(decodeRunBitpacked(cursor, _count, opts)));
|
|
86
|
+
const count = (header >> 1) * 8;
|
|
87
|
+
values.push(...decodeRunBitpacked(cursor, count, opts));
|
|
95
88
|
} else {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
var _count2 = header >> 1;
|
|
99
|
-
|
|
100
|
-
(_values2 = values).push.apply(_values2, (0, _toConsumableArray2.default)(decodeRunRepeated(cursor, _count2, opts)));
|
|
89
|
+
const count = header >> 1;
|
|
90
|
+
values.push(...decodeRunRepeated(cursor, count, opts));
|
|
101
91
|
}
|
|
102
92
|
}
|
|
103
93
|
|
|
@@ -111,15 +101,15 @@ function decodeValues(type, cursor, count, opts) {
|
|
|
111
101
|
}
|
|
112
102
|
|
|
113
103
|
function decodeRunBitpacked(cursor, count, opts) {
|
|
114
|
-
|
|
104
|
+
const bitWidth = opts.bitWidth;
|
|
115
105
|
|
|
116
106
|
if (count % 8 !== 0) {
|
|
117
107
|
throw new Error('must be a multiple of 8');
|
|
118
108
|
}
|
|
119
109
|
|
|
120
|
-
|
|
110
|
+
const values = new Array(count).fill(0);
|
|
121
111
|
|
|
122
|
-
for (
|
|
112
|
+
for (let b = 0; b < bitWidth * count; b++) {
|
|
123
113
|
if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
|
|
124
114
|
values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
|
|
125
115
|
}
|
|
@@ -130,10 +120,10 @@ function decodeRunBitpacked(cursor, count, opts) {
|
|
|
130
120
|
}
|
|
131
121
|
|
|
132
122
|
function decodeRunRepeated(cursor, count, opts) {
|
|
133
|
-
|
|
134
|
-
|
|
123
|
+
const bitWidth = opts.bitWidth;
|
|
124
|
+
let value = 0;
|
|
135
125
|
|
|
136
|
-
for (
|
|
126
|
+
for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
|
|
137
127
|
value << 8;
|
|
138
128
|
value += cursor.buffer[cursor.offset];
|
|
139
129
|
cursor.offset += 1;
|
|
@@ -143,15 +133,15 @@ function decodeRunRepeated(cursor, count, opts) {
|
|
|
143
133
|
}
|
|
144
134
|
|
|
145
135
|
function encodeRunBitpacked(values, opts) {
|
|
146
|
-
|
|
136
|
+
const bitWidth = opts.bitWidth;
|
|
147
137
|
|
|
148
|
-
for (
|
|
138
|
+
for (let i = 0; i < values.length % 8; i++) {
|
|
149
139
|
values.push(0);
|
|
150
140
|
}
|
|
151
141
|
|
|
152
|
-
|
|
142
|
+
const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
|
|
153
143
|
|
|
154
|
-
for (
|
|
144
|
+
for (let b = 0; b < bitWidth * values.length; b++) {
|
|
155
145
|
if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
|
|
156
146
|
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
157
147
|
}
|
|
@@ -161,10 +151,10 @@ function encodeRunBitpacked(values, opts) {
|
|
|
161
151
|
}
|
|
162
152
|
|
|
163
153
|
function encodeRunRepeated(value, count, opts) {
|
|
164
|
-
|
|
165
|
-
|
|
154
|
+
const bitWidth = opts.bitWidth;
|
|
155
|
+
const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
|
|
166
156
|
|
|
167
|
-
for (
|
|
157
|
+
for (let i = 0; i < buf.length; i++) {
|
|
168
158
|
buf.writeUInt8(value & 0xff, i);
|
|
169
159
|
value >> 8;
|
|
170
160
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAEhD6B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AAEA6B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n // eslint-disable-next-line\n value << 8; // TODO - this looks wrong\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n // eslint-disable-next-line\n value >> 8; // TODO - this looks wrong\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
|
|
1
|
+
{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,kBAAkB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGgB,iBAAiB,CAACT,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAEhD6B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,QAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AAEA6B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n // eslint-disable-next-line\n value << 8; // TODO - this looks wrong\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n // eslint-disable-next-line\n value >> 8; // TODO - this looks wrong\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
|