@loaders.gl/parquet 3.3.0-alpha.4 → 3.3.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +14 -14
- package/dist/dist.min.js.map +2 -2
- package/dist/es5/bundle.js +0 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/constants.js +3 -1
- package/dist/es5/constants.js.map +1 -1
- package/dist/es5/index.js +23 -39
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/convert-schema.js +2 -11
- package/dist/es5/lib/convert-schema.js.map +1 -1
- package/dist/es5/lib/parse-parquet.js +29 -72
- package/dist/es5/lib/parse-parquet.js.map +1 -1
- package/dist/es5/lib/read-array-buffer.js +0 -10
- package/dist/es5/lib/read-array-buffer.js.map +1 -1
- package/dist/es5/lib/wasm/encode-parquet-wasm.js +0 -11
- package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -1
- package/dist/es5/lib/wasm/load-wasm/index.js +0 -1
- package/dist/es5/lib/wasm/load-wasm/index.js.map +1 -1
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +0 -14
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +0 -10
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
- package/dist/es5/lib/wasm/parse-parquet-wasm.js +1 -19
- package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -1
- package/dist/es5/parquet-loader.js +2 -1
- package/dist/es5/parquet-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-loader.js +2 -1
- package/dist/es5/parquet-wasm-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-writer.js +1 -3
- package/dist/es5/parquet-wasm-writer.js.map +1 -1
- package/dist/es5/parquet-writer.js +1 -2
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/es5/parquetjs/codecs/declare.js.map +1 -1
- package/dist/es5/parquetjs/codecs/dictionary.js +2 -9
- package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/es5/parquetjs/codecs/index.js +0 -8
- package/dist/es5/parquetjs/codecs/index.js.map +1 -1
- package/dist/es5/parquetjs/codecs/plain.js +1 -77
- package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
- package/dist/es5/parquetjs/codecs/rle.js +1 -39
- package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
- package/dist/es5/parquetjs/compression.js +5 -30
- package/dist/es5/parquetjs/compression.js.map +1 -1
- package/dist/es5/parquetjs/encoder/writer.js +31 -149
- package/dist/es5/parquetjs/encoder/writer.js.map +1 -1
- package/dist/es5/parquetjs/file.js +3 -12
- package/dist/es5/parquetjs/file.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +0 -48
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +0 -47
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +0 -82
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +0 -25
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +0 -39
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -51
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DateType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +0 -26
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +0 -59
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IntType.js +0 -26
- package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +0 -26
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ListType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +0 -85
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MapType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/NullType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +0 -25
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +0 -54
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +0 -31
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +0 -41
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +0 -59
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js +0 -42
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/StringType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js +0 -27
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +0 -30
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +0 -27
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/index.js +0 -86
- package/dist/es5/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/es5/parquetjs/parser/decoders.js +3 -82
- package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-cursor.js +5 -37
- package/dist/es5/parquetjs/parser/parquet-cursor.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +2 -88
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
- package/dist/es5/parquetjs/parser/parquet-reader.js +14 -67
- package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/es5/parquetjs/schema/declare.js +3 -7
- package/dist/es5/parquetjs/schema/declare.js.map +1 -1
- package/dist/es5/parquetjs/schema/schema.js +6 -34
- package/dist/es5/parquetjs/schema/schema.js.map +1 -1
- package/dist/es5/parquetjs/schema/shred.js +11 -41
- package/dist/es5/parquetjs/schema/shred.js.map +1 -1
- package/dist/es5/parquetjs/schema/types.js +3 -84
- package/dist/es5/parquetjs/schema/types.js.map +1 -1
- package/dist/es5/parquetjs/utils/buffer-utils.js +0 -2
- package/dist/es5/parquetjs/utils/buffer-utils.js.map +1 -1
- package/dist/es5/parquetjs/utils/file-utils.js +1 -7
- package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/es5/parquetjs/utils/read-utils.js +6 -38
- package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/es5/workers/parquet-worker.js +0 -2
- package/dist/es5/workers/parquet-worker.js.map +1 -1
- package/dist/esm/bundle.js +1 -1
- package/dist/esm/bundle.js.map +1 -1
- package/dist/esm/constants.js +3 -0
- package/dist/esm/constants.js.map +1 -1
- package/dist/esm/index.js +10 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/convert-schema.js +1 -7
- package/dist/esm/lib/convert-schema.js.map +1 -1
- package/dist/esm/lib/parse-parquet.js +2 -5
- package/dist/esm/lib/parse-parquet.js.map +1 -1
- package/dist/esm/lib/read-array-buffer.js +2 -1
- package/dist/esm/lib/read-array-buffer.js.map +1 -1
- package/dist/esm/lib/wasm/encode-parquet-wasm.js +1 -1
- package/dist/esm/lib/wasm/encode-parquet-wasm.js.map +1 -1
- package/dist/esm/lib/wasm/load-wasm/index.js.map +1 -1
- package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
- package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
- package/dist/esm/lib/wasm/parse-parquet-wasm.js +2 -3
- package/dist/esm/lib/wasm/parse-parquet-wasm.js.map +1 -1
- package/dist/esm/parquet-loader.js +3 -1
- package/dist/esm/parquet-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-loader.js +3 -1
- package/dist/esm/parquet-wasm-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-writer.js +2 -1
- package/dist/esm/parquet-wasm-writer.js.map +1 -1
- package/dist/esm/parquet-writer.js +2 -2
- package/dist/esm/parquet-writer.js.map +1 -1
- package/dist/esm/parquetjs/codecs/declare.js.map +1 -1
- package/dist/esm/parquetjs/codecs/dictionary.js +2 -1
- package/dist/esm/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/esm/parquetjs/codecs/index.js +2 -0
- package/dist/esm/parquetjs/codecs/index.js.map +1 -1
- package/dist/esm/parquetjs/codecs/plain.js +2 -68
- package/dist/esm/parquetjs/codecs/plain.js.map +1 -1
- package/dist/esm/parquetjs/codecs/rle.js +3 -29
- package/dist/esm/parquetjs/codecs/rle.js.map +1 -1
- package/dist/esm/parquetjs/compression.js +9 -5
- package/dist/esm/parquetjs/compression.js.map +1 -1
- package/dist/esm/parquetjs/encoder/writer.js +21 -51
- package/dist/esm/parquetjs/encoder/writer.js.map +1 -1
- package/dist/esm/parquetjs/file.js +1 -0
- package/dist/esm/parquetjs/file.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +0 -44
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +0 -42
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +0 -82
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +0 -18
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +0 -34
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -49
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DateType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +2 -53
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/IntType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/ListType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +0 -90
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/MapType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/NullType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +0 -16
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +0 -49
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageType.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +0 -33
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +0 -59
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js +0 -38
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/StringType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +0 -24
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +0 -19
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Type.js +1 -1
- package/dist/esm/parquetjs/parquet-thrift/Type.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +1 -8
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/esm/parquetjs/parquet-thrift/index.js +1 -0
- package/dist/esm/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/esm/parquetjs/parser/decoders.js +9 -39
- package/dist/esm/parquetjs/parser/decoders.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-cursor.js +1 -13
- package/dist/esm/parquetjs/parser/parquet-cursor.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +6 -32
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
- package/dist/esm/parquetjs/parser/parquet-reader.js +1 -18
- package/dist/esm/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/esm/parquetjs/schema/declare.js +4 -4
- package/dist/esm/parquetjs/schema/declare.js.map +1 -1
- package/dist/esm/parquetjs/schema/schema.js +3 -29
- package/dist/esm/parquetjs/schema/schema.js.map +1 -1
- package/dist/esm/parquetjs/schema/shred.js +7 -22
- package/dist/esm/parquetjs/schema/shred.js.map +1 -1
- package/dist/esm/parquetjs/schema/types.js +3 -78
- package/dist/esm/parquetjs/schema/types.js.map +1 -1
- package/dist/esm/parquetjs/utils/buffer-utils.js +2 -1
- package/dist/esm/parquetjs/utils/buffer-utils.js.map +1 -1
- package/dist/esm/parquetjs/utils/file-utils.js +1 -0
- package/dist/esm/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/esm/parquetjs/utils/read-utils.js +5 -12
- package/dist/esm/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/esm/workers/parquet-worker.js.map +1 -1
- package/dist/lib/wasm/load-wasm/load-wasm-browser.js +5 -1
- package/dist/lib/wasm/load-wasm/load-wasm-node.js +5 -1
- package/dist/parquet-worker.js +14 -14
- package/dist/parquet-worker.js.map +2 -2
- package/dist/parquetjs/codecs/index.js +5 -1
- package/dist/parquetjs/encoder/writer.d.ts +1 -0
- package/dist/parquetjs/encoder/writer.d.ts.map +1 -1
- package/dist/parquetjs/encoder/writer.js +5 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +5 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +5 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +5 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +5 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +5 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +5 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +5 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +5 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +5 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +5 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +5 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +5 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +5 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +5 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +5 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +5 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +5 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +5 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +5 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +5 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +5 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +5 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +5 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +5 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +5 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +5 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +5 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +5 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +5 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +5 -1
- package/dist/parquetjs/parquet-thrift/index.js +5 -1
- package/dist/parquetjs/schema/shred.js +5 -1
- package/dist/parquetjs/utils/file-utils.d.ts +2 -0
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/package.json +6 -6
|
@@ -1,69 +1,44 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
var _typeof = require("@babel/runtime/helpers/typeof");
|
|
6
|
-
|
|
7
5
|
Object.defineProperty(exports, "__esModule", {
|
|
8
6
|
value: true
|
|
9
7
|
});
|
|
10
|
-
exports.
|
|
11
|
-
|
|
8
|
+
exports.ParquetWriter = exports.ParquetTransformer = exports.ParquetEnvelopeWriter = void 0;
|
|
12
9
|
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
|
|
13
|
-
|
|
14
10
|
var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
|
|
15
|
-
|
|
16
11
|
var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
|
|
17
|
-
|
|
18
12
|
var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
|
|
19
|
-
|
|
20
13
|
var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
|
|
21
|
-
|
|
22
14
|
var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
|
|
23
|
-
|
|
24
15
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
25
|
-
|
|
26
16
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
27
|
-
|
|
28
17
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
29
|
-
|
|
30
18
|
var _stream = require("stream");
|
|
31
|
-
|
|
32
19
|
var _codecs = require("../codecs");
|
|
33
|
-
|
|
34
20
|
var Compression = _interopRequireWildcard(require("../compression"));
|
|
35
|
-
|
|
36
21
|
var Shred = _interopRequireWildcard(require("../schema/shred"));
|
|
37
|
-
|
|
38
22
|
var _parquetThrift = require("../parquet-thrift");
|
|
39
|
-
|
|
40
23
|
var _fileUtils = require("../utils/file-utils");
|
|
41
|
-
|
|
42
24
|
var _readUtils = require("../utils/read-utils");
|
|
43
|
-
|
|
44
25
|
var _nodeInt = _interopRequireDefault(require("node-int64"));
|
|
45
|
-
|
|
46
26
|
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
|
47
|
-
|
|
48
27
|
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
|
|
49
|
-
|
|
50
28
|
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
|
|
51
|
-
|
|
52
29
|
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
53
|
-
|
|
54
30
|
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
55
|
-
|
|
56
31
|
function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
|
|
57
|
-
|
|
58
32
|
function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
|
|
59
|
-
|
|
60
33
|
var PARQUET_MAGIC = 'PAR1';
|
|
34
|
+
|
|
61
35
|
var PARQUET_VERSION = 1;
|
|
36
|
+
|
|
62
37
|
var PARQUET_DEFAULT_PAGE_SIZE = 8192;
|
|
63
38
|
var PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
|
|
39
|
+
|
|
64
40
|
var PARQUET_RDLVL_TYPE = 'INT32';
|
|
65
41
|
var PARQUET_RDLVL_ENCODING = 'RLE';
|
|
66
|
-
|
|
67
42
|
var ParquetWriter = function () {
|
|
68
43
|
function ParquetWriter(schema, envelopeWriter, opts) {
|
|
69
44
|
(0, _classCallCheck2.default)(this, ParquetWriter);
|
|
@@ -79,9 +54,9 @@ var ParquetWriter = function () {
|
|
|
79
54
|
this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
|
|
80
55
|
this.closed = false;
|
|
81
56
|
this.userMetadata = {};
|
|
57
|
+
|
|
82
58
|
this.writeHeader();
|
|
83
59
|
}
|
|
84
|
-
|
|
85
60
|
(0, _createClass2.default)(ParquetWriter, [{
|
|
86
61
|
key: "writeHeader",
|
|
87
62
|
value: function () {
|
|
@@ -93,20 +68,16 @@ var ParquetWriter = function () {
|
|
|
93
68
|
_context.prev = 0;
|
|
94
69
|
_context.next = 3;
|
|
95
70
|
return this.envelopeWriter.writeHeader();
|
|
96
|
-
|
|
97
71
|
case 3:
|
|
98
72
|
_context.next = 10;
|
|
99
73
|
break;
|
|
100
|
-
|
|
101
74
|
case 5:
|
|
102
75
|
_context.prev = 5;
|
|
103
76
|
_context.t0 = _context["catch"](0);
|
|
104
77
|
_context.next = 9;
|
|
105
78
|
return this.envelopeWriter.close();
|
|
106
|
-
|
|
107
79
|
case 9:
|
|
108
80
|
throw _context.t0;
|
|
109
|
-
|
|
110
81
|
case 10:
|
|
111
82
|
case "end":
|
|
112
83
|
return _context.stop();
|
|
@@ -114,11 +85,9 @@ var ParquetWriter = function () {
|
|
|
114
85
|
}
|
|
115
86
|
}, _callee, this, [[0, 5]]);
|
|
116
87
|
}));
|
|
117
|
-
|
|
118
88
|
function writeHeader() {
|
|
119
89
|
return _writeHeader.apply(this, arguments);
|
|
120
90
|
}
|
|
121
|
-
|
|
122
91
|
return writeHeader;
|
|
123
92
|
}()
|
|
124
93
|
}, {
|
|
@@ -133,16 +102,12 @@ var ParquetWriter = function () {
|
|
|
133
102
|
_context2.next = 2;
|
|
134
103
|
break;
|
|
135
104
|
}
|
|
136
|
-
|
|
137
105
|
throw new Error('writer was closed');
|
|
138
|
-
|
|
139
106
|
case 2:
|
|
140
107
|
Shred.shredRecord(this.schema, row, this.rowBuffer);
|
|
141
|
-
|
|
142
108
|
if (this.rowBuffer.rowCount >= this.rowGroupSize) {
|
|
143
109
|
this.rowBuffer = {};
|
|
144
110
|
}
|
|
145
|
-
|
|
146
111
|
case 4:
|
|
147
112
|
case "end":
|
|
148
113
|
return _context2.stop();
|
|
@@ -150,11 +115,9 @@ var ParquetWriter = function () {
|
|
|
150
115
|
}
|
|
151
116
|
}, _callee2, this);
|
|
152
117
|
}));
|
|
153
|
-
|
|
154
118
|
function appendRow(_x) {
|
|
155
119
|
return _appendRow.apply(this, arguments);
|
|
156
120
|
}
|
|
157
|
-
|
|
158
121
|
return appendRow;
|
|
159
122
|
}()
|
|
160
123
|
}, {
|
|
@@ -169,28 +132,22 @@ var ParquetWriter = function () {
|
|
|
169
132
|
_context3.next = 2;
|
|
170
133
|
break;
|
|
171
134
|
}
|
|
172
|
-
|
|
173
135
|
throw new Error('writer was closed');
|
|
174
|
-
|
|
175
136
|
case 2:
|
|
176
137
|
this.closed = true;
|
|
177
|
-
|
|
178
138
|
if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
|
|
179
139
|
this.rowBuffer = {};
|
|
180
140
|
}
|
|
181
|
-
|
|
182
141
|
_context3.next = 6;
|
|
183
142
|
return this.envelopeWriter.writeFooter(this.userMetadata);
|
|
184
|
-
|
|
185
143
|
case 6:
|
|
186
144
|
_context3.next = 8;
|
|
187
145
|
return this.envelopeWriter.close();
|
|
188
|
-
|
|
189
146
|
case 8:
|
|
147
|
+
|
|
190
148
|
if (callback) {
|
|
191
149
|
callback();
|
|
192
150
|
}
|
|
193
|
-
|
|
194
151
|
case 9:
|
|
195
152
|
case "end":
|
|
196
153
|
return _context3.stop();
|
|
@@ -198,26 +155,29 @@ var ParquetWriter = function () {
|
|
|
198
155
|
}
|
|
199
156
|
}, _callee3, this);
|
|
200
157
|
}));
|
|
201
|
-
|
|
202
158
|
function close(_x2) {
|
|
203
159
|
return _close.apply(this, arguments);
|
|
204
160
|
}
|
|
205
|
-
|
|
206
161
|
return close;
|
|
207
162
|
}()
|
|
208
163
|
}, {
|
|
209
164
|
key: "setMetadata",
|
|
210
|
-
value:
|
|
165
|
+
value:
|
|
166
|
+
function setMetadata(key, value) {
|
|
211
167
|
this.userMetadata[String(key)] = String(value);
|
|
212
168
|
}
|
|
169
|
+
|
|
213
170
|
}, {
|
|
214
171
|
key: "setRowGroupSize",
|
|
215
|
-
value:
|
|
172
|
+
value:
|
|
173
|
+
function setRowGroupSize(cnt) {
|
|
216
174
|
this.rowGroupSize = cnt;
|
|
217
175
|
}
|
|
176
|
+
|
|
218
177
|
}, {
|
|
219
178
|
key: "setPageSize",
|
|
220
|
-
value:
|
|
179
|
+
value:
|
|
180
|
+
function setPageSize(cnt) {
|
|
221
181
|
this.envelopeWriter.setPageSize(cnt);
|
|
222
182
|
}
|
|
223
183
|
}], [{
|
|
@@ -231,11 +191,9 @@ var ParquetWriter = function () {
|
|
|
231
191
|
case 0:
|
|
232
192
|
_context4.next = 2;
|
|
233
193
|
return (0, _fileUtils.osopen)(path, opts);
|
|
234
|
-
|
|
235
194
|
case 2:
|
|
236
195
|
outputStream = _context4.sent;
|
|
237
196
|
return _context4.abrupt("return", ParquetWriter.openStream(schema, outputStream, opts));
|
|
238
|
-
|
|
239
197
|
case 4:
|
|
240
198
|
case "end":
|
|
241
199
|
return _context4.stop();
|
|
@@ -243,11 +201,9 @@ var ParquetWriter = function () {
|
|
|
243
201
|
}
|
|
244
202
|
}, _callee4);
|
|
245
203
|
}));
|
|
246
|
-
|
|
247
204
|
function openFile(_x3, _x4, _x5) {
|
|
248
205
|
return _openFile.apply(this, arguments);
|
|
249
206
|
}
|
|
250
|
-
|
|
251
207
|
return openFile;
|
|
252
208
|
}()
|
|
253
209
|
}, {
|
|
@@ -262,14 +218,11 @@ var ParquetWriter = function () {
|
|
|
262
218
|
if (!opts) {
|
|
263
219
|
opts = {};
|
|
264
220
|
}
|
|
265
|
-
|
|
266
221
|
_context5.next = 3;
|
|
267
222
|
return ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
|
|
268
|
-
|
|
269
223
|
case 3:
|
|
270
224
|
envelopeWriter = _context5.sent;
|
|
271
225
|
return _context5.abrupt("return", new ParquetWriter(schema, envelopeWriter, opts));
|
|
272
|
-
|
|
273
226
|
case 5:
|
|
274
227
|
case "end":
|
|
275
228
|
return _context5.stop();
|
|
@@ -277,19 +230,15 @@ var ParquetWriter = function () {
|
|
|
277
230
|
}
|
|
278
231
|
}, _callee5);
|
|
279
232
|
}));
|
|
280
|
-
|
|
281
233
|
function openStream(_x6, _x7, _x8) {
|
|
282
234
|
return _openStream.apply(this, arguments);
|
|
283
235
|
}
|
|
284
|
-
|
|
285
236
|
return openStream;
|
|
286
237
|
}()
|
|
287
238
|
}]);
|
|
288
239
|
return ParquetWriter;
|
|
289
240
|
}();
|
|
290
|
-
|
|
291
241
|
exports.ParquetWriter = ParquetWriter;
|
|
292
|
-
|
|
293
242
|
var ParquetEnvelopeWriter = function () {
|
|
294
243
|
function ParquetEnvelopeWriter(schema, writeFn, closeFn, fileOffset, opts) {
|
|
295
244
|
(0, _classCallCheck2.default)(this, ParquetEnvelopeWriter);
|
|
@@ -310,18 +259,20 @@ var ParquetEnvelopeWriter = function () {
|
|
|
310
259
|
this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
|
|
311
260
|
this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
|
|
312
261
|
}
|
|
313
|
-
|
|
314
262
|
(0, _createClass2.default)(ParquetEnvelopeWriter, [{
|
|
315
263
|
key: "writeSection",
|
|
316
264
|
value: function writeSection(buf) {
|
|
317
265
|
this.offset += buf.length;
|
|
318
266
|
return this.write(buf);
|
|
319
267
|
}
|
|
268
|
+
|
|
320
269
|
}, {
|
|
321
270
|
key: "writeHeader",
|
|
322
|
-
value:
|
|
271
|
+
value:
|
|
272
|
+
function writeHeader() {
|
|
323
273
|
return this.writeSection(Buffer.from(PARQUET_MAGIC));
|
|
324
274
|
}
|
|
275
|
+
|
|
325
276
|
}, {
|
|
326
277
|
key: "writeRowGroup",
|
|
327
278
|
value: function () {
|
|
@@ -337,17 +288,14 @@ var ParquetEnvelopeWriter = function () {
|
|
|
337
288
|
pageSize: this.pageSize,
|
|
338
289
|
useDataPageV2: this.useDataPageV2
|
|
339
290
|
});
|
|
340
|
-
|
|
341
291
|
case 2:
|
|
342
292
|
rgroup = _context6.sent;
|
|
343
293
|
this.rowCount += records.rowCount;
|
|
344
294
|
this.rowGroups.push(rgroup.metadata);
|
|
345
295
|
_context6.next = 7;
|
|
346
296
|
return this.writeSection(rgroup.body);
|
|
347
|
-
|
|
348
297
|
case 7:
|
|
349
298
|
return _context6.abrupt("return", _context6.sent);
|
|
350
|
-
|
|
351
299
|
case 8:
|
|
352
300
|
case "end":
|
|
353
301
|
return _context6.stop();
|
|
@@ -355,25 +303,25 @@ var ParquetEnvelopeWriter = function () {
|
|
|
355
303
|
}
|
|
356
304
|
}, _callee6, this);
|
|
357
305
|
}));
|
|
358
|
-
|
|
359
306
|
function writeRowGroup(_x9) {
|
|
360
307
|
return _writeRowGroup.apply(this, arguments);
|
|
361
308
|
}
|
|
362
|
-
|
|
363
309
|
return writeRowGroup;
|
|
364
310
|
}()
|
|
365
311
|
}, {
|
|
366
312
|
key: "writeFooter",
|
|
367
|
-
value:
|
|
313
|
+
value:
|
|
314
|
+
function writeFooter(userMetadata) {
|
|
368
315
|
if (!userMetadata) {
|
|
369
316
|
userMetadata = {};
|
|
370
317
|
}
|
|
371
|
-
|
|
372
318
|
return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
|
|
373
319
|
}
|
|
320
|
+
|
|
374
321
|
}, {
|
|
375
322
|
key: "setPageSize",
|
|
376
|
-
value:
|
|
323
|
+
value:
|
|
324
|
+
function setPageSize(cnt) {
|
|
377
325
|
this.pageSize = cnt;
|
|
378
326
|
}
|
|
379
327
|
}], [{
|
|
@@ -388,7 +336,6 @@ var ParquetEnvelopeWriter = function () {
|
|
|
388
336
|
writeFn = _fileUtils.oswrite.bind(undefined, outputStream);
|
|
389
337
|
closeFn = _fileUtils.osclose.bind(undefined, outputStream);
|
|
390
338
|
return _context7.abrupt("return", new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts));
|
|
391
|
-
|
|
392
339
|
case 3:
|
|
393
340
|
case "end":
|
|
394
341
|
return _context7.stop();
|
|
@@ -396,34 +343,26 @@ var ParquetEnvelopeWriter = function () {
|
|
|
396
343
|
}
|
|
397
344
|
}, _callee7);
|
|
398
345
|
}));
|
|
399
|
-
|
|
400
346
|
function openStream(_x10, _x11, _x12) {
|
|
401
347
|
return _openStream2.apply(this, arguments);
|
|
402
348
|
}
|
|
403
|
-
|
|
404
349
|
return openStream;
|
|
405
350
|
}()
|
|
406
351
|
}]);
|
|
407
352
|
return ParquetEnvelopeWriter;
|
|
408
353
|
}();
|
|
409
|
-
|
|
410
354
|
exports.ParquetEnvelopeWriter = ParquetEnvelopeWriter;
|
|
411
|
-
|
|
412
355
|
var ParquetTransformer = function (_Transform) {
|
|
413
356
|
(0, _inherits2.default)(ParquetTransformer, _Transform);
|
|
414
|
-
|
|
415
357
|
var _super = _createSuper(ParquetTransformer);
|
|
416
|
-
|
|
417
358
|
function ParquetTransformer(schema) {
|
|
418
359
|
var _this;
|
|
419
|
-
|
|
420
360
|
var opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
|
|
421
361
|
(0, _classCallCheck2.default)(this, ParquetTransformer);
|
|
422
362
|
_this = _super.call(this, {
|
|
423
363
|
objectMode: true
|
|
424
364
|
});
|
|
425
365
|
(0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "writer", void 0);
|
|
426
|
-
|
|
427
366
|
var writeProxy = function (t) {
|
|
428
367
|
return function () {
|
|
429
368
|
var _ref = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee8(b) {
|
|
@@ -432,7 +371,6 @@ var ParquetTransformer = function (_Transform) {
|
|
|
432
371
|
switch (_context8.prev = _context8.next) {
|
|
433
372
|
case 0:
|
|
434
373
|
t.push(b);
|
|
435
|
-
|
|
436
374
|
case 1:
|
|
437
375
|
case "end":
|
|
438
376
|
return _context8.stop();
|
|
@@ -440,13 +378,11 @@ var ParquetTransformer = function (_Transform) {
|
|
|
440
378
|
}
|
|
441
379
|
}, _callee8);
|
|
442
380
|
}));
|
|
443
|
-
|
|
444
381
|
return function (_x13) {
|
|
445
382
|
return _ref.apply(this, arguments);
|
|
446
383
|
};
|
|
447
384
|
}();
|
|
448
385
|
}((0, _assertThisInitialized2.default)(_this));
|
|
449
|
-
|
|
450
386
|
_this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee9() {
|
|
451
387
|
return _regenerator.default.wrap(function _callee9$(_context9) {
|
|
452
388
|
while (1) {
|
|
@@ -463,14 +399,15 @@ var ParquetTransformer = function (_Transform) {
|
|
|
463
399
|
|
|
464
400
|
(0, _createClass2.default)(ParquetTransformer, [{
|
|
465
401
|
key: "_transform",
|
|
466
|
-
value:
|
|
402
|
+
value:
|
|
403
|
+
function _transform(row, encoding, callback) {
|
|
467
404
|
if (row) {
|
|
468
405
|
return this.writer.appendRow(row).then(callback);
|
|
469
406
|
}
|
|
470
|
-
|
|
471
407
|
callback();
|
|
472
408
|
return Promise.resolve();
|
|
473
409
|
}
|
|
410
|
+
|
|
474
411
|
}, {
|
|
475
412
|
key: "_flush",
|
|
476
413
|
value: function () {
|
|
@@ -481,7 +418,6 @@ var ParquetTransformer = function (_Transform) {
|
|
|
481
418
|
case 0:
|
|
482
419
|
_context10.next = 2;
|
|
483
420
|
return this.writer.close(callback);
|
|
484
|
-
|
|
485
421
|
case 2:
|
|
486
422
|
case "end":
|
|
487
423
|
return _context10.stop();
|
|
@@ -489,31 +425,25 @@ var ParquetTransformer = function (_Transform) {
|
|
|
489
425
|
}
|
|
490
426
|
}, _callee10, this);
|
|
491
427
|
}));
|
|
492
|
-
|
|
493
428
|
function _flush(_x14) {
|
|
494
429
|
return _flush2.apply(this, arguments);
|
|
495
430
|
}
|
|
496
|
-
|
|
497
431
|
return _flush;
|
|
498
432
|
}()
|
|
499
433
|
}]);
|
|
500
434
|
return ParquetTransformer;
|
|
501
435
|
}(_stream.Transform);
|
|
502
|
-
|
|
503
436
|
exports.ParquetTransformer = ParquetTransformer;
|
|
504
|
-
|
|
505
437
|
function encodeValues(type, encoding, values, opts) {
|
|
506
438
|
if (!(encoding in _codecs.PARQUET_CODECS)) {
|
|
507
439
|
throw new Error("invalid encoding: ".concat(encoding));
|
|
508
440
|
}
|
|
509
|
-
|
|
510
441
|
return _codecs.PARQUET_CODECS[encoding].encodeValues(type, values, opts);
|
|
511
442
|
}
|
|
512
443
|
|
|
513
444
|
function encodeDataPage(_x15, _x16) {
|
|
514
445
|
return _encodeDataPage.apply(this, arguments);
|
|
515
446
|
}
|
|
516
|
-
|
|
517
447
|
function _encodeDataPage() {
|
|
518
448
|
_encodeDataPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee11(column, data) {
|
|
519
449
|
var rLevelsBuf, dLevelsBuf, valuesBuf, dataBuf, compressedBuf, header, headerBuf, page;
|
|
@@ -522,15 +452,12 @@ function _encodeDataPage() {
|
|
|
522
452
|
switch (_context11.prev = _context11.next) {
|
|
523
453
|
case 0:
|
|
524
454
|
rLevelsBuf = Buffer.alloc(0);
|
|
525
|
-
|
|
526
455
|
if (column.rLevelMax > 0) {
|
|
527
456
|
rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
|
|
528
457
|
bitWidth: (0, _readUtils.getBitWidth)(column.rLevelMax)
|
|
529
458
|
});
|
|
530
459
|
}
|
|
531
|
-
|
|
532
460
|
dLevelsBuf = Buffer.alloc(0);
|
|
533
|
-
|
|
534
461
|
if (column.dLevelMax > 0) {
|
|
535
462
|
dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
|
|
536
463
|
bitWidth: (0, _readUtils.getBitWidth)(column.dLevelMax)
|
|
@@ -544,7 +471,6 @@ function _encodeDataPage() {
|
|
|
544
471
|
dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
|
|
545
472
|
_context11.next = 8;
|
|
546
473
|
return Compression.deflate(column.compression, dataBuf);
|
|
547
|
-
|
|
548
474
|
case 8:
|
|
549
475
|
compressedBuf = _context11.sent;
|
|
550
476
|
header = new _parquetThrift.PageHeader({
|
|
@@ -555,6 +481,7 @@ function _encodeDataPage() {
|
|
|
555
481
|
definition_level_encoding: _parquetThrift.Encoding[PARQUET_RDLVL_ENCODING],
|
|
556
482
|
repetition_level_encoding: _parquetThrift.Encoding[PARQUET_RDLVL_ENCODING]
|
|
557
483
|
}),
|
|
484
|
+
|
|
558
485
|
uncompressed_page_size: dataBuf.length,
|
|
559
486
|
compressed_page_size: compressedBuf.length
|
|
560
487
|
});
|
|
@@ -565,7 +492,6 @@ function _encodeDataPage() {
|
|
|
565
492
|
headerSize: headerBuf.length,
|
|
566
493
|
page: page
|
|
567
494
|
});
|
|
568
|
-
|
|
569
495
|
case 13:
|
|
570
496
|
case "end":
|
|
571
497
|
return _context11.stop();
|
|
@@ -575,11 +501,9 @@ function _encodeDataPage() {
|
|
|
575
501
|
}));
|
|
576
502
|
return _encodeDataPage.apply(this, arguments);
|
|
577
503
|
}
|
|
578
|
-
|
|
579
504
|
function encodeDataPageV2(_x17, _x18, _x19) {
|
|
580
505
|
return _encodeDataPageV.apply(this, arguments);
|
|
581
506
|
}
|
|
582
|
-
|
|
583
507
|
function _encodeDataPageV() {
|
|
584
508
|
_encodeDataPageV = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee12(column, data, rowCount) {
|
|
585
509
|
var valuesBuf, compressedBuf, rLevelsBuf, dLevelsBuf, header, headerBuf, page;
|
|
@@ -593,20 +517,16 @@ function _encodeDataPageV() {
|
|
|
593
517
|
});
|
|
594
518
|
_context12.next = 3;
|
|
595
519
|
return Compression.deflate(column.compression, valuesBuf);
|
|
596
|
-
|
|
597
520
|
case 3:
|
|
598
521
|
compressedBuf = _context12.sent;
|
|
599
522
|
rLevelsBuf = Buffer.alloc(0);
|
|
600
|
-
|
|
601
523
|
if (column.rLevelMax > 0) {
|
|
602
524
|
rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
|
|
603
525
|
bitWidth: (0, _readUtils.getBitWidth)(column.rLevelMax),
|
|
604
526
|
disableEnvelope: true
|
|
605
527
|
});
|
|
606
528
|
}
|
|
607
|
-
|
|
608
529
|
dLevelsBuf = Buffer.alloc(0);
|
|
609
|
-
|
|
610
530
|
if (column.dLevelMax > 0) {
|
|
611
531
|
dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
|
|
612
532
|
bitWidth: (0, _readUtils.getBitWidth)(column.dLevelMax),
|
|
@@ -635,7 +555,6 @@ function _encodeDataPageV() {
|
|
|
635
555
|
headerSize: headerBuf.length,
|
|
636
556
|
page: page
|
|
637
557
|
});
|
|
638
|
-
|
|
639
558
|
case 12:
|
|
640
559
|
case "end":
|
|
641
560
|
return _context12.stop();
|
|
@@ -645,11 +564,9 @@ function _encodeDataPageV() {
|
|
|
645
564
|
}));
|
|
646
565
|
return _encodeDataPageV.apply(this, arguments);
|
|
647
566
|
}
|
|
648
|
-
|
|
649
567
|
function encodeColumnChunk(_x20, _x21, _x22, _x23) {
|
|
650
568
|
return _encodeColumnChunk.apply(this, arguments);
|
|
651
569
|
}
|
|
652
|
-
|
|
653
570
|
function _encodeColumnChunk() {
|
|
654
571
|
_encodeColumnChunk = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee13(column, buffer, offset, opts) {
|
|
655
572
|
var data, baseOffset, pageBuf, total_uncompressed_size, total_compressed_size, result, metadata, metadataOffset, body;
|
|
@@ -661,27 +578,21 @@ function _encodeColumnChunk() {
|
|
|
661
578
|
baseOffset = (opts.baseOffset || 0) + offset;
|
|
662
579
|
total_uncompressed_size = 0;
|
|
663
580
|
total_compressed_size = 0;
|
|
664
|
-
|
|
665
581
|
if (!opts.useDataPageV2) {
|
|
666
582
|
_context13.next = 10;
|
|
667
583
|
break;
|
|
668
584
|
}
|
|
669
|
-
|
|
670
585
|
_context13.next = 7;
|
|
671
586
|
return encodeDataPageV2(column, data, buffer.rowCount);
|
|
672
|
-
|
|
673
587
|
case 7:
|
|
674
588
|
_context13.t0 = _context13.sent;
|
|
675
589
|
_context13.next = 13;
|
|
676
590
|
break;
|
|
677
|
-
|
|
678
591
|
case 10:
|
|
679
592
|
_context13.next = 12;
|
|
680
593
|
return encodeDataPage(column, data);
|
|
681
|
-
|
|
682
594
|
case 12:
|
|
683
595
|
_context13.t0 = _context13.sent;
|
|
684
|
-
|
|
685
596
|
case 13:
|
|
686
597
|
result = _context13.t0;
|
|
687
598
|
pageBuf = result.page;
|
|
@@ -699,6 +610,7 @@ function _encodeColumnChunk() {
|
|
|
699
610
|
});
|
|
700
611
|
metadata.encodings.push(_parquetThrift.Encoding[PARQUET_RDLVL_ENCODING]);
|
|
701
612
|
metadata.encodings.push(_parquetThrift.Encoding[column.encoding]);
|
|
613
|
+
|
|
702
614
|
metadataOffset = baseOffset + pageBuf.length;
|
|
703
615
|
body = Buffer.concat([pageBuf, (0, _readUtils.serializeThrift)(metadata)]);
|
|
704
616
|
return _context13.abrupt("return", {
|
|
@@ -706,7 +618,6 @@ function _encodeColumnChunk() {
|
|
|
706
618
|
metadata: metadata,
|
|
707
619
|
metadataOffset: metadataOffset
|
|
708
620
|
});
|
|
709
|
-
|
|
710
621
|
case 23:
|
|
711
622
|
case "end":
|
|
712
623
|
return _context13.stop();
|
|
@@ -716,15 +627,12 @@ function _encodeColumnChunk() {
|
|
|
716
627
|
}));
|
|
717
628
|
return _encodeColumnChunk.apply(this, arguments);
|
|
718
629
|
}
|
|
719
|
-
|
|
720
630
|
function encodeRowGroup(_x24, _x25, _x26) {
|
|
721
631
|
return _encodeRowGroup.apply(this, arguments);
|
|
722
632
|
}
|
|
723
|
-
|
|
724
633
|
function _encodeRowGroup() {
|
|
725
634
|
_encodeRowGroup = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee14(schema, data, opts) {
|
|
726
635
|
var metadata, body, _iterator2, _step2, field, cchunkData, cchunk;
|
|
727
|
-
|
|
728
636
|
return _regenerator.default.wrap(function _callee14$(_context14) {
|
|
729
637
|
while (1) {
|
|
730
638
|
switch (_context14.prev = _context14.next) {
|
|
@@ -737,28 +645,21 @@ function _encodeRowGroup() {
|
|
|
737
645
|
body = Buffer.alloc(0);
|
|
738
646
|
_iterator2 = _createForOfIteratorHelper(schema.fieldList);
|
|
739
647
|
_context14.prev = 3;
|
|
740
|
-
|
|
741
648
|
_iterator2.s();
|
|
742
|
-
|
|
743
649
|
case 5:
|
|
744
650
|
if ((_step2 = _iterator2.n()).done) {
|
|
745
651
|
_context14.next = 18;
|
|
746
652
|
break;
|
|
747
653
|
}
|
|
748
|
-
|
|
749
654
|
field = _step2.value;
|
|
750
|
-
|
|
751
655
|
if (!field.isNested) {
|
|
752
656
|
_context14.next = 9;
|
|
753
657
|
break;
|
|
754
658
|
}
|
|
755
|
-
|
|
756
659
|
return _context14.abrupt("continue", 16);
|
|
757
|
-
|
|
758
660
|
case 9:
|
|
759
661
|
_context14.next = 11;
|
|
760
662
|
return encodeColumnChunk(field, data, body.length, opts);
|
|
761
|
-
|
|
762
663
|
case 11:
|
|
763
664
|
cchunkData = _context14.sent;
|
|
764
665
|
cchunk = new _parquetThrift.ColumnChunk({
|
|
@@ -768,34 +669,25 @@ function _encodeRowGroup() {
|
|
|
768
669
|
metadata.columns.push(cchunk);
|
|
769
670
|
metadata.total_byte_size = new _nodeInt.default(Number(metadata.total_byte_size) + cchunkData.body.length);
|
|
770
671
|
body = Buffer.concat([body, cchunkData.body]);
|
|
771
|
-
|
|
772
672
|
case 16:
|
|
773
673
|
_context14.next = 5;
|
|
774
674
|
break;
|
|
775
|
-
|
|
776
675
|
case 18:
|
|
777
676
|
_context14.next = 23;
|
|
778
677
|
break;
|
|
779
|
-
|
|
780
678
|
case 20:
|
|
781
679
|
_context14.prev = 20;
|
|
782
680
|
_context14.t0 = _context14["catch"](3);
|
|
783
|
-
|
|
784
681
|
_iterator2.e(_context14.t0);
|
|
785
|
-
|
|
786
682
|
case 23:
|
|
787
683
|
_context14.prev = 23;
|
|
788
|
-
|
|
789
684
|
_iterator2.f();
|
|
790
|
-
|
|
791
685
|
return _context14.finish(23);
|
|
792
|
-
|
|
793
686
|
case 26:
|
|
794
687
|
return _context14.abrupt("return", {
|
|
795
688
|
body: body,
|
|
796
689
|
metadata: metadata
|
|
797
690
|
});
|
|
798
|
-
|
|
799
691
|
case 27:
|
|
800
692
|
case "end":
|
|
801
693
|
return _context14.stop();
|
|
@@ -805,7 +697,6 @@ function _encodeRowGroup() {
|
|
|
805
697
|
}));
|
|
806
698
|
return _encodeRowGroup.apply(this, arguments);
|
|
807
699
|
}
|
|
808
|
-
|
|
809
700
|
function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
810
701
|
var metadata = new _parquetThrift.FileMetaData({
|
|
811
702
|
version: PARQUET_VERSION,
|
|
@@ -815,17 +706,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
|
815
706
|
schema: [],
|
|
816
707
|
key_value_metadata: []
|
|
817
708
|
});
|
|
818
|
-
|
|
819
709
|
for (var key in userMetadata) {
|
|
820
|
-
var _metadata$key_value_m, _metadata$key_value_m2;
|
|
821
|
-
|
|
710
|
+
var _metadata$key_value_m, _metadata$key_value_m2, _metadata$key_value_m3;
|
|
822
711
|
var kv = new _parquetThrift.KeyValue({
|
|
823
712
|
key: key,
|
|
824
713
|
value: userMetadata[key]
|
|
825
714
|
});
|
|
826
|
-
(_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = _metadata$key_value_m.push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$
|
|
715
|
+
(_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = (_metadata$key_value_m3 = _metadata$key_value_m).push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m3, kv);
|
|
827
716
|
}
|
|
828
|
-
|
|
829
717
|
{
|
|
830
718
|
var schemaRoot = new _parquetThrift.SchemaElement({
|
|
831
719
|
name: 'root',
|
|
@@ -833,10 +721,8 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
|
833
721
|
});
|
|
834
722
|
metadata.schema.push(schemaRoot);
|
|
835
723
|
}
|
|
836
|
-
|
|
837
724
|
var _iterator = _createForOfIteratorHelper(schema.fieldList),
|
|
838
|
-
|
|
839
|
-
|
|
725
|
+
_step;
|
|
840
726
|
try {
|
|
841
727
|
for (_iterator.s(); !(_step = _iterator.n()).done;) {
|
|
842
728
|
var field = _step.value;
|
|
@@ -845,17 +731,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
|
845
731
|
name: field.name,
|
|
846
732
|
repetition_type: relt
|
|
847
733
|
});
|
|
848
|
-
|
|
849
734
|
if (field.isNested) {
|
|
850
735
|
schemaElem.num_children = field.fieldCount;
|
|
851
736
|
} else {
|
|
852
737
|
schemaElem.type = _parquetThrift.Type[field.primitiveType];
|
|
853
738
|
}
|
|
854
|
-
|
|
855
739
|
if (field.originalType) {
|
|
856
740
|
schemaElem.converted_type = _parquetThrift.ConvertedType[field.originalType];
|
|
857
741
|
}
|
|
858
|
-
|
|
859
742
|
schemaElem.type_length = field.typeLength;
|
|
860
743
|
metadata.schema.push(schemaElem);
|
|
861
744
|
}
|
|
@@ -864,7 +747,6 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
|
864
747
|
} finally {
|
|
865
748
|
_iterator.f();
|
|
866
749
|
}
|
|
867
|
-
|
|
868
750
|
var metadataEncoded = (0, _readUtils.serializeThrift)(metadata);
|
|
869
751
|
var footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
|
|
870
752
|
metadataEncoded.copy(footerEncoded);
|