@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.js +2 -2
- package/dist/bundle.js.map +1 -0
- package/dist/constants.js +6 -18
- package/dist/constants.js.map +1 -0
- package/dist/dist.min.js +17 -8
- package/dist/dist.min.js.map +3 -3
- package/dist/index.js +14 -29
- package/dist/index.js.map +1 -0
- package/dist/lib/convert-schema.js +63 -62
- package/dist/lib/convert-schema.js.map +1 -0
- package/dist/lib/parse-parquet.js +25 -25
- package/dist/lib/parse-parquet.js.map +1 -0
- package/dist/lib/read-array-buffer.js +8 -28
- package/dist/lib/read-array-buffer.js.map +1 -0
- package/dist/parquet-loader.js +19 -24
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-worker.js +18 -9
- package/dist/parquet-worker.js.map +3 -3
- package/dist/parquet-writer.js +14 -17
- package/dist/parquet-writer.js.map +1 -0
- package/dist/{es5/parquetjs → parquetjs}/LICENSE +0 -0
- package/dist/parquetjs/codecs/declare.js +2 -2
- package/dist/{es5/parquetjs → parquetjs}/codecs/declare.js.map +0 -0
- package/dist/parquetjs/codecs/dictionary.js +10 -12
- package/dist/parquetjs/codecs/dictionary.js.map +1 -0
- package/dist/parquetjs/codecs/index.js +22 -50
- package/dist/parquetjs/codecs/index.js.map +1 -0
- package/dist/parquetjs/codecs/plain.js +232 -173
- package/dist/parquetjs/codecs/plain.js.map +1 -0
- package/dist/parquetjs/codecs/rle.js +140 -134
- package/dist/parquetjs/codecs/rle.js.map +1 -0
- package/dist/parquetjs/compression.js +48 -154
- package/dist/parquetjs/compression.js.map +1 -0
- package/dist/parquetjs/encoder/writer.js +383 -440
- package/dist/parquetjs/encoder/writer.js.map +1 -0
- package/dist/parquetjs/file.js +66 -85
- package/dist/parquetjs/file.js.map +1 -0
- package/dist/{es5/parquetjs → parquetjs}/modules.d.ts +0 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +7 -14
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/BsonType.js +37 -56
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +215 -205
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +212 -207
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +422 -391
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +90 -99
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +12 -19
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +26 -33
- package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +162 -162
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +234 -224
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DateType.js +37 -56
- package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.js +91 -101
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +113 -118
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js +12 -19
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/EnumType.js +37 -56
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +7 -14
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +264 -250
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +37 -56
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IntType.js +91 -101
- package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/JsonType.js +37 -56
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.js +89 -98
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ListType.js +37 -56
- package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.js +450 -363
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/MapType.js +37 -56
- package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +37 -56
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +37 -56
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.js +37 -56
- package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +80 -92
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +115 -123
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.js +231 -214
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.js +124 -137
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageType.js +8 -15
- package/dist/parquetjs/parquet-thrift/PageType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.js +172 -176
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +268 -237
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +115 -123
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Statistics.js +179 -172
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/StringType.js +37 -56
- package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimeType.js +92 -102
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +120 -121
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.js +92 -102
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Type.js +12 -19
- package/dist/parquetjs/parquet-thrift/Type.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +37 -56
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.js +37 -56
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.js +44 -61
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -0
- package/dist/parquetjs/parser/decoders.js +283 -301
- package/dist/{es5/parquetjs → parquetjs}/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-cursor.js +85 -69
- package/dist/parquetjs/parser/parquet-cursor.js.map +1 -0
- package/dist/parquetjs/parser/parquet-envelope-reader.js +146 -127
- package/dist/parquetjs/parser/parquet-envelope-reader.js.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.js +113 -127
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -0
- package/dist/parquetjs/schema/declare.js +12 -9
- package/dist/parquetjs/schema/declare.js.map +1 -0
- package/dist/parquetjs/schema/schema.js +162 -148
- package/dist/{es5/parquetjs → parquetjs}/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.js +151 -214
- package/dist/parquetjs/schema/shred.js.map +1 -0
- package/dist/parquetjs/schema/types.js +415 -357
- package/dist/parquetjs/schema/types.js.map +1 -0
- package/dist/parquetjs/utils/buffer-utils.js +10 -20
- package/dist/parquetjs/utils/buffer-utils.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.js +28 -40
- package/dist/parquetjs/utils/file-utils.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.js +95 -99
- package/dist/parquetjs/utils/read-utils.js.map +1 -0
- package/dist/workers/parquet-worker.js +4 -5
- package/dist/workers/parquet-worker.js.map +1 -0
- package/package.json +8 -8
- package/dist/es5/bundle.js +0 -7
- package/dist/es5/bundle.js.map +0 -1
- package/dist/es5/constants.js +0 -17
- package/dist/es5/constants.js.map +0 -1
- package/dist/es5/index.js +0 -82
- package/dist/es5/index.js.map +0 -1
- package/dist/es5/lib/convert-schema.js +0 -82
- package/dist/es5/lib/convert-schema.js.map +0 -1
- package/dist/es5/lib/parse-parquet.js +0 -173
- package/dist/es5/lib/parse-parquet.js.map +0 -1
- package/dist/es5/lib/read-array-buffer.js +0 -53
- package/dist/es5/lib/read-array-buffer.js.map +0 -1
- package/dist/es5/parquet-loader.js +0 -30
- package/dist/es5/parquet-loader.js.map +0 -1
- package/dist/es5/parquet-writer.js +0 -25
- package/dist/es5/parquet-writer.js.map +0 -1
- package/dist/es5/parquetjs/codecs/declare.js +0 -2
- package/dist/es5/parquetjs/codecs/dictionary.js +0 -30
- package/dist/es5/parquetjs/codecs/dictionary.js.map +0 -1
- package/dist/es5/parquetjs/codecs/index.js +0 -56
- package/dist/es5/parquetjs/codecs/index.js.map +0 -1
- package/dist/es5/parquetjs/codecs/plain.js +0 -287
- package/dist/es5/parquetjs/codecs/plain.js.map +0 -1
- package/dist/es5/parquetjs/codecs/rle.js +0 -174
- package/dist/es5/parquetjs/codecs/rle.js.map +0 -1
- package/dist/es5/parquetjs/compression.js +0 -167
- package/dist/es5/parquetjs/compression.js.map +0 -1
- package/dist/es5/parquetjs/encoder/writer.js +0 -875
- package/dist/es5/parquetjs/encoder/writer.js.map +0 -1
- package/dist/es5/parquetjs/file.js +0 -103
- package/dist/es5/parquetjs/file.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +0 -241
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +0 -245
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +0 -449
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +0 -124
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +0 -20
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +0 -34
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +0 -191
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -258
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/DateType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +0 -122
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -143
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js +0 -20
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -15
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +0 -298
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/IntType.js +0 -122
- package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +0 -120
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/ListType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +0 -508
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/MapType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/NullType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +0 -114
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +0 -145
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +0 -258
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +0 -155
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js +0 -16
- package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +0 -206
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +0 -290
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +0 -145
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js +0 -207
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/StringType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js +0 -124
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +0 -156
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +0 -124
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js +0 -20
- package/dist/es5/parquetjs/parquet-thrift/Type.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +0 -67
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
- package/dist/es5/parquetjs/parquet-thrift/index.js +0 -565
- package/dist/es5/parquetjs/parquet-thrift/index.js.map +0 -1
- package/dist/es5/parquetjs/parser/decoders.js +0 -489
- package/dist/es5/parquetjs/parser/parquet-cursor.js +0 -215
- package/dist/es5/parquetjs/parser/parquet-cursor.js.map +0 -1
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +0 -413
- package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
- package/dist/es5/parquetjs/parser/parquet-reader.js +0 -364
- package/dist/es5/parquetjs/parser/parquet-reader.js.map +0 -1
- package/dist/es5/parquetjs/schema/declare.js +0 -25
- package/dist/es5/parquetjs/schema/declare.js.map +0 -1
- package/dist/es5/parquetjs/schema/schema.js +0 -203
- package/dist/es5/parquetjs/schema/shred.js +0 -223
- package/dist/es5/parquetjs/schema/shred.js.map +0 -1
- package/dist/es5/parquetjs/schema/types.js +0 -492
- package/dist/es5/parquetjs/schema/types.js.map +0 -1
- package/dist/es5/parquetjs/utils/buffer-utils.js +0 -21
- package/dist/es5/parquetjs/utils/buffer-utils.js.map +0 -1
- package/dist/es5/parquetjs/utils/file-utils.js +0 -55
- package/dist/es5/parquetjs/utils/file-utils.js.map +0 -1
- package/dist/es5/parquetjs/utils/read-utils.js +0 -159
- package/dist/es5/parquetjs/utils/read-utils.js.map +0 -1
- package/dist/es5/workers/parquet-worker.js +0 -8
- package/dist/es5/workers/parquet-worker.js.map +0 -1
- package/dist/esm/bundle.js +0 -5
- package/dist/esm/bundle.js.map +0 -1
- package/dist/esm/constants.js +0 -6
- package/dist/esm/constants.js.map +0 -1
- package/dist/esm/index.js +0 -15
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/lib/convert-schema.js +0 -71
- package/dist/esm/lib/convert-schema.js.map +0 -1
- package/dist/esm/lib/parse-parquet.js +0 -28
- package/dist/esm/lib/parse-parquet.js.map +0 -1
- package/dist/esm/lib/read-array-buffer.js +0 -9
- package/dist/esm/lib/read-array-buffer.js.map +0 -1
- package/dist/esm/parquet-loader.js +0 -22
- package/dist/esm/parquet-loader.js.map +0 -1
- package/dist/esm/parquet-writer.js +0 -18
- package/dist/esm/parquet-writer.js.map +0 -1
- package/dist/esm/parquetjs/LICENSE +0 -20
- package/dist/esm/parquetjs/codecs/declare.js +0 -2
- package/dist/esm/parquetjs/codecs/declare.js.map +0 -1
- package/dist/esm/parquetjs/codecs/dictionary.js +0 -12
- package/dist/esm/parquetjs/codecs/dictionary.js.map +0 -1
- package/dist/esm/parquetjs/codecs/index.js +0 -23
- package/dist/esm/parquetjs/codecs/index.js.map +0 -1
- package/dist/esm/parquetjs/codecs/plain.js +0 -270
- package/dist/esm/parquetjs/codecs/plain.js.map +0 -1
- package/dist/esm/parquetjs/codecs/rle.js +0 -151
- package/dist/esm/parquetjs/codecs/rle.js.map +0 -1
- package/dist/esm/parquetjs/compression.js +0 -62
- package/dist/esm/parquetjs/compression.js.map +0 -1
- package/dist/esm/parquetjs/encoder/writer.js +0 -421
- package/dist/esm/parquetjs/encoder/writer.js.map +0 -1
- package/dist/esm/parquetjs/file.js +0 -80
- package/dist/esm/parquetjs/file.js.map +0 -1
- package/dist/esm/parquetjs/modules.d.ts +0 -21
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +0 -8
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +0 -217
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +0 -218
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +0 -429
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +0 -95
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +0 -13
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +0 -27
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +0 -166
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -236
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/DateType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +0 -95
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -117
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js +0 -13
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -8
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +0 -270
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/IntType.js +0 -95
- package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +0 -93
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/ListType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +0 -467
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/MapType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/NullType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +0 -85
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +0 -119
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +0 -233
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +0 -128
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/PageType.js +0 -9
- package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +0 -178
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +0 -270
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +0 -119
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js +0 -183
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/StringType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js +0 -96
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +0 -126
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +0 -96
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/Type.js +0 -13
- package/dist/esm/parquetjs/parquet-thrift/Type.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +0 -39
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
- package/dist/esm/parquetjs/parquet-thrift/index.js +0 -44
- package/dist/esm/parquetjs/parquet-thrift/index.js.map +0 -1
- package/dist/esm/parquetjs/parser/decoders.js +0 -300
- package/dist/esm/parquetjs/parser/decoders.js.map +0 -1
- package/dist/esm/parquetjs/parser/parquet-cursor.js +0 -90
- package/dist/esm/parquetjs/parser/parquet-cursor.js.map +0 -1
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js +0 -155
- package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
- package/dist/esm/parquetjs/parser/parquet-reader.js +0 -120
- package/dist/esm/parquetjs/parser/parquet-reader.js.map +0 -1
- package/dist/esm/parquetjs/schema/declare.js +0 -13
- package/dist/esm/parquetjs/schema/declare.js.map +0 -1
- package/dist/esm/parquetjs/schema/schema.js +0 -176
- package/dist/esm/parquetjs/schema/schema.js.map +0 -1
- package/dist/esm/parquetjs/schema/shred.js +0 -162
- package/dist/esm/parquetjs/schema/shred.js.map +0 -1
- package/dist/esm/parquetjs/schema/types.js +0 -476
- package/dist/esm/parquetjs/schema/types.js.map +0 -1
- package/dist/esm/parquetjs/utils/buffer-utils.js +0 -12
- package/dist/esm/parquetjs/utils/buffer-utils.js.map +0 -1
- package/dist/esm/parquetjs/utils/file-utils.js +0 -34
- package/dist/esm/parquetjs/utils/file-utils.js.map +0 -1
- package/dist/esm/parquetjs/utils/read-utils.js +0 -105
- package/dist/esm/parquetjs/utils/read-utils.js.map +0 -1
- package/dist/esm/workers/parquet-worker.js +0 -4
- package/dist/esm/workers/parquet-worker.js.map +0 -1
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", {
|
|
6
|
-
value: true
|
|
7
|
-
});
|
|
8
|
-
exports.serializeThrift = serializeThrift;
|
|
9
|
-
exports.decodeThrift = decodeThrift;
|
|
10
|
-
exports.getThriftEnum = getThriftEnum;
|
|
11
|
-
exports.decodeFileMetadata = decodeFileMetadata;
|
|
12
|
-
exports.decodePageHeader = decodePageHeader;
|
|
13
|
-
exports.getBitWidth = getBitWidth;
|
|
14
|
-
exports.fieldIndexOf = fieldIndexOf;
|
|
15
|
-
|
|
16
|
-
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
17
|
-
|
|
18
|
-
var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
|
|
19
|
-
|
|
20
|
-
var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
|
|
21
|
-
|
|
22
|
-
var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
|
|
23
|
-
|
|
24
|
-
var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
|
|
25
|
-
|
|
26
|
-
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
27
|
-
|
|
28
|
-
var _thrift = require("thrift");
|
|
29
|
-
|
|
30
|
-
var _parquetThrift = require("../parquet-thrift");
|
|
31
|
-
|
|
32
|
-
function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
|
|
33
|
-
|
|
34
|
-
function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
|
|
35
|
-
|
|
36
|
-
var UFramedTransport = function (_TFramedTransport) {
|
|
37
|
-
(0, _inherits2.default)(UFramedTransport, _TFramedTransport);
|
|
38
|
-
|
|
39
|
-
var _super = _createSuper(UFramedTransport);
|
|
40
|
-
|
|
41
|
-
function UFramedTransport() {
|
|
42
|
-
var _this;
|
|
43
|
-
|
|
44
|
-
(0, _classCallCheck2.default)(this, UFramedTransport);
|
|
45
|
-
|
|
46
|
-
for (var _len = arguments.length, args = new Array(_len), _key = 0; _key < _len; _key++) {
|
|
47
|
-
args[_key] = arguments[_key];
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
_this = _super.call.apply(_super, [this].concat(args));
|
|
51
|
-
(0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "readPos", 0);
|
|
52
|
-
return _this;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
return UFramedTransport;
|
|
56
|
-
}(_thrift.TFramedTransport);
|
|
57
|
-
|
|
58
|
-
function serializeThrift(obj) {
|
|
59
|
-
var output = [];
|
|
60
|
-
var transport = new _thrift.TBufferedTransport(undefined, function (buf) {
|
|
61
|
-
output.push(buf);
|
|
62
|
-
});
|
|
63
|
-
var protocol = new _thrift.TCompactProtocol(transport);
|
|
64
|
-
obj.write(protocol);
|
|
65
|
-
transport.flush();
|
|
66
|
-
return Buffer.concat(output);
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function decodeThrift(obj, buf, offset) {
|
|
70
|
-
if (!offset) {
|
|
71
|
-
offset = 0;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
var transport = new UFramedTransport(buf);
|
|
75
|
-
transport.readPos = offset;
|
|
76
|
-
var protocol = new _thrift.TCompactProtocol(transport);
|
|
77
|
-
obj.read(protocol);
|
|
78
|
-
return transport.readPos - offset;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
function getThriftEnum(klass, value) {
|
|
82
|
-
for (var k in klass) {
|
|
83
|
-
if (klass[k] === value) {
|
|
84
|
-
return k;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
throw new Error('Invalid ENUM value');
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function decodeFileMetadata(buf, offset) {
|
|
92
|
-
if (!offset) {
|
|
93
|
-
offset = 0;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
var transport = new UFramedTransport(buf);
|
|
97
|
-
transport.readPos = offset;
|
|
98
|
-
var protocol = new _thrift.TCompactProtocol(transport);
|
|
99
|
-
|
|
100
|
-
var metadata = _parquetThrift.FileMetaData.read(protocol);
|
|
101
|
-
|
|
102
|
-
return {
|
|
103
|
-
length: transport.readPos - offset,
|
|
104
|
-
metadata: metadata
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
function decodePageHeader(buf, offset) {
|
|
109
|
-
if (!offset) {
|
|
110
|
-
offset = 0;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
var transport = new UFramedTransport(buf);
|
|
114
|
-
transport.readPos = offset;
|
|
115
|
-
var protocol = new _thrift.TCompactProtocol(transport);
|
|
116
|
-
|
|
117
|
-
var pageHeader = _parquetThrift.PageHeader.read(protocol);
|
|
118
|
-
|
|
119
|
-
return {
|
|
120
|
-
length: transport.readPos - offset,
|
|
121
|
-
pageHeader: pageHeader
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function getBitWidth(val) {
|
|
126
|
-
if (val === 0) {
|
|
127
|
-
return 0;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
return Math.ceil(Math.log2(val + 1));
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function fieldIndexOf(arr, elem) {
|
|
134
|
-
for (var j = 0; j < arr.length; j++) {
|
|
135
|
-
if (arr[j].length > elem.length) {
|
|
136
|
-
continue;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
var m = true;
|
|
140
|
-
|
|
141
|
-
for (var i = 0; i < elem.length; i++) {
|
|
142
|
-
if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
|
|
143
|
-
continue;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
m = false;
|
|
151
|
-
break;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
if (m) return j;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
return -1;
|
|
158
|
-
}
|
|
159
|
-
//# sourceMappingURL=read-utils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/utils/read-utils.ts"],"names":["UFramedTransport","TFramedTransport","serializeThrift","obj","output","transport","TBufferedTransport","undefined","buf","push","protocol","TCompactProtocol","write","flush","Buffer","concat","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","FileMetaData","length","decodePageHeader","pageHeader","PageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;AACA;;;;;;IAEMA,gB;;;;;;;;;;;;;;;0FACqB,C;;;;;EADIC,wB;;AAOxB,SAASC,eAAT,CAAyBC,GAAzB,EAA2C;AAChD,MAAMC,MAAgB,GAAG,EAAzB;AAEA,MAAMC,SAAS,GAAG,IAAIC,0BAAJ,CAAuBC,SAAvB,EAAkC,UAACC,GAAD,EAAS;AAC3DJ,IAAAA,MAAM,CAACK,IAAP,CAAYD,GAAZ;AACD,GAFiB,CAAlB;AAIA,MAAME,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;AACAF,EAAAA,GAAG,CAACS,KAAJ,CAAUF,QAAV;AACAL,EAAAA,SAAS,CAACQ,KAAV;AAEA,SAAOC,MAAM,CAACC,MAAP,CAAcX,MAAd,CAAP;AACD;;AAEM,SAASY,YAAT,CAAsBb,GAAtB,EAAgCK,GAAhC,EAA6CS,MAA7C,EAA8D;AACnE,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,MAAMZ,SAAS,GAAG,IAAIL,gBAAJ,CAAqBQ,GAArB,CAAlB;AACAH,EAAAA,SAAS,CAACa,OAAV,GAAoBD,MAApB;AACA,MAAMP,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;AACAF,EAAAA,GAAG,CAACgB,IAAJ,CAAST,QAAT;AACA,SAAOL,SAAS,CAACa,OAAV,GAAoBD,MAA3B;AACD;;AAKM,SAASG,aAAT,CAAuBC,KAAvB,EAAmCC,KAAnC,EAAmE;AACxE,OAAK,IAAMC,CAAX,IAAgBF,KAAhB,EAAuB;AACrB,QAAIA,KAAK,CAACE,CAAD,CAAL,KAAaD,KAAjB,EAAwB;AACtB,aAAOC,CAAP;AACD;AACF;;AACD,QAAM,IAAIC,KAAJ,CAAU,oBAAV,CAAN;AACD;;AAEM,SAASC,kBAAT,CAA4BjB,GAA5B,EAAyCS,MAAzC,EAA0D;AAC/D,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,MAAMZ,SAAS,GAAG,IAAIL,gBAAJ,CAAqBQ,GAArB,CAAlB;AACAH,EAAAA,SAAS,CAACa,OAAV,GAAoBD,MAApB;AACA,MAAMP,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;;AACA,MAAMqB,QAAQ,GAAGC,4BAAaR,IAAb,CAAkBT,QAAlB,CAAjB;;AACA,SAAO;AAACkB,IAAAA,MAAM,EAAEvB,SAAS,CAACa,OAAV,GAAoBD,MAA7B;AAAqCS,IAAAA,QAAQ,EAARA;AAArC,GAAP;AACD;;AAEM,SAASG,gBAAT,CAA0BrB,GAA1B,EAAuCS,MAAvC,EAAwD;AAC7D,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,MAAMZ,SAAS,GAAG,IAAIL,gBAAJ,CAAqBQ,GAArB,CAAlB;AACAH,EAAAA,SAAS,CAACa,OAAV,GAAoBD,MAApB;AACA,MAAMP,QAAQ,GAAG,IAAIC,wBAAJ,CAAqBN,SAArB,CAAjB;;AACA,MAAMyB,UAAU,GAAGC,0BAAWZ,IAAX,CAAgBT,QAAhB,CAAnB;;AACA,SAAO;AAACkB,IAAAA,MAAM,EAAEvB,SAAS,CAACa,OAAV,GAAoBD,MAA7B;AAAqCa,IAAAA,UAAU,EAAVA;AAArC,GAAP;AACD;;AAKM,SAASE,WAAT,CAAqBC,GAArB,EAA0C;AAC/C,MAAIA,GAAG,KAAK,CAAZ,EAAe;AACb,WAAO,CAAP;AAED;;AACD,SAAOC,IAAI,CAACC,IAAL,CAAUD,IAAI,CAACE,IAAL,CAAUH,GAAG,GAAG,CAAhB,CAAV,CAAP;AACD;;AAKM,SAASI,YAAT,CAAsBC,GAAtB,EAAuCC,IAAvC,EAA+D;AACpE,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGF,GAAG,CAACV,MAAxB,EAAgCY,CAAC,EAAjC,EAAqC;AACnC,QAAIF,GAAG,CAACE,CAAD,CAAH,CAAOZ,MAAP,GAAgBW,IAAI,CAACX,MAAzB,EAAiC;AAC/B;AACD;;AACD,QAAIa,CAAC,GAAG,IAAR;;AACA,SAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,IAAI,CAACX,MAAzB,EAAiCc,CAAC,EAAlC,EAAsC;AACpC,UAAIJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAcH,IAAI,CAACG,CAAD,CAAlB,IAAyBJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAc,GAAvC,IAA8CJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAc,GAAhE,EAAqE;AACnE;AACD;;AACD,UAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAD,CAAH,CAAOZ,MAAZ,IAAsBU,GAAG,CAACE,CAAD,CAAH,CAAOF,GAAG,CAACE,CAAD,CAAH,CAAOZ,MAAP,GAAgB,CAAvB,MAA8B,GAAxD,EAA6D;AAC3D;AACD;;AACDa,MAAAA,CAAC,GAAG,KAAJ;AACA;AACD;;AACD,QAAIA,CAAJ,EAAO,OAAOD,CAAP;AACR;;AACD,SAAO,CAAC,CAAR;AACD","sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n const output: Buffer[] = [];\n\n const transport = new TBufferedTransport(undefined, (buf) => {\n output.push(buf as Buffer);\n });\n\n const protocol = new TCompactProtocol(transport);\n obj.write(protocol);\n transport.flush();\n\n return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n obj.read(protocol);\n return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n for (const k in klass) {\n if (klass[k] === value) {\n return k;\n }\n }\n throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n const metadata = FileMetaData.read(protocol);\n return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n const pageHeader = PageHeader.read(protocol);\n return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n if (val === 0) {\n return 0;\n // tslint:disable-next-line:no-else-after-return\n }\n return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n for (let j = 0; j < arr.length; j++) {\n if (arr[j].length > elem.length) {\n continue; // eslint-disable-line no-continue\n }\n let m = true;\n for (let i = 0; i < elem.length; i++) {\n if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n continue; // eslint-disable-line no-continue\n }\n if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n continue; // eslint-disable-line no-continue\n }\n m = false;\n break;\n }\n if (m) return j;\n }\n return -1;\n}\n"],"file":"read-utils.js"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/workers/parquet-worker.ts"],"names":["ParquetLoader"],"mappings":";;AAAA;;AACA;;AAEA,qCAAmBA,oBAAnB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ParquetLoader} from '../index';\n\ncreateLoaderWorker(ParquetLoader);\n"],"file":"parquet-worker.js"}
|
package/dist/esm/bundle.js
DELETED
package/dist/esm/bundle.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}
|
package/dist/esm/constants.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/constants.ts"],"names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"mappings":"AAIA,OAAO,MAAMA,aAAa,GAAG,MAAtB;AACP,OAAO,MAAMC,uBAAuB,GAAG,MAAhC;AAKP,OAAO,MAAMC,eAAe,GAAG,CAAxB;AAKP,OAAO,MAAMC,kBAAkB,GAAG,OAA3B;AACP,OAAO,MAAMC,sBAAsB,GAAG,KAA/B","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"file":"constants.js"}
|
package/dist/esm/index.js
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
|
|
2
|
-
import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
|
|
3
|
-
export { ParquetWorkerLoader };
|
|
4
|
-
export const ParquetLoader = { ...ParquetWorkerLoader,
|
|
5
|
-
parse: parseParquet,
|
|
6
|
-
parseFileInBatches: parseParquetFileInBatches
|
|
7
|
-
};
|
|
8
|
-
export { ParquetWriter as _ParquetWriter } from './parquet-writer';
|
|
9
|
-
export { preloadCompressions } from './parquetjs/compression';
|
|
10
|
-
export { ParquetSchema } from './parquetjs/schema/schema';
|
|
11
|
-
export { ParquetReader } from './parquetjs/parser/parquet-reader';
|
|
12
|
-
export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
|
|
13
|
-
export { convertParquetToArrowSchema } from './lib/convert-schema';
|
|
14
|
-
export const _typecheckParquetLoader = ParquetLoader;
|
|
15
|
-
//# sourceMappingURL=index.js.map
|
package/dist/esm/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEnvelopeReader","convertParquetToArrowSchema","_typecheckParquetLoader"],"mappings":"AAIA,SAAQA,aAAa,IAAIC,mBAAzB,QAAmD,kBAAnD;AACA,SAAQC,YAAR,EAAsBC,yBAAtB,QAAsD,qBAAtD;AAEA,SAAQF,mBAAR;AAGA,OAAO,MAAMD,aAAa,GAAG,EAC3B,GAAGC,mBADwB;AAE3BG,EAAAA,KAAK,EAAEF,YAFoB;AAG3BG,EAAAA,kBAAkB,EAAEF;AAHO,CAAtB;AAQP,SAAQG,aAAa,IAAIC,cAAzB,QAA8C,kBAA9C;AAIA,SAAQC,mBAAR,QAAkC,yBAAlC;AAEA,SAAQC,aAAR,QAA4B,2BAA5B;AACA,SAAQC,aAAR,QAA4B,mCAA5B;AACA,SAAQC,qBAAR,QAAoC,4CAApC;AAEA,SAAQC,2BAAR,QAA0C,sBAA1C;AAGA,OAAO,MAAMC,uBAAyC,GAAGb,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n ...ParquetWorkerLoader,\n parse: parseParquet,\n parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
import { Schema, Struct, Field, Bool, Float64, Int32, Float32, Binary, Utf8, Int64, Uint16, Uint32, Uint64, Int8, Int16 } from '@loaders.gl/schema';
|
|
2
|
-
export const PARQUET_TYPE_MAPPING = {
|
|
3
|
-
BOOLEAN: Bool,
|
|
4
|
-
INT32: Int32,
|
|
5
|
-
INT64: Float64,
|
|
6
|
-
INT96: Float64,
|
|
7
|
-
FLOAT: Float32,
|
|
8
|
-
DOUBLE: Float64,
|
|
9
|
-
BYTE_ARRAY: Binary,
|
|
10
|
-
FIXED_LEN_BYTE_ARRAY: Binary,
|
|
11
|
-
UTF8: Utf8,
|
|
12
|
-
DATE: Int32,
|
|
13
|
-
TIME_MILLIS: Int64,
|
|
14
|
-
TIME_MICROS: Int64,
|
|
15
|
-
TIMESTAMP_MILLIS: Int64,
|
|
16
|
-
TIMESTAMP_MICROS: Int64,
|
|
17
|
-
UINT_8: Int32,
|
|
18
|
-
UINT_16: Uint16,
|
|
19
|
-
UINT_32: Uint32,
|
|
20
|
-
UINT_64: Uint64,
|
|
21
|
-
INT_8: Int8,
|
|
22
|
-
INT_16: Int16,
|
|
23
|
-
INT_32: Int32,
|
|
24
|
-
INT_64: Int64,
|
|
25
|
-
JSON: Binary,
|
|
26
|
-
BSON: Binary,
|
|
27
|
-
INTERVAL: Binary,
|
|
28
|
-
DECIMAL_INT32: Float32,
|
|
29
|
-
DECIMAL_INT64: Float64,
|
|
30
|
-
DECIMAL_BYTE_ARRAY: Float64,
|
|
31
|
-
DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
|
|
32
|
-
};
|
|
33
|
-
export function convertParquetToArrowSchema(parquetSchema) {
|
|
34
|
-
const fields = getFields(parquetSchema.schema);
|
|
35
|
-
return new Schema(fields);
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
function getFieldMetadata(field) {
|
|
39
|
-
const metadata = new Map();
|
|
40
|
-
|
|
41
|
-
for (const key in field) {
|
|
42
|
-
if (key !== 'name') {
|
|
43
|
-
const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
44
|
-
metadata.set(key, value);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
return metadata;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function getFields(schema) {
|
|
52
|
-
const fields = [];
|
|
53
|
-
|
|
54
|
-
for (const name in schema) {
|
|
55
|
-
const field = schema[name];
|
|
56
|
-
|
|
57
|
-
if (field.fields) {
|
|
58
|
-
const childField = getFields(field.fields);
|
|
59
|
-
const nestedField = new Field(name, new Struct(childField), field.optional);
|
|
60
|
-
fields.push(nestedField);
|
|
61
|
-
} else {
|
|
62
|
-
const FieldType = PARQUET_TYPE_MAPPING[field.type];
|
|
63
|
-
const metadata = getFieldMetadata(field);
|
|
64
|
-
const arrowField = new Field(name, new FieldType(), field.optional, metadata);
|
|
65
|
-
fields.push(arrowField);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return fields;
|
|
70
|
-
}
|
|
71
|
-
//# sourceMappingURL=convert-schema.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/convert-schema.ts"],"names":["Schema","Struct","Field","Bool","Float64","Int32","Float32","Binary","Utf8","Int64","Uint16","Uint32","Uint64","Int8","Int16","PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetToArrowSchema","parquetSchema","fields","getFields","schema","getFieldMetadata","field","metadata","Map","key","value","stringify","set","name","childField","nestedField","optional","push","FieldType","type","arrowField"],"mappings":"AAGA,SACEA,MADF,EAEEC,MAFF,EAGEC,KAHF,EAKEC,IALF,EAMEC,OANF,EAOEC,KAPF,EAQEC,OARF,EASEC,MATF,EAUEC,IAVF,EAWEC,KAXF,EAYEC,MAZF,EAaEC,MAbF,EAcEC,MAdF,EAeEC,IAfF,EAgBEC,KAhBF,QAiBO,oBAjBP;AAmBA,OAAO,MAAMC,oBAA8D,GAAG;AAC5EC,EAAAA,OAAO,EAAEb,IADmE;AAE5Ec,EAAAA,KAAK,EAAEZ,KAFqE;AAG5Ea,EAAAA,KAAK,EAAEd,OAHqE;AAI5Ee,EAAAA,KAAK,EAAEf,OAJqE;AAK5EgB,EAAAA,KAAK,EAAEd,OALqE;AAM5Ee,EAAAA,MAAM,EAAEjB,OANoE;AAO5EkB,EAAAA,UAAU,EAAEf,MAPgE;AAQ5EgB,EAAAA,oBAAoB,EAAEhB,MARsD;AAS5EiB,EAAAA,IAAI,EAAEhB,IATsE;AAU5EiB,EAAAA,IAAI,EAAEpB,KAVsE;AAW5EqB,EAAAA,WAAW,EAAEjB,KAX+D;AAY5EkB,EAAAA,WAAW,EAAElB,KAZ+D;AAa5EmB,EAAAA,gBAAgB,EAAEnB,KAb0D;AAc5EoB,EAAAA,gBAAgB,EAAEpB,KAd0D;AAe5EqB,EAAAA,MAAM,EAAEzB,KAfoE;AAgB5E0B,EAAAA,OAAO,EAAErB,MAhBmE;AAiB5EsB,EAAAA,OAAO,EAAErB,MAjBmE;AAkB5EsB,EAAAA,OAAO,EAAErB,MAlBmE;AAmB5EsB,EAAAA,KAAK,EAAErB,IAnBqE;AAoB5EsB,EAAAA,MAAM,EAAErB,KApBoE;AAqB5EsB,EAAAA,MAAM,EAAE/B,KArBoE;AAsB5EgC,EAAAA,MAAM,EAAE5B,KAtBoE;AAuB5E6B,EAAAA,IAAI,EAAE/B,MAvBsE;AAwB5EgC,EAAAA,IAAI,EAAEhC,MAxBsE;AA0B5EiC,EAAAA,QAAQ,EAAEjC,MA1BkE;AA2B5EkC,EAAAA,aAAa,EAAEnC,OA3B6D;AA4B5EoC,EAAAA,aAAa,EAAEtC,OA5B6D;AA6B5EuC,EAAAA,kBAAkB,EAAEvC,OA7BwD;AA8B5EwC,EAAAA,4BAA4B,EAAExC;AA9B8C,CAAvE;AAiCP,OAAO,SAASyC,2BAAT,CAAqCC,aAArC,EAA2E;AAChF,QAAMC,MAAM,GAAGC,SAAS,CAACF,aAAa,CAACG,MAAf,CAAxB;AAGA,SAAO,IAAIjD,MAAJ,CAAW+C,MAAX,CAAP;AACD;;AAED,SAASG,gBAAT,CAA0BC,KAA1B,EAAoE;AAClE,QAAMC,QAAQ,GAAG,IAAIC,GAAJ,EAAjB;;AAEA,OAAK,MAAMC,GAAX,IAAkBH,KAAlB,EAAyB;AACvB,QAAIG,GAAG,KAAK,MAAZ,EAAoB;AAClB,YAAMC,KAAK,GAAG,OAAOJ,KAAK,CAACG,GAAD,CAAZ,KAAsB,QAAtB,GAAiChB,IAAI,CAACkB,SAAL,CAAeL,KAAK,CAACG,GAAD,CAApB,CAAjC,GAA8DH,KAAK,CAACG,GAAD,CAAjF;AACAF,MAAAA,QAAQ,CAACK,GAAT,CAAaH,GAAb,EAAkBC,KAAlB;AACD;AACF;;AAED,SAAOH,QAAP;AACD;;AAED,SAASJ,SAAT,CAAmBC,MAAnB,EAAqD;AACnD,QAAMF,MAAe,GAAG,EAAxB;;AAEA,OAAK,MAAMW,IAAX,IAAmBT,MAAnB,EAA2B;AACzB,UAAME,KAAK,GAAGF,MAAM,CAACS,IAAD,CAApB;;AAEA,QAAIP,KAAK,CAACJ,MAAV,EAAkB;AAChB,YAAMY,UAAU,GAAGX,SAAS,CAACG,KAAK,CAACJ,MAAP,CAA5B;AACA,YAAMa,WAAW,GAAG,IAAI1D,KAAJ,CAAUwD,IAAV,EAAgB,IAAIzD,MAAJ,CAAW0D,UAAX,CAAhB,EAAwCR,KAAK,CAACU,QAA9C,CAApB;AACAd,MAAAA,MAAM,CAACe,IAAP,CAAYF,WAAZ;AACD,KAJD,MAIO;AACL,YAAMG,SAAS,GAAGhD,oBAAoB,CAACoC,KAAK,CAACa,IAAP,CAAtC;AACA,YAAMZ,QAAQ,GAAGF,gBAAgB,CAACC,KAAD,CAAjC;AACA,YAAMc,UAAU,GAAG,IAAI/D,KAAJ,CAAUwD,IAAV,EAAgB,IAAIK,SAAJ,EAAhB,EAAiCZ,KAAK,CAACU,QAAvC,EAAiDT,QAAjD,CAAnB;AACAL,MAAAA,MAAM,CAACe,IAAP,CAAYG,UAAZ;AACD;AACF;;AAED,SAAOlB,MAAP;AACD","sourcesContent":["import type {ParquetSchema} from '../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';\n\nimport {\n Schema,\n Struct,\n Field,\n DataType,\n Bool,\n Float64,\n Int32,\n Float32,\n Binary,\n Utf8,\n Int64,\n Uint16,\n Uint32,\n Uint64,\n Int8,\n Int16\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {\n BOOLEAN: Bool,\n INT32: Int32,\n INT64: Float64,\n INT96: Float64,\n FLOAT: Float32,\n DOUBLE: Float64,\n BYTE_ARRAY: Binary,\n FIXED_LEN_BYTE_ARRAY: Binary,\n UTF8: Utf8,\n DATE: Int32,\n TIME_MILLIS: Int64,\n TIME_MICROS: Int64,\n TIMESTAMP_MILLIS: Int64,\n TIMESTAMP_MICROS: Int64,\n UINT_8: Int32,\n UINT_16: Uint16,\n UINT_32: Uint32,\n UINT_64: Uint64,\n INT_8: Int8,\n INT_16: Int16,\n INT_32: Int32,\n INT_64: Int64,\n JSON: Binary,\n BSON: Binary,\n // TODO check interal type\n INTERVAL: Binary,\n DECIMAL_INT32: Float32,\n DECIMAL_INT64: Float64,\n DECIMAL_BYTE_ARRAY: Float64,\n DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64\n};\n\nexport function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {\n const fields = getFields(parquetSchema.schema);\n\n // TODO add metadata if needed.\n return new Schema(fields);\n}\n\nfunction getFieldMetadata(field: ParquetField): Map<string, string> {\n const metadata = new Map();\n\n for (const key in field) {\n if (key !== 'name') {\n const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n metadata.set(key, value);\n }\n }\n\n return metadata;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n const fields: Field[] = [];\n\n for (const name in schema) {\n const field = schema[name];\n\n if (field.fields) {\n const childField = getFields(field.fields);\n const nestedField = new Field(name, new Struct(childField), field.optional);\n fields.push(nestedField);\n } else {\n const FieldType = PARQUET_TYPE_MAPPING[field.type];\n const metadata = getFieldMetadata(field);\n const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n fields.push(arrowField);\n }\n }\n\n return fields;\n}\n"],"file":"convert-schema.js"}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { ParquetReader } from '../parquetjs/parser/parquet-reader';
|
|
2
|
-
export async function parseParquet(arrayBuffer, options) {
|
|
3
|
-
const blob = new Blob([arrayBuffer]);
|
|
4
|
-
|
|
5
|
-
for await (const batch of parseParquetFileInBatches(blob, options)) {
|
|
6
|
-
return batch;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
return null;
|
|
10
|
-
}
|
|
11
|
-
export async function* parseParquetFileInBatches(blob, options) {
|
|
12
|
-
const reader = await ParquetReader.openBlob(blob);
|
|
13
|
-
const rows = [];
|
|
14
|
-
|
|
15
|
-
try {
|
|
16
|
-
const cursor = reader.getCursor();
|
|
17
|
-
let record;
|
|
18
|
-
|
|
19
|
-
while (record = await cursor.next()) {
|
|
20
|
-
rows.push(record);
|
|
21
|
-
}
|
|
22
|
-
} finally {
|
|
23
|
-
await reader.close();
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
yield rows;
|
|
27
|
-
}
|
|
28
|
-
//# sourceMappingURL=parse-parquet.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/parse-parquet.ts"],"names":["ParquetReader","parseParquet","arrayBuffer","options","blob","Blob","batch","parseParquetFileInBatches","reader","openBlob","rows","cursor","getCursor","record","next","push","close"],"mappings":"AAGA,SAAQA,aAAR,QAA4B,oCAA5B;AAEA,OAAO,eAAeC,YAAf,CAA4BC,WAA5B,EAAsDC,OAAtD,EAAsF;AAC3F,QAAMC,IAAI,GAAG,IAAIC,IAAJ,CAAS,CAACH,WAAD,CAAT,CAAb;;AACA,aAAW,MAAMI,KAAjB,IAA0BC,yBAAyB,CAACH,IAAD,EAAOD,OAAP,CAAnD,EAAoE;AAClE,WAAOG,KAAP;AACD;;AACD,SAAO,IAAP;AACD;AAED,OAAO,gBAAgBC,yBAAhB,CAA0CH,IAA1C,EAAsDD,OAAtD,EAAsF;AAC3F,QAAMK,MAAM,GAAG,MAAMR,aAAa,CAACS,QAAd,CAAuBL,IAAvB,CAArB;AACA,QAAMM,IAAa,GAAG,EAAtB;;AACA,MAAI;AACF,UAAMC,MAAM,GAAGH,MAAM,CAACI,SAAP,EAAf;AACA,QAAIC,MAAJ;;AACA,WAAQA,MAAM,GAAG,MAAMF,MAAM,CAACG,IAAP,EAAvB,EAAuC;AACrCJ,MAAAA,IAAI,CAACK,IAAL,CAAUF,MAAV;AACD;AACF,GAND,SAMU;AACR,UAAML,MAAM,CAACQ,KAAP,EAAN;AACD;;AACD,QAAMN,IAAN;AACD","sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n const blob = new Blob([arrayBuffer]);\n for await (const batch of parseParquetFileInBatches(blob, options)) {\n return batch;\n }\n return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n const reader = await ParquetReader.openBlob(blob);\n const rows: any[][] = [];\n try {\n const cursor = reader.getCursor();\n let record: any[] | null;\n while ((record = await cursor.next())) {\n rows.push(record);\n }\n } finally {\n await reader.close();\n }\n yield rows;\n}\n"],"file":"parse-parquet.js"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export async function readArrayBuffer(file, start, length) {
|
|
2
|
-
if (file instanceof Blob) {
|
|
3
|
-
const slice = file.slice(start, start + length);
|
|
4
|
-
return await slice.arrayBuffer();
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
return await file.read(start, start + length);
|
|
8
|
-
}
|
|
9
|
-
//# sourceMappingURL=read-array-buffer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/read-array-buffer.ts"],"names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"mappings":"AAEA,OAAO,eAAeA,eAAf,CACLC,IADK,EAELC,KAFK,EAGLC,MAHK,EAIiB;AACtB,MAAIF,IAAI,YAAYG,IAApB,EAA0B;AACxB,UAAMC,KAAK,GAAGJ,IAAI,CAACI,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,CAAd;AACA,WAAO,MAAME,KAAK,CAACC,WAAN,EAAb;AACD;;AACD,SAAO,MAAML,IAAI,CAACM,IAAL,CAAUL,KAAV,EAAiBA,KAAK,GAAGC,MAAzB,CAAb;AACD","sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n file: Blob | ArrayBuffer | any,\n start: number,\n length: number\n): Promise<ArrayBuffer> {\n if (file instanceof Blob) {\n const slice = file.slice(start, start + length);\n return await slice.arrayBuffer();\n }\n return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n return await new Promise((resolve, reject) => {\n const fileReader = new FileReader();\n fileReader.onload = (event: ProgressEvent<FileReader>) =>\n resolve(event?.target?.result as ArrayBuffer);\n // TODO - reject with a proper Error\n fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n fileReader.readAsArrayBuffer(blob);\n });\n}\n*/\n"],"file":"read-array-buffer.js"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
|
|
2
|
-
const DEFAULT_PARQUET_LOADER_OPTIONS = {
|
|
3
|
-
parquet: {
|
|
4
|
-
type: 'object-row-table',
|
|
5
|
-
url: undefined
|
|
6
|
-
}
|
|
7
|
-
};
|
|
8
|
-
export const ParquetLoader = {
|
|
9
|
-
name: 'Apache Parquet',
|
|
10
|
-
id: 'parquet',
|
|
11
|
-
module: 'parquet',
|
|
12
|
-
version: VERSION,
|
|
13
|
-
worker: true,
|
|
14
|
-
category: 'table',
|
|
15
|
-
extensions: ['parquet'],
|
|
16
|
-
mimeTypes: ['application/octet-stream'],
|
|
17
|
-
binary: true,
|
|
18
|
-
tests: ['PAR1', 'PARE'],
|
|
19
|
-
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
20
|
-
};
|
|
21
|
-
export const _typecheckParquetLoader = ParquetLoader;
|
|
22
|
-
//# sourceMappingURL=parquet-loader.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQA,OAAO,MAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;AAcP,OAAO,MAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'object-row-table';\n url?: string;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'object-row-table',\n url: undefined\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
|
|
2
|
-
const DEFAULT_PARQUET_LOADER_OPTIONS = {};
|
|
3
|
-
export const ParquetWriter = {
|
|
4
|
-
name: 'Apache Parquet',
|
|
5
|
-
id: 'parquet',
|
|
6
|
-
module: 'parquet',
|
|
7
|
-
version: VERSION,
|
|
8
|
-
extensions: ['parquet'],
|
|
9
|
-
mimeTypes: ['application/octet-stream'],
|
|
10
|
-
encodeSync,
|
|
11
|
-
binary: true,
|
|
12
|
-
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
function encodeSync(data, options) {
|
|
16
|
-
return new ArrayBuffer(0);
|
|
17
|
-
}
|
|
18
|
-
//# sourceMappingURL=parquet-writer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":"AAIA,MAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEA,OAAO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
The code in the parquetjs is forked from https://github.com/kbajalc/parquets under MIT license
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2017 ironSource Ltd.
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
6
|
-
this software and associated documentation files (the "Software"), to deal in the
|
|
7
|
-
Software without restriction, including without limitation the rights to use,
|
|
8
|
-
copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
|
9
|
-
Software, and to permit persons to whom the Software is furnished to do so,
|
|
10
|
-
subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
16
|
-
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
|
17
|
-
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
18
|
-
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
20
|
-
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":[],"names":[],"mappings":"","sourcesContent":[],"file":"declare.js"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { decodeValues as decodeRleValues } from './rle';
|
|
2
|
-
export function decodeValues(type, cursor, count, opts) {
|
|
3
|
-
opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
|
|
4
|
-
cursor.offset += 1;
|
|
5
|
-
return decodeRleValues(type, cursor, count, { ...opts,
|
|
6
|
-
disableEnvelope: true
|
|
7
|
-
});
|
|
8
|
-
}
|
|
9
|
-
export function encodeValues(type, cursor, count, opts) {
|
|
10
|
-
throw new Error('Encode dictionary functionality is not supported');
|
|
11
|
-
}
|
|
12
|
-
//# sourceMappingURL=dictionary.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/dictionary.ts"],"names":["decodeValues","decodeRleValues","type","cursor","count","opts","bitWidth","buffer","slice","offset","readInt8","disableEnvelope","encodeValues","Error"],"mappings":"AAAA,SAAQA,YAAY,IAAIC,eAAxB,QAA8C,OAA9C;AAEA,OAAO,SAASD,YAAT,CAAsBE,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtDA,EAAAA,IAAI,CAACC,QAAL,GAAgBH,MAAM,CAACI,MAAP,CAAcC,KAAd,CAAoBL,MAAM,CAACM,MAA3B,EAAmCN,MAAM,CAACM,MAAP,GAAgB,CAAnD,EAAsDC,QAAtD,CAA+D,CAA/D,CAAhB;AACAP,EAAAA,MAAM,CAACM,MAAP,IAAiB,CAAjB;AACA,SAAOR,eAAe,CAACC,IAAD,EAAOC,MAAP,EAAeC,KAAf,EAAsB,EAAC,GAAGC,IAAJ;AAAUM,IAAAA,eAAe,EAAE;AAA3B,GAAtB,CAAtB;AACD;AAED,OAAO,SAASC,YAAT,CAAsBV,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtD,QAAM,IAAIQ,KAAJ,CAAU,kDAAV,CAAN;AACD","sourcesContent":["import {decodeValues as decodeRleValues} from './rle';\n\nexport function decodeValues(type, cursor, count, opts) {\n opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);\n cursor.offset += 1;\n return decodeRleValues(type, cursor, count, {...opts, disableEnvelope: true});\n}\n\nexport function encodeValues(type, cursor, count, opts) {\n throw new Error('Encode dictionary functionality is not supported');\n}\n"],"file":"dictionary.js"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import * as PLAIN from './plain';
|
|
2
|
-
import * as RLE from './rle';
|
|
3
|
-
import * as DICTIONARY from './dictionary';
|
|
4
|
-
export * from './declare';
|
|
5
|
-
export const PARQUET_CODECS = {
|
|
6
|
-
PLAIN: {
|
|
7
|
-
encodeValues: PLAIN.encodeValues,
|
|
8
|
-
decodeValues: PLAIN.decodeValues
|
|
9
|
-
},
|
|
10
|
-
RLE: {
|
|
11
|
-
encodeValues: RLE.encodeValues,
|
|
12
|
-
decodeValues: RLE.decodeValues
|
|
13
|
-
},
|
|
14
|
-
PLAIN_DICTIONARY: {
|
|
15
|
-
encodeValues: DICTIONARY.encodeValues,
|
|
16
|
-
decodeValues: DICTIONARY.decodeValues
|
|
17
|
-
},
|
|
18
|
-
RLE_DICTIONARY: {
|
|
19
|
-
encodeValues: DICTIONARY.encodeValues,
|
|
20
|
-
decodeValues: DICTIONARY.decodeValues
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PLAIN","RLE","DICTIONARY","PARQUET_CODECS","encodeValues","decodeValues","PLAIN_DICTIONARY","RLE_DICTIONARY"],"mappings":"AAGA,OAAO,KAAKA,KAAZ,MAAuB,SAAvB;AACA,OAAO,KAAKC,GAAZ,MAAqB,OAArB;AACA,OAAO,KAAKC,UAAZ,MAA4B,cAA5B;AAEA,cAAc,WAAd;AAEA,OAAO,MAAMC,cAAqD,GAAG;AACnEH,EAAAA,KAAK,EAAE;AACLI,IAAAA,YAAY,EAAEJ,KAAK,CAACI,YADf;AAELC,IAAAA,YAAY,EAAEL,KAAK,CAACK;AAFf,GAD4D;AAKnEJ,EAAAA,GAAG,EAAE;AACHG,IAAAA,YAAY,EAAEH,GAAG,CAACG,YADf;AAEHC,IAAAA,YAAY,EAAEJ,GAAG,CAACI;AAFf,GAL8D;AAUnEC,EAAAA,gBAAgB,EAAE;AAEhBF,IAAAA,YAAY,EAAEF,UAAU,CAACE,YAFT;AAGhBC,IAAAA,YAAY,EAAEH,UAAU,CAACG;AAHT,GAViD;AAgBnEE,EAAAA,cAAc,EAAE;AAEdH,IAAAA,YAAY,EAAEF,UAAU,CAACE,YAFX;AAGdC,IAAAA,YAAY,EAAEH,UAAU,CAACG;AAHX;AAhBmD,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\nimport * as DICTIONARY from './dictionary';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n PLAIN: {\n encodeValues: PLAIN.encodeValues,\n decodeValues: PLAIN.decodeValues\n },\n RLE: {\n encodeValues: RLE.encodeValues,\n decodeValues: RLE.decodeValues\n },\n // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.\n PLAIN_DICTIONARY: {\n // @ts-ignore\n encodeValues: DICTIONARY.encodeValues,\n decodeValues: DICTIONARY.decodeValues\n },\n // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.\n RLE_DICTIONARY: {\n // @ts-ignore\n encodeValues: DICTIONARY.encodeValues,\n decodeValues: DICTIONARY.decodeValues\n }\n};\n"],"file":"index.js"}
|