@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.js +2 -2
- package/dist/constants.js +18 -6
- package/dist/dist.min.js +27 -25
- package/dist/dist.min.js.map +3 -3
- package/dist/es5/bundle.js +6 -0
- package/dist/es5/bundle.js.map +1 -0
- package/dist/es5/constants.js +17 -0
- package/dist/es5/constants.js.map +1 -0
- package/dist/es5/index.js +128 -0
- package/dist/es5/index.js.map +1 -0
- package/dist/es5/lib/arrow/convert-columns-to-row-group.js +2 -0
- package/dist/es5/lib/arrow/convert-columns-to-row-group.js.map +1 -0
- package/dist/es5/lib/arrow/convert-row-group-to-columns.js +19 -0
- package/dist/es5/lib/arrow/convert-row-group-to-columns.js.map +1 -0
- package/dist/es5/lib/arrow/convert-schema-from-parquet.js +114 -0
- package/dist/es5/lib/arrow/convert-schema-from-parquet.js.map +1 -0
- package/dist/es5/lib/arrow/convert-schema-to-parquet.js +47 -0
- package/dist/es5/lib/arrow/convert-schema-to-parquet.js.map +1 -0
- package/dist/es5/lib/geo/decode-geo-metadata.js +81 -0
- package/dist/es5/lib/geo/decode-geo-metadata.js.map +1 -0
- package/dist/es5/lib/geo/geoparquet-schema.js +83 -0
- package/dist/es5/lib/geo/geoparquet-schema.js.map +1 -0
- package/dist/es5/lib/parsers/parse-parquet-to-columns.js +177 -0
- package/dist/es5/lib/parsers/parse-parquet-to-columns.js.map +1 -0
- package/dist/es5/lib/parsers/parse-parquet-to-rows.js +172 -0
- package/dist/es5/lib/parsers/parse-parquet-to-rows.js.map +1 -0
- package/dist/es5/lib/wasm/encode-parquet-wasm.js +43 -0
- package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -0
- package/dist/es5/lib/wasm/load-wasm/index.js +13 -0
- package/dist/es5/lib/wasm/load-wasm/index.js.map +1 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +42 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +31 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -0
- package/dist/es5/lib/wasm/parse-parquet-wasm.js +60 -0
- package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -0
- package/dist/es5/lib/wip/convert-schema-deep.java.disabled +910 -0
- package/dist/es5/lib/wip/convert-schema-deep.rs.disabled +976 -0
- package/dist/es5/parquet-loader.js +44 -0
- package/dist/es5/parquet-loader.js.map +1 -0
- package/dist/es5/parquet-wasm-loader.js +30 -0
- package/dist/es5/parquet-wasm-loader.js.map +1 -0
- package/dist/es5/parquet-wasm-writer.js +26 -0
- package/dist/es5/parquet-wasm-writer.js.map +1 -0
- package/dist/es5/parquet-writer.js +24 -0
- package/dist/es5/parquet-writer.js.map +1 -0
- package/dist/es5/parquetjs/codecs/declare.js +2 -0
- package/dist/es5/parquetjs/codecs/declare.js.map +1 -0
- package/dist/es5/parquetjs/codecs/dictionary.js +23 -0
- package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -0
- package/dist/es5/parquetjs/codecs/index.js +47 -0
- package/dist/es5/parquetjs/codecs/index.js.map +1 -0
- package/dist/es5/parquetjs/codecs/plain.js +208 -0
- package/dist/es5/parquetjs/codecs/plain.js.map +1 -0
- package/dist/es5/parquetjs/codecs/rle.js +132 -0
- package/dist/es5/parquetjs/codecs/rle.js.map +1 -0
- package/dist/es5/parquetjs/compression.js +137 -0
- package/dist/es5/parquetjs/compression.js.map +1 -0
- package/dist/es5/parquetjs/encoder/parquet-encoder.js +625 -0
- package/dist/es5/parquetjs/encoder/parquet-encoder.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +14 -0
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +193 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +198 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +367 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +99 -0
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +19 -0
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +33 -0
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +152 -0
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +207 -0
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/DateType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +96 -0
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +113 -0
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js +19 -0
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +14 -0
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +239 -0
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/IntType.js +96 -0
- package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +94 -0
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/ListType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +423 -0
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/MapType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/NullType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +89 -0
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +115 -0
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +204 -0
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +124 -0
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/PageType.js +15 -0
- package/dist/es5/parquetjs/parquet-thrift/PageType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +165 -0
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +231 -0
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +115 -0
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js +165 -0
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/StringType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js +97 -0
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +126 -0
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +97 -0
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/Type.js +19 -0
- package/dist/es5/parquetjs/parquet-thrift/Type.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +52 -0
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
- package/dist/es5/parquetjs/parquet-thrift/index.js +479 -0
- package/dist/es5/parquetjs/parquet-thrift/index.js.map +1 -0
- package/dist/es5/parquetjs/parser/decoders.js +393 -0
- package/dist/es5/parquetjs/parser/decoders.js.map +1 -0
- package/dist/es5/parquetjs/parser/parquet-reader.js +610 -0
- package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -0
- package/dist/es5/parquetjs/schema/declare.js +21 -0
- package/dist/es5/parquetjs/schema/declare.js.map +1 -0
- package/dist/es5/parquetjs/schema/schema.js +165 -0
- package/dist/es5/parquetjs/schema/schema.js.map +1 -0
- package/dist/es5/parquetjs/schema/shred.js +282 -0
- package/dist/es5/parquetjs/schema/shred.js.map +1 -0
- package/dist/es5/parquetjs/schema/types.js +406 -0
- package/dist/es5/parquetjs/schema/types.js.map +1 -0
- package/dist/es5/parquetjs/utils/file-utils.js +47 -0
- package/dist/es5/parquetjs/utils/file-utils.js.map +1 -0
- package/dist/es5/parquetjs/utils/read-utils.js +120 -0
- package/dist/es5/parquetjs/utils/read-utils.js.map +1 -0
- package/dist/es5/workers/parquet-worker.js +6 -0
- package/dist/es5/workers/parquet-worker.js.map +1 -0
- package/dist/esm/bundle.js +4 -0
- package/dist/esm/bundle.js.map +1 -0
- package/dist/esm/constants.js +6 -0
- package/dist/esm/constants.js.map +1 -0
- package/dist/esm/index.js +31 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/lib/arrow/convert-columns-to-row-group.js +2 -0
- package/dist/esm/lib/arrow/convert-columns-to-row-group.js.map +1 -0
- package/dist/esm/lib/arrow/convert-row-group-to-columns.js +8 -0
- package/dist/esm/lib/arrow/convert-row-group-to-columns.js.map +1 -0
- package/dist/esm/lib/arrow/convert-schema-from-parquet.js +95 -0
- package/dist/esm/lib/arrow/convert-schema-from-parquet.js.map +1 -0
- package/dist/esm/lib/arrow/convert-schema-to-parquet.js +39 -0
- package/dist/esm/lib/arrow/convert-schema-to-parquet.js.map +1 -0
- package/dist/esm/lib/geo/decode-geo-metadata.js +62 -0
- package/dist/esm/lib/geo/decode-geo-metadata.js.map +1 -0
- package/dist/esm/lib/geo/geoparquet-schema.js +76 -0
- package/dist/esm/lib/geo/geoparquet-schema.js.map +1 -0
- package/dist/esm/lib/parsers/parse-parquet-to-columns.js +39 -0
- package/dist/esm/lib/parsers/parse-parquet-to-columns.js.map +1 -0
- package/dist/esm/lib/parsers/parse-parquet-to-rows.js +29 -0
- package/dist/esm/lib/parsers/parse-parquet-to-rows.js.map +1 -0
- package/dist/esm/lib/wasm/encode-parquet-wasm.js +15 -0
- package/dist/esm/lib/wasm/encode-parquet-wasm.js.map +1 -0
- package/dist/esm/lib/wasm/load-wasm/index.js +2 -0
- package/dist/esm/lib/wasm/load-wasm/index.js.map +1 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js +11 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js +5 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map +1 -0
- package/dist/esm/lib/wasm/parse-parquet-wasm.js +21 -0
- package/dist/esm/lib/wasm/parse-parquet-wasm.js.map +1 -0
- package/dist/esm/lib/wip/convert-schema-deep.java.disabled +910 -0
- package/dist/esm/lib/wip/convert-schema-deep.rs.disabled +976 -0
- package/dist/esm/parquet-loader.js +36 -0
- package/dist/esm/parquet-loader.js.map +1 -0
- package/dist/esm/parquet-wasm-loader.js +22 -0
- package/dist/esm/parquet-wasm-loader.js.map +1 -0
- package/dist/esm/parquet-wasm-writer.js +19 -0
- package/dist/esm/parquet-wasm-writer.js.map +1 -0
- package/dist/esm/parquet-writer.js +17 -0
- package/dist/esm/parquet-writer.js.map +1 -0
- package/dist/esm/parquetjs/LICENSE +20 -0
- package/dist/esm/parquetjs/codecs/declare.js +2 -0
- package/dist/esm/parquetjs/codecs/declare.js.map +1 -0
- package/dist/esm/parquetjs/codecs/dictionary.js +13 -0
- package/dist/esm/parquetjs/codecs/dictionary.js.map +1 -0
- package/dist/esm/parquetjs/codecs/index.js +23 -0
- package/dist/esm/parquetjs/codecs/index.js.map +1 -0
- package/dist/esm/parquetjs/codecs/plain.js +200 -0
- package/dist/esm/parquetjs/codecs/plain.js.map +1 -0
- package/dist/esm/parquetjs/codecs/rle.js +119 -0
- package/dist/esm/parquetjs/codecs/rle.js.map +1 -0
- package/dist/esm/parquetjs/compression.js +61 -0
- package/dist/esm/parquetjs/compression.js.map +1 -0
- package/dist/{parquetjs/encoder/writer.js → esm/parquetjs/encoder/parquet-encoder.js} +8 -106
- package/dist/esm/parquetjs/encoder/parquet-encoder.js.map +1 -0
- package/dist/esm/parquetjs/modules.d.ts +21 -0
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js +7 -0
- package/dist/esm/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/BsonType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js +173 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js +176 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js +347 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js +77 -0
- package/dist/esm/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js +12 -0
- package/dist/esm/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js +26 -0
- package/dist/esm/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js +132 -0
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js +187 -0
- package/dist/esm/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/DateType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/DateType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js +76 -0
- package/dist/esm/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js +93 -0
- package/dist/esm/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js +12 -0
- package/dist/esm/parquetjs/parquet-thrift/Encoding.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/EnumType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js +7 -0
- package/dist/esm/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js +219 -0
- package/dist/esm/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/IntType.js +76 -0
- package/dist/esm/parquetjs/parquet-thrift/IntType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/JsonType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js +74 -0
- package/dist/esm/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/ListType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/ListType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js +377 -0
- package/dist/esm/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/MapType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/MapType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/NullType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/NullType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js +69 -0
- package/dist/esm/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js +95 -0
- package/dist/esm/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js +184 -0
- package/dist/esm/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js +104 -0
- package/dist/esm/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/PageType.js +8 -0
- package/dist/esm/parquetjs/parquet-thrift/PageType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js +145 -0
- package/dist/esm/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js +211 -0
- package/dist/esm/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js +95 -0
- package/dist/esm/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js +145 -0
- package/dist/esm/parquetjs/parquet-thrift/Statistics.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/StringType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/StringType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js +77 -0
- package/dist/esm/parquetjs/parquet-thrift/TimeType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js +102 -0
- package/dist/esm/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js +77 -0
- package/dist/esm/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/Type.js +12 -0
- package/dist/esm/parquetjs/parquet-thrift/Type.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js +31 -0
- package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
- package/dist/esm/parquetjs/parquet-thrift/index.js +44 -0
- package/dist/esm/parquetjs/parquet-thrift/index.js.map +1 -0
- package/dist/esm/parquetjs/parser/decoders.js +253 -0
- package/dist/esm/parquetjs/parser/decoders.js.map +1 -0
- package/dist/{parquetjs/parser/parquet-envelope-reader.js → esm/parquetjs/parser/parquet-reader.js} +95 -74
- package/dist/esm/parquetjs/parser/parquet-reader.js.map +1 -0
- package/dist/esm/parquetjs/schema/declare.js +12 -0
- package/dist/esm/parquetjs/schema/declare.js.map +1 -0
- package/dist/esm/parquetjs/schema/schema.js +140 -0
- package/dist/esm/parquetjs/schema/schema.js.map +1 -0
- package/dist/esm/parquetjs/schema/shred.js +228 -0
- package/dist/esm/parquetjs/schema/shred.js.map +1 -0
- package/dist/esm/parquetjs/schema/types.js +397 -0
- package/dist/esm/parquetjs/schema/types.js.map +1 -0
- package/dist/esm/parquetjs/utils/file-utils.js +34 -0
- package/dist/esm/parquetjs/utils/file-utils.js.map +1 -0
- package/dist/esm/parquetjs/utils/read-utils.js +90 -0
- package/dist/esm/parquetjs/utils/read-utils.js.map +1 -0
- package/dist/esm/workers/parquet-worker.js +4 -0
- package/dist/esm/workers/parquet-worker.js.map +1 -0
- package/dist/index.d.ts +16 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +58 -15
- package/dist/lib/arrow/convert-columns-to-row-group.d.ts +1 -0
- package/dist/lib/arrow/convert-columns-to-row-group.d.ts.map +1 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js +1 -0
- package/dist/lib/arrow/convert-row-group-to-columns.d.ts +4 -0
- package/dist/lib/arrow/convert-row-group-to-columns.d.ts.map +1 -0
- package/dist/lib/arrow/convert-row-group-to-columns.js +12 -0
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +9 -0
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -0
- package/dist/lib/arrow/convert-schema-from-parquet.js +86 -0
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts +7 -0
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -0
- package/dist/lib/arrow/convert-schema-to-parquet.js +71 -0
- package/dist/lib/geo/decode-geo-metadata.d.ts +31 -0
- package/dist/lib/geo/decode-geo-metadata.d.ts.map +1 -0
- package/dist/lib/geo/decode-geo-metadata.js +77 -0
- package/dist/lib/geo/geoparquet-schema.d.ts +80 -0
- package/dist/lib/geo/geoparquet-schema.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-schema.js +69 -0
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +5 -0
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-columns.js +46 -0
- package/dist/lib/parsers/parse-parquet-to-rows.d.ts +5 -0
- package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-rows.js +37 -0
- package/dist/lib/wasm/encode-parquet-wasm.d.ts +21 -0
- package/dist/lib/wasm/encode-parquet-wasm.d.ts.map +1 -0
- package/dist/lib/wasm/encode-parquet-wasm.js +30 -0
- package/dist/lib/wasm/load-wasm/index.d.ts +2 -0
- package/dist/lib/wasm/load-wasm/index.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm/index.js +5 -0
- package/dist/lib/wasm/load-wasm/load-wasm-browser.d.ts +3 -0
- package/dist/lib/wasm/load-wasm/load-wasm-browser.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm/load-wasm-browser.js +38 -0
- package/dist/lib/wasm/load-wasm/load-wasm-node.d.ts +3 -0
- package/dist/lib/wasm/load-wasm/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm/load-wasm-node.js +31 -0
- package/dist/lib/wasm/parse-parquet-wasm.d.ts +10 -0
- package/dist/lib/wasm/parse-parquet-wasm.d.ts.map +1 -0
- package/dist/lib/wasm/parse-parquet-wasm.js +27 -0
- package/dist/parquet-loader.d.ts +6 -15
- package/dist/parquet-loader.d.ts.map +1 -1
- package/dist/parquet-loader.js +38 -19
- package/dist/parquet-wasm-loader.d.ts +23 -0
- package/dist/parquet-wasm-loader.d.ts.map +1 -0
- package/dist/parquet-wasm-loader.js +27 -0
- package/dist/parquet-wasm-writer.d.ts +3 -0
- package/dist/parquet-wasm-writer.d.ts.map +1 -0
- package/dist/parquet-wasm-writer.js +23 -0
- package/dist/parquet-worker.js +27 -25
- package/dist/parquet-worker.js.map +3 -3
- package/dist/parquet-writer.d.ts +3 -2
- package/dist/parquet-writer.d.ts.map +1 -1
- package/dist/parquet-writer.js +18 -14
- package/dist/parquetjs/codecs/declare.js +2 -2
- package/dist/parquetjs/codecs/dictionary.js +12 -10
- package/dist/parquetjs/codecs/index.js +54 -22
- package/dist/parquetjs/codecs/plain.js +173 -232
- package/dist/parquetjs/codecs/rle.js +134 -140
- package/dist/parquetjs/compression.d.ts +3 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +169 -48
- package/dist/parquetjs/encoder/{writer.d.ts → parquet-encoder.d.ts} +15 -23
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.js +484 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +14 -7
- package/dist/parquetjs/parquet-thrift/BsonType.js +60 -37
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +209 -215
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +210 -211
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +394 -421
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +102 -89
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +19 -12
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +33 -26
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +165 -161
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +228 -234
- package/dist/parquetjs/parquet-thrift/DateType.js +60 -37
- package/dist/parquetjs/parquet-thrift/DecimalType.js +104 -90
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +121 -112
- package/dist/parquetjs/parquet-thrift/Encoding.js +19 -12
- package/dist/parquetjs/parquet-thrift/EnumType.js +60 -37
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +14 -7
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +253 -263
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +60 -37
- package/dist/parquetjs/parquet-thrift/IntType.js +104 -90
- package/dist/parquetjs/parquet-thrift/JsonType.js +60 -37
- package/dist/parquetjs/parquet-thrift/KeyValue.js +101 -88
- package/dist/parquetjs/parquet-thrift/ListType.js +60 -37
- package/dist/parquetjs/parquet-thrift/LogicalType.js +366 -449
- package/dist/parquetjs/parquet-thrift/MapType.js +60 -37
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +60 -37
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +60 -37
- package/dist/parquetjs/parquet-thrift/NullType.js +60 -37
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +96 -80
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +126 -114
- package/dist/parquetjs/parquet-thrift/PageHeader.js +218 -231
- package/dist/parquetjs/parquet-thrift/PageLocation.js +140 -123
- package/dist/parquetjs/parquet-thrift/PageType.js +15 -8
- package/dist/parquetjs/parquet-thrift/RowGroup.js +179 -171
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +241 -268
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +126 -114
- package/dist/parquetjs/parquet-thrift/Statistics.js +175 -178
- package/dist/parquetjs/parquet-thrift/StringType.js +60 -37
- package/dist/parquetjs/parquet-thrift/TimeType.js +105 -91
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +124 -119
- package/dist/parquetjs/parquet-thrift/TimestampType.js +105 -91
- package/dist/parquetjs/parquet-thrift/Type.js +19 -12
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +60 -37
- package/dist/parquetjs/parquet-thrift/UUIDType.js +60 -37
- package/dist/parquetjs/parquet-thrift/index.js +65 -44
- package/dist/parquetjs/parser/decoders.d.ts +2 -2
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +301 -283
- package/dist/parquetjs/parser/parquet-reader.d.ts +47 -57
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +193 -113
- package/dist/parquetjs/schema/declare.d.ts +26 -18
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js +11 -12
- package/dist/parquetjs/schema/schema.d.ts +4 -4
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +148 -162
- package/dist/parquetjs/schema/shred.d.ts +33 -12
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +340 -147
- package/dist/parquetjs/schema/types.d.ts +2 -2
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +355 -415
- package/dist/parquetjs/utils/file-utils.d.ts +5 -4
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +37 -28
- package/dist/parquetjs/utils/read-utils.js +99 -95
- package/dist/workers/parquet-worker.js +5 -4
- package/package.json +17 -12
- package/src/index.ts +58 -7
- package/src/lib/arrow/convert-columns-to-row-group.ts +0 -0
- package/src/lib/arrow/convert-row-group-to-columns.ts +15 -0
- package/src/lib/arrow/convert-schema-from-parquet.ts +104 -0
- package/src/lib/arrow/convert-schema-to-parquet.ts +90 -0
- package/src/lib/geo/decode-geo-metadata.ts +108 -0
- package/src/lib/geo/geoparquet-schema.ts +69 -0
- package/src/lib/parsers/parse-parquet-to-columns.ts +60 -0
- package/src/lib/parsers/parse-parquet-to-rows.ts +45 -0
- package/src/lib/wasm/encode-parquet-wasm.ts +40 -0
- package/src/lib/wasm/load-wasm/index.ts +1 -0
- package/src/lib/wasm/load-wasm/load-wasm-browser.ts +15 -0
- package/src/lib/wasm/load-wasm/load-wasm-node.ts +5 -0
- package/src/lib/wasm/parse-parquet-wasm.ts +42 -0
- package/src/lib/wip/convert-schema-deep.java.disabled +910 -0
- package/src/lib/wip/convert-schema-deep.rs.disabled +976 -0
- package/src/parquet-loader.ts +30 -3
- package/src/parquet-wasm-loader.ts +36 -0
- package/src/parquet-wasm-writer.ts +24 -0
- package/src/parquet-writer.ts +4 -1
- package/src/parquetjs/compression.ts +24 -7
- package/src/parquetjs/encoder/{writer.ts → parquet-encoder.ts} +33 -38
- package/src/parquetjs/parser/decoders.ts +3 -3
- package/src/parquetjs/parser/parquet-reader.ts +239 -122
- package/src/parquetjs/schema/declare.ts +22 -13
- package/src/parquetjs/schema/schema.ts +8 -8
- package/src/parquetjs/schema/shred.ts +239 -71
- package/src/parquetjs/schema/types.ts +25 -30
- package/src/parquetjs/utils/file-utils.ts +3 -4
- package/dist/bundle.js.map +0 -1
- package/dist/constants.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/convert-schema.d.ts +0 -8
- package/dist/lib/convert-schema.d.ts.map +0 -1
- package/dist/lib/convert-schema.js +0 -71
- package/dist/lib/convert-schema.js.map +0 -1
- package/dist/lib/parse-parquet.d.ts +0 -4
- package/dist/lib/parse-parquet.d.ts.map +0 -1
- package/dist/lib/parse-parquet.js +0 -28
- package/dist/lib/parse-parquet.js.map +0 -1
- package/dist/lib/read-array-buffer.d.ts +0 -19
- package/dist/lib/read-array-buffer.d.ts.map +0 -1
- package/dist/lib/read-array-buffer.js +0 -9
- package/dist/lib/read-array-buffer.js.map +0 -1
- package/dist/parquet-loader.js.map +0 -1
- package/dist/parquet-writer.js.map +0 -1
- package/dist/parquetjs/codecs/declare.js.map +0 -1
- package/dist/parquetjs/codecs/dictionary.js.map +0 -1
- package/dist/parquetjs/codecs/index.js.map +0 -1
- package/dist/parquetjs/codecs/plain.js.map +0 -1
- package/dist/parquetjs/codecs/rle.js.map +0 -1
- package/dist/parquetjs/compression.js.map +0 -1
- package/dist/parquetjs/encoder/writer.d.ts.map +0 -1
- package/dist/parquetjs/encoder/writer.js.map +0 -1
- package/dist/parquetjs/file.d.ts +0 -10
- package/dist/parquetjs/file.d.ts.map +0 -1
- package/dist/parquetjs/file.js +0 -80
- package/dist/parquetjs/file.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DateType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/IntType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ListType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/MapType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/NullType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/StringType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/Type.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/index.js.map +0 -1
- package/dist/parquetjs/parser/decoders.js.map +0 -1
- package/dist/parquetjs/parser/parquet-cursor.d.ts +0 -36
- package/dist/parquetjs/parser/parquet-cursor.d.ts.map +0 -1
- package/dist/parquetjs/parser/parquet-cursor.js +0 -90
- package/dist/parquetjs/parser/parquet-cursor.js.map +0 -1
- package/dist/parquetjs/parser/parquet-envelope-reader.d.ts +0 -40
- package/dist/parquetjs/parser/parquet-envelope-reader.d.ts.map +0 -1
- package/dist/parquetjs/parser/parquet-envelope-reader.js.map +0 -1
- package/dist/parquetjs/parser/parquet-reader.js.map +0 -1
- package/dist/parquetjs/schema/declare.js.map +0 -1
- package/dist/parquetjs/schema/schema.js.map +0 -1
- package/dist/parquetjs/schema/shred.js.map +0 -1
- package/dist/parquetjs/schema/types.js.map +0 -1
- package/dist/parquetjs/utils/buffer-utils.d.ts +0 -10
- package/dist/parquetjs/utils/buffer-utils.d.ts.map +0 -1
- package/dist/parquetjs/utils/buffer-utils.js +0 -12
- package/dist/parquetjs/utils/buffer-utils.js.map +0 -1
- package/dist/parquetjs/utils/file-utils.js.map +0 -1
- package/dist/parquetjs/utils/read-utils.js.map +0 -1
- package/dist/workers/parquet-worker.js.map +0 -1
- package/src/lib/convert-schema.ts +0 -95
- package/src/lib/parse-parquet.ts +0 -27
- package/src/lib/read-array-buffer.ts +0 -31
- package/src/parquetjs/file.ts +0 -90
- package/src/parquetjs/parser/parquet-cursor.ts +0 -94
- package/src/parquetjs/parser/parquet-envelope-reader.ts +0 -199
- package/src/parquetjs/utils/buffer-utils.ts +0 -18
- /package/dist/{parquetjs → es5/parquetjs}/LICENSE +0 -0
- /package/dist/{parquetjs → es5/parquetjs}/modules.d.ts +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/schema/shred.ts"],"names":["ParquetBuffer","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"mappings":"AAEA,SAAQA,aAAR,QAAsE,WAAtE;AAEA,OAAO,KAAKC,KAAZ,MAAuB,SAAvB;AAEA,SAAQD,aAAR;AAEA,OAAO,SAASE,WAAT,CAAqBC,MAArB,EAA2D;AAChE,QAAMC,UAAuC,GAAG,EAAhD;;AACA,OAAK,MAAMC,KAAX,IAAoBF,MAAM,CAACG,SAA3B,EAAsC;AACpCF,IAAAA,UAAU,CAACC,KAAK,CAACE,GAAP,CAAV,GAAwB;AACtBC,MAAAA,OAAO,EAAE,EADa;AAEtBC,MAAAA,OAAO,EAAE,EAFa;AAGtBC,MAAAA,MAAM,EAAE,EAHc;AAItBC,MAAAA,WAAW,EAAE,EAJS;AAKtBC,MAAAA,KAAK,EAAE;AALe,KAAxB;AAOD;;AACD,SAAO;AAACC,IAAAA,QAAQ,EAAE,CAAX;AAAcT,IAAAA;AAAd,GAAP;AACD;AAwBD,OAAO,SAASU,WAAT,CAAqBX,MAArB,EAA4CY,MAA5C,EAAyDC,MAAzD,EAAsF;AAE3F,QAAMC,IAAI,GAAGf,WAAW,CAACC,MAAD,CAAX,CAAoBC,UAAjC;AAEAc,EAAAA,iBAAiB,CAACf,MAAM,CAACgB,MAAR,EAAgBJ,MAAhB,EAAwBE,IAAxB,EAA8B,CAA9B,EAAiC,CAAjC,CAAjB;;AAGA,MAAID,MAAM,CAACH,QAAP,KAAoB,CAAxB,EAA2B;AACzBG,IAAAA,MAAM,CAACH,QAAP,GAAkB,CAAlB;AACAG,IAAAA,MAAM,CAACZ,UAAP,GAAoBa,IAApB;AACA;AACD;;AACDD,EAAAA,MAAM,CAACH,QAAP,IAAmB,CAAnB;;AACA,OAAK,MAAMR,KAAX,IAAoBF,MAAM,CAACG,SAA3B,EAAsC;AACpCc,IAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BE,OAAxD,EAAiEQ,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBE,OAAjF;AACAW,IAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BC,OAAxD,EAAiES,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBC,OAAjF;AACAY,IAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BG,MAAxD,EAAgEO,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBG,MAAhF;AACAM,IAAAA,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BK,KAA7B,IAAsCK,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBK,KAAtD;AACD;AACF;;AAGD,SAASM,iBAAT,CACEC,MADF,EAEEJ,MAFF,EAGEE,IAHF,EAIEO,MAJF,EAKEC,MALF,EAME;AACA,OAAK,MAAMC,IAAX,IAAmBP,MAAnB,EAA2B;AACzB,UAAMd,KAAK,GAAGc,MAAM,CAACO,IAAD,CAApB;AAGA,QAAIhB,MAAa,GAAG,EAApB;;AACA,QACEK,MAAM,IACNV,KAAK,CAACqB,IAAN,IAAcX,MADd,IAEAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,KAAuBC,SAFvB,IAGAZ,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,KAAuB,IAJzB,EAKE;AACA,UAAIX,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmBE,WAAnB,KAAmCR,KAAvC,EAA8C;AAC5CV,QAAAA,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAf;AACD,OAFD,MAEO;AACLhB,QAAAA,MAAM,CAACY,IAAP,CAAYP,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAlB;AACD;AACF;;AAED,QAAIhB,MAAM,CAACmB,MAAP,KAAkB,CAAlB,IAAuBC,OAAO,CAACf,MAAD,CAA9B,IAA0CV,KAAK,CAAC0B,cAAN,KAAyB,UAAvE,EAAmF;AACjF,YAAM,IAAIC,KAAJ,mCAAqC3B,KAAK,CAACqB,IAA3C,EAAN;AACD;;AACD,QAAIhB,MAAM,CAACmB,MAAP,GAAgB,CAAhB,IAAqBxB,KAAK,CAAC0B,cAAN,KAAyB,UAAlD,EAA8D;AAC5D,YAAM,IAAIC,KAAJ,sCAAwC3B,KAAK,CAACqB,IAA9C,EAAN;AACD;;AAGD,QAAIhB,MAAM,CAACmB,MAAP,KAAkB,CAAtB,EAAyB;AACvB,UAAIxB,KAAK,CAAC4B,QAAV,EAAoB;AAClBf,QAAAA,iBAAiB,CAACb,KAAK,CAACc,MAAP,EAAgB,IAAhB,EAAsBF,IAAtB,EAA4BO,MAA5B,EAAoCC,MAApC,CAAjB;AACD,OAFD,MAEO;AACLR,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBK,KAAhB,IAAyB,CAAzB;AACAK,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBE,OAAhB,CAAwBa,IAAxB,CAA6BE,MAA7B;AACAP,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBC,OAAhB,CAAwBc,IAAxB,CAA6BG,MAA7B;AACD;;AACD;AACD;;AAGD,SAAK,IAAIS,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGxB,MAAM,CAACmB,MAA3B,EAAmCK,CAAC,EAApC,EAAwC;AACtC,YAAMC,IAAI,GAAGD,CAAC,KAAK,CAAN,GAAUV,MAAV,GAAmBnB,KAAK,CAAC+B,SAAtC;;AACA,UAAI/B,KAAK,CAAC4B,QAAV,EAAoB;AAClBf,QAAAA,iBAAiB,CAACb,KAAK,CAACc,MAAP,EAAgBT,MAAM,CAACwB,CAAD,CAAtB,EAA2BjB,IAA3B,EAAiCkB,IAAjC,EAAuC9B,KAAK,CAACgC,SAA7C,CAAjB;AACD,OAFD,MAEO;AACLpB,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBK,KAAhB,IAAyB,CAAzB;AACAK,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBE,OAAhB,CAAwBa,IAAxB,CAA6Ba,IAA7B;AACAlB,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBC,OAAhB,CAAwBc,IAAxB,CAA6BjB,KAAK,CAACgC,SAAnC;AACApB,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBG,MAAhB,CAAuBY,IAAvB,CACErB,KAAK,CAACqC,WAAN,CAAmBjC,KAAK,CAACkC,YAAN,IAAsBlC,KAAK,CAACmC,aAA/C,EAAgE9B,MAAM,CAACwB,CAAD,CAAtE,CADF;AAGD;AACF;AACF;AACF;;AAqBD,OAAO,SAASO,kBAAT,CAA4BtC,MAA5B,EAAmDa,MAAnD,EAA2F;AAChG,QAAM0B,OAAwB,GAAG,EAAjC;;AACA,OAAK,IAAIR,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGlB,MAAM,CAACH,QAA3B,EAAqCqB,CAAC,EAAtC,EAA0CQ,OAAO,CAACpB,IAAR,CAAa,EAAb;;AAC1C,OAAK,MAAMf,GAAX,IAAkBS,MAAM,CAACZ,UAAzB,EAAqC;AACnCuC,IAAAA,iBAAiB,CAACxC,MAAD,EAASa,MAAT,EAAiBT,GAAjB,EAAsBmC,OAAtB,CAAjB;AACD;;AACD,SAAOA,OAAP;AACD;;AAGD,SAASC,iBAAT,CACExC,MADF,EAEEa,MAFF,EAGET,GAHF,EAIEmC,OAJF,EAKE;AACA,QAAMzB,IAAI,GAAGD,MAAM,CAACZ,UAAP,CAAkBG,GAAlB,CAAb;AACA,MAAI,CAACU,IAAI,CAACL,KAAV,EAAiB;AAEjB,QAAMP,KAAK,GAAGF,MAAM,CAACyC,SAAP,CAAiBrC,GAAjB,CAAd;AACA,QAAMsC,MAAM,GAAG1C,MAAM,CAAC2C,eAAP,CAAuBvC,GAAvB,CAAf;AAGA,QAAMwC,OAAiB,GAAG,IAAI3B,KAAJ,CAAUf,KAAK,CAAC+B,SAAN,GAAkB,CAA5B,EAA+BY,IAA/B,CAAoC,CAApC,CAA1B;AACA,MAAIC,MAAM,GAAG,CAAb;;AACA,OAAK,IAAIf,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGjB,IAAI,CAACL,KAAzB,EAAgCsB,CAAC,EAAjC,EAAqC;AACnC,UAAMT,MAAM,GAAGR,IAAI,CAACT,OAAL,CAAa0B,CAAb,CAAf;AACA,UAAMV,MAAM,GAAGP,IAAI,CAACR,OAAL,CAAayB,CAAb,CAAf;AACAa,IAAAA,OAAO,CAACvB,MAAD,CAAP;AACAuB,IAAAA,OAAO,CAACC,IAAR,CAAa,CAAb,EAAgBxB,MAAM,GAAG,CAAzB;AAEA,QAAI0B,MAAM,GAAG,CAAb;AACA,QAAInC,MAAM,GAAG2B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAP,CAAP,GAAoB,CAArB,CAApB;;AAGA,SAAK,MAAMC,IAAX,IAAmBN,MAAnB,EAA2B;AACzB,UAAIM,IAAI,KAAK9C,KAAb,EAAoB;AACpB,UAAIoB,MAAM,GAAG0B,IAAI,CAACd,SAAlB,EAA6B;;AAC7B,UAAIc,IAAI,CAACpB,cAAL,KAAwB,UAA5B,EAAwC;AACtC,YAAI,EAAEoB,IAAI,CAACzB,IAAL,IAAaX,MAAf,CAAJ,EAA4B;AAE1BA,UAAAA,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAN,GAAoB,EAApB;AACD;;AACD,cAAM0B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAP,CAAlB;;AACA,eAAOnC,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAN,CAAkBG,MAAlB,IAA4BuB,EAAnC,EAAuC;AAErCrC,UAAAA,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAN,CAAkBJ,IAAlB,CAAuB,EAAvB;AACD;;AACDP,QAAAA,MAAM,GAAGA,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAN,CAAkB0B,EAAlB,CAAT;AACD,OAXD,MAWO;AACLrC,QAAAA,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAN,GAAoBX,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAN,IAAqB,EAAzC;AACAX,QAAAA,MAAM,GAAGA,MAAM,CAACoC,IAAI,CAACzB,IAAN,CAAf;AACD;AACF;;AAGD,QAAID,MAAM,KAAKpB,KAAK,CAACgC,SAArB,EAAgC;AAC9B,YAAMgB,KAAK,GAAGpD,KAAK,CAACqD,aAAN,CAEZjD,KAAK,CAACkC,YAAN,IAAsBlC,KAAK,CAACmC,aAFhB,EAGZvB,IAAI,CAACP,MAAL,CAAYuC,MAAZ,CAHY,EAIZ5C,KAJY,CAAd;AAMA4C,MAAAA,MAAM;;AACN,UAAI5C,KAAK,CAAC0B,cAAN,KAAyB,UAA7B,EAAyC;AACvC,YAAI,EAAE1B,KAAK,CAACqB,IAAN,IAAcX,MAAhB,CAAJ,EAA6B;AAE3BA,UAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,GAAqB,EAArB;AACD;;AACD,cAAM0B,EAAE,GAAGL,OAAO,CAACG,MAAD,CAAlB;;AACA,eAAOnC,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmBG,MAAnB,IAA6BuB,EAApC,EAAwC;AAEtCrC,UAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmBJ,IAAnB,CAAwB,IAAxB;AACD;;AACDP,QAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmB0B,EAAnB,IAAyBC,KAAzB;AACD,OAXD,MAWO;AACLtC,QAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,GAAqB2B,KAArB;AACD;AACF;AACF;AACF","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n const columnData: Record<string, ParquetData> = {};\n for (const field of schema.fieldList) {\n columnData[field.key] = {\n dlevels: [],\n rlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n }\n return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n * buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n /* shred the record, this may raise an exception */\n const data = shredBuffer(schema).columnData;\n\n shredRecordFields(schema.fields, record, data, 0, 0);\n\n /* if no error during shredding, add the shredded record to the buffer */\n if (buffer.rowCount === 0) {\n buffer.rowCount = 1;\n buffer.columnData = data;\n return;\n }\n buffer.rowCount += 1;\n for (const field of schema.fieldList) {\n Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n buffer.columnData[field.key].count += data[field.key].count;\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n fields: Record<string, ParquetField>,\n record: any,\n data: Record<string, ParquetData>,\n rLevel: number,\n dLevel: number\n) {\n for (const name in fields) {\n const field = fields[name];\n\n // fetch values\n let values: any[] = [];\n if (\n record &&\n field.name in record &&\n record[field.name] !== undefined &&\n record[field.name] !== null\n ) {\n if (record[field.name].constructor === Array) {\n values = record[field.name];\n } else {\n values.push(record[field.name]);\n }\n }\n // check values\n if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n throw new Error(`missing required field: ${field.name}`);\n }\n if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n throw new Error(`too many values for field: ${field.name}`);\n }\n\n // push null\n if (values.length === 0) {\n if (field.isNested) {\n shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n } else {\n data[field.key].count += 1;\n data[field.key].rlevels.push(rLevel);\n data[field.key].dlevels.push(dLevel);\n }\n continue; // eslint-disable-line no-continue\n }\n\n // push values\n for (let i = 0; i < values.length; i++) {\n const rlvl = i === 0 ? rLevel : field.rLevelMax;\n if (field.isNested) {\n shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n } else {\n data[field.key].count += 1;\n data[field.key].rlevels.push(rlvl);\n data[field.key].dlevels.push(field.dLevelMax);\n data[field.key].values.push(\n Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n );\n }\n }\n }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n * buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n const records: ParquetRecord[] = [];\n for (let i = 0; i < buffer.rowCount; i++) records.push({});\n for (const key in buffer.columnData) {\n materializeColumn(schema, buffer, key, records);\n }\n return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n schema: ParquetSchema,\n buffer: ParquetBuffer,\n key: string,\n records: ParquetRecord[]\n) {\n const data = buffer.columnData[key];\n if (!data.count) return;\n\n const field = schema.findField(key);\n const branch = schema.findFieldBranch(key);\n\n // tslint:disable-next-line:prefer-array-literal\n const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n let vIndex = 0;\n for (let i = 0; i < data.count; i++) {\n const dLevel = data.dlevels[i];\n const rLevel = data.rlevels[i];\n rLevels[rLevel]++;\n rLevels.fill(0, rLevel + 1);\n\n let rIndex = 0;\n let record = records[rLevels[rIndex++] - 1];\n\n // Internal nodes\n for (const step of branch) {\n if (step === field) break;\n if (dLevel < step.dLevelMax) break;\n if (step.repetitionType === 'REPEATED') {\n if (!(step.name in record)) {\n // eslint-disable max-depth\n record[step.name] = [];\n }\n const ix = rLevels[rIndex++];\n while (record[step.name].length <= ix) {\n // eslint-disable max-depth\n record[step.name].push({});\n }\n record = record[step.name][ix];\n } else {\n record[step.name] = record[step.name] || {};\n record = record[step.name];\n }\n }\n\n // Leaf node\n if (dLevel === field.dLevelMax) {\n const value = Types.fromPrimitive(\n // @ts-ignore\n field.originalType || field.primitiveType,\n data.values[vIndex],\n field\n );\n vIndex++;\n if (field.repetitionType === 'REPEATED') {\n if (!(field.name in record)) {\n // eslint-disable max-depth\n record[field.name] = [];\n }\n const ix = rLevels[rIndex];\n while (record[field.name].length <= ix) {\n // eslint-disable max-depth\n record[field.name].push(null);\n }\n record[field.name][ix] = value;\n } else {\n record[field.name] = value;\n }\n }\n }\n}\n"],"file":"shred.js"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/schema/types.ts"],"names":["BSON","PARQUET_LOGICAL_TYPES","BOOLEAN","primitiveType","toPrimitive","toPrimitive_BOOLEAN","fromPrimitive","fromPrimitive_BOOLEAN","INT32","toPrimitive_INT32","INT64","toPrimitive_INT64","INT96","toPrimitive_INT96","FLOAT","toPrimitive_FLOAT","DOUBLE","toPrimitive_DOUBLE","BYTE_ARRAY","toPrimitive_BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","originalType","toPrimitive_UTF8","fromPrimitive_UTF8","TIME_MILLIS","toPrimitive_TIME_MILLIS","TIME_MICROS","toPrimitive_TIME_MICROS","DATE","toPrimitive_DATE","fromPrimitive_DATE","TIMESTAMP_MILLIS","toPrimitive_TIMESTAMP_MILLIS","fromPrimitive_TIMESTAMP_MILLIS","TIMESTAMP_MICROS","toPrimitive_TIMESTAMP_MICROS","fromPrimitive_TIMESTAMP_MICROS","UINT_8","toPrimitive_UINT8","UINT_16","toPrimitive_UINT16","UINT_32","toPrimitive_UINT32","UINT_64","toPrimitive_UINT64","INT_8","toPrimitive_INT8","INT_16","toPrimitive_INT16","INT_32","INT_64","JSON","toPrimitive_JSON","fromPrimitive_JSON","toPrimitive_BSON","fromPrimitive_BSON","INTERVAL","typeLength","toPrimitive_INTERVAL","fromPrimitive_INTERVAL","DECIMAL_INT32","decimalToPrimitive_INT32","decimalFromPrimitive_INT","DECIMAL_INT64","decimalToPrimitive_INT64","DECIMAL_BYTE_ARRAY","decimalToPrimitive_BYTE_ARRAY","decimalFromPrimitive_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","type","value","field","Error","Boolean","v","parseFloat","isNaN","parseInt","primitiveValue","scale","Math","round","presision","Buffer","from","toString","stringify","parse","serialize","deserialize","kMillisPerDay","Date","getTime","months","days","milliseconds","buf","alloc","writeUInt32LE","readUInt32LE","millis","presisionInt","number","length","i","component"],"mappings":"AAEA,OAAOA,IAAP,MAAiB,MAAjB;AAWA,OAAO,MAAMC,qBAA0D,GAAG;AACxEC,EAAAA,OAAO,EAAE;AACPC,IAAAA,aAAa,EAAE,SADR;AAEPC,IAAAA,WAAW,EAAEC,mBAFN;AAGPC,IAAAA,aAAa,EAAEC;AAHR,GAD+D;AAMxEC,EAAAA,KAAK,EAAE;AACLL,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEK;AAFR,GANiE;AAUxEC,EAAAA,KAAK,EAAE;AACLP,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEO;AAFR,GAViE;AAcxEC,EAAAA,KAAK,EAAE;AACLT,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAES;AAFR,GAdiE;AAkBxEC,EAAAA,KAAK,EAAE;AACLX,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEW;AAFR,GAlBiE;AAsBxEC,EAAAA,MAAM,EAAE;AACNb,IAAAA,aAAa,EAAE,QADT;AAENC,IAAAA,WAAW,EAAEa;AAFP,GAtBgE;AA0BxEC,EAAAA,UAAU,EAAE;AACVf,IAAAA,aAAa,EAAE,YADL;AAEVC,IAAAA,WAAW,EAAEe;AAFH,GA1B4D;AA8BxEC,EAAAA,oBAAoB,EAAE;AACpBjB,IAAAA,aAAa,EAAE,sBADK;AAEpBC,IAAAA,WAAW,EAAEe;AAFO,GA9BkD;AAkCxEE,EAAAA,IAAI,EAAE;AACJlB,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEmB,gBAHT;AAIJjB,IAAAA,aAAa,EAAEkB;AAJX,GAlCkE;AAwCxEC,EAAAA,WAAW,EAAE;AACXtB,IAAAA,aAAa,EAAE,OADJ;AAEXmB,IAAAA,YAAY,EAAE,aAFH;AAGXlB,IAAAA,WAAW,EAAEsB;AAHF,GAxC2D;AA6CxEC,EAAAA,WAAW,EAAE;AACXxB,IAAAA,aAAa,EAAE,OADJ;AAEXmB,IAAAA,YAAY,EAAE,aAFH;AAGXlB,IAAAA,WAAW,EAAEwB;AAHF,GA7C2D;AAkDxEC,EAAAA,IAAI,EAAE;AACJ1B,IAAAA,aAAa,EAAE,OADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAE0B,gBAHT;AAIJxB,IAAAA,aAAa,EAAEyB;AAJX,GAlDkE;AAwDxEC,EAAAA,gBAAgB,EAAE;AAChB7B,IAAAA,aAAa,EAAE,OADC;AAEhBmB,IAAAA,YAAY,EAAE,kBAFE;AAGhBlB,IAAAA,WAAW,EAAE6B,4BAHG;AAIhB3B,IAAAA,aAAa,EAAE4B;AAJC,GAxDsD;AA8DxEC,EAAAA,gBAAgB,EAAE;AAChBhC,IAAAA,aAAa,EAAE,OADC;AAEhBmB,IAAAA,YAAY,EAAE,kBAFE;AAGhBlB,IAAAA,WAAW,EAAEgC,4BAHG;AAIhB9B,IAAAA,aAAa,EAAE+B;AAJC,GA9DsD;AAoExEC,EAAAA,MAAM,EAAE;AACNnC,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEmC;AAHP,GApEgE;AAyExEC,EAAAA,OAAO,EAAE;AACPrC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEqC;AAHN,GAzE+D;AA8ExEC,EAAAA,OAAO,EAAE;AACPvC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEuC;AAHN,GA9E+D;AAmFxEC,EAAAA,OAAO,EAAE;AACPzC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEyC;AAHN,GAnF+D;AAwFxEC,EAAAA,KAAK,EAAE;AACL3C,IAAAA,aAAa,EAAE,OADV;AAELmB,IAAAA,YAAY,EAAE,OAFT;AAGLlB,IAAAA,WAAW,EAAE2C;AAHR,GAxFiE;AA6FxEC,EAAAA,MAAM,EAAE;AACN7C,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAE6C;AAHP,GA7FgE;AAkGxEC,EAAAA,MAAM,EAAE;AACN/C,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEK;AAHP,GAlGgE;AAuGxE0C,EAAAA,MAAM,EAAE;AACNhD,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEO;AAHP,GAvGgE;AA4GxEyC,EAAAA,IAAI,EAAE;AACJjD,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEiD,gBAHT;AAIJ/C,IAAAA,aAAa,EAAEgD;AAJX,GA5GkE;AAkHxEtD,EAAAA,IAAI,EAAE;AACJG,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEmD,gBAHT;AAIJjD,IAAAA,aAAa,EAAEkD;AAJX,GAlHkE;AAwHxEC,EAAAA,QAAQ,EAAE;AACRtD,IAAAA,aAAa,EAAE,sBADP;AAERmB,IAAAA,YAAY,EAAE,UAFN;AAGRoC,IAAAA,UAAU,EAAE,EAHJ;AAIRtD,IAAAA,WAAW,EAAEuD,oBAJL;AAKRrD,IAAAA,aAAa,EAAEsD;AALP,GAxH8D;AA+HxEC,EAAAA,aAAa,EAAE;AACb1D,IAAAA,aAAa,EAAE,OADF;AAEbmB,IAAAA,YAAY,EAAE,eAFD;AAGblB,IAAAA,WAAW,EAAE0D,wBAHA;AAIbxD,IAAAA,aAAa,EAAEyD;AAJF,GA/HyD;AAqIxEC,EAAAA,aAAa,EAAE;AACb7D,IAAAA,aAAa,EAAE,OADF;AAEbmB,IAAAA,YAAY,EAAE,eAFD;AAGblB,IAAAA,WAAW,EAAE6D,wBAHA;AAIb3D,IAAAA,aAAa,EAAEyD;AAJF,GArIyD;AA2IxEG,EAAAA,kBAAkB,EAAE;AAClB/D,IAAAA,aAAa,EAAE,YADG;AAElBmB,IAAAA,YAAY,EAAE,oBAFI;AAGlBlB,IAAAA,WAAW,EAAE+D,6BAHK;AAIlB7D,IAAAA,aAAa,EAAE8D;AAJG,GA3IoD;AAiJxEC,EAAAA,4BAA4B,EAAE;AAC5BlE,IAAAA,aAAa,EAAE,sBADa;AAE5BmB,IAAAA,YAAY,EAAE,8BAFc;AAG5BlB,IAAAA,WAAW,EAAE+D,6BAHe;AAI5B7D,IAAAA,aAAa,EAAE8D;AAJa;AAjJ0C,CAAnE;AA6JP,OAAO,SAAShE,WAAT,CAAqBkE,IAArB,EAAwCC,KAAxC,EAAoDC,KAApD,EAA0E;AAC/E,MAAI,EAAEF,IAAI,IAAIrE,qBAAV,CAAJ,EAAsC;AACpC,UAAM,IAAIwE,KAAJ,yBAA2BH,IAA3B,EAAN;AACD;;AAED,SAAOrE,qBAAqB,CAACqE,IAAD,CAArB,CAA4BlE,WAA5B,CAAwCmE,KAAxC,EAA+CC,KAA/C,CAAP;AACD;AAMD,OAAO,SAASlE,aAAT,CAAuBgE,IAAvB,EAA0CC,KAA1C,EAAsDC,KAAtD,EAA4E;AACjF,MAAI,EAAEF,IAAI,IAAIrE,qBAAV,CAAJ,EAAsC;AACpC,UAAM,IAAIwE,KAAJ,yBAA2BH,IAA3B,EAAN;AACD;;AAED,MAAI,mBAAmBrE,qBAAqB,CAACqE,IAAD,CAA5C,EAAoD;AAAA;;AAClD,oCAAO,0BAAArE,qBAAqB,CAACqE,IAAD,CAArB,EAA4BhE,aAAnC,0DAAO,mDAA4CiE,KAA5C,EAAmDC,KAAnD,CAAP;AAED;;AACD,SAAOD,KAAP;AACD;;AAED,SAASlE,mBAAT,CAA6BkE,KAA7B,EAAyC;AACvC,SAAOG,OAAO,CAACH,KAAD,CAAd;AACD;;AAED,SAAShE,qBAAT,CAA+BgE,KAA/B,EAA2C;AACzC,SAAOG,OAAO,CAACH,KAAD,CAAd;AACD;;AAED,SAASxD,iBAAT,CAA2BwD,KAA3B,EAAuC;AACrC,QAAMI,CAAC,GAAGC,UAAU,CAACL,KAAD,CAApB;;AACA,MAAIM,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAS1D,kBAAT,CAA4BsD,KAA5B,EAAwC;AACtC,QAAMI,CAAC,GAAGC,UAAU,CAACL,KAAD,CAApB;;AACA,MAAIM,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,qCAAuCF,KAAvC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAS5B,gBAAT,CAA0BwB,KAA1B,EAAsC;AACpC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAC,IAAL,IAAaA,CAAC,GAAG,IAAjB,IAAyBE,KAAK,CAACF,CAAD,CAAlC,EAAuC;AACrC,UAAM,IAAIF,KAAJ,mCAAqCF,KAArC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAASpC,iBAAT,CAA2BgC,KAA3B,EAAuC;AACrC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,IAAb,IAAqBE,KAAK,CAACF,CAAD,CAA9B,EAAmC;AACjC,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAS1B,iBAAT,CAA2BsB,KAA3B,EAAuC;AACrC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAC,MAAL,IAAeA,CAAC,GAAG,MAAnB,IAA6BE,KAAK,CAACF,CAAD,CAAtC,EAA2C;AACzC,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAASlC,kBAAT,CAA4B8B,KAA5B,EAAwC;AACtC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,MAAb,IAAuBE,KAAK,CAACF,CAAD,CAAhC,EAAqC;AACnC,UAAM,IAAIF,KAAJ,qCAAuCF,KAAvC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAASlE,iBAAT,CAA2B8D,KAA3B,EAAuC;AACrC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAC,UAAL,IAAmBA,CAAC,GAAG,UAAvB,IAAqCE,KAAK,CAACF,CAAD,CAA9C,EAAmD;AACjD,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAASb,wBAAT,CAAkCS,KAAlC,EAAiDC,KAAjD,EAAsE;AACpE,QAAMO,cAAc,GAAGR,KAAK,GAAG,OAAOC,KAAK,CAACQ,KAAN,IAAe,CAAtB,CAA/B;AACA,QAAML,CAAC,GAAGM,IAAI,CAACC,KAAL,CAAaH,cAAc,GAAG,MAAM,CAACP,KAAK,CAACW,SAA/B,GAA6C,CAA9C,GAAmD,MAAMX,KAAK,CAACW,SAA1E,CAAV;;AACA,MAAIR,CAAC,GAAG,CAAC,UAAL,IAAmBA,CAAC,GAAG,UAAvB,IAAqCE,KAAK,CAACF,CAAD,CAA9C,EAAmD;AACjD,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAShC,kBAAT,CAA4B4B,KAA5B,EAAwC;AACtC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,cAAb,IAA+BE,KAAK,CAACF,CAAD,CAAxC,EAA6C;AAC3C,UAAM,IAAIF,KAAJ,qCAAuCF,KAAvC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAShE,iBAAT,CAA2B4D,KAA3B,EAAuC;AACrC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIM,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAASV,wBAAT,CAAkCM,KAAlC,EAAiDC,KAAjD,EAAsE;AACpE,QAAMO,cAAc,GAAGR,KAAK,GAAG,OAAOC,KAAK,CAACQ,KAAN,IAAe,CAAtB,CAA/B;AACA,QAAML,CAAC,GAAGM,IAAI,CAACC,KAAL,CAAaH,cAAc,GAAG,MAAM,CAACP,KAAK,CAACW,SAA/B,GAA6C,CAA9C,GAAmD,MAAMX,KAAK,CAACW,SAA1E,CAAV;;AACA,MAAIN,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAS9B,kBAAT,CAA4B0B,KAA5B,EAAwC;AACtC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,UAAM,IAAIF,KAAJ,qCAAuCF,KAAvC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAS9D,iBAAT,CAA2B0D,KAA3B,EAAuC;AACrC,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIM,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCF,KAAtC,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAASxD,sBAAT,CAAgCoD,KAAhC,EAA4C;AAC1C,SAAOa,MAAM,CAACC,IAAP,CAAYd,KAAZ,CAAP;AACD;;AAED,SAASJ,6BAAT,CAAuCI,KAAvC,EAAmD;AAEjD,SAAOa,MAAM,CAACC,IAAP,CAAYd,KAAZ,CAAP;AACD;;AAED,SAAShD,gBAAT,CAA0BgD,KAA1B,EAAsC;AACpC,SAAOa,MAAM,CAACC,IAAP,CAAYd,KAAZ,EAAmB,MAAnB,CAAP;AACD;;AAED,SAAS/C,kBAAT,CAA4B+C,KAA5B,EAAwC;AACtC,SAAOA,KAAK,CAACe,QAAN,EAAP;AACD;;AAED,SAASjC,gBAAT,CAA0BkB,KAA1B,EAAsC;AACpC,SAAOa,MAAM,CAACC,IAAP,CAAYjC,IAAI,CAACmC,SAAL,CAAehB,KAAf,CAAZ,CAAP;AACD;;AAED,SAASjB,kBAAT,CAA4BiB,KAA5B,EAAwC;AACtC,SAAOnB,IAAI,CAACoC,KAAL,CAAWjB,KAAX,CAAP;AACD;;AAED,SAAShB,gBAAT,CAA0BgB,KAA1B,EAAsC;AACpC,SAAOa,MAAM,CAACC,IAAP,CAAYrF,IAAI,CAACyF,SAAL,CAAelB,KAAf,CAAZ,CAAP;AACD;;AAED,SAASf,kBAAT,CAA4Be,KAA5B,EAAwC;AACtC,SAAOvE,IAAI,CAAC0F,WAAL,CAAiBnB,KAAjB,CAAP;AACD;;AAED,SAAS7C,uBAAT,CAAiC6C,KAAjC,EAA6C;AAC3C,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,kBAAb,IAAmCE,KAAK,CAACF,CAAD,CAA5C,EAAiD;AAC/C,UAAM,IAAIF,KAAJ,0CAA4CF,KAA5C,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,SAAS/C,uBAAT,CAAiC2C,KAAjC,EAA6C;AAC3C,QAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAII,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,UAAM,IAAIF,KAAJ,0CAA4CF,KAA5C,EAAN;AACD;;AAED,SAAOI,CAAP;AACD;;AAED,MAAMgB,aAAa,GAAG,QAAtB;;AAEA,SAAS7D,gBAAT,CAA0ByC,KAA1B,EAAsC;AAEpC,MAAIA,KAAK,YAAYqB,IAArB,EAA2B;AACzB,WAAOrB,KAAK,CAACsB,OAAN,KAAkBF,aAAzB;AACD;;AAGD;AACE,UAAMhB,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAII,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,mCAAqCF,KAArC,EAAN;AACD;;AAED,WAAOI,CAAP;AACD;AACF;;AAED,SAAS5C,kBAAT,CAA4BwC,KAA5B,EAAwC;AACtC,SAAO,IAAIqB,IAAJ,CAASrB,KAAK,GAAGoB,aAAjB,CAAP;AACD;;AAED,SAAS1D,4BAAT,CAAsCsC,KAAtC,EAAkD;AAEhD,MAAIA,KAAK,YAAYqB,IAArB,EAA2B;AACzB,WAAOrB,KAAK,CAACsB,OAAN,EAAP;AACD;;AAGD;AACE,UAAMlB,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAII,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,+CAAiDF,KAAjD,EAAN;AACD;;AAED,WAAOI,CAAP;AACD;AACF;;AAED,SAASzC,8BAAT,CAAwCqC,KAAxC,EAAoD;AAClD,SAAO,IAAIqB,IAAJ,CAASrB,KAAT,CAAP;AACD;;AAED,SAASnC,4BAAT,CAAsCmC,KAAtC,EAAkD;AAEhD,MAAIA,KAAK,YAAYqB,IAArB,EAA2B;AACzB,WAAOrB,KAAK,CAACsB,OAAN,KAAkB,IAAzB;AACD;;AAGD;AACE,UAAMlB,CAAC,GAAGG,QAAQ,CAACP,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAII,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,+CAAiDF,KAAjD,EAAN;AACD;;AAED,WAAOI,CAAP;AACD;AACF;;AAED,SAAStC,8BAAT,CAAwCkC,KAAxC,EAAoD;AAClD,SAAO,IAAIqB,IAAJ,CAASrB,KAAK,GAAG,IAAjB,CAAP;AACD;;AAED,SAASZ,oBAAT,CAA8BY,KAA9B,EAA0C;AACxC,MAAI,CAACA,KAAK,CAACuB,MAAP,IAAiB,CAACvB,KAAK,CAACwB,IAAxB,IAAgC,CAACxB,KAAK,CAACyB,YAA3C,EAAyD;AACvD,UAAM,IAAIvB,KAAJ,CACJ,iFADI,CAAN;AAGD;;AAED,QAAMwB,GAAG,GAAGb,MAAM,CAACc,KAAP,CAAa,EAAb,CAAZ;AAEAD,EAAAA,GAAG,CAACE,aAAJ,CAAkB5B,KAAK,CAACuB,MAAxB,EAAgC,CAAhC;AACAG,EAAAA,GAAG,CAACE,aAAJ,CAAkB5B,KAAK,CAACwB,IAAxB,EAA8B,CAA9B;AACAE,EAAAA,GAAG,CAACE,aAAJ,CAAkB5B,KAAK,CAACyB,YAAxB,EAAsC,CAAtC;AACA,SAAOC,GAAP;AACD;;AAED,SAASrC,sBAAT,CAAgCW,KAAhC,EAA4C;AAC1C,QAAM0B,GAAG,GAAGb,MAAM,CAACC,IAAP,CAAYd,KAAZ,CAAZ;AACA,QAAMuB,MAAM,GAAGG,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAf;AACA,QAAML,IAAI,GAAGE,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAb;AACA,QAAMC,MAAM,GAAGJ,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAf;AAEA,SAAO;AAACN,IAAAA,MAAD;AAASC,IAAAA,IAAT;AAAeC,IAAAA,YAAY,EAAEK;AAA7B,GAAP;AACD;;AAED,SAAStC,wBAAT,CAAkCQ,KAAlC,EAA8CC,KAA9C,EAAmE;AACjE,QAAM8B,YAAY,GAAGrB,IAAI,CAACC,KAAL,CAAaX,KAAK,GAAG,MAAM,CAACC,KAAK,CAACW,SAAtB,GAAoC,CAArC,GAA0C,MAAMX,KAAK,CAACW,SAAjE,CAArB;AACA,SAAOmB,YAAY,GAAG,MAAM,EAAE9B,KAAK,CAACQ,KAAN,IAAe,CAAjB,CAA5B;AACD;;AAED,SAASZ,+BAAT,CAAyCG,KAAzC,EAAqDC,KAArD,EAA0E;AACxE,MAAI+B,MAAM,GAAG,CAAb;;AACA,MAAIhC,KAAK,CAACiC,MAAN,IAAgB,CAApB,EAAuB;AAErB,SAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGlC,KAAK,CAACiC,MAA1B,EAAkCC,CAAC,EAAnC,EAAuC;AAErC,YAAMC,SAAS,GAAGnC,KAAK,CAACkC,CAAD,CAAL,IAAa,KAAKlC,KAAK,CAACiC,MAAN,GAAeC,CAAf,GAAmB,CAAxB,CAA/B;AACAF,MAAAA,MAAM,IAAIG,SAAV;AACD;AACF,GAPD,MAOO;AACL,SAAK,IAAID,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGlC,KAAK,CAACiC,MAA1B,EAAkCC,CAAC,EAAnC,EAAuC;AAErC,YAAMC,SAAS,GAAGnC,KAAK,CAACkC,CAAD,CAAL,GAAW,MAAM,KAAKlC,KAAK,CAACiC,MAAN,GAAe,CAAf,GAAmBC,CAAxB,CAAN,CAA7B;AACAF,MAAAA,MAAM,IAAIG,SAAV;AACD;AACF;;AAED,QAAMJ,YAAY,GAAGrB,IAAI,CAACC,KAAL,CACjBqB,MAAM,GAAG,MAAM,CAAC/B,KAAK,CAACW,SAAvB,GAAqC,CAAtC,GAA2C,MAAMX,KAAK,CAACW,SADpC,CAArB;AAGA,SAAOmB,YAAY,GAAG,MAAM,EAAE9B,KAAK,CAACQ,KAAN,IAAe,CAAjB,CAA5B;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport BSON from 'bson';\nimport {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';\n\nexport interface ParquetTypeKit {\n primitiveType: PrimitiveType;\n originalType?: OriginalType;\n typeLength?: number;\n toPrimitive: Function;\n fromPrimitive?: Function;\n}\n\nexport const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {\n BOOLEAN: {\n primitiveType: 'BOOLEAN',\n toPrimitive: toPrimitive_BOOLEAN,\n fromPrimitive: fromPrimitive_BOOLEAN\n },\n INT32: {\n primitiveType: 'INT32',\n toPrimitive: toPrimitive_INT32\n },\n INT64: {\n primitiveType: 'INT64',\n toPrimitive: toPrimitive_INT64\n },\n INT96: {\n primitiveType: 'INT96',\n toPrimitive: toPrimitive_INT96\n },\n FLOAT: {\n primitiveType: 'FLOAT',\n toPrimitive: toPrimitive_FLOAT\n },\n DOUBLE: {\n primitiveType: 'DOUBLE',\n toPrimitive: toPrimitive_DOUBLE\n },\n BYTE_ARRAY: {\n primitiveType: 'BYTE_ARRAY',\n toPrimitive: toPrimitive_BYTE_ARRAY\n },\n FIXED_LEN_BYTE_ARRAY: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n toPrimitive: toPrimitive_BYTE_ARRAY\n },\n UTF8: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'UTF8',\n toPrimitive: toPrimitive_UTF8,\n fromPrimitive: fromPrimitive_UTF8\n },\n TIME_MILLIS: {\n primitiveType: 'INT32',\n originalType: 'TIME_MILLIS',\n toPrimitive: toPrimitive_TIME_MILLIS\n },\n TIME_MICROS: {\n primitiveType: 'INT64',\n originalType: 'TIME_MICROS',\n toPrimitive: toPrimitive_TIME_MICROS\n },\n DATE: {\n primitiveType: 'INT32',\n originalType: 'DATE',\n toPrimitive: toPrimitive_DATE,\n fromPrimitive: fromPrimitive_DATE\n },\n TIMESTAMP_MILLIS: {\n primitiveType: 'INT64',\n originalType: 'TIMESTAMP_MILLIS',\n toPrimitive: toPrimitive_TIMESTAMP_MILLIS,\n fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS\n },\n TIMESTAMP_MICROS: {\n primitiveType: 'INT64',\n originalType: 'TIMESTAMP_MICROS',\n toPrimitive: toPrimitive_TIMESTAMP_MICROS,\n fromPrimitive: fromPrimitive_TIMESTAMP_MICROS\n },\n UINT_8: {\n primitiveType: 'INT32',\n originalType: 'UINT_8',\n toPrimitive: toPrimitive_UINT8\n },\n UINT_16: {\n primitiveType: 'INT32',\n originalType: 'UINT_16',\n toPrimitive: toPrimitive_UINT16\n },\n UINT_32: {\n primitiveType: 'INT32',\n originalType: 'UINT_32',\n toPrimitive: toPrimitive_UINT32\n },\n UINT_64: {\n primitiveType: 'INT64',\n originalType: 'UINT_64',\n toPrimitive: toPrimitive_UINT64\n },\n INT_8: {\n primitiveType: 'INT32',\n originalType: 'INT_8',\n toPrimitive: toPrimitive_INT8\n },\n INT_16: {\n primitiveType: 'INT32',\n originalType: 'INT_16',\n toPrimitive: toPrimitive_INT16\n },\n INT_32: {\n primitiveType: 'INT32',\n originalType: 'INT_32',\n toPrimitive: toPrimitive_INT32\n },\n INT_64: {\n primitiveType: 'INT64',\n originalType: 'INT_64',\n toPrimitive: toPrimitive_INT64\n },\n JSON: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'JSON',\n toPrimitive: toPrimitive_JSON,\n fromPrimitive: fromPrimitive_JSON\n },\n BSON: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'BSON',\n toPrimitive: toPrimitive_BSON,\n fromPrimitive: fromPrimitive_BSON\n },\n INTERVAL: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n originalType: 'INTERVAL',\n typeLength: 12,\n toPrimitive: toPrimitive_INTERVAL,\n fromPrimitive: fromPrimitive_INTERVAL\n },\n DECIMAL_INT32: {\n primitiveType: 'INT32',\n originalType: 'DECIMAL_INT32',\n toPrimitive: decimalToPrimitive_INT32,\n fromPrimitive: decimalFromPrimitive_INT\n },\n DECIMAL_INT64: {\n primitiveType: 'INT64',\n originalType: 'DECIMAL_INT64',\n toPrimitive: decimalToPrimitive_INT64,\n fromPrimitive: decimalFromPrimitive_INT\n },\n DECIMAL_BYTE_ARRAY: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'DECIMAL_BYTE_ARRAY',\n toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n },\n DECIMAL_FIXED_LEN_BYTE_ARRAY: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n originalType: 'DECIMAL_FIXED_LEN_BYTE_ARRAY',\n toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n }\n};\n\n/**\n * Convert a value from it's native representation to the internal/underlying\n * primitive type\n */\nexport function toPrimitive(type: ParquetType, value: any, field?: ParquetField) {\n if (!(type in PARQUET_LOGICAL_TYPES)) {\n throw new Error(`invalid type: ${type}`);\n }\n\n return PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);\n}\n\n/**\n * Convert a value from it's internal/underlying primitive representation to\n * the native representation\n */\nexport function fromPrimitive(type: ParquetType, value: any, field?: ParquetField) {\n if (!(type in PARQUET_LOGICAL_TYPES)) {\n throw new Error(`invalid type: ${type}`);\n }\n\n if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {\n return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value, field);\n // tslint:disable-next-line:no-else-after-return\n }\n return value;\n}\n\nfunction toPrimitive_BOOLEAN(value: any) {\n return Boolean(value);\n}\n\nfunction fromPrimitive_BOOLEAN(value: any) {\n return Boolean(value);\n}\n\nfunction toPrimitive_FLOAT(value: any) {\n const v = parseFloat(value);\n if (isNaN(v)) {\n throw new Error(`invalid value for FLOAT: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_DOUBLE(value: any) {\n const v = parseFloat(value);\n if (isNaN(v)) {\n throw new Error(`invalid value for DOUBLE: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT8(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x80 || v > 0x7f || isNaN(v)) {\n throw new Error(`invalid value for INT8: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT8(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 || v > 0xff || isNaN(v)) {\n throw new Error(`invalid value for UINT8: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT16(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x8000 || v > 0x7fff || isNaN(v)) {\n throw new Error(`invalid value for INT16: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT16(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 || v > 0xffff || isNaN(v)) {\n throw new Error(`invalid value for UINT16: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT32(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n throw new Error(`invalid value for INT32: ${value}`);\n }\n\n return v;\n}\n\nfunction decimalToPrimitive_INT32(value: number, field: ParquetField) {\n const primitiveValue = value * 10 ** (field.scale || 0);\n const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n throw new Error(`invalid value for INT32: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT32(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 || v > 0xffffffffffff || isNaN(v)) {\n throw new Error(`invalid value for UINT32: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT64(value: any) {\n const v = parseInt(value, 10);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT64: ${value}`);\n }\n\n return v;\n}\n\nfunction decimalToPrimitive_INT64(value: number, field: ParquetField) {\n const primitiveValue = value * 10 ** (field.scale || 0);\n const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT64: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT64(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 || isNaN(v)) {\n throw new Error(`invalid value for UINT64: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT96(value: any) {\n const v = parseInt(value, 10);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT96: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_BYTE_ARRAY(value: any) {\n return Buffer.from(value);\n}\n\nfunction decimalToPrimitive_BYTE_ARRAY(value: any) {\n // TBD\n return Buffer.from(value);\n}\n\nfunction toPrimitive_UTF8(value: any) {\n return Buffer.from(value, 'utf8');\n}\n\nfunction fromPrimitive_UTF8(value: any) {\n return value.toString();\n}\n\nfunction toPrimitive_JSON(value: any) {\n return Buffer.from(JSON.stringify(value));\n}\n\nfunction fromPrimitive_JSON(value: any) {\n return JSON.parse(value);\n}\n\nfunction toPrimitive_BSON(value: any) {\n return Buffer.from(BSON.serialize(value));\n}\n\nfunction fromPrimitive_BSON(value: any) {\n return BSON.deserialize(value);\n}\n\nfunction toPrimitive_TIME_MILLIS(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {\n throw new Error(`invalid value for TIME_MILLIS: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_TIME_MICROS(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 || isNaN(v)) {\n throw new Error(`invalid value for TIME_MICROS: ${value}`);\n }\n\n return v;\n}\n\nconst kMillisPerDay = 86400000;\n\nfunction toPrimitive_DATE(value: any) {\n /* convert from date */\n if (value instanceof Date) {\n return value.getTime() / kMillisPerDay;\n }\n\n /* convert from integer */\n {\n const v = parseInt(value, 10);\n if (v < 0 || isNaN(v)) {\n throw new Error(`invalid value for DATE: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_DATE(value: any) {\n return new Date(value * kMillisPerDay);\n}\n\nfunction toPrimitive_TIMESTAMP_MILLIS(value: any) {\n /* convert from date */\n if (value instanceof Date) {\n return value.getTime();\n }\n\n /* convert from integer */\n {\n const v = parseInt(value, 10);\n if (v < 0 || isNaN(v)) {\n throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_TIMESTAMP_MILLIS(value: any) {\n return new Date(value);\n}\n\nfunction toPrimitive_TIMESTAMP_MICROS(value: any) {\n /* convert from date */\n if (value instanceof Date) {\n return value.getTime() * 1000;\n }\n\n /* convert from integer */\n {\n const v = parseInt(value, 10);\n if (v < 0 || isNaN(v)) {\n throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_TIMESTAMP_MICROS(value: any) {\n return new Date(value / 1000);\n}\n\nfunction toPrimitive_INTERVAL(value: any) {\n if (!value.months || !value.days || !value.milliseconds) {\n throw new Error(\n 'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }'\n );\n }\n\n const buf = Buffer.alloc(12);\n\n buf.writeUInt32LE(value.months, 0);\n buf.writeUInt32LE(value.days, 4);\n buf.writeUInt32LE(value.milliseconds, 8);\n return buf;\n}\n\nfunction fromPrimitive_INTERVAL(value: any) {\n const buf = Buffer.from(value);\n const months = buf.readUInt32LE(0);\n const days = buf.readUInt32LE(4);\n const millis = buf.readUInt32LE(8);\n\n return {months, days, milliseconds: millis};\n}\n\nfunction decimalFromPrimitive_INT(value: any, field: ParquetField) {\n const presisionInt = Math.round(((value * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n return presisionInt * 10 ** -(field.scale || 0);\n}\n\nfunction decimalFromPrimitive_BYTE_ARRAY(value: any, field: ParquetField) {\n let number = 0;\n if (value.length <= 4) {\n // Bytewise operators faster. Use them if it is possible\n for (let i = 0; i < value.length; i++) {\n // `value.length - i - 1` bytes have reverse order (big-endian)\n const component = value[i] << (8 * (value.length - i - 1));\n number += component;\n }\n } else {\n for (let i = 0; i < value.length; i++) {\n // `value.length - i - 1` bytes have reverse order (big-endian)\n const component = value[i] * 2 ** (8 * (value.length - 1 - i));\n number += component;\n }\n }\n\n const presisionInt = Math.round(\n ((number * 10 ** -field.presision!) % 1) * 10 ** field.presision!\n );\n return presisionInt * 10 ** -(field.scale || 0);\n}\n"],"file":"types.js"}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
/**
|
|
3
|
-
* Convert Buffer to ArrayBuffer
|
|
4
|
-
*/
|
|
5
|
-
export declare function toArrayBuffer(buffer: Buffer): ArrayBuffer;
|
|
6
|
-
/**
|
|
7
|
-
* Convert (copy) ArrayBuffer to Buffer
|
|
8
|
-
*/
|
|
9
|
-
export declare function toBuffer(arrayBuffer: ArrayBuffer): Buffer;
|
|
10
|
-
//# sourceMappingURL=buffer-utils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"buffer-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/buffer-utils.ts"],"names":[],"mappings":";AAAA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAOzD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAEzD"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
export function toArrayBuffer(buffer) {
|
|
2
|
-
if (Buffer.isBuffer(buffer)) {
|
|
3
|
-
const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
|
|
4
|
-
return typedArray.slice().buffer;
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
return buffer;
|
|
8
|
-
}
|
|
9
|
-
export function toBuffer(arrayBuffer) {
|
|
10
|
-
return Buffer.from(arrayBuffer);
|
|
11
|
-
}
|
|
12
|
-
//# sourceMappingURL=buffer-utils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/utils/buffer-utils.ts"],"names":["toArrayBuffer","buffer","Buffer","isBuffer","typedArray","Uint8Array","byteOffset","length","slice","toBuffer","arrayBuffer","from"],"mappings":"AAGA,OAAO,SAASA,aAAT,CAAuBC,MAAvB,EAAoD;AAEzD,MAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,UAAMG,UAAU,GAAG,IAAIC,UAAJ,CAAeJ,MAAM,CAACA,MAAtB,EAA8BA,MAAM,CAACK,UAArC,EAAiDL,MAAM,CAACM,MAAxD,CAAnB;AACA,WAAOH,UAAU,CAACI,KAAX,GAAmBP,MAA1B;AACD;;AACD,SAAOA,MAAP;AACD;AAKD,OAAO,SAASQ,QAAT,CAAkBC,WAAlB,EAAoD;AACzD,SAAOR,MAAM,CAACS,IAAP,CAAYD,WAAZ,CAAP;AACD","sourcesContent":["/**\n * Convert Buffer to ArrayBuffer\n */\nexport function toArrayBuffer(buffer: Buffer): ArrayBuffer {\n // TODO - per docs we should just be able to call buffer.buffer, but there are issues\n if (Buffer.isBuffer(buffer)) {\n const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);\n return typedArray.slice().buffer;\n }\n return buffer;\n}\n\n/**\n * Convert (copy) ArrayBuffer to Buffer\n */\nexport function toBuffer(arrayBuffer: ArrayBuffer): Buffer {\n return Buffer.from(arrayBuffer);\n}\n"],"file":"buffer-utils.js"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":["fs","load","name","module","global","require","oswrite","os","buf","Promise","resolve","reject","write","err","osclose","close","osopen","path","opts","outputStream","createWriteStream","once","fd"],"mappings":"AACA,OAAOA,EAAP,MAAe,IAAf;AAGA,OAAO,SAASC,IAAT,CAAcC,IAAd,EAAiC;AACtC,SAAO,CAACC,MAAM,IAAKC,MAAZ,EAA4BC,OAA5B,CAAoCH,IAApC,CAAP;AACD;AAUD,OAAO,SAASI,OAAT,CAAiBC,EAAjB,EAA+BC,GAA/B,EAA2D;AAChE,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCJ,IAAAA,EAAE,CAACK,KAAH,CAASJ,GAAT,EAAeK,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASI,OAAT,CAAiBP,EAAjB,EAA8C;AACnD,SAAO,IAAIE,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACrCJ,IAAAA,EAAD,CAAYQ,KAAZ,CAAmBF,GAAD,IAAc;AAC9B,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASM,MAAT,CAAgBC,IAAhB,EAA8BC,IAA9B,EAAkF;AACvF,SAAO,IAAIT,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAMQ,YAAY,GAAGnB,EAAE,CAACoB,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;AACAC,IAAAA,YAAY,CAACE,IAAb,CAAkB,MAAlB,EAA2BC,EAAD,IAAQZ,OAAO,CAACS,YAAD,CAAzC;AACAA,IAAAA,YAAY,CAACE,IAAb,CAAkB,OAAlB,EAA4BR,GAAD,IAASF,MAAM,CAACE,GAAD,CAA1C;AACD,GAJM,CAAP;AAKD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\nimport {Writable} from 'stream';\n\nexport function load(name: string): any {\n return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n flags?: string;\n encoding?: string;\n fd?: number;\n mode?: number;\n autoClose?: boolean;\n start?: number;\n}\n\nexport function oswrite(os: Writable, buf: Buffer): Promise<void> {\n return new Promise((resolve, reject) => {\n os.write(buf, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osclose(os: Writable): Promise<void> {\n return new Promise((resolve, reject) => {\n (os as any).close((err: any) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n return new Promise((resolve, reject) => {\n const outputStream = fs.createWriteStream(path, opts as any);\n outputStream.once('open', (fd) => resolve(outputStream));\n outputStream.once('error', (err) => reject(err));\n });\n}\n"],"file":"file-utils.js"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/utils/read-utils.ts"],"names":["TBufferedTransport","TCompactProtocol","TFramedTransport","FileMetaData","PageHeader","UFramedTransport","serializeThrift","obj","output","transport","undefined","buf","push","protocol","write","flush","Buffer","concat","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","length","decodePageHeader","pageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"mappings":";AAAA,SAAQA,kBAAR,EAA4BC,gBAA5B,EAA8CC,gBAA9C,QAAqE,QAArE;AACA,SAAQC,YAAR,EAAsBC,UAAtB,QAAuC,mBAAvC;;AAEA,MAAMC,gBAAN,SAA+BH,gBAA/B,CAAgD;AAAA;AAAA;;AAAA,qCACrB,CADqB;AAAA;;AAAA;;AAOhD,OAAO,SAASI,eAAT,CAAyBC,GAAzB,EAA2C;AAChD,QAAMC,MAAgB,GAAG,EAAzB;AAEA,QAAMC,SAAS,GAAG,IAAIT,kBAAJ,CAAuBU,SAAvB,EAAmCC,GAAD,IAAS;AAC3DH,IAAAA,MAAM,CAACI,IAAP,CAAYD,GAAZ;AACD,GAFiB,CAAlB;AAIA,QAAME,QAAQ,GAAG,IAAIZ,gBAAJ,CAAqBQ,SAArB,CAAjB;AACAF,EAAAA,GAAG,CAACO,KAAJ,CAAUD,QAAV;AACAJ,EAAAA,SAAS,CAACM,KAAV;AAEA,SAAOC,MAAM,CAACC,MAAP,CAAcT,MAAd,CAAP;AACD;AAED,OAAO,SAASU,YAAT,CAAsBX,GAAtB,EAAgCI,GAAhC,EAA6CQ,MAA7C,EAA8D;AACnE,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,QAAMV,SAAS,GAAG,IAAIJ,gBAAJ,CAAqBM,GAArB,CAAlB;AACAF,EAAAA,SAAS,CAACW,OAAV,GAAoBD,MAApB;AACA,QAAMN,QAAQ,GAAG,IAAIZ,gBAAJ,CAAqBQ,SAArB,CAAjB;AACAF,EAAAA,GAAG,CAACc,IAAJ,CAASR,QAAT;AACA,SAAOJ,SAAS,CAACW,OAAV,GAAoBD,MAA3B;AACD;AAKD,OAAO,SAASG,aAAT,CAAuBC,KAAvB,EAAmCC,KAAnC,EAAmE;AACxE,OAAK,MAAMC,CAAX,IAAgBF,KAAhB,EAAuB;AACrB,QAAIA,KAAK,CAACE,CAAD,CAAL,KAAaD,KAAjB,EAAwB;AACtB,aAAOC,CAAP;AACD;AACF;;AACD,QAAM,IAAIC,KAAJ,CAAU,oBAAV,CAAN;AACD;AAED,OAAO,SAASC,kBAAT,CAA4BhB,GAA5B,EAAyCQ,MAAzC,EAA0D;AAC/D,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,QAAMV,SAAS,GAAG,IAAIJ,gBAAJ,CAAqBM,GAArB,CAAlB;AACAF,EAAAA,SAAS,CAACW,OAAV,GAAoBD,MAApB;AACA,QAAMN,QAAQ,GAAG,IAAIZ,gBAAJ,CAAqBQ,SAArB,CAAjB;AACA,QAAMmB,QAAQ,GAAGzB,YAAY,CAACkB,IAAb,CAAkBR,QAAlB,CAAjB;AACA,SAAO;AAACgB,IAAAA,MAAM,EAAEpB,SAAS,CAACW,OAAV,GAAoBD,MAA7B;AAAqCS,IAAAA;AAArC,GAAP;AACD;AAED,OAAO,SAASE,gBAAT,CAA0BnB,GAA1B,EAAuCQ,MAAvC,EAAwD;AAC7D,MAAI,CAACA,MAAL,EAAa;AAEXA,IAAAA,MAAM,GAAG,CAAT;AACD;;AAED,QAAMV,SAAS,GAAG,IAAIJ,gBAAJ,CAAqBM,GAArB,CAAlB;AACAF,EAAAA,SAAS,CAACW,OAAV,GAAoBD,MAApB;AACA,QAAMN,QAAQ,GAAG,IAAIZ,gBAAJ,CAAqBQ,SAArB,CAAjB;AACA,QAAMsB,UAAU,GAAG3B,UAAU,CAACiB,IAAX,CAAgBR,QAAhB,CAAnB;AACA,SAAO;AAACgB,IAAAA,MAAM,EAAEpB,SAAS,CAACW,OAAV,GAAoBD,MAA7B;AAAqCY,IAAAA;AAArC,GAAP;AACD;AAKD,OAAO,SAASC,WAAT,CAAqBC,GAArB,EAA0C;AAC/C,MAAIA,GAAG,KAAK,CAAZ,EAAe;AACb,WAAO,CAAP;AAED;;AACD,SAAOC,IAAI,CAACC,IAAL,CAAUD,IAAI,CAACE,IAAL,CAAUH,GAAG,GAAG,CAAhB,CAAV,CAAP;AACD;AAKD,OAAO,SAASI,YAAT,CAAsBC,GAAtB,EAAuCC,IAAvC,EAA+D;AACpE,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGF,GAAG,CAACT,MAAxB,EAAgCW,CAAC,EAAjC,EAAqC;AACnC,QAAIF,GAAG,CAACE,CAAD,CAAH,CAAOX,MAAP,GAAgBU,IAAI,CAACV,MAAzB,EAAiC;AAC/B;AACD;;AACD,QAAIY,CAAC,GAAG,IAAR;;AACA,SAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,IAAI,CAACV,MAAzB,EAAiCa,CAAC,EAAlC,EAAsC;AACpC,UAAIJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAcH,IAAI,CAACG,CAAD,CAAlB,IAAyBJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAc,GAAvC,IAA8CJ,GAAG,CAACE,CAAD,CAAH,CAAOE,CAAP,MAAc,GAAhE,EAAqE;AACnE;AACD;;AACD,UAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAD,CAAH,CAAOX,MAAZ,IAAsBS,GAAG,CAACE,CAAD,CAAH,CAAOF,GAAG,CAACE,CAAD,CAAH,CAAOX,MAAP,GAAgB,CAAvB,MAA8B,GAAxD,EAA6D;AAC3D;AACD;;AACDY,MAAAA,CAAC,GAAG,KAAJ;AACA;AACD;;AACD,QAAIA,CAAJ,EAAO,OAAOD,CAAP;AACR;;AACD,SAAO,CAAC,CAAR;AACD","sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n const output: Buffer[] = [];\n\n const transport = new TBufferedTransport(undefined, (buf) => {\n output.push(buf as Buffer);\n });\n\n const protocol = new TCompactProtocol(transport);\n obj.write(protocol);\n transport.flush();\n\n return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n obj.read(protocol);\n return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n for (const k in klass) {\n if (klass[k] === value) {\n return k;\n }\n }\n throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n const metadata = FileMetaData.read(protocol);\n return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n if (!offset) {\n // tslint:disable-next-line:no-parameter-reassignment\n offset = 0;\n }\n\n const transport = new UFramedTransport(buf);\n transport.readPos = offset;\n const protocol = new TCompactProtocol(transport);\n const pageHeader = PageHeader.read(protocol);\n return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n if (val === 0) {\n return 0;\n // tslint:disable-next-line:no-else-after-return\n }\n return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n for (let j = 0; j < arr.length; j++) {\n if (arr[j].length > elem.length) {\n continue; // eslint-disable-line no-continue\n }\n let m = true;\n for (let i = 0; i < elem.length; i++) {\n if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n continue; // eslint-disable-line no-continue\n }\n if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n continue; // eslint-disable-line no-continue\n }\n m = false;\n break;\n }\n if (m) return j;\n }\n return -1;\n}\n"],"file":"read-utils.js"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/workers/parquet-worker.ts"],"names":["createLoaderWorker","ParquetLoader"],"mappings":"AAAA,SAAQA,kBAAR,QAAiC,0BAAjC;AACA,SAAQC,aAAR,QAA4B,UAA5B;AAEAD,kBAAkB,CAACC,aAAD,CAAlB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ParquetLoader} from '../index';\n\ncreateLoaderWorker(ParquetLoader);\n"],"file":"parquet-worker.js"}
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import type {ParquetSchema} from '../parquetjs/schema/schema';
|
|
2
|
-
import type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
Schema,
|
|
6
|
-
Struct,
|
|
7
|
-
Field,
|
|
8
|
-
DataType,
|
|
9
|
-
Bool,
|
|
10
|
-
Float64,
|
|
11
|
-
Int32,
|
|
12
|
-
Float32,
|
|
13
|
-
Binary,
|
|
14
|
-
Utf8,
|
|
15
|
-
Int64,
|
|
16
|
-
Uint16,
|
|
17
|
-
Uint32,
|
|
18
|
-
Uint64,
|
|
19
|
-
Int8,
|
|
20
|
-
Int16
|
|
21
|
-
} from '@loaders.gl/schema';
|
|
22
|
-
|
|
23
|
-
export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
|
|
24
|
-
BOOLEAN: Bool,
|
|
25
|
-
INT32: Int32,
|
|
26
|
-
INT64: Float64,
|
|
27
|
-
INT96: Float64,
|
|
28
|
-
FLOAT: Float32,
|
|
29
|
-
DOUBLE: Float64,
|
|
30
|
-
BYTE_ARRAY: Binary,
|
|
31
|
-
FIXED_LEN_BYTE_ARRAY: Binary,
|
|
32
|
-
UTF8: Utf8,
|
|
33
|
-
DATE: Int32,
|
|
34
|
-
TIME_MILLIS: Int64,
|
|
35
|
-
TIME_MICROS: Int64,
|
|
36
|
-
TIMESTAMP_MILLIS: Int64,
|
|
37
|
-
TIMESTAMP_MICROS: Int64,
|
|
38
|
-
UINT_8: Int32,
|
|
39
|
-
UINT_16: Uint16,
|
|
40
|
-
UINT_32: Uint32,
|
|
41
|
-
UINT_64: Uint64,
|
|
42
|
-
INT_8: Int8,
|
|
43
|
-
INT_16: Int16,
|
|
44
|
-
INT_32: Int32,
|
|
45
|
-
INT_64: Int64,
|
|
46
|
-
JSON: Binary,
|
|
47
|
-
BSON: Binary,
|
|
48
|
-
// TODO check interal type
|
|
49
|
-
INTERVAL: Binary,
|
|
50
|
-
DECIMAL_INT32: Float32,
|
|
51
|
-
DECIMAL_INT64: Float64,
|
|
52
|
-
DECIMAL_BYTE_ARRAY: Float64,
|
|
53
|
-
DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
export function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {
|
|
57
|
-
const fields = getFields(parquetSchema.schema);
|
|
58
|
-
|
|
59
|
-
// TODO add metadata if needed.
|
|
60
|
-
return new Schema(fields);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function getFieldMetadata(field: ParquetField): Map<string, string> {
|
|
64
|
-
const metadata = new Map();
|
|
65
|
-
|
|
66
|
-
for (const key in field) {
|
|
67
|
-
if (key !== 'name') {
|
|
68
|
-
const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
69
|
-
metadata.set(key, value);
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
return metadata;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function getFields(schema: FieldDefinition): Field[] {
|
|
77
|
-
const fields: Field[] = [];
|
|
78
|
-
|
|
79
|
-
for (const name in schema) {
|
|
80
|
-
const field = schema[name];
|
|
81
|
-
|
|
82
|
-
if (field.fields) {
|
|
83
|
-
const childField = getFields(field.fields);
|
|
84
|
-
const nestedField = new Field(name, new Struct(childField), field.optional);
|
|
85
|
-
fields.push(nestedField);
|
|
86
|
-
} else {
|
|
87
|
-
const FieldType = PARQUET_TYPE_MAPPING[field.type];
|
|
88
|
-
const metadata = getFieldMetadata(field);
|
|
89
|
-
const arrowField = new Field(name, new FieldType(), field.optional, metadata);
|
|
90
|
-
fields.push(arrowField);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
return fields;
|
|
95
|
-
}
|
package/src/lib/parse-parquet.ts
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
2
|
-
import type {ParquetLoaderOptions} from '../parquet-loader';
|
|
3
|
-
|
|
4
|
-
import {ParquetReader} from '../parquetjs/parser/parquet-reader';
|
|
5
|
-
|
|
6
|
-
export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
|
|
7
|
-
const blob = new Blob([arrayBuffer]);
|
|
8
|
-
for await (const batch of parseParquetFileInBatches(blob, options)) {
|
|
9
|
-
return batch;
|
|
10
|
-
}
|
|
11
|
-
return null;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {
|
|
15
|
-
const reader = await ParquetReader.openBlob(blob);
|
|
16
|
-
const rows: any[][] = [];
|
|
17
|
-
try {
|
|
18
|
-
const cursor = reader.getCursor();
|
|
19
|
-
let record: any[] | null;
|
|
20
|
-
while ((record = await cursor.next())) {
|
|
21
|
-
rows.push(record);
|
|
22
|
-
}
|
|
23
|
-
} finally {
|
|
24
|
-
await reader.close();
|
|
25
|
-
}
|
|
26
|
-
yield rows;
|
|
27
|
-
}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
// Random-Access read
|
|
2
|
-
|
|
3
|
-
export async function readArrayBuffer(
|
|
4
|
-
file: Blob | ArrayBuffer | any,
|
|
5
|
-
start: number,
|
|
6
|
-
length: number
|
|
7
|
-
): Promise<ArrayBuffer> {
|
|
8
|
-
if (file instanceof Blob) {
|
|
9
|
-
const slice = file.slice(start, start + length);
|
|
10
|
-
return await slice.arrayBuffer();
|
|
11
|
-
}
|
|
12
|
-
return await file.read(start, start + length);
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
* Read a slice of a Blob or File, without loading the entire file into memory
|
|
17
|
-
* The trick when reading File objects is to read successive "slices" of the File
|
|
18
|
-
* Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
|
|
19
|
-
* Actually reading from file happens in `readAsArrayBuffer`
|
|
20
|
-
* @param blob to read
|
|
21
|
-
export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
|
|
22
|
-
return await new Promise((resolve, reject) => {
|
|
23
|
-
const fileReader = new FileReader();
|
|
24
|
-
fileReader.onload = (event: ProgressEvent<FileReader>) =>
|
|
25
|
-
resolve(event?.target?.result as ArrayBuffer);
|
|
26
|
-
// TODO - reject with a proper Error
|
|
27
|
-
fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
|
|
28
|
-
fileReader.readAsArrayBuffer(blob);
|
|
29
|
-
});
|
|
30
|
-
}
|
|
31
|
-
*/
|
package/src/parquetjs/file.ts
DELETED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
|
|
2
|
-
import fs from 'fs';
|
|
3
|
-
|
|
4
|
-
export function fopen(filePath) {
|
|
5
|
-
return new Promise((resolve, reject) => {
|
|
6
|
-
fs.open(filePath, 'r', (err, fd) => {
|
|
7
|
-
if (err) {
|
|
8
|
-
reject(err);
|
|
9
|
-
} else {
|
|
10
|
-
resolve(fd);
|
|
11
|
-
}
|
|
12
|
-
});
|
|
13
|
-
});
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export function fstat(filePath) {
|
|
17
|
-
return new Promise<fs.Stats>((resolve, reject) => {
|
|
18
|
-
fs.stat(filePath, (err, stat) => {
|
|
19
|
-
if (err) {
|
|
20
|
-
reject(err);
|
|
21
|
-
} else {
|
|
22
|
-
resolve(stat);
|
|
23
|
-
}
|
|
24
|
-
});
|
|
25
|
-
});
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export function fread(fd, position, length) {
|
|
29
|
-
const buffer = Buffer.alloc(length);
|
|
30
|
-
|
|
31
|
-
return new Promise((resolve, reject) => {
|
|
32
|
-
fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
|
|
33
|
-
if (err || bytesRead !== length) {
|
|
34
|
-
reject(err || Error('read failed'));
|
|
35
|
-
} else {
|
|
36
|
-
resolve(buf);
|
|
37
|
-
}
|
|
38
|
-
});
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export function fclose(fd) {
|
|
43
|
-
return new Promise((resolve, reject) => {
|
|
44
|
-
fs.close(fd, (err) => {
|
|
45
|
-
if (err) {
|
|
46
|
-
reject(err);
|
|
47
|
-
} else {
|
|
48
|
-
resolve(err);
|
|
49
|
-
}
|
|
50
|
-
});
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export function oswrite(os, buf): Promise<void> {
|
|
55
|
-
return new Promise((resolve, reject) => {
|
|
56
|
-
os.write(buf, (err) => {
|
|
57
|
-
if (err) {
|
|
58
|
-
reject(err);
|
|
59
|
-
} else {
|
|
60
|
-
resolve();
|
|
61
|
-
}
|
|
62
|
-
});
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export function osclose(os): Promise<void> {
|
|
67
|
-
return new Promise((resolve, reject) => {
|
|
68
|
-
os.close((err) => {
|
|
69
|
-
if (err) {
|
|
70
|
-
reject(err);
|
|
71
|
-
} else {
|
|
72
|
-
resolve();
|
|
73
|
-
}
|
|
74
|
-
});
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
export function osopen(path, opts) {
|
|
79
|
-
return new Promise((resolve, reject) => {
|
|
80
|
-
const outputStream = fs.createWriteStream(path, opts);
|
|
81
|
-
|
|
82
|
-
outputStream.on('open', function (fd) {
|
|
83
|
-
resolve(outputStream);
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
outputStream.on('error', function (err) {
|
|
87
|
-
reject(err);
|
|
88
|
-
});
|
|
89
|
-
});
|
|
90
|
-
}
|
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
|
|
2
|
-
import {FileMetaData} from '../parquet-thrift';
|
|
3
|
-
import {ParquetEnvelopeReader} from './parquet-envelope-reader';
|
|
4
|
-
import {ParquetSchema} from '../schema/schema';
|
|
5
|
-
import {ParquetRecord} from '../schema/declare';
|
|
6
|
-
import {materializeRecords} from '../schema/shred';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* A parquet cursor is used to retrieve rows from a parquet file in order
|
|
10
|
-
*/
|
|
11
|
-
export class ParquetCursor<T> implements AsyncIterable<T> {
|
|
12
|
-
public metadata: FileMetaData;
|
|
13
|
-
public envelopeReader: ParquetEnvelopeReader;
|
|
14
|
-
public schema: ParquetSchema;
|
|
15
|
-
public columnList: string[][];
|
|
16
|
-
public rowGroup: ParquetRecord[] = [];
|
|
17
|
-
public rowGroupIndex: number;
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Create a new parquet reader from the file metadata and an envelope reader.
|
|
21
|
-
* It is usually not recommended to call this constructor directly except for
|
|
22
|
-
* advanced and internal use cases. Consider using getCursor() on the
|
|
23
|
-
* ParquetReader instead
|
|
24
|
-
*/
|
|
25
|
-
constructor(
|
|
26
|
-
metadata: FileMetaData,
|
|
27
|
-
envelopeReader: ParquetEnvelopeReader,
|
|
28
|
-
schema: ParquetSchema,
|
|
29
|
-
columnList: string[][]
|
|
30
|
-
) {
|
|
31
|
-
this.metadata = metadata;
|
|
32
|
-
this.envelopeReader = envelopeReader;
|
|
33
|
-
this.schema = schema;
|
|
34
|
-
this.columnList = columnList;
|
|
35
|
-
this.rowGroupIndex = 0;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Retrieve the next row from the cursor. Returns a row or NULL if the end
|
|
40
|
-
* of the file was reached
|
|
41
|
-
*/
|
|
42
|
-
async next<T = any>(): Promise<T> {
|
|
43
|
-
if (this.rowGroup.length === 0) {
|
|
44
|
-
if (this.rowGroupIndex >= this.metadata.row_groups.length) {
|
|
45
|
-
// @ts-ignore
|
|
46
|
-
return null;
|
|
47
|
-
}
|
|
48
|
-
const rowBuffer = await this.envelopeReader.readRowGroup(
|
|
49
|
-
this.schema,
|
|
50
|
-
this.metadata.row_groups[this.rowGroupIndex],
|
|
51
|
-
this.columnList
|
|
52
|
-
);
|
|
53
|
-
this.rowGroup = materializeRecords(this.schema, rowBuffer);
|
|
54
|
-
this.rowGroupIndex++;
|
|
55
|
-
}
|
|
56
|
-
return this.rowGroup.shift() as any;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Rewind the cursor the the beginning of the file
|
|
61
|
-
*/
|
|
62
|
-
rewind(): void {
|
|
63
|
-
this.rowGroup = [];
|
|
64
|
-
this.rowGroupIndex = 0;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Implement AsyncIterable
|
|
69
|
-
*/
|
|
70
|
-
// tslint:disable-next-line:function-name
|
|
71
|
-
[Symbol.asyncIterator](): AsyncIterator<T> {
|
|
72
|
-
let done = false;
|
|
73
|
-
return {
|
|
74
|
-
next: async () => {
|
|
75
|
-
if (done) {
|
|
76
|
-
return {done, value: null};
|
|
77
|
-
}
|
|
78
|
-
const value = await this.next();
|
|
79
|
-
if (value === null) {
|
|
80
|
-
return {done: true, value};
|
|
81
|
-
}
|
|
82
|
-
return {done: false, value};
|
|
83
|
-
},
|
|
84
|
-
return: async () => {
|
|
85
|
-
done = true;
|
|
86
|
-
return {done, value: null};
|
|
87
|
-
},
|
|
88
|
-
throw: async () => {
|
|
89
|
-
done = true;
|
|
90
|
-
return {done: true, value: null};
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
}
|