@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/bundled.d.ts +9 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +10 -0
- package/dist/bundled.js.map +1 -0
- package/dist/geoparquet-loader-with-parser.d.ts +33 -0
- package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
- package/dist/geoparquet-loader-with-parser.js +52 -0
- package/dist/geoparquet-loader-with-parser.js.map +1 -0
- package/dist/geoparquet-loader.d.ts +59 -0
- package/dist/geoparquet-loader.d.ts.map +1 -0
- package/dist/geoparquet-loader.js +30 -0
- package/dist/geoparquet-loader.js.map +1 -0
- package/dist/index.cjs +25873 -4776
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -11
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-columns-to-row-group.js +3 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +86 -26
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.js +16 -7
- package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts +8 -0
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts.map +1 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js +396 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -0
- package/dist/lib/constants.d.ts +0 -1
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +1 -2
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js +199 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
- package/dist/lib/geo/geospatial-metadata.js +338 -0
- package/dist/lib/geo/geospatial-metadata.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +62 -0
- package/dist/lib/parquet-page-index.d.ts.map +1 -0
- package/dist/lib/parquet-page-index.js +377 -0
- package/dist/lib/parquet-page-index.js.map +1 -0
- package/dist/lib/parquet-predicate.d.ts +17 -0
- package/dist/lib/parquet-predicate.d.ts.map +1 -0
- package/dist/lib/parquet-predicate.js +300 -0
- package/dist/lib/parquet-predicate.js.map +1 -0
- package/dist/lib/parquet-source-worker-client.d.ts +6 -0
- package/dist/lib/parquet-source-worker-client.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-client.js +22 -0
- package/dist/lib/parquet-source-worker-client.js.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.js +115 -0
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -0
- package/dist/lib/parquet-source-worker-file.d.ts +24 -0
- package/dist/lib/parquet-source-worker-file.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-file.js +40 -0
- package/dist/lib/parquet-source-worker-file.js.map +1 -0
- package/dist/lib/parquet-source-worker-types.d.ts +97 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-types.js +5 -0
- package/dist/lib/parquet-source-worker-types.js.map +1 -0
- package/dist/lib/parquet-worker-transport.d.ts +8 -0
- package/dist/lib/parquet-worker-transport.d.ts.map +1 -0
- package/dist/lib/parquet-worker-transport.js +40 -0
- package/dist/lib/parquet-worker-transport.js.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts +16 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.js +30 -0
- package/dist/lib/parsers/convert-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-tables.d.ts +37 -0
- package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-tables.js +50 -0
- package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +19 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +608 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -4
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +77 -26
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts +80 -0
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -0
- package/dist/lib/sources/parquet-range-file.js +261 -0
- package/dist/lib/sources/parquet-range-file.js.map +1 -0
- package/dist/lib/sources/single-batch-queue.d.ts +24 -0
- package/dist/lib/sources/single-batch-queue.d.ts.map +1 -0
- package/dist/lib/sources/single-batch-queue.js +91 -0
- package/dist/lib/sources/single-batch-queue.js.map +1 -0
- package/dist/lib/utils/load-wasm-node.d.ts +3 -0
- package/dist/lib/utils/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/utils/load-wasm-node.js +23 -0
- package/dist/lib/utils/load-wasm-node.js.map +1 -0
- package/dist/lib/utils/load-wasm.d.ts +2 -2
- package/dist/lib/utils/load-wasm.d.ts.map +1 -1
- package/dist/lib/utils/load-wasm.js +28 -12
- package/dist/lib/utils/load-wasm.js.map +1 -1
- package/dist/lib/utils/make-stream-iterator.d.ts +2 -0
- package/dist/lib/utils/make-stream-iterator.d.ts.map +1 -1
- package/dist/lib/utils/make-stream-iterator.js +123 -14
- package/dist/lib/utils/make-stream-iterator.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
- package/dist/lib/utils/normalize-parquet-options.js +11 -0
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
- package/dist/parquet-dataset-source.cjs +26413 -0
- package/dist/parquet-dataset-source.cjs.map +6 -0
- package/dist/parquet-dataset-source.d.ts +54 -0
- package/dist/parquet-dataset-source.d.ts.map +1 -0
- package/dist/parquet-dataset-source.js +380 -0
- package/dist/parquet-dataset-source.js.map +1 -0
- package/dist/parquet-format.d.ts +2 -0
- package/dist/parquet-format.d.ts.map +1 -1
- package/dist/parquet-format.js +2 -0
- package/dist/parquet-format.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +57 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -0
- package/dist/parquet-js-loader-types.js +33 -0
- package/dist/parquet-js-loader-types.js.map +1 -0
- package/dist/parquet-js-loader.d.ts +32 -0
- package/dist/parquet-js-loader.d.ts.map +1 -0
- package/dist/parquet-js-loader.js +58 -0
- package/dist/parquet-js-loader.js.map +1 -0
- package/dist/parquet-js-writer.d.ts +44 -0
- package/dist/parquet-js-writer.d.ts.map +1 -0
- package/dist/parquet-js-writer.js +28 -0
- package/dist/parquet-js-writer.js.map +1 -0
- package/dist/parquet-loader-base.d.ts +25 -0
- package/dist/parquet-loader-base.d.ts.map +1 -0
- package/dist/parquet-loader-base.js +23 -0
- package/dist/parquet-loader-base.js.map +1 -0
- package/dist/parquet-loader-options.d.ts +44 -0
- package/dist/parquet-loader-options.d.ts.map +1 -0
- package/dist/parquet-loader-options.js +10 -0
- package/dist/parquet-loader-options.js.map +1 -0
- package/dist/parquet-loader-types.d.ts +61 -0
- package/dist/parquet-loader-types.d.ts.map +1 -0
- package/dist/parquet-loader-types.js +17 -0
- package/dist/parquet-loader-types.js.map +1 -0
- package/dist/parquet-loader.d.ts +42 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +68 -0
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-source-capabilities.d.ts +36 -0
- package/dist/parquet-source-capabilities.d.ts.map +1 -0
- package/dist/parquet-source-capabilities.js +22 -0
- package/dist/parquet-source-capabilities.js.map +1 -0
- package/dist/parquet-source-loader-types.d.ts +72 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -0
- package/dist/parquet-source-loader-types.js +74 -0
- package/dist/parquet-source-loader-types.js.map +1 -0
- package/dist/parquet-source-loader.cjs +25975 -0
- package/dist/parquet-source-loader.cjs.map +6 -0
- package/dist/parquet-source-loader.d.ts +120 -0
- package/dist/parquet-source-loader.d.ts.map +1 -0
- package/dist/parquet-source-loader.js +812 -0
- package/dist/parquet-source-loader.js.map +1 -0
- package/dist/parquet-source-types.d.ts +384 -0
- package/dist/parquet-source-types.d.ts.map +1 -0
- package/dist/parquet-source-types.js +5 -0
- package/dist/parquet-source-types.js.map +1 -0
- package/dist/parquet-source-worker-url.d.ts +3 -0
- package/dist/parquet-source-worker-url.d.ts.map +1 -0
- package/dist/parquet-source-worker-url.js +22 -0
- package/dist/parquet-source-worker-url.js.map +1 -0
- package/dist/parquet-source-worker.js +38491 -0
- package/dist/parquet-source-worker.js.map +7 -0
- package/dist/parquet-worker-node.cjs +54779 -0
- package/dist/parquet-worker-node.cjs.map +7 -0
- package/dist/parquet-worker-url.d.ts +3 -0
- package/dist/parquet-worker-url.d.ts.map +1 -0
- package/dist/parquet-worker-url.js +15 -0
- package/dist/parquet-worker-url.js.map +1 -0
- package/dist/parquet-worker.js +19920 -0
- package/dist/parquet-worker.js.map +7 -0
- package/dist/parquet-writer.d.ts +10 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +27 -0
- package/dist/parquet-writer.js.map +1 -0
- package/dist/parquet_wasm_bg.wasm +0 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts +9 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts.map +1 -0
- package/dist/parquetjs/codecs/byte-stream-split.js +159 -0
- package/dist/parquetjs/codecs/byte-stream-split.js.map +1 -0
- package/dist/parquetjs/codecs/declare.d.ts +20 -3
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +7 -1
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts +17 -0
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -0
- package/dist/parquetjs/codecs/delta.js +439 -0
- package/dist/parquetjs/codecs/delta.js.map +1 -0
- package/dist/parquetjs/codecs/dictionary.d.ts +5 -2
- package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +16 -4
- package/dist/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +18 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +3 -3
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +88 -76
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +3 -3
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +196 -68
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +7 -8
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +95 -141
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts +17 -0
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/dictionary-planner.js +64 -0
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -0
- package/dist/parquetjs/encoder/page-planner.d.ts +11 -0
- package/dist/parquetjs/encoder/page-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/page-planner.js +60 -0
- package/dist/parquetjs/encoder/page-planner.js.map +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +11 -4
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +259 -52
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +6 -6
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +3 -3
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts +8 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js +14 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts +2 -1
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +2 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js +27 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts +14 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js +51 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js +41 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +17 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +97 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js +27 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts +14 -10
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +38 -2
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +5 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +25 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js +41 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -2
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -2
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +14 -3
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +381 -117
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +35 -10
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +179 -59
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +62 -8
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts +3 -0
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +21 -3
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts +2 -0
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +71 -23
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +272 -51
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/utils/binary-utils.d.ts +41 -0
- package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/binary-utils.js +106 -0
- package/dist/parquetjs/utils/binary-utils.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.d.ts +1 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +3 -3
- package/dist/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts +14 -6
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +46 -39
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/parquetjs/utils/thrift-runtime.d.ts +106 -0
- package/dist/parquetjs/utils/thrift-runtime.d.ts.map +1 -0
- package/dist/parquetjs/utils/thrift-runtime.js +50 -0
- package/dist/parquetjs/utils/thrift-runtime.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +80 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
- package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
- package/dist/polyfills/process.d.ts +12 -0
- package/dist/polyfills/process.d.ts.map +1 -0
- package/dist/polyfills/process.js +17 -0
- package/dist/polyfills/process.js.map +1 -0
- package/dist/unbundled.d.ts +9 -0
- package/dist/unbundled.d.ts.map +1 -0
- package/dist/unbundled.js +10 -0
- package/dist/unbundled.js.map +1 -0
- package/dist/wasm.d.ts +3 -0
- package/dist/wasm.d.ts.map +1 -0
- package/dist/wasm.js +6 -0
- package/dist/wasm.js.map +1 -0
- package/dist/workers/parquet-source-worker.d.ts +2 -0
- package/dist/workers/parquet-source-worker.d.ts.map +1 -0
- package/dist/workers/parquet-source-worker.js +17 -0
- package/dist/workers/parquet-source-worker.js.map +1 -0
- package/dist/workers/parquet-worker-node.d.ts +2 -0
- package/dist/workers/parquet-worker-node.d.ts.map +1 -0
- package/dist/workers/parquet-worker-node.js +5 -0
- package/dist/workers/parquet-worker-node.js.map +1 -0
- package/dist/workers/parquet-worker.d.ts +2 -0
- package/dist/workers/parquet-worker.d.ts.map +1 -0
- package/dist/workers/parquet-worker.js +27 -0
- package/dist/workers/parquet-worker.js.map +1 -0
- package/package.json +63 -33
- package/src/bundled.ts +33 -0
- package/src/geoparquet-loader-with-parser.ts +95 -0
- package/src/geoparquet-loader.ts +54 -0
- package/src/index.ts +49 -23
- package/src/lib/arrow/convert-columns-to-row-group.ts +3 -0
- package/src/lib/arrow/convert-schema-from-parquet.ts +97 -30
- package/src/lib/arrow/convert-schema-to-parquet.ts +16 -7
- package/src/lib/arrow/create-nested-arrow-vector.ts +589 -0
- package/src/lib/constants.ts +0 -2
- package/src/lib/encoders/encode-arrow-to-parquet.ts +10 -16
- package/src/lib/encoders/encode-table-to-parquet-js.ts +261 -0
- package/src/lib/geo/geospatial-metadata.ts +457 -0
- package/src/lib/parquet-page-index.ts +581 -0
- package/src/lib/parquet-predicate.ts +374 -0
- package/src/lib/parquet-source-worker-client.ts +39 -0
- package/src/lib/parquet-source-worker-decoder.ts +156 -0
- package/src/lib/parquet-source-worker-file.ts +56 -0
- package/src/lib/parquet-source-worker-types.ts +107 -0
- package/src/lib/parquet-worker-transport.ts +57 -0
- package/src/lib/parsers/convert-parquet-tables.ts +39 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
- package/src/lib/parsers/parse-parquet-tables.ts +82 -0
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +829 -0
- package/src/lib/parsers/parse-parquet-to-arrow.ts +63 -36
- package/src/lib/parsers/parse-parquet-to-columns.ts +6 -7
- package/src/lib/parsers/parse-parquet-to-json.ts +101 -33
- package/src/lib/sources/parquet-range-file.ts +359 -0
- package/src/lib/sources/single-batch-queue.ts +95 -0
- package/src/lib/utils/load-wasm-node.ts +25 -0
- package/src/lib/utils/load-wasm.ts +36 -13
- package/src/lib/utils/make-stream-iterator.ts +126 -14
- package/src/lib/utils/normalize-parquet-options.ts +14 -0
- package/src/parquet-dataset-source.ts +488 -0
- package/src/parquet-format.ts +2 -0
- package/src/parquet-js-loader-types.ts +52 -0
- package/src/parquet-js-loader.ts +83 -0
- package/src/parquet-js-writer.ts +68 -0
- package/src/parquet-loader-base.ts +34 -0
- package/src/parquet-loader-options.ts +51 -0
- package/src/parquet-loader-types.ts +35 -0
- package/src/parquet-loader.ts +115 -0
- package/src/parquet-source-capabilities.ts +56 -0
- package/src/parquet-source-loader-types.ts +87 -0
- package/src/parquet-source-loader.ts +1265 -0
- package/src/parquet-source-types.ts +458 -0
- package/src/parquet-source-worker-url.ts +23 -0
- package/src/parquet-worker-url.ts +18 -0
- package/src/parquet-writer.ts +42 -0
- package/src/parquetjs/LICENSE +23 -1
- package/src/parquetjs/codecs/byte-stream-split.ts +221 -0
- package/src/parquetjs/codecs/declare.ts +35 -3
- package/src/parquetjs/codecs/delta.ts +587 -0
- package/src/parquetjs/codecs/dictionary.ts +22 -4
- package/src/parquetjs/codecs/index.ts +18 -0
- package/src/parquetjs/codecs/plain.ts +149 -87
- package/src/parquetjs/codecs/rle.ts +268 -82
- package/src/parquetjs/compression.ts +110 -155
- package/src/parquetjs/encoder/dictionary-planner.ts +99 -0
- package/src/parquetjs/encoder/page-planner.ts +83 -0
- package/src/parquetjs/encoder/parquet-encoder.ts +378 -77
- package/src/parquetjs/parquet-thrift/BsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +2 -2
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +13 -13
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +2 -2
- package/src/parquetjs/parquet-thrift/ColumnOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeaderV2.ts +1 -1
- package/src/parquetjs/parquet-thrift/DateType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DecimalType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DictionaryPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.ts +13 -0
- package/src/parquetjs/parquet-thrift/Encoding.ts +2 -1
- package/src/parquetjs/parquet-thrift/EnumType.ts +1 -1
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +2 -2
- package/src/parquetjs/parquet-thrift/Float16Type.ts +28 -0
- package/src/parquetjs/parquet-thrift/GeographyType.ts +54 -0
- package/src/parquetjs/parquet-thrift/GeometryType.ts +43 -0
- package/src/parquetjs/parquet-thrift/IndexPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/IntType.ts +1 -1
- package/src/parquetjs/parquet-thrift/JsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/KeyValue.ts +1 -1
- package/src/parquetjs/parquet-thrift/ListType.ts +1 -1
- package/src/parquetjs/parquet-thrift/LogicalType.ts +97 -1
- package/src/parquetjs/parquet-thrift/MapType.ts +1 -1
- package/src/parquetjs/parquet-thrift/MicroSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/MilliSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/NanoSeconds.ts +28 -0
- package/src/parquetjs/parquet-thrift/NullType.ts +1 -1
- package/src/parquetjs/parquet-thrift/OffsetIndex.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageEncodingStats.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageLocation.ts +2 -2
- package/src/parquetjs/parquet-thrift/RowGroup.ts +2 -2
- package/src/parquetjs/parquet-thrift/SchemaElement.ts +1 -1
- package/src/parquetjs/parquet-thrift/SortingColumn.ts +1 -1
- package/src/parquetjs/parquet-thrift/Statistics.ts +50 -14
- package/src/parquetjs/parquet-thrift/StringType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeUnit.ts +25 -1
- package/src/parquetjs/parquet-thrift/TimestampType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TypeDefinedOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/UUIDType.ts +1 -1
- package/src/parquetjs/parquet-thrift/VariantType.ts +43 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -4
- package/src/parquetjs/parser/decoders.ts +487 -133
- package/src/parquetjs/parser/parquet-reader.ts +297 -71
- package/src/parquetjs/schema/declare.ts +99 -7
- package/src/parquetjs/schema/schema.ts +23 -3
- package/src/parquetjs/schema/shred.ts +112 -35
- package/src/parquetjs/schema/types.ts +294 -61
- package/src/parquetjs/utils/binary-utils.ts +132 -0
- package/src/parquetjs/utils/file-utils.ts +4 -4
- package/src/parquetjs/utils/read-utils.ts +56 -55
- package/src/parquetjs/utils/thrift-runtime.ts +109 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
- package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
- package/src/polyfills/process.ts +17 -0
- package/src/unbundled.ts +33 -0
- package/src/wasm.ts +6 -0
- package/src/workers/parquet-source-worker.ts +21 -0
- package/src/workers/parquet-worker-node.ts +5 -0
- package/src/workers/parquet-worker.ts +31 -0
- package/dist/parquet-arrow-loader.d.ts +0 -43
- package/dist/parquet-arrow-loader.d.ts.map +0 -1
- package/dist/parquet-arrow-loader.js +0 -49
- package/dist/parquet-arrow-loader.js.map +0 -1
- package/dist/parquet-arrow-writer.d.ts +0 -9
- package/dist/parquet-arrow-writer.d.ts.map +0 -1
- package/dist/parquet-arrow-writer.js +0 -23
- package/dist/parquet-arrow-writer.js.map +0 -1
- package/dist/parquet-json-loader.d.ts +0 -148
- package/dist/parquet-json-loader.d.ts.map +0 -1
- package/dist/parquet-json-loader.js +0 -106
- package/dist/parquet-json-loader.js.map +0 -1
- package/dist/parquet-json-writer.d.ts +0 -19
- package/dist/parquet-json-writer.d.ts.map +0 -1
- package/dist/parquet-json-writer.js +0 -20
- package/dist/parquet-json-writer.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.d.ts +0 -222
- package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer.js +0 -1868
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.d.ts +0 -4
- package/dist/polyfills/buffer/index.d.ts.map +0 -1
- package/dist/polyfills/buffer/index.js +0 -9
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/workers/parquet-json-worker.d.ts +0 -2
- package/dist/workers/parquet-json-worker.d.ts.map +0 -1
- package/dist/workers/parquet-json-worker.js +0 -7
- package/dist/workers/parquet-json-worker.js.map +0 -1
- package/src/parquet-arrow-loader.ts +0 -74
- package/src/parquet-arrow-writer.ts +0 -33
- package/src/parquet-json-loader.ts +0 -160
- package/src/parquet-json-writer.ts +0 -27
- package/src/parquetjs/modules.d.ts +0 -25
- package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
- package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
- package/src/polyfills/buffer/buffer.ts +0 -2205
- package/src/polyfills/buffer/index.ts +0 -11
- package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
- package/src/workers/parquet-json-worker.ts +0 -8
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import Int64 from
|
|
1
|
+
import { CompactInt64 as Int64 } from "../utils/uint8-array-compact-protocol.js";
|
|
2
2
|
import type { PageHeader } from "../parquet-thrift.js";
|
|
3
|
-
|
|
3
|
+
import type { ParquetValueBuffer } from "../codecs/declare.js";
|
|
4
|
+
export type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY' | 'RLE_DICTIONARY' | 'DELTA_BINARY_PACKED' | 'DELTA_LENGTH_BYTE_ARRAY' | 'DELTA_BYTE_ARRAY' | 'BYTE_STREAM_SPLIT';
|
|
4
5
|
export type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4' | 'LZ4_RAW' | 'ZSTD';
|
|
5
6
|
export type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';
|
|
6
7
|
export type ParquetType = PrimitiveType | OriginalType;
|
|
@@ -11,16 +12,52 @@ export type PrimitiveType = 'BOOLEAN' | 'INT32' | 'INT64' | 'INT96' | 'FLOAT' |
|
|
|
11
12
|
/**
|
|
12
13
|
* Logical type
|
|
13
14
|
*/
|
|
14
|
-
export type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'JSON' | 'BSON' | 'INTERVAL';
|
|
15
|
-
|
|
15
|
+
export type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIME_NANOS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'TIMESTAMP_NANOS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'ENUM' | 'UUID' | 'FLOAT16' | 'UNKNOWN' | 'VARIANT' | 'GEOMETRY' | 'GEOGRAPHY' | 'JSON' | 'BSON' | 'INTERVAL';
|
|
16
|
+
/** Units carried by Parquet TIME and TIMESTAMP logical type annotations. */
|
|
17
|
+
export type ParquetTimeUnit = 'MILLIS' | 'MICROS' | 'NANOS';
|
|
18
|
+
/** Logical type names defined by the Parquet 2.13 format. */
|
|
19
|
+
export type ParquetLogicalTypeName = 'STRING' | 'MAP' | 'LIST' | 'ENUM' | 'DECIMAL' | 'DATE' | 'TIME' | 'TIMESTAMP' | 'INTEGER' | 'UNKNOWN' | 'JSON' | 'BSON' | 'UUID' | 'FLOAT16' | 'VARIANT' | 'GEOMETRY' | 'GEOGRAPHY';
|
|
20
|
+
/** Serializable representation of one Parquet logical type annotation. */
|
|
21
|
+
export interface ParquetLogicalType {
|
|
22
|
+
/** Logical type discriminator. */
|
|
23
|
+
type: ParquetLogicalTypeName;
|
|
24
|
+
/** Integer width for INTEGER annotations. */
|
|
25
|
+
bitWidth?: 8 | 16 | 32 | 64;
|
|
26
|
+
/** Whether an INTEGER annotation is signed. */
|
|
27
|
+
isSigned?: boolean;
|
|
28
|
+
/** Time unit for TIME and TIMESTAMP annotations. */
|
|
29
|
+
unit?: ParquetTimeUnit;
|
|
30
|
+
/** Whether a TIME or TIMESTAMP value represents a UTC-normalized instant. */
|
|
31
|
+
isAdjustedToUTC?: boolean;
|
|
32
|
+
/** Decimal precision. */
|
|
33
|
+
precision?: number;
|
|
34
|
+
/** Decimal scale. */
|
|
35
|
+
scale?: number;
|
|
36
|
+
/** Variant specification version. */
|
|
37
|
+
specificationVersion?: number;
|
|
38
|
+
/** Coordinate reference system for geospatial logical types. */
|
|
39
|
+
crs?: string;
|
|
40
|
+
/** Edge interpolation algorithm for GEOGRAPHY values. */
|
|
41
|
+
algorithm?: string;
|
|
42
|
+
}
|
|
43
|
+
export type ParquetDictionary = any[];
|
|
16
44
|
export interface SchemaDefinition {
|
|
17
45
|
[string: string]: FieldDefinition;
|
|
18
46
|
}
|
|
19
47
|
export interface FieldDefinition {
|
|
20
48
|
type?: ParquetType;
|
|
49
|
+
/** Physical type declared by the file, retained independently from its logical type. */
|
|
50
|
+
physicalType?: PrimitiveType;
|
|
21
51
|
typeLength?: number;
|
|
52
|
+
/** @deprecated Use `precision`. */
|
|
22
53
|
presision?: number;
|
|
54
|
+
/** Decimal precision. */
|
|
55
|
+
precision?: number;
|
|
23
56
|
scale?: number;
|
|
57
|
+
/** Modern Parquet logical type annotation. */
|
|
58
|
+
logicalType?: ParquetLogicalType;
|
|
59
|
+
/** Stable field identifier declared by the Parquet schema. */
|
|
60
|
+
fieldId?: number;
|
|
24
61
|
encoding?: ParquetCodec;
|
|
25
62
|
compression?: ParquetCompression;
|
|
26
63
|
optional?: boolean;
|
|
@@ -33,9 +70,16 @@ export interface ParquetField {
|
|
|
33
70
|
key: string;
|
|
34
71
|
primitiveType?: PrimitiveType;
|
|
35
72
|
originalType?: OriginalType;
|
|
73
|
+
/** Modern Parquet logical type annotation. */
|
|
74
|
+
logicalType?: ParquetLogicalType;
|
|
75
|
+
/** Stable field identifier declared by the Parquet schema. */
|
|
76
|
+
fieldId?: number;
|
|
36
77
|
repetitionType: RepetitionType;
|
|
37
78
|
typeLength?: number;
|
|
79
|
+
/** @deprecated Use `precision`. */
|
|
38
80
|
presision?: number;
|
|
81
|
+
/** Decimal precision. */
|
|
82
|
+
precision?: number;
|
|
39
83
|
scale?: number;
|
|
40
84
|
encoding?: ParquetCodec;
|
|
41
85
|
compression?: ParquetCompression;
|
|
@@ -56,12 +100,22 @@ export interface ParquetReaderContext {
|
|
|
56
100
|
dictionary?: ParquetDictionary;
|
|
57
101
|
/** If true, binary values are not converted to strings */
|
|
58
102
|
preserveBinary?: boolean;
|
|
103
|
+
/** Retain byte arrays as views into decoded page buffers for direct materialization. */
|
|
104
|
+
retainByteArrayViews?: boolean;
|
|
105
|
+
/** Decode primitive values into typed column buffers when their physical type permits it. */
|
|
106
|
+
useTypedValueBuffers?: boolean;
|
|
107
|
+
/** Decode repetition and definition levels into compact unsigned typed arrays. */
|
|
108
|
+
useTypedLevelBuffers?: boolean;
|
|
59
109
|
}
|
|
110
|
+
/** Mutable storage for decoded Parquet repetition and definition levels. */
|
|
111
|
+
export type ParquetLevelBuffer = number[] | Uint8Array | Uint16Array | Uint32Array;
|
|
60
112
|
export interface ParquetPageData {
|
|
61
113
|
dlevels: number[];
|
|
62
114
|
rlevels: number[];
|
|
63
115
|
/** Actual column chunks */
|
|
64
|
-
values:
|
|
116
|
+
values: ParquetValueBuffer;
|
|
117
|
+
/** Number of values written directly into the column destination, if one was supplied. */
|
|
118
|
+
directValuesWritten?: number;
|
|
65
119
|
count: number;
|
|
66
120
|
dictionary?: ParquetDictionary;
|
|
67
121
|
/** The "raw" page header from the file */
|
|
@@ -81,9 +135,9 @@ export declare class ParquetRowGroup {
|
|
|
81
135
|
}
|
|
82
136
|
/** Holds the data for one column chunk */
|
|
83
137
|
export interface ParquetColumnChunk {
|
|
84
|
-
dlevels:
|
|
85
|
-
rlevels:
|
|
86
|
-
values:
|
|
138
|
+
dlevels: ParquetLevelBuffer;
|
|
139
|
+
rlevels: ParquetLevelBuffer;
|
|
140
|
+
values: ParquetValueBuffer;
|
|
87
141
|
count: number;
|
|
88
142
|
pageHeaders: PageHeader[];
|
|
89
143
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,YAAY,IAAI,KAAK,EAAC,iDAA8C;AAC5E,OAAO,KAAK,EAAC,UAAU,EAAC,6BAA0B;AAClD,OAAO,KAAK,EAAC,kBAAkB,EAAC,6BAA0B;AAE1D,MAAM,MAAM,YAAY,GACpB,OAAO,GACP,KAAK,GACL,kBAAkB,GAClB,gBAAgB,GAChB,qBAAqB,GACrB,yBAAyB,GACzB,kBAAkB,GAClB,mBAAmB,CAAC;AACxB,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,YAAY,GACZ,kBAAkB,GAClB,kBAAkB,GAClB,iBAAiB,GACjB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,SAAS,GACT,SAAS,GACT,SAAS,GACT,UAAU,GACV,WAAW,GACX,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE5D,6DAA6D;AAC7D,MAAM,MAAM,sBAAsB,GAC9B,QAAQ,GACR,KAAK,GACL,MAAM,GACN,MAAM,GACN,SAAS,GACT,MAAM,GACN,MAAM,GACN,WAAW,GACX,SAAS,GACT,SAAS,GACT,MAAM,GACN,MAAM,GACN,MAAM,GACN,SAAS,GACT,SAAS,GACT,UAAU,GACV,WAAW,CAAC;AAEhB,0EAA0E;AAC1E,MAAM,WAAW,kBAAkB;IACjC,kCAAkC;IAClC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IAC5B,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,oDAAoD;IACpD,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,6EAA6E;IAC7E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,yBAAyB;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC;AAEtC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,wFAAwF;IACxF,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,8CAA8C;IAC9C,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0DAA0D;IAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wFAAwF;IACxF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,6FAA6F;IAC7F,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,kFAAkF;IAClF,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,4EAA4E;AAC5E,MAAM,MAAM,kBAAkB,GAAG,MAAM,EAAE,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AAEnF,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2BAA2B;IAC3B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,0FAA0F;IAC1F,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0CAA0C;IAC1C,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;kDACkD;AAClD,qBAAa,eAAe;IAC1B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;gBAEnC,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAM;CAItF;AAED,0CAA0C;AAC1C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,kBAAkB,CAAC;IAC5B,OAAO,EAAE,kBAAkB,CAAC;IAC5B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"declare.js","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;
|
|
1
|
+
{"version":3,"file":"declare.js","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AA6NpE;kDACkD;AAClD,MAAM,OAAO,eAAe;IAC1B,kCAAkC;IAClC,QAAQ,CAAS;IACjB,2BAA2B;IAC3B,UAAU,CAAqC;IAE/C,YAAY,WAAmB,CAAC,EAAE,aAAiD,EAAE;QACnF,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FieldDefinition, ParquetRowGroup, ParquetCompression, ParquetField, ParquetRow, SchemaDefinition } from "./declare.js";
|
|
2
|
+
import type { ArrayType } from '@loaders.gl/schema';
|
|
2
3
|
/**
|
|
3
4
|
* A parquet file schema
|
|
4
5
|
*/
|
|
@@ -20,6 +21,8 @@ export declare class ParquetSchema {
|
|
|
20
21
|
findFieldBranch(path: string | string[]): ParquetField[];
|
|
21
22
|
shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void;
|
|
22
23
|
materializeRows(rowGroup: ParquetRowGroup): ParquetRow[];
|
|
24
|
+
/** Materializes one decoded row group as top-level column arrays. */
|
|
25
|
+
materializeColumns(rowGroup: ParquetRowGroup): Record<string, ArrayType>;
|
|
23
26
|
compress(type: ParquetCompression): this;
|
|
24
27
|
rowGroup(): ParquetRowGroup;
|
|
25
28
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,UAAU,EAEV,gBAAgB,EACjB,qBAAkB;AACnB,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAIlD;;GAEG;AACH,qBAAa,aAAa;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACrC,SAAS,EAAE,YAAY,EAAE,CAAC;IAEjC;;OAEG;gBACS,MAAM,EAAE,gBAAgB;IAMpC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY;IAiBhD;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY,EAAE;IAgBxD,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI;IAI7D,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE;IAIxD,qEAAqE;IACrE,kBAAkB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC;IAIxE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI;IAMxC,QAAQ,IAAI,eAAe;CAG5B"}
|
|
@@ -4,8 +4,10 @@
|
|
|
4
4
|
// Copyright (c) 2017 ironSource Ltd.
|
|
5
5
|
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
6
6
|
import { PARQUET_CODECS } from "../codecs/index.js";
|
|
7
|
+
import { isByteStreamSplitType } from "../codecs/byte-stream-split.js";
|
|
8
|
+
import { isDeltaEncodingType } from "../codecs/delta.js";
|
|
7
9
|
import { PARQUET_COMPRESSION_METHODS } from "../compression.js";
|
|
8
|
-
import { materializeRows, shredBuffer, shredRecord } from "./shred.js";
|
|
10
|
+
import { materializeColumns, materializeRows, shredBuffer, shredRecord } from "./shred.js";
|
|
9
11
|
import { PARQUET_LOGICAL_TYPES } from "./types.js";
|
|
10
12
|
/**
|
|
11
13
|
* A parquet file schema
|
|
@@ -64,6 +66,10 @@ export class ParquetSchema {
|
|
|
64
66
|
materializeRows(rowGroup) {
|
|
65
67
|
return materializeRows(this, rowGroup);
|
|
66
68
|
}
|
|
69
|
+
/** Materializes one decoded row group as top-level column arrays. */
|
|
70
|
+
materializeColumns(rowGroup) {
|
|
71
|
+
return materializeColumns(this, rowGroup);
|
|
72
|
+
}
|
|
67
73
|
compress(type) {
|
|
68
74
|
setCompress(this.schema, type);
|
|
69
75
|
setCompress(this.fields, type);
|
|
@@ -112,6 +118,8 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
|
|
|
112
118
|
name,
|
|
113
119
|
path: cpath,
|
|
114
120
|
key: cpath.join(),
|
|
121
|
+
logicalType: opts.logicalType,
|
|
122
|
+
fieldId: opts.fieldId,
|
|
115
123
|
repetitionType,
|
|
116
124
|
rLevelMax,
|
|
117
125
|
dLevelMax,
|
|
@@ -129,6 +137,13 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
|
|
|
129
137
|
if (!(opts.encoding in PARQUET_CODECS)) {
|
|
130
138
|
throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
|
|
131
139
|
}
|
|
140
|
+
const primitiveType = opts.physicalType || typeDef.primitiveType;
|
|
141
|
+
if (opts.encoding === 'BYTE_STREAM_SPLIT' && !isByteStreamSplitType(primitiveType)) {
|
|
142
|
+
throw new Error(`BYTE_STREAM_SPLIT does not support ${primitiveType}`);
|
|
143
|
+
}
|
|
144
|
+
if (opts.encoding.startsWith('DELTA_') && !isDeltaEncodingType(opts.encoding, primitiveType)) {
|
|
145
|
+
throw new Error(`${opts.encoding} does not support ${primitiveType}`);
|
|
146
|
+
}
|
|
132
147
|
opts.compression = opts.compression || 'UNCOMPRESSED';
|
|
133
148
|
if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
|
|
134
149
|
throw new Error(`unsupported compression method: ${opts.compression}`);
|
|
@@ -137,15 +152,18 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
|
|
|
137
152
|
const cpath = path.concat([name]);
|
|
138
153
|
fieldList[name] = {
|
|
139
154
|
name,
|
|
140
|
-
primitiveType
|
|
155
|
+
primitiveType,
|
|
141
156
|
originalType: typeDef.originalType,
|
|
157
|
+
logicalType: opts.logicalType,
|
|
158
|
+
fieldId: opts.fieldId,
|
|
142
159
|
path: cpath,
|
|
143
160
|
key: cpath.join(),
|
|
144
161
|
repetitionType,
|
|
145
162
|
encoding: opts.encoding,
|
|
146
163
|
compression: opts.compression,
|
|
147
164
|
typeLength: opts.typeLength || typeDef.typeLength,
|
|
148
|
-
presision: opts.presision,
|
|
165
|
+
presision: opts.precision ?? opts.presision,
|
|
166
|
+
precision: opts.precision ?? opts.presision,
|
|
149
167
|
scale: opts.scale,
|
|
150
168
|
rLevelMax,
|
|
151
169
|
dLevelMax
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,OAAO,EAAC,cAAc,EAAC,2BAAwB;AAC/C,OAAO,EAAC,2BAA2B,EAAC,0BAAuB;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,OAAO,EAAC,cAAc,EAAC,2BAAwB;AAC/C,OAAO,EAAC,qBAAqB,EAAC,uCAAoC;AAClE,OAAO,EAAC,mBAAmB,EAAC,2BAAwB;AACpD,OAAO,EAAC,2BAA2B,EAAC,0BAAuB;AAW3D,OAAO,EAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,EAAE,WAAW,EAAC,mBAAgB;AACtF,OAAO,EAAC,qBAAqB,EAAC,mBAAgB;AAE9C;;GAEG;AACH,MAAM,OAAO,aAAa;IACjB,MAAM,CAAkC;IACxC,MAAM,CAA+B;IACrC,SAAS,CAAiB;IAEjC;;OAEG;IACH,YAAY,MAAwB;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,IAAuB;QAC/B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,qDAAqD;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,qDAAqD;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc;QACtC,CAAC;QAED,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACpB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;YACrC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAsC,CAAC;QACxD,CAAC;QAED,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,IAAuB;QACrC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,qDAAqD;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACpB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAsC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,WAAW,CAAC,GAAe,EAAE,QAAyB;QACpD,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,eAAe,CAAC,QAAyB;QACvC,OAAO,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED,qEAAqE;IACrE,kBAAkB,CAAC,QAAyB;QAC1C,OAAO,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,QAAQ,CAAC,IAAwB;QAC/B,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/B,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ;QACN,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;CACF;AAED,SAAS,WAAW,CAAC,MAAW,EAAE,IAAwB;IACxD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,sDAAsD;AACtD,SAAS,WAAW,CAClB,MAAwB,EACxB,eAAuB,EACvB,eAAuB,EACvB,IAAc;IAEd,MAAM,SAAS,GAAiC,EAAE,CAAC;IAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAE1B,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,SAAS,GAAG,eAAe,CAAC;QAChC,IAAI,SAAS,GAAG,eAAe,CAAC;QAEhC,IAAI,cAAc,GAAmB,UAAU,CAAC;QAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,cAAc,GAAG,UAAU,CAAC;YAC5B,SAAS,EAAE,CAAC;QACd,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,cAAc,GAAG,UAAU,CAAC;YAC5B,SAAS,EAAE,CAAC;YACZ,IAAI,QAAQ;gBAAE,SAAS,EAAE,CAAC;QAC5B,CAAC;QAED,kBAAkB;QAClB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAChB,IAAI;gBACJ,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE;gBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,cAAc;gBACd,SAAS;gBACT,SAAS;gBACT,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM;gBAC3C,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC;aAC9D,CAAC;YACF,SAAS,CAAC,kCAAkC;QAC9C,CAAC;QAED,MAAM,OAAO,GAAQ,qBAAqB,CAAC,IAAI,CAAC,IAAK,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;QACzC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,cAAc,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,aAAa,CAAC;QACjE,IAAI,IAAI,CAAC,QAAQ,KAAK,mBAAmB,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,KAAK,CAAC,sCAAsC,aAAa,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,qBAAqB,aAAa,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,cAAc,CAAC;QACtD,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,2BAA2B,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,mBAAmB;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,SAAS,CAAC,IAAI,CAAC,GAAG;YAChB,IAAI;YACJ,aAAa;YACb,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,KAAK;YACX,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE;YACjB,cAAc;YACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU;YACjD,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS;YAC3C,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS;YAC3C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS;YACT,SAAS;SACV,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,UAAU,CAAC,MAAoC;IACtD,IAAI,IAAI,GAAmB,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACvB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAO,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -66,4 +66,6 @@ export declare function materializeRows(schema: ParquetSchema, rowGroup: Parquet
|
|
|
66
66
|
* }
|
|
67
67
|
*/
|
|
68
68
|
export declare function materializeColumns(schema: ParquetSchema, rowGroup: ParquetRowGroup): Record<string, ArrayType>;
|
|
69
|
+
/** Materializes one decoded Parquet column into its top-level columnar representation. */
|
|
70
|
+
export declare function materializeColumn(schema: ParquetSchema, rowGroup: ParquetRowGroup, key: string): ArrayType | undefined;
|
|
69
71
|
//# sourceMappingURL=shred.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAoC,UAAU,EAAC,qBAAkB;AACxF,OAAO,EAAC,aAAa,EAAC,oBAAiB;AAGvC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,eAAe,GACxB,IAAI,
|
|
1
|
+
{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAoC,UAAU,EAAC,qBAAkB;AACxF,OAAO,EAAC,aAAa,EAAC,oBAAiB;AAGvC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,eAAe,GACxB,IAAI,CA4BN;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE,CAY9F;AAqHD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAS3B;AAED,0FAA0F;AAC1F,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,eAAe,EACzB,GAAG,EAAE,MAAM,GACV,SAAS,GAAG,SAAS,CASvB"}
|
|
@@ -105,7 +105,7 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
|
|
|
105
105
|
data[field.key].count += 1;
|
|
106
106
|
data[field.key].rlevels.push(rlvl);
|
|
107
107
|
data[field.key].dlevels.push(field.dLevelMax);
|
|
108
|
-
data[field.key].values.push(Types.toPrimitive((field.originalType || field.primitiveType), values[i]));
|
|
108
|
+
data[field.key].values.push(Types.toPrimitive((field.originalType || field.primitiveType), values[i], field));
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
}
|
|
@@ -130,10 +130,9 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
|
|
|
130
130
|
* }
|
|
131
131
|
*/
|
|
132
132
|
export function materializeRows(schema, rowGroup) {
|
|
133
|
-
const rows =
|
|
134
|
-
// rows = new Array(rowGroup.rowCount).fill({})'
|
|
133
|
+
const rows = new Array(rowGroup.rowCount);
|
|
135
134
|
for (let i = 0; i < rowGroup.rowCount; i++) {
|
|
136
|
-
rows
|
|
135
|
+
rows[i] = {};
|
|
137
136
|
}
|
|
138
137
|
for (const key in rowGroup.columnData) {
|
|
139
138
|
const columnData = rowGroup.columnData[key];
|
|
@@ -148,6 +147,12 @@ export function materializeRows(schema, rowGroup) {
|
|
|
148
147
|
function materializeColumnAsRows(schema, columnData, key, rows) {
|
|
149
148
|
const field = schema.findField(key);
|
|
150
149
|
const branch = schema.findFieldBranch(key);
|
|
150
|
+
if (branch.length === 1 && field.repetitionType !== 'REPEATED') {
|
|
151
|
+
materializeFlatColumnAsRows(field, columnData, rows);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const logicalType = field.originalType || field.primitiveType;
|
|
155
|
+
const fromPrimitive = Types.PARQUET_LOGICAL_TYPES[logicalType].fromPrimitive;
|
|
151
156
|
// tslint:disable-next-line:prefer-array-literal
|
|
152
157
|
const rLevels = new Array(field.rLevelMax + 1).fill(0);
|
|
153
158
|
let vIndex = 0;
|
|
@@ -183,9 +188,8 @@ function materializeColumnAsRows(schema, columnData, key, rows) {
|
|
|
183
188
|
}
|
|
184
189
|
// Leaf node - Add the value
|
|
185
190
|
if (dLevel === field.dLevelMax) {
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
field.originalType || field.primitiveType, columnData.values[vIndex], field);
|
|
191
|
+
const primitiveValue = columnData.values[vIndex];
|
|
192
|
+
const value = fromPrimitive ? fromPrimitive(primitiveValue, field) : primitiveValue;
|
|
189
193
|
vIndex++;
|
|
190
194
|
switch (field.repetitionType) {
|
|
191
195
|
case 'REPEATED':
|
|
@@ -206,6 +210,28 @@ function materializeColumnAsRows(schema, columnData, key, rows) {
|
|
|
206
210
|
}
|
|
207
211
|
}
|
|
208
212
|
}
|
|
213
|
+
/** Materializes a top-level required or optional primitive directly into row objects. */
|
|
214
|
+
function materializeFlatColumnAsRows(field, columnData, rows) {
|
|
215
|
+
const logicalType = field.originalType || field.primitiveType;
|
|
216
|
+
const fromPrimitive = Types.PARQUET_LOGICAL_TYPES[logicalType].fromPrimitive;
|
|
217
|
+
const count = Math.min(columnData.count, rows.length);
|
|
218
|
+
let valueIndex = 0;
|
|
219
|
+
if (field.repetitionType === 'REQUIRED' && !fromPrimitive) {
|
|
220
|
+
for (let rowIndex = 0; rowIndex < count; rowIndex++) {
|
|
221
|
+
rows[rowIndex][field.name] = columnData.values[rowIndex];
|
|
222
|
+
}
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
for (let rowIndex = 0; rowIndex < count; rowIndex++) {
|
|
226
|
+
if (columnData.dlevels[rowIndex] === field.dLevelMax) {
|
|
227
|
+
const primitiveValue = columnData.values[valueIndex];
|
|
228
|
+
rows[rowIndex][field.name] = fromPrimitive
|
|
229
|
+
? fromPrimitive(primitiveValue, field)
|
|
230
|
+
: primitiveValue;
|
|
231
|
+
valueIndex++;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
209
235
|
// Columnar export
|
|
210
236
|
/**
|
|
211
237
|
* 'Materialize' a list of <value, repetition_level, definition_level>
|
|
@@ -229,13 +255,23 @@ function materializeColumnAsRows(schema, columnData, key, rows) {
|
|
|
229
255
|
export function materializeColumns(schema, rowGroup) {
|
|
230
256
|
const columns = {};
|
|
231
257
|
for (const key in rowGroup.columnData) {
|
|
232
|
-
const
|
|
233
|
-
if (
|
|
234
|
-
|
|
258
|
+
const column = materializeColumn(schema, rowGroup, key);
|
|
259
|
+
if (column) {
|
|
260
|
+
columns[schema.findFieldBranch(key)[0].name] = column;
|
|
235
261
|
}
|
|
236
262
|
}
|
|
237
263
|
return columns;
|
|
238
264
|
}
|
|
265
|
+
/** Materializes one decoded Parquet column into its top-level columnar representation. */
|
|
266
|
+
export function materializeColumn(schema, rowGroup, key) {
|
|
267
|
+
const columnData = rowGroup.columnData[key];
|
|
268
|
+
if (!columnData?.count) {
|
|
269
|
+
return undefined;
|
|
270
|
+
}
|
|
271
|
+
const columns = {};
|
|
272
|
+
materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
|
|
273
|
+
return columns[schema.findFieldBranch(key)[0].name];
|
|
274
|
+
}
|
|
239
275
|
// eslint-disable-next-line max-statements, complexity
|
|
240
276
|
function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
|
|
241
277
|
if (columnData.count <= 0) {
|
|
@@ -244,21 +280,11 @@ function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, col
|
|
|
244
280
|
const field = schema.findField(key);
|
|
245
281
|
const branch = schema.findFieldBranch(key);
|
|
246
282
|
const columnName = branch[0].name;
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
if (values.length === rowCount && branch[0].primitiveType) {
|
|
250
|
-
// if (branch[0].repetitionType === `REQUIRED`) {
|
|
251
|
-
// switch (branch[0].primitiveType) {
|
|
252
|
-
// case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);
|
|
253
|
-
// }
|
|
254
|
-
// }
|
|
255
|
-
column = values;
|
|
256
|
-
}
|
|
257
|
-
if (column) {
|
|
258
|
-
columns[columnName] = column;
|
|
283
|
+
if (branch.length === 1 && field.repetitionType !== 'REPEATED') {
|
|
284
|
+
columns[columnName] = materializeFlatColumn(field, columnData, rowCount);
|
|
259
285
|
return;
|
|
260
286
|
}
|
|
261
|
-
column = new Array(rowCount);
|
|
287
|
+
const column = new Array(rowCount);
|
|
262
288
|
for (let i = 0; i < rowCount; i++) {
|
|
263
289
|
column[i] = {};
|
|
264
290
|
}
|
|
@@ -327,4 +353,26 @@ function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, col
|
|
|
327
353
|
}
|
|
328
354
|
}
|
|
329
355
|
}
|
|
356
|
+
/** Materializes a required or optional top-level primitive column with logical type conversion. */
|
|
357
|
+
function materializeFlatColumn(field, columnData, rowCount) {
|
|
358
|
+
const logicalType = field.originalType || field.primitiveType;
|
|
359
|
+
const fromPrimitive = Types.PARQUET_LOGICAL_TYPES[logicalType].fromPrimitive;
|
|
360
|
+
const count = Math.min(columnData.count, rowCount);
|
|
361
|
+
if (field.repetitionType === 'REQUIRED' &&
|
|
362
|
+
!fromPrimitive &&
|
|
363
|
+
count === rowCount &&
|
|
364
|
+
columnData.values.length === rowCount) {
|
|
365
|
+
return columnData.values;
|
|
366
|
+
}
|
|
367
|
+
const column = new Array(rowCount).fill(null);
|
|
368
|
+
let valueIndex = 0;
|
|
369
|
+
for (let rowIndex = 0; rowIndex < count; rowIndex++) {
|
|
370
|
+
if (columnData.dlevels[rowIndex] === field.dLevelMax) {
|
|
371
|
+
const primitiveValue = columnData.values[valueIndex];
|
|
372
|
+
column[rowIndex] = fromPrimitive ? fromPrimitive(primitiveValue, field) : primitiveValue;
|
|
373
|
+
valueIndex++;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return column;
|
|
377
|
+
}
|
|
330
378
|
//# sourceMappingURL=shred.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shred.js","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAGpE,OAAO,EAAC,eAAe,EAA+C,qBAAkB;AAExF,OAAO,KAAK,KAAK,mBAAgB;AAEjC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,MAAM,UAAU,WAAW,CAAC,MAAqB;IAC/C,MAAM,UAAU,GAAuC,EAAE,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG;YACtB,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,EAAE;YACV,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,CAAC;SACT,CAAC;IACJ,CAAC;IACD,OAAO,EAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,WAAW,CACzB,MAAqB,EACrB,MAAkB,EAClB,QAAyB;IAEzB,mDAAmD;IACnD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC;IAE5C,iBAAiB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAErD,2EAA2E;IAC3E,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC;QACtB,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,OAAO;IACT,CAAC;IACD,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC5F,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC5F,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC1F,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;AACH,CAAC;AAED,sDAAsD;AACtD,SAAS,iBAAiB,CACxB,MAAoC,EACpC,MAAkB,EAClB,IAAwC,EACxC,MAAc,EACd,MAAc;IAEd,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3B,eAAe;QACf,IAAI,MAAM,GAAU,EAAE,CAAC;QACvB,IACE,MAAM;YACN,KAAK,CAAC,IAAI,IAAI,MAAM;YACpB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS;YAChC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAC3B,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBAC7C,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,eAAe;QACf,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,2BAA2B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,YAAY;QACZ,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,iBAAiB,CAAC,KAAK,CAAC,MAAO,EAAE,IAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAChE,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;YACD,SAAS,CAAC,kCAAkC;QAC9C,CAAC;QAED,cAAc;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;YAChD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,iBAAiB,CAAC,KAAK,CAAC,MAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3E,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CACzB,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,CAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAC3E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,eAAe,CAAC,MAAqB,EAAE,QAAyB;IAC9E,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,gDAAgD;IAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,uBAAuB,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4CAA4C;AAC5C,sDAAsD;AACtD,SAAS,uBAAuB,CAC9B,MAAqB,EACrB,UAA8B,EAC9B,GAAW,EACX,IAAkB;IAElB,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAE3C,gDAAgD;IAChD,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAE5B,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzC,6CAA6C;QAC7C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC9C,MAAM;YACR,CAAC;YAED,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC5B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC3B,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzB,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACtC,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7B,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC5C,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,MAAM,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa;YAC/B,aAAa;YACb,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,EACzC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EACzB,KAAK,CACN,CAAC;YACF,MAAM,EAAE,CAAC;YAET,QAAQ,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC7B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC5B,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC1B,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC3B,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACvC,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChC,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,kBAAkB;AAElB;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAqB,EACrB,QAAyB;IAEzB,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,gCAAgC,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,sDAAsD;AACtD,SAAS,gCAAgC,CACvC,MAAqB,EACrB,UAA8B,EAC9B,QAAgB,EAChB,GAAW,EACX,OAAuC;IAEvC,IAAI,UAAU,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAElC,IAAI,MAA6B,CAAC;IAClC,MAAM,EAAC,MAAM,EAAC,GAAG,UAAU,CAAC;IAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QAC1D,iDAAiD;QACjD,uCAAuC;QACvC,2FAA2F;QAC3F,MAAM;QACN,IAAI;QACJ,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;QAC7B,OAAO;IACT,CAAC;IAED,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;IAE7B,gDAAgD;IAChD,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAE5B,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAe,CAAC;QAEzD,6CAA6C;QAC7C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC9C,MAAM;YACR,CAAC;YAED,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC5B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC3B,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzB,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACtC,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7B,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC5C,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,MAAM,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa;YAC/B,aAAa;YACb,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,EACzC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EACzB,KAAK,CACN,CAAC;YACF,MAAM,EAAE,CAAC;YAET,QAAQ,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC7B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC5B,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC1B,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC3B,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACvC,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChC,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;QAClC,IAAI,UAAU,IAAK,MAAM,CAAC,CAAC,CAAY,EAAE,CAAC;YACxC,MAAM,CAAC,CAAC,CAAC,GAAI,MAAM,CAAC,CAAC,CAAY,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"shred.js","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAGpE,OAAO,EAAC,eAAe,EAA+C,qBAAkB;AAExF,OAAO,KAAK,KAAK,mBAAgB;AAEjC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,MAAM,UAAU,WAAW,CAAC,MAAqB;IAC/C,MAAM,UAAU,GAAuC,EAAE,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG;YACtB,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,EAAE;YACV,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,CAAC;SACT,CAAC;IACJ,CAAC;IACD,OAAO,EAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,WAAW,CACzB,MAAqB,EACrB,MAAkB,EAClB,QAAyB;IAEzB,mDAAmD;IACnD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC;IAE5C,iBAAiB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAErD,2EAA2E;IAC3E,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC;QACtB,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,OAAO;IACT,CAAC;IACD,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CACxB,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAmB,EAClD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAmB,CACpC,CAAC;QACF,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CACxB,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAmB,EAClD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAmB,CACpC,CAAC;QACF,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CACxB,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAmB,EAClD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAA8B,CAC/C,CAAC;QACF,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;AACH,CAAC;AAED,sDAAsD;AACtD,SAAS,iBAAiB,CACxB,MAAoC,EACpC,MAAkB,EAClB,IAAwC,EACxC,MAAc,EACd,MAAc;IAEd,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3B,eAAe;QACf,IAAI,MAAM,GAAU,EAAE,CAAC;QACvB,IACE,MAAM;YACN,KAAK,CAAC,IAAI,IAAI,MAAM;YACpB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS;YAChC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAC3B,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBAC7C,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,eAAe;QACf,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,2BAA2B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,YAAY;QACZ,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,iBAAiB,CAAC,KAAK,CAAC,MAAO,EAAE,IAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAChE,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,CAAC;YACD,SAAS,CAAC,kCAAkC;QAC9C,CAAC;QAED,cAAc;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;YAChD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,iBAAiB,CAAC,KAAK,CAAC,MAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3E,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC3D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAoB,CAAC,IAAI,CACxC,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,CAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAClF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,eAAe,CAAC,MAAqB,EAAE,QAAyB;IAC9E,MAAM,IAAI,GAAG,IAAI,KAAK,CAAa,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,uBAAuB,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4CAA4C;AAC5C,sDAAsD;AACtD,SAAS,uBAAuB,CAC9B,MAAqB,EACrB,UAA8B,EAC9B,GAAW,EACX,IAAkB;IAElB,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAE3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;QAC/D,2BAA2B,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAc,CAAC;IAC/D,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC;IAE7E,gDAAgD;IAChD,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAE5B,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzC,6CAA6C;QAC7C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC9C,MAAM;YACR,CAAC;YAED,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC5B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC3B,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzB,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACtC,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7B,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC5C,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,MAAM,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YACpF,MAAM,EAAE,CAAC;YAET,QAAQ,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC7B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC5B,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC1B,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC3B,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACvC,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChC,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,yFAAyF;AACzF,SAAS,2BAA2B,CAClC,KAAmB,EACnB,UAA8B,EAC9B,IAAkB;IAElB,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAc,CAAC;IAC/D,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1D,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;YACpD,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO;IACT,CAAC;IAED,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;QACpD,IAAI,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa;gBACxC,CAAC,CAAC,aAAa,CAAC,cAAc,EAAE,KAAK,CAAC;gBACtC,CAAC,CAAC,cAAc,CAAC;YACnB,UAAU,EAAE,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC;AAED,kBAAkB;AAElB;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAqB,EACrB,QAAyB;IAEzB,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACxD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;QACxD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,iBAAiB,CAC/B,MAAqB,EACrB,QAAyB,EACzB,GAAW;IAEX,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,gCAAgC,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACtF,OAAO,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC;AAED,sDAAsD;AACtD,SAAS,gCAAgC,CACvC,MAAqB,EACrB,UAA8B,EAC9B,QAAgB,EAChB,GAAW,EACX,OAAuC;IAEvC,IAAI,UAAU,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAElC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;QAC/D,OAAO,CAAC,UAAU,CAAC,GAAG,qBAAqB,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACzE,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAc,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;IAE7B,gDAAgD;IAChD,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAE5B,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAe,CAAC;QAEzD,6CAA6C;QAC7C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC9C,MAAM;YACR,CAAC;YAED,QAAQ,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC5B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC3B,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzB,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACtC,2BAA2B;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7B,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC5C,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,MAAM,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa;YAC/B,aAAa;YACb,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,EACzC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EACzB,KAAK,CACN,CAAC;YACF,MAAM,EAAE,CAAC;YAET,QAAQ,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC7B,KAAK,UAAU;oBACb,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC5B,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC1B,CAAC;oBACD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC3B,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBACvC,2BAA2B;wBAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChC,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;oBAC/B,MAAM;gBAER;oBACE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;QAClC,IAAI,UAAU,IAAK,MAAM,CAAC,CAAC,CAAY,EAAE,CAAC;YACxC,MAAM,CAAC,CAAC,CAAC,GAAI,MAAM,CAAC,CAAC,CAAY,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,mGAAmG;AACnG,SAAS,qBAAqB,CAC5B,KAAmB,EACnB,UAA8B,EAC9B,QAAgB;IAEhB,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAc,CAAC;IAC/D,MAAM,aAAa,GAAG,KAAK,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAEnD,IACE,KAAK,CAAC,cAAc,KAAK,UAAU;QACnC,CAAC,aAAa;QACd,KAAK,KAAK,QAAQ;QAClB,UAAU,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EACrC,CAAC;QACD,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;QACpD,IAAI,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YACzF,UAAU,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAQA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,qBAAkB;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAQA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,qBAAkB;AAUjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAwMrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,GAAG,OAAO,CAM5F;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,OAUpF"}
|