@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/bundled.d.ts +9 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +10 -0
- package/dist/bundled.js.map +1 -0
- package/dist/geoparquet-loader-with-parser.d.ts +33 -0
- package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
- package/dist/geoparquet-loader-with-parser.js +52 -0
- package/dist/geoparquet-loader-with-parser.js.map +1 -0
- package/dist/geoparquet-loader.d.ts +59 -0
- package/dist/geoparquet-loader.d.ts.map +1 -0
- package/dist/geoparquet-loader.js +30 -0
- package/dist/geoparquet-loader.js.map +1 -0
- package/dist/index.cjs +25873 -4776
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -11
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-columns-to-row-group.js +3 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +86 -26
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.js +16 -7
- package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts +8 -0
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts.map +1 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js +396 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -0
- package/dist/lib/constants.d.ts +0 -1
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +1 -2
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js +199 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
- package/dist/lib/geo/geospatial-metadata.js +338 -0
- package/dist/lib/geo/geospatial-metadata.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +62 -0
- package/dist/lib/parquet-page-index.d.ts.map +1 -0
- package/dist/lib/parquet-page-index.js +377 -0
- package/dist/lib/parquet-page-index.js.map +1 -0
- package/dist/lib/parquet-predicate.d.ts +17 -0
- package/dist/lib/parquet-predicate.d.ts.map +1 -0
- package/dist/lib/parquet-predicate.js +300 -0
- package/dist/lib/parquet-predicate.js.map +1 -0
- package/dist/lib/parquet-source-worker-client.d.ts +6 -0
- package/dist/lib/parquet-source-worker-client.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-client.js +22 -0
- package/dist/lib/parquet-source-worker-client.js.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.js +115 -0
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -0
- package/dist/lib/parquet-source-worker-file.d.ts +24 -0
- package/dist/lib/parquet-source-worker-file.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-file.js +40 -0
- package/dist/lib/parquet-source-worker-file.js.map +1 -0
- package/dist/lib/parquet-source-worker-types.d.ts +97 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-types.js +5 -0
- package/dist/lib/parquet-source-worker-types.js.map +1 -0
- package/dist/lib/parquet-worker-transport.d.ts +8 -0
- package/dist/lib/parquet-worker-transport.d.ts.map +1 -0
- package/dist/lib/parquet-worker-transport.js +40 -0
- package/dist/lib/parquet-worker-transport.js.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts +16 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.js +30 -0
- package/dist/lib/parsers/convert-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-tables.d.ts +37 -0
- package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-tables.js +50 -0
- package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +19 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +608 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -4
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +77 -26
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts +80 -0
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -0
- package/dist/lib/sources/parquet-range-file.js +261 -0
- package/dist/lib/sources/parquet-range-file.js.map +1 -0
- package/dist/lib/sources/single-batch-queue.d.ts +24 -0
- package/dist/lib/sources/single-batch-queue.d.ts.map +1 -0
- package/dist/lib/sources/single-batch-queue.js +91 -0
- package/dist/lib/sources/single-batch-queue.js.map +1 -0
- package/dist/lib/utils/load-wasm-node.d.ts +3 -0
- package/dist/lib/utils/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/utils/load-wasm-node.js +23 -0
- package/dist/lib/utils/load-wasm-node.js.map +1 -0
- package/dist/lib/utils/load-wasm.d.ts +2 -2
- package/dist/lib/utils/load-wasm.d.ts.map +1 -1
- package/dist/lib/utils/load-wasm.js +28 -12
- package/dist/lib/utils/load-wasm.js.map +1 -1
- package/dist/lib/utils/make-stream-iterator.d.ts +2 -0
- package/dist/lib/utils/make-stream-iterator.d.ts.map +1 -1
- package/dist/lib/utils/make-stream-iterator.js +123 -14
- package/dist/lib/utils/make-stream-iterator.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
- package/dist/lib/utils/normalize-parquet-options.js +11 -0
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
- package/dist/parquet-dataset-source.cjs +26413 -0
- package/dist/parquet-dataset-source.cjs.map +6 -0
- package/dist/parquet-dataset-source.d.ts +54 -0
- package/dist/parquet-dataset-source.d.ts.map +1 -0
- package/dist/parquet-dataset-source.js +380 -0
- package/dist/parquet-dataset-source.js.map +1 -0
- package/dist/parquet-format.d.ts +2 -0
- package/dist/parquet-format.d.ts.map +1 -1
- package/dist/parquet-format.js +2 -0
- package/dist/parquet-format.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +57 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -0
- package/dist/parquet-js-loader-types.js +33 -0
- package/dist/parquet-js-loader-types.js.map +1 -0
- package/dist/parquet-js-loader.d.ts +32 -0
- package/dist/parquet-js-loader.d.ts.map +1 -0
- package/dist/parquet-js-loader.js +58 -0
- package/dist/parquet-js-loader.js.map +1 -0
- package/dist/parquet-js-writer.d.ts +44 -0
- package/dist/parquet-js-writer.d.ts.map +1 -0
- package/dist/parquet-js-writer.js +28 -0
- package/dist/parquet-js-writer.js.map +1 -0
- package/dist/parquet-loader-base.d.ts +25 -0
- package/dist/parquet-loader-base.d.ts.map +1 -0
- package/dist/parquet-loader-base.js +23 -0
- package/dist/parquet-loader-base.js.map +1 -0
- package/dist/parquet-loader-options.d.ts +44 -0
- package/dist/parquet-loader-options.d.ts.map +1 -0
- package/dist/parquet-loader-options.js +10 -0
- package/dist/parquet-loader-options.js.map +1 -0
- package/dist/parquet-loader-types.d.ts +61 -0
- package/dist/parquet-loader-types.d.ts.map +1 -0
- package/dist/parquet-loader-types.js +17 -0
- package/dist/parquet-loader-types.js.map +1 -0
- package/dist/parquet-loader.d.ts +42 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +68 -0
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-source-capabilities.d.ts +36 -0
- package/dist/parquet-source-capabilities.d.ts.map +1 -0
- package/dist/parquet-source-capabilities.js +22 -0
- package/dist/parquet-source-capabilities.js.map +1 -0
- package/dist/parquet-source-loader-types.d.ts +72 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -0
- package/dist/parquet-source-loader-types.js +74 -0
- package/dist/parquet-source-loader-types.js.map +1 -0
- package/dist/parquet-source-loader.cjs +25975 -0
- package/dist/parquet-source-loader.cjs.map +6 -0
- package/dist/parquet-source-loader.d.ts +120 -0
- package/dist/parquet-source-loader.d.ts.map +1 -0
- package/dist/parquet-source-loader.js +812 -0
- package/dist/parquet-source-loader.js.map +1 -0
- package/dist/parquet-source-types.d.ts +384 -0
- package/dist/parquet-source-types.d.ts.map +1 -0
- package/dist/parquet-source-types.js +5 -0
- package/dist/parquet-source-types.js.map +1 -0
- package/dist/parquet-source-worker-url.d.ts +3 -0
- package/dist/parquet-source-worker-url.d.ts.map +1 -0
- package/dist/parquet-source-worker-url.js +22 -0
- package/dist/parquet-source-worker-url.js.map +1 -0
- package/dist/parquet-source-worker.js +38491 -0
- package/dist/parquet-source-worker.js.map +7 -0
- package/dist/parquet-worker-node.cjs +54779 -0
- package/dist/parquet-worker-node.cjs.map +7 -0
- package/dist/parquet-worker-url.d.ts +3 -0
- package/dist/parquet-worker-url.d.ts.map +1 -0
- package/dist/parquet-worker-url.js +15 -0
- package/dist/parquet-worker-url.js.map +1 -0
- package/dist/parquet-worker.js +19920 -0
- package/dist/parquet-worker.js.map +7 -0
- package/dist/parquet-writer.d.ts +10 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +27 -0
- package/dist/parquet-writer.js.map +1 -0
- package/dist/parquet_wasm_bg.wasm +0 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts +9 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts.map +1 -0
- package/dist/parquetjs/codecs/byte-stream-split.js +159 -0
- package/dist/parquetjs/codecs/byte-stream-split.js.map +1 -0
- package/dist/parquetjs/codecs/declare.d.ts +20 -3
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +7 -1
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts +17 -0
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -0
- package/dist/parquetjs/codecs/delta.js +439 -0
- package/dist/parquetjs/codecs/delta.js.map +1 -0
- package/dist/parquetjs/codecs/dictionary.d.ts +5 -2
- package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +16 -4
- package/dist/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +18 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +3 -3
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +88 -76
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +3 -3
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +196 -68
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +7 -8
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +95 -141
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts +17 -0
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/dictionary-planner.js +64 -0
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -0
- package/dist/parquetjs/encoder/page-planner.d.ts +11 -0
- package/dist/parquetjs/encoder/page-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/page-planner.js +60 -0
- package/dist/parquetjs/encoder/page-planner.js.map +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +11 -4
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +259 -52
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +6 -6
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +3 -3
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts +8 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js +14 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts +2 -1
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +2 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js +27 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts +14 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js +51 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js +41 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +17 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +97 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js +27 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts +14 -10
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +38 -2
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +5 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +25 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js +41 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -2
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -2
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +14 -3
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +381 -117
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +35 -10
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +179 -59
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +62 -8
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts +3 -0
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +21 -3
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts +2 -0
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +71 -23
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +272 -51
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/utils/binary-utils.d.ts +41 -0
- package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/binary-utils.js +106 -0
- package/dist/parquetjs/utils/binary-utils.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.d.ts +1 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +3 -3
- package/dist/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts +14 -6
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +46 -39
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/parquetjs/utils/thrift-runtime.d.ts +106 -0
- package/dist/parquetjs/utils/thrift-runtime.d.ts.map +1 -0
- package/dist/parquetjs/utils/thrift-runtime.js +50 -0
- package/dist/parquetjs/utils/thrift-runtime.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +80 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
- package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
- package/dist/polyfills/process.d.ts +12 -0
- package/dist/polyfills/process.d.ts.map +1 -0
- package/dist/polyfills/process.js +17 -0
- package/dist/polyfills/process.js.map +1 -0
- package/dist/unbundled.d.ts +9 -0
- package/dist/unbundled.d.ts.map +1 -0
- package/dist/unbundled.js +10 -0
- package/dist/unbundled.js.map +1 -0
- package/dist/wasm.d.ts +3 -0
- package/dist/wasm.d.ts.map +1 -0
- package/dist/wasm.js +6 -0
- package/dist/wasm.js.map +1 -0
- package/dist/workers/parquet-source-worker.d.ts +2 -0
- package/dist/workers/parquet-source-worker.d.ts.map +1 -0
- package/dist/workers/parquet-source-worker.js +17 -0
- package/dist/workers/parquet-source-worker.js.map +1 -0
- package/dist/workers/parquet-worker-node.d.ts +2 -0
- package/dist/workers/parquet-worker-node.d.ts.map +1 -0
- package/dist/workers/parquet-worker-node.js +5 -0
- package/dist/workers/parquet-worker-node.js.map +1 -0
- package/dist/workers/parquet-worker.d.ts +2 -0
- package/dist/workers/parquet-worker.d.ts.map +1 -0
- package/dist/workers/parquet-worker.js +27 -0
- package/dist/workers/parquet-worker.js.map +1 -0
- package/package.json +63 -33
- package/src/bundled.ts +33 -0
- package/src/geoparquet-loader-with-parser.ts +95 -0
- package/src/geoparquet-loader.ts +54 -0
- package/src/index.ts +49 -23
- package/src/lib/arrow/convert-columns-to-row-group.ts +3 -0
- package/src/lib/arrow/convert-schema-from-parquet.ts +97 -30
- package/src/lib/arrow/convert-schema-to-parquet.ts +16 -7
- package/src/lib/arrow/create-nested-arrow-vector.ts +589 -0
- package/src/lib/constants.ts +0 -2
- package/src/lib/encoders/encode-arrow-to-parquet.ts +10 -16
- package/src/lib/encoders/encode-table-to-parquet-js.ts +261 -0
- package/src/lib/geo/geospatial-metadata.ts +457 -0
- package/src/lib/parquet-page-index.ts +581 -0
- package/src/lib/parquet-predicate.ts +374 -0
- package/src/lib/parquet-source-worker-client.ts +39 -0
- package/src/lib/parquet-source-worker-decoder.ts +156 -0
- package/src/lib/parquet-source-worker-file.ts +56 -0
- package/src/lib/parquet-source-worker-types.ts +107 -0
- package/src/lib/parquet-worker-transport.ts +57 -0
- package/src/lib/parsers/convert-parquet-tables.ts +39 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
- package/src/lib/parsers/parse-parquet-tables.ts +82 -0
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +829 -0
- package/src/lib/parsers/parse-parquet-to-arrow.ts +63 -36
- package/src/lib/parsers/parse-parquet-to-columns.ts +6 -7
- package/src/lib/parsers/parse-parquet-to-json.ts +101 -33
- package/src/lib/sources/parquet-range-file.ts +359 -0
- package/src/lib/sources/single-batch-queue.ts +95 -0
- package/src/lib/utils/load-wasm-node.ts +25 -0
- package/src/lib/utils/load-wasm.ts +36 -13
- package/src/lib/utils/make-stream-iterator.ts +126 -14
- package/src/lib/utils/normalize-parquet-options.ts +14 -0
- package/src/parquet-dataset-source.ts +488 -0
- package/src/parquet-format.ts +2 -0
- package/src/parquet-js-loader-types.ts +52 -0
- package/src/parquet-js-loader.ts +83 -0
- package/src/parquet-js-writer.ts +68 -0
- package/src/parquet-loader-base.ts +34 -0
- package/src/parquet-loader-options.ts +51 -0
- package/src/parquet-loader-types.ts +35 -0
- package/src/parquet-loader.ts +115 -0
- package/src/parquet-source-capabilities.ts +56 -0
- package/src/parquet-source-loader-types.ts +87 -0
- package/src/parquet-source-loader.ts +1265 -0
- package/src/parquet-source-types.ts +458 -0
- package/src/parquet-source-worker-url.ts +23 -0
- package/src/parquet-worker-url.ts +18 -0
- package/src/parquet-writer.ts +42 -0
- package/src/parquetjs/LICENSE +23 -1
- package/src/parquetjs/codecs/byte-stream-split.ts +221 -0
- package/src/parquetjs/codecs/declare.ts +35 -3
- package/src/parquetjs/codecs/delta.ts +587 -0
- package/src/parquetjs/codecs/dictionary.ts +22 -4
- package/src/parquetjs/codecs/index.ts +18 -0
- package/src/parquetjs/codecs/plain.ts +149 -87
- package/src/parquetjs/codecs/rle.ts +268 -82
- package/src/parquetjs/compression.ts +110 -155
- package/src/parquetjs/encoder/dictionary-planner.ts +99 -0
- package/src/parquetjs/encoder/page-planner.ts +83 -0
- package/src/parquetjs/encoder/parquet-encoder.ts +378 -77
- package/src/parquetjs/parquet-thrift/BsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +2 -2
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +13 -13
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +2 -2
- package/src/parquetjs/parquet-thrift/ColumnOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeaderV2.ts +1 -1
- package/src/parquetjs/parquet-thrift/DateType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DecimalType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DictionaryPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.ts +13 -0
- package/src/parquetjs/parquet-thrift/Encoding.ts +2 -1
- package/src/parquetjs/parquet-thrift/EnumType.ts +1 -1
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +2 -2
- package/src/parquetjs/parquet-thrift/Float16Type.ts +28 -0
- package/src/parquetjs/parquet-thrift/GeographyType.ts +54 -0
- package/src/parquetjs/parquet-thrift/GeometryType.ts +43 -0
- package/src/parquetjs/parquet-thrift/IndexPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/IntType.ts +1 -1
- package/src/parquetjs/parquet-thrift/JsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/KeyValue.ts +1 -1
- package/src/parquetjs/parquet-thrift/ListType.ts +1 -1
- package/src/parquetjs/parquet-thrift/LogicalType.ts +97 -1
- package/src/parquetjs/parquet-thrift/MapType.ts +1 -1
- package/src/parquetjs/parquet-thrift/MicroSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/MilliSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/NanoSeconds.ts +28 -0
- package/src/parquetjs/parquet-thrift/NullType.ts +1 -1
- package/src/parquetjs/parquet-thrift/OffsetIndex.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageEncodingStats.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageLocation.ts +2 -2
- package/src/parquetjs/parquet-thrift/RowGroup.ts +2 -2
- package/src/parquetjs/parquet-thrift/SchemaElement.ts +1 -1
- package/src/parquetjs/parquet-thrift/SortingColumn.ts +1 -1
- package/src/parquetjs/parquet-thrift/Statistics.ts +50 -14
- package/src/parquetjs/parquet-thrift/StringType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeUnit.ts +25 -1
- package/src/parquetjs/parquet-thrift/TimestampType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TypeDefinedOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/UUIDType.ts +1 -1
- package/src/parquetjs/parquet-thrift/VariantType.ts +43 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -4
- package/src/parquetjs/parser/decoders.ts +487 -133
- package/src/parquetjs/parser/parquet-reader.ts +297 -71
- package/src/parquetjs/schema/declare.ts +99 -7
- package/src/parquetjs/schema/schema.ts +23 -3
- package/src/parquetjs/schema/shred.ts +112 -35
- package/src/parquetjs/schema/types.ts +294 -61
- package/src/parquetjs/utils/binary-utils.ts +132 -0
- package/src/parquetjs/utils/file-utils.ts +4 -4
- package/src/parquetjs/utils/read-utils.ts +56 -55
- package/src/parquetjs/utils/thrift-runtime.ts +109 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
- package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
- package/src/polyfills/process.ts +17 -0
- package/src/unbundled.ts +33 -0
- package/src/wasm.ts +6 -0
- package/src/workers/parquet-source-worker.ts +21 -0
- package/src/workers/parquet-worker-node.ts +5 -0
- package/src/workers/parquet-worker.ts +31 -0
- package/dist/parquet-arrow-loader.d.ts +0 -43
- package/dist/parquet-arrow-loader.d.ts.map +0 -1
- package/dist/parquet-arrow-loader.js +0 -49
- package/dist/parquet-arrow-loader.js.map +0 -1
- package/dist/parquet-arrow-writer.d.ts +0 -9
- package/dist/parquet-arrow-writer.d.ts.map +0 -1
- package/dist/parquet-arrow-writer.js +0 -23
- package/dist/parquet-arrow-writer.js.map +0 -1
- package/dist/parquet-json-loader.d.ts +0 -148
- package/dist/parquet-json-loader.d.ts.map +0 -1
- package/dist/parquet-json-loader.js +0 -106
- package/dist/parquet-json-loader.js.map +0 -1
- package/dist/parquet-json-writer.d.ts +0 -19
- package/dist/parquet-json-writer.d.ts.map +0 -1
- package/dist/parquet-json-writer.js +0 -20
- package/dist/parquet-json-writer.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.d.ts +0 -222
- package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer.js +0 -1868
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.d.ts +0 -4
- package/dist/polyfills/buffer/index.d.ts.map +0 -1
- package/dist/polyfills/buffer/index.js +0 -9
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/workers/parquet-json-worker.d.ts +0 -2
- package/dist/workers/parquet-json-worker.d.ts.map +0 -1
- package/dist/workers/parquet-json-worker.js +0 -7
- package/dist/workers/parquet-json-worker.js.map +0 -1
- package/src/parquet-arrow-loader.ts +0 -74
- package/src/parquet-arrow-writer.ts +0 -33
- package/src/parquet-json-loader.ts +0 -160
- package/src/parquet-json-writer.ts +0 -27
- package/src/parquetjs/modules.d.ts +0 -25
- package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
- package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
- package/src/polyfills/buffer/buffer.ts +0 -2205
- package/src/polyfills/buffer/index.ts +0 -11
- package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
- package/src/workers/parquet-json-worker.ts +0 -8
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Loader} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {
|
|
7
|
+
ArrowTable,
|
|
8
|
+
ArrowTableBatch,
|
|
9
|
+
ObjectRowTable,
|
|
10
|
+
ObjectRowTableBatch
|
|
11
|
+
} from '@loaders.gl/schema';
|
|
12
|
+
|
|
13
|
+
import {ParquetFormat} from './parquet-format';
|
|
14
|
+
import type {ParquetJSLoaderOptions} from './parquet-loader-options';
|
|
15
|
+
|
|
16
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
17
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
18
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
19
|
+
|
|
20
|
+
/** Default option bag for the experimental parquetjs plain-row loader. */
|
|
21
|
+
const DEFAULT_PARQUET_JS_OPTIONS = {
|
|
22
|
+
columns: undefined,
|
|
23
|
+
preserveBinary: false,
|
|
24
|
+
shape: 'object-row-table' as const
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** Preloads the parser-bearing TypeScript Parquet loader implementation. */
|
|
28
|
+
async function preloadParquetJSLoader() {
|
|
29
|
+
const {ParquetJSLoaderWithParser} = await import('@loaders.gl/parquet/parquet-js-loader');
|
|
30
|
+
return ParquetJSLoaderWithParser;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Metadata-only Parquet loader backed by the experimental TypeScript implementation. */
|
|
34
|
+
export const ParquetJSLoader = {
|
|
35
|
+
...ParquetFormat,
|
|
36
|
+
|
|
37
|
+
dataType: null as unknown as ObjectRowTable | ArrowTable,
|
|
38
|
+
batchType: null as unknown as ObjectRowTableBatch | ArrowTableBatch,
|
|
39
|
+
|
|
40
|
+
id: 'parquet-js',
|
|
41
|
+
module: 'parquet',
|
|
42
|
+
version: VERSION,
|
|
43
|
+
worker: false,
|
|
44
|
+
options: {
|
|
45
|
+
parquet: DEFAULT_PARQUET_JS_OPTIONS
|
|
46
|
+
},
|
|
47
|
+
preload: preloadParquetJSLoader
|
|
48
|
+
} as const satisfies Loader<
|
|
49
|
+
ObjectRowTable | ArrowTable,
|
|
50
|
+
ObjectRowTableBatch | ArrowTableBatch,
|
|
51
|
+
ParquetJSLoaderOptions
|
|
52
|
+
>;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {LoaderWithParser, ReadableFile} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {ArrayBufferFile, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
7
|
+
import type {
|
|
8
|
+
ArrowTable,
|
|
9
|
+
ArrowTableBatch,
|
|
10
|
+
ObjectRowTable,
|
|
11
|
+
ObjectRowTableBatch
|
|
12
|
+
} from '@loaders.gl/schema';
|
|
13
|
+
import {parseParquetFile, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-json';
|
|
14
|
+
import {
|
|
15
|
+
parseParquetFileToArrowInBatchesWithJs,
|
|
16
|
+
parseParquetFileToArrowWithJs
|
|
17
|
+
} from './lib/parsers/parse-parquet-to-arrow-js';
|
|
18
|
+
import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
|
|
19
|
+
import {ParquetJSLoader as ParquetJSLoaderMetadata} from './parquet-js-loader-types';
|
|
20
|
+
import type {ParquetJSLoaderOptions} from './parquet-loader-options';
|
|
21
|
+
|
|
22
|
+
const {preload: _ParquetJSLoaderPreload, ...ParquetJSLoaderMetadataWithoutPreload} =
|
|
23
|
+
ParquetJSLoaderMetadata;
|
|
24
|
+
|
|
25
|
+
/** Default option bag for the experimental parquetjs plain-row loader. */
|
|
26
|
+
const DEFAULT_PARQUET_JS_OPTIONS = {
|
|
27
|
+
columns: undefined,
|
|
28
|
+
preserveBinary: false
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** Parser-bearing TypeScript-only Parquet loader implementation. */
|
|
32
|
+
export const ParquetJSLoaderWithParser = {
|
|
33
|
+
...ParquetJSLoaderMetadataWithoutPreload,
|
|
34
|
+
worker: false,
|
|
35
|
+
parse(arrayBuffer: ArrayBuffer, options?: ParquetJSLoaderOptions) {
|
|
36
|
+
const parquetOptions = getParquetOptions(options);
|
|
37
|
+
const file = new ArrayBufferFile(arrayBuffer);
|
|
38
|
+
return parquetOptions.parquet?.shape === 'arrow-table'
|
|
39
|
+
? parseParquetFileToArrowWithJs(file, parquetOptions)
|
|
40
|
+
: parseParquetFile(file, parquetOptions);
|
|
41
|
+
},
|
|
42
|
+
parseFile(file: ReadableFile, options?: ParquetJSLoaderOptions) {
|
|
43
|
+
const parquetOptions = getParquetOptions(options);
|
|
44
|
+
return parquetOptions.parquet?.shape === 'arrow-table'
|
|
45
|
+
? parseParquetFileToArrowWithJs(file, parquetOptions)
|
|
46
|
+
: parseParquetFile(file, parquetOptions);
|
|
47
|
+
},
|
|
48
|
+
parseFileInBatches(file: ReadableFile, options?: ParquetJSLoaderOptions) {
|
|
49
|
+
const parquetOptions = getParquetOptions(options);
|
|
50
|
+
return parquetOptions.parquet?.shape === 'arrow-table'
|
|
51
|
+
? parseParquetFileToArrowInBatchesWithJs(file, parquetOptions)
|
|
52
|
+
: parseParquetFileInBatches(file, parquetOptions);
|
|
53
|
+
},
|
|
54
|
+
async *parseInBatches(
|
|
55
|
+
asyncIterator:
|
|
56
|
+
| AsyncIterable<ArrayBufferLike | ArrayBufferView>
|
|
57
|
+
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
58
|
+
options?: ParquetJSLoaderOptions,
|
|
59
|
+
_context?: unknown
|
|
60
|
+
) {
|
|
61
|
+
const arrayBuffer = await concatenateArrayBuffersAsync(asyncIterator);
|
|
62
|
+
const parquetOptions = getParquetOptions(options);
|
|
63
|
+
const file = new ArrayBufferFile(arrayBuffer);
|
|
64
|
+
if (parquetOptions.parquet?.shape === 'arrow-table') {
|
|
65
|
+
yield* parseParquetFileToArrowInBatchesWithJs(file, parquetOptions);
|
|
66
|
+
} else {
|
|
67
|
+
yield* parseParquetFileInBatches(file, parquetOptions);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
} as const satisfies LoaderWithParser<
|
|
71
|
+
ObjectRowTable | ArrowTable,
|
|
72
|
+
ObjectRowTableBatch | ArrowTableBatch,
|
|
73
|
+
ParquetJSLoaderOptions
|
|
74
|
+
>;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Normalizes caller options for the TypeScript-backed Parquet loader.
|
|
78
|
+
* @param options caller-supplied loader options
|
|
79
|
+
* @returns normalized options with TypeScript defaults applied
|
|
80
|
+
*/
|
|
81
|
+
function getParquetOptions(options?: ParquetJSLoaderOptions): ParquetJSLoaderOptions {
|
|
82
|
+
return normalizeParquetOptions(options, DEFAULT_PARQUET_JS_OPTIONS);
|
|
83
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {WriterOptions, WriterWithEncoder} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {ObjectRowTable, Table, TableBatch} from '@loaders.gl/schema';
|
|
7
|
+
import {convertTable, deduceTableSchema} from '@loaders.gl/schema-utils';
|
|
8
|
+
|
|
9
|
+
import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
|
|
10
|
+
import {encodeTableToParquetJs} from './lib/encoders/encode-table-to-parquet-js';
|
|
11
|
+
import {ParquetFormat} from './parquet-format';
|
|
12
|
+
|
|
13
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
14
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
15
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
16
|
+
|
|
17
|
+
/** Stable value encodings selectable by the TypeScript Parquet writer. */
|
|
18
|
+
export type ParquetJSWriterEncoding =
|
|
19
|
+
| 'PLAIN'
|
|
20
|
+
| 'BYTE_STREAM_SPLIT'
|
|
21
|
+
| 'DELTA_BINARY_PACKED'
|
|
22
|
+
| 'DELTA_LENGTH_BYTE_ARRAY'
|
|
23
|
+
| 'DELTA_BYTE_ARRAY';
|
|
24
|
+
|
|
25
|
+
/** Encoder-specific options for the experimental parquetjs writer. */
|
|
26
|
+
type ParquetJSWriterEncoderOptions = {
|
|
27
|
+
/** Value encoding overrides keyed by top-level column name. */
|
|
28
|
+
columnEncodings?: Record<string, ParquetJSWriterEncoding>;
|
|
29
|
+
/** Enables dictionary pages globally when forced or size-beneficial. */
|
|
30
|
+
dictionary?: boolean | 'auto';
|
|
31
|
+
/** Dictionary policy overrides keyed by top-level column name. */
|
|
32
|
+
columnDictionaries?: Record<string, boolean | 'auto'>;
|
|
33
|
+
/** Maximum uncompressed PLAIN dictionary payload per column chunk. */
|
|
34
|
+
dictionaryPageSizeLimit?: number;
|
|
35
|
+
rowGroupSize?: number;
|
|
36
|
+
pageSize?: number;
|
|
37
|
+
useDataPageV2?: boolean;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/** Public options for the parquetjs-backed plain-table writer. */
|
|
41
|
+
export type ParquetJSWriterOptions = WriterOptions & {
|
|
42
|
+
parquet?: {
|
|
43
|
+
[Key in keyof ParquetJSWriterEncoderOptions]?: ParquetJSWriterEncoderOptions[Key];
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/** Default option bag for the experimental parquetjs writer. */
|
|
48
|
+
const DEFAULT_PARQUET_JS_OPTIONS = {};
|
|
49
|
+
|
|
50
|
+
/** Plain-row Parquet writer backed by the experimental parquetjs implementation. */
|
|
51
|
+
export const ParquetJSWriter = {
|
|
52
|
+
...ParquetFormat,
|
|
53
|
+
id: 'parquet-js',
|
|
54
|
+
module: 'parquet',
|
|
55
|
+
version: VERSION,
|
|
56
|
+
options: {
|
|
57
|
+
parquet: DEFAULT_PARQUET_JS_OPTIONS
|
|
58
|
+
},
|
|
59
|
+
async encode(table: Table, options?: ParquetJSWriterOptions) {
|
|
60
|
+
const schema = table.schema || deduceTableSchema(table);
|
|
61
|
+
const objectRowTable = convertTable({...table, schema}, 'object-row-table') as ObjectRowTable;
|
|
62
|
+
return await encodeTableToParquetJs(
|
|
63
|
+
{...table, schema},
|
|
64
|
+
objectRowTable,
|
|
65
|
+
normalizeParquetOptions(options, DEFAULT_PARQUET_JS_OPTIONS)
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
} as const satisfies WriterWithEncoder<Table, TableBatch, ParquetJSWriterOptions>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
ArrowTable,
|
|
7
|
+
ArrowTableBatch,
|
|
8
|
+
ObjectRowTable,
|
|
9
|
+
ObjectRowTableBatch
|
|
10
|
+
} from '@loaders.gl/schema';
|
|
11
|
+
|
|
12
|
+
import {ParquetFormat} from './parquet-format';
|
|
13
|
+
import {PARQUET_LOADER_DEFAULT_OPTIONS} from './parquet-loader-options';
|
|
14
|
+
|
|
15
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
16
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
17
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
18
|
+
|
|
19
|
+
/** Shared Parquet loader metadata without parser preloading or worker delivery policy. */
|
|
20
|
+
export const PARQUET_LOADER_BASE = {
|
|
21
|
+
...ParquetFormat,
|
|
22
|
+
|
|
23
|
+
dataType: null as unknown as ObjectRowTable | ArrowTable,
|
|
24
|
+
batchType: null as unknown as ObjectRowTableBatch | ArrowTableBatch,
|
|
25
|
+
|
|
26
|
+
id: 'parquet',
|
|
27
|
+
module: 'parquet',
|
|
28
|
+
version: VERSION,
|
|
29
|
+
options: {
|
|
30
|
+
parquet: {
|
|
31
|
+
...PARQUET_LOADER_DEFAULT_OPTIONS
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
} as const;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {LoaderOptions} from '@loaders.gl/loader-utils';
|
|
6
|
+
|
|
7
|
+
/** Shared row-level options supported by both Parquet backends. */
|
|
8
|
+
type ParquetCommonLoaderOptions = {
|
|
9
|
+
limit?: number;
|
|
10
|
+
offset?: number;
|
|
11
|
+
batchSize?: number;
|
|
12
|
+
columns?: string[];
|
|
13
|
+
preserveBinary?: boolean;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
/** Additional options supported by the wasm-backed Parquet reader. */
|
|
17
|
+
type ParquetWasmLoaderOptions = ParquetCommonLoaderOptions & {
|
|
18
|
+
rowGroups?: number[];
|
|
19
|
+
concurrency?: number;
|
|
20
|
+
/** Cancels an active worker parse by terminating its worker. */
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
/** Overrides the packaged Parquet worker asset URL. */
|
|
23
|
+
workerUrl?: string;
|
|
24
|
+
/** Overrides the package-local parquet-wasm binary URL. */
|
|
25
|
+
wasmUrl?: string;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** Public options for the wasm-backed `ParquetLoader` and `GeoParquetLoader`. */
|
|
29
|
+
export type ParquetLoaderOptions = LoaderOptions & {
|
|
30
|
+
parquet?: {
|
|
31
|
+
shape?: 'object-row-table' | 'arrow-table';
|
|
32
|
+
} & {
|
|
33
|
+
[Key in keyof ParquetWasmLoaderOptions]?: ParquetWasmLoaderOptions[Key];
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/** Public options for the experimental parquetjs-backed `ParquetJSLoader`. */
|
|
38
|
+
export type ParquetJSLoaderOptions = LoaderOptions & {
|
|
39
|
+
parquet?: {
|
|
40
|
+
shape?: 'object-row-table' | 'arrow-table';
|
|
41
|
+
} & {
|
|
42
|
+
[Key in keyof ParquetCommonLoaderOptions]?: ParquetCommonLoaderOptions[Key];
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** Shared default option bag for the wasm-backed Parquet loaders. */
|
|
47
|
+
export const PARQUET_LOADER_DEFAULT_OPTIONS = {
|
|
48
|
+
columns: undefined,
|
|
49
|
+
preserveBinary: false,
|
|
50
|
+
shape: 'object-row-table'
|
|
51
|
+
} as const;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Loader} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {
|
|
7
|
+
ObjectRowTable,
|
|
8
|
+
ObjectRowTableBatch,
|
|
9
|
+
ArrowTable,
|
|
10
|
+
ArrowTableBatch
|
|
11
|
+
} from '@loaders.gl/schema';
|
|
12
|
+
|
|
13
|
+
import {PARQUET_LOADER_BASE} from './parquet-loader-base';
|
|
14
|
+
import type {ParquetLoaderOptions as SharedParquetLoaderOptions} from './parquet-loader-options';
|
|
15
|
+
import {PARQUET_WORKER_URL} from './parquet-worker-url';
|
|
16
|
+
|
|
17
|
+
/** Options for the Parquet loader. */
|
|
18
|
+
export type ParquetLoaderOptions = SharedParquetLoaderOptions;
|
|
19
|
+
|
|
20
|
+
/** Preloads the parser-bearing WASM Parquet loader implementation. */
|
|
21
|
+
async function preloadParquetLoader() {
|
|
22
|
+
const {ParquetLoaderWithParser} = await import('@loaders.gl/parquet/parquet-loader');
|
|
23
|
+
return ParquetLoaderWithParser;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Metadata-only Parquet table loader supporting object-row and Arrow table output. */
|
|
27
|
+
export const ParquetLoader = {
|
|
28
|
+
...PARQUET_LOADER_BASE,
|
|
29
|
+
worker: PARQUET_WORKER_URL,
|
|
30
|
+
preload: preloadParquetLoader
|
|
31
|
+
} as const satisfies Loader<
|
|
32
|
+
ObjectRowTable | ArrowTable,
|
|
33
|
+
ObjectRowTableBatch | ArrowTableBatch,
|
|
34
|
+
ParquetLoaderOptions
|
|
35
|
+
>;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {LoaderWithParser} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {BlobFile, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
7
|
+
import type {
|
|
8
|
+
ObjectRowTable,
|
|
9
|
+
ObjectRowTableBatch,
|
|
10
|
+
ArrowTable,
|
|
11
|
+
ArrowTableBatch
|
|
12
|
+
} from '@loaders.gl/schema';
|
|
13
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
convertArrowBatchToObjectRows,
|
|
17
|
+
convertArrowTableToObjectRows
|
|
18
|
+
} from './lib/parsers/convert-parquet-tables';
|
|
19
|
+
import {
|
|
20
|
+
parseParquetFileToArrow,
|
|
21
|
+
parseParquetFileToArrowInBatches
|
|
22
|
+
} from './lib/parsers/parse-parquet-to-arrow';
|
|
23
|
+
import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
|
|
24
|
+
import {
|
|
25
|
+
deserializeParquetWorkerResult,
|
|
26
|
+
serializeParquetWorkerResult
|
|
27
|
+
} from './lib/parquet-worker-transport';
|
|
28
|
+
import {ParquetLoader as ParquetLoaderMetadata} from './parquet-loader-types';
|
|
29
|
+
import type {ParquetLoaderOptions} from './parquet-loader-options';
|
|
30
|
+
|
|
31
|
+
export type {ParquetLoaderOptions} from './parquet-loader-options';
|
|
32
|
+
|
|
33
|
+
const {preload: _ParquetLoaderPreload, ...ParquetLoaderMetadataWithoutPreload} =
|
|
34
|
+
ParquetLoaderMetadata;
|
|
35
|
+
|
|
36
|
+
/** WASM-backed Parquet table loader supporting object-row and Arrow table output. */
|
|
37
|
+
export const ParquetLoaderWithParser = {
|
|
38
|
+
...ParquetLoaderMetadataWithoutPreload,
|
|
39
|
+
parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
|
|
40
|
+
return parseParquetTable(new BlobFile(arrayBuffer), options);
|
|
41
|
+
},
|
|
42
|
+
parseFile(file, options?: ParquetLoaderOptions) {
|
|
43
|
+
return parseParquetTable(file, options);
|
|
44
|
+
},
|
|
45
|
+
parseFileInBatches(file, options?: ParquetLoaderOptions) {
|
|
46
|
+
return parseParquetTableInBatches(file, options);
|
|
47
|
+
},
|
|
48
|
+
async *parseInBatches(
|
|
49
|
+
asyncIterator:
|
|
50
|
+
| AsyncIterable<ArrayBufferLike | ArrayBufferView>
|
|
51
|
+
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
52
|
+
options?: ParquetLoaderOptions,
|
|
53
|
+
_context?: unknown
|
|
54
|
+
) {
|
|
55
|
+
const arrayBuffer = await concatenateArrayBuffersAsync(asyncIterator);
|
|
56
|
+
yield* parseParquetTableInBatches(new BlobFile(arrayBuffer), options);
|
|
57
|
+
},
|
|
58
|
+
serializeWorkerResult: serializeParquetWorkerResult,
|
|
59
|
+
deserializeWorkerResult: deserializeParquetWorkerResult
|
|
60
|
+
} as const satisfies LoaderWithParser<
|
|
61
|
+
ObjectRowTable | ArrowTable,
|
|
62
|
+
ObjectRowTableBatch | ArrowTableBatch,
|
|
63
|
+
ParquetLoaderOptions
|
|
64
|
+
>;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Parses a Parquet file using the canonical WASM-backed table loader.
|
|
68
|
+
* @param file readable file abstraction
|
|
69
|
+
* @param options optional loader options
|
|
70
|
+
* @returns object-row or Arrow table output depending on `parquet.shape`
|
|
71
|
+
*/
|
|
72
|
+
async function parseParquetTable(
|
|
73
|
+
file: BlobFile | ReadableFile,
|
|
74
|
+
options?: ParquetLoaderOptions
|
|
75
|
+
): Promise<ObjectRowTable | ArrowTable> {
|
|
76
|
+
const parquetOptions = getParquetOptions(options);
|
|
77
|
+
|
|
78
|
+
if (parquetOptions.parquet?.shape === 'arrow-table') {
|
|
79
|
+
return await parseParquetFileToArrow(file, parquetOptions.parquet);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const arrowTable = await parseParquetFileToArrow(file, parquetOptions.parquet);
|
|
83
|
+
return convertArrowTableToObjectRows(arrowTable);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Parses a Parquet file into streamed table batches using the canonical WASM-backed loader.
|
|
88
|
+
* @param file readable file abstraction
|
|
89
|
+
* @param options optional loader options
|
|
90
|
+
* @returns async iterable of object-row or Arrow batches
|
|
91
|
+
*/
|
|
92
|
+
async function* parseParquetTableInBatches(
|
|
93
|
+
file: BlobFile | ReadableFile,
|
|
94
|
+
options?: ParquetLoaderOptions
|
|
95
|
+
): AsyncIterable<ObjectRowTableBatch | ArrowTableBatch> {
|
|
96
|
+
const parquetOptions = getParquetOptions(options);
|
|
97
|
+
|
|
98
|
+
if (parquetOptions.parquet?.shape === 'arrow-table') {
|
|
99
|
+
yield* parseParquetFileToArrowInBatches(file, parquetOptions.parquet);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
for await (const batch of parseParquetFileToArrowInBatches(file, parquetOptions.parquet)) {
|
|
104
|
+
yield convertArrowBatchToObjectRows(batch);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Normalizes caller options for the canonical WASM-backed Parquet loaders.
|
|
110
|
+
* @param options caller-supplied loader options
|
|
111
|
+
* @returns normalized loader options
|
|
112
|
+
*/
|
|
113
|
+
export function getParquetOptions(options?: ParquetLoaderOptions): ParquetLoaderOptions {
|
|
114
|
+
return normalizeParquetOptions(options, ParquetLoaderWithParser.options.parquet);
|
|
115
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
/** Feature support advertised by a {@link ParquetSource}. */
|
|
6
|
+
export type ParquetSourceCapabilities = Readonly<{
|
|
7
|
+
/** Schema and footer metadata are cached for the lifetime of the source. */
|
|
8
|
+
supportsCachedMetadata: boolean;
|
|
9
|
+
/** Reads can select explicit Parquet row groups. */
|
|
10
|
+
supportsRowGroupSelection: boolean;
|
|
11
|
+
/** Reads can project explicit Parquet column paths. */
|
|
12
|
+
supportsColumnProjection: boolean;
|
|
13
|
+
/** Every emitted batch identifies its source row group and row offsets. */
|
|
14
|
+
supportsBatchProvenance: boolean;
|
|
15
|
+
/** Read cancellation is propagated cooperatively through the active stream. */
|
|
16
|
+
supportsCooperativeReadCancellation: boolean;
|
|
17
|
+
/** The package includes a local parquet-wasm binary for self-hosted delivery. */
|
|
18
|
+
supportsLocalWasmAsset: boolean;
|
|
19
|
+
/** Column min/max/null statistics are exposed in public metadata. */
|
|
20
|
+
supportsColumnStatistics: boolean;
|
|
21
|
+
/** Serializable predicates can prune impossible row groups using column statistics. */
|
|
22
|
+
supportsPredicatePushdown: boolean;
|
|
23
|
+
/** Column and offset indexes can avoid irrelevant data-page range reads for flat columns. */
|
|
24
|
+
supportsPageIndexPruning: boolean;
|
|
25
|
+
/** Surviving decoded rows are filtered exactly on the caller thread or worker. */
|
|
26
|
+
supportsExactPredicateFiltering: boolean;
|
|
27
|
+
/** Callers can supply the random-access transport used by the source. */
|
|
28
|
+
supportsCustomRangeTransport: boolean;
|
|
29
|
+
/** Range requests validate that the source object version remains unchanged. */
|
|
30
|
+
supportsObjectVersionValidation: boolean;
|
|
31
|
+
/** Requested bytes, downloaded bytes, request counts, and network time are reported. */
|
|
32
|
+
supportsNetworkTelemetry: boolean;
|
|
33
|
+
/** Decoder time is reported separately from network and Arrow conversion time. */
|
|
34
|
+
supportsDecodeTelemetry: boolean;
|
|
35
|
+
/** Parquet decoding can run in a worker and transfer Arrow output. */
|
|
36
|
+
supportsWorkerDecoding: boolean;
|
|
37
|
+
}>;
|
|
38
|
+
|
|
39
|
+
/** Capabilities of the current range-backed TypeScript {@link ParquetSource}. */
|
|
40
|
+
export const PARQUET_SOURCE_CAPABILITIES: ParquetSourceCapabilities = Object.freeze({
|
|
41
|
+
supportsCachedMetadata: true,
|
|
42
|
+
supportsRowGroupSelection: true,
|
|
43
|
+
supportsColumnProjection: true,
|
|
44
|
+
supportsBatchProvenance: true,
|
|
45
|
+
supportsCooperativeReadCancellation: true,
|
|
46
|
+
supportsLocalWasmAsset: true,
|
|
47
|
+
supportsColumnStatistics: true,
|
|
48
|
+
supportsPredicatePushdown: true,
|
|
49
|
+
supportsPageIndexPruning: true,
|
|
50
|
+
supportsExactPredicateFiltering: true,
|
|
51
|
+
supportsCustomRangeTransport: true,
|
|
52
|
+
supportsObjectVersionValidation: true,
|
|
53
|
+
supportsNetworkTelemetry: true,
|
|
54
|
+
supportsDecodeTelemetry: true,
|
|
55
|
+
supportsWorkerDecoding: true
|
|
56
|
+
});
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {CoreAPI, SourceLoader} from '@loaders.gl/loader-utils';
|
|
6
|
+
|
|
7
|
+
import {ParquetFormat} from './parquet-format';
|
|
8
|
+
import type {ParquetSource} from './parquet-source-loader';
|
|
9
|
+
import type {ParquetSourceLoaderOptions} from './parquet-source-types';
|
|
10
|
+
|
|
11
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
12
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
13
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
14
|
+
|
|
15
|
+
/** Loads the runtime Parquet source implementation from its explicit package subpath. */
|
|
16
|
+
async function preloadParquetSourceLoader(): Promise<SourceLoader<ParquetSource>> {
|
|
17
|
+
const {ParquetSourceLoaderWithParser} = await import('@loaders.gl/parquet/parquet-source-loader');
|
|
18
|
+
return ParquetSourceLoaderWithParser;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Lightweight Parquet source metadata.
|
|
23
|
+
*
|
|
24
|
+
* Use with the async `load()` API, which preloads the implementation, or import the runtime source
|
|
25
|
+
* loader from `@loaders.gl/parquet/parquet-source-loader` for synchronous `createDataSource()`.
|
|
26
|
+
*/
|
|
27
|
+
export const ParquetSourceLoader = {
|
|
28
|
+
...ParquetFormat,
|
|
29
|
+
dataType: null as unknown as ParquetSource,
|
|
30
|
+
batchType: null as never,
|
|
31
|
+
name: 'ParquetSourceLoader',
|
|
32
|
+
id: 'parquet-source',
|
|
33
|
+
module: 'parquet',
|
|
34
|
+
version: VERSION,
|
|
35
|
+
type: 'parquet',
|
|
36
|
+
fromUrl: true,
|
|
37
|
+
fromBlob: true,
|
|
38
|
+
options: {
|
|
39
|
+
parquet: {
|
|
40
|
+
rowGroups: undefined,
|
|
41
|
+
columns: undefined,
|
|
42
|
+
batchSize: undefined,
|
|
43
|
+
concurrency: undefined,
|
|
44
|
+
rowGroupFilter: undefined,
|
|
45
|
+
headers: undefined,
|
|
46
|
+
preserveBinary: false,
|
|
47
|
+
onTelemetry: undefined,
|
|
48
|
+
workerUrl: undefined,
|
|
49
|
+
wasmUrl: undefined
|
|
50
|
+
},
|
|
51
|
+
rangeRequests: {
|
|
52
|
+
batchDelayMs: 0,
|
|
53
|
+
maxGapBytes: 0,
|
|
54
|
+
rangeExpansionBytes: 0
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
defaultOptions: {
|
|
58
|
+
parquet: {
|
|
59
|
+
rowGroups: undefined!,
|
|
60
|
+
columns: undefined!,
|
|
61
|
+
batchSize: undefined!,
|
|
62
|
+
concurrency: undefined!,
|
|
63
|
+
rowGroupFilter: undefined!,
|
|
64
|
+
headers: undefined!,
|
|
65
|
+
preserveBinary: false,
|
|
66
|
+
onTelemetry: undefined!,
|
|
67
|
+
workerUrl: undefined!,
|
|
68
|
+
wasmUrl: undefined!
|
|
69
|
+
},
|
|
70
|
+
rangeRequests: {
|
|
71
|
+
batchDelayMs: 0,
|
|
72
|
+
maxGapBytes: 0,
|
|
73
|
+
rangeExpansionBytes: 0
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
testURL: (url: string): boolean => /\.parquet(?:$|[?#])/i.test(url),
|
|
77
|
+
preload: preloadParquetSourceLoader,
|
|
78
|
+
createDataSource(
|
|
79
|
+
_data: string | Blob,
|
|
80
|
+
_options: ParquetSourceLoaderOptions,
|
|
81
|
+
_coreApi?: CoreAPI
|
|
82
|
+
): ParquetSource {
|
|
83
|
+
throw new Error(
|
|
84
|
+
'ParquetSourceLoader requires async load() or an explicit @loaders.gl/parquet/parquet-source-loader import'
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
} as const satisfies SourceLoader<ParquetSource>;
|