@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/bundled.d.ts +9 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +10 -0
- package/dist/bundled.js.map +1 -0
- package/dist/geoparquet-loader-with-parser.d.ts +33 -0
- package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
- package/dist/geoparquet-loader-with-parser.js +52 -0
- package/dist/geoparquet-loader-with-parser.js.map +1 -0
- package/dist/geoparquet-loader.d.ts +59 -0
- package/dist/geoparquet-loader.d.ts.map +1 -0
- package/dist/geoparquet-loader.js +30 -0
- package/dist/geoparquet-loader.js.map +1 -0
- package/dist/index.cjs +25873 -4776
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -11
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-columns-to-row-group.js +3 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +86 -26
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.js +16 -7
- package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts +8 -0
- package/dist/lib/arrow/create-nested-arrow-vector.d.ts.map +1 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js +396 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -0
- package/dist/lib/constants.d.ts +0 -1
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +1 -2
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js +199 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
- package/dist/lib/geo/geospatial-metadata.js +338 -0
- package/dist/lib/geo/geospatial-metadata.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +62 -0
- package/dist/lib/parquet-page-index.d.ts.map +1 -0
- package/dist/lib/parquet-page-index.js +377 -0
- package/dist/lib/parquet-page-index.js.map +1 -0
- package/dist/lib/parquet-predicate.d.ts +17 -0
- package/dist/lib/parquet-predicate.d.ts.map +1 -0
- package/dist/lib/parquet-predicate.js +300 -0
- package/dist/lib/parquet-predicate.js.map +1 -0
- package/dist/lib/parquet-source-worker-client.d.ts +6 -0
- package/dist/lib/parquet-source-worker-client.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-client.js +22 -0
- package/dist/lib/parquet-source-worker-client.js.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -0
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-decoder.js +115 -0
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -0
- package/dist/lib/parquet-source-worker-file.d.ts +24 -0
- package/dist/lib/parquet-source-worker-file.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-file.js +40 -0
- package/dist/lib/parquet-source-worker-file.js.map +1 -0
- package/dist/lib/parquet-source-worker-types.d.ts +97 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -0
- package/dist/lib/parquet-source-worker-types.js +5 -0
- package/dist/lib/parquet-source-worker-types.js.map +1 -0
- package/dist/lib/parquet-worker-transport.d.ts +8 -0
- package/dist/lib/parquet-worker-transport.d.ts.map +1 -0
- package/dist/lib/parquet-worker-transport.js +40 -0
- package/dist/lib/parquet-worker-transport.js.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts +16 -0
- package/dist/lib/parsers/convert-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/convert-parquet-tables.js +30 -0
- package/dist/lib/parsers/convert-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-tables.d.ts +37 -0
- package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-tables.js +50 -0
- package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +19 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +608 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -4
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +77 -26
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts +80 -0
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -0
- package/dist/lib/sources/parquet-range-file.js +261 -0
- package/dist/lib/sources/parquet-range-file.js.map +1 -0
- package/dist/lib/sources/single-batch-queue.d.ts +24 -0
- package/dist/lib/sources/single-batch-queue.d.ts.map +1 -0
- package/dist/lib/sources/single-batch-queue.js +91 -0
- package/dist/lib/sources/single-batch-queue.js.map +1 -0
- package/dist/lib/utils/load-wasm-node.d.ts +3 -0
- package/dist/lib/utils/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/utils/load-wasm-node.js +23 -0
- package/dist/lib/utils/load-wasm-node.js.map +1 -0
- package/dist/lib/utils/load-wasm.d.ts +2 -2
- package/dist/lib/utils/load-wasm.d.ts.map +1 -1
- package/dist/lib/utils/load-wasm.js +28 -12
- package/dist/lib/utils/load-wasm.js.map +1 -1
- package/dist/lib/utils/make-stream-iterator.d.ts +2 -0
- package/dist/lib/utils/make-stream-iterator.d.ts.map +1 -1
- package/dist/lib/utils/make-stream-iterator.js +123 -14
- package/dist/lib/utils/make-stream-iterator.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
- package/dist/lib/utils/normalize-parquet-options.js +11 -0
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
- package/dist/parquet-dataset-source.cjs +26413 -0
- package/dist/parquet-dataset-source.cjs.map +6 -0
- package/dist/parquet-dataset-source.d.ts +54 -0
- package/dist/parquet-dataset-source.d.ts.map +1 -0
- package/dist/parquet-dataset-source.js +380 -0
- package/dist/parquet-dataset-source.js.map +1 -0
- package/dist/parquet-format.d.ts +2 -0
- package/dist/parquet-format.d.ts.map +1 -1
- package/dist/parquet-format.js +2 -0
- package/dist/parquet-format.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +57 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -0
- package/dist/parquet-js-loader-types.js +33 -0
- package/dist/parquet-js-loader-types.js.map +1 -0
- package/dist/parquet-js-loader.d.ts +32 -0
- package/dist/parquet-js-loader.d.ts.map +1 -0
- package/dist/parquet-js-loader.js +58 -0
- package/dist/parquet-js-loader.js.map +1 -0
- package/dist/parquet-js-writer.d.ts +44 -0
- package/dist/parquet-js-writer.d.ts.map +1 -0
- package/dist/parquet-js-writer.js +28 -0
- package/dist/parquet-js-writer.js.map +1 -0
- package/dist/parquet-loader-base.d.ts +25 -0
- package/dist/parquet-loader-base.d.ts.map +1 -0
- package/dist/parquet-loader-base.js +23 -0
- package/dist/parquet-loader-base.js.map +1 -0
- package/dist/parquet-loader-options.d.ts +44 -0
- package/dist/parquet-loader-options.d.ts.map +1 -0
- package/dist/parquet-loader-options.js +10 -0
- package/dist/parquet-loader-options.js.map +1 -0
- package/dist/parquet-loader-types.d.ts +61 -0
- package/dist/parquet-loader-types.d.ts.map +1 -0
- package/dist/parquet-loader-types.js +17 -0
- package/dist/parquet-loader-types.js.map +1 -0
- package/dist/parquet-loader.d.ts +42 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +68 -0
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-source-capabilities.d.ts +36 -0
- package/dist/parquet-source-capabilities.d.ts.map +1 -0
- package/dist/parquet-source-capabilities.js +22 -0
- package/dist/parquet-source-capabilities.js.map +1 -0
- package/dist/parquet-source-loader-types.d.ts +72 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -0
- package/dist/parquet-source-loader-types.js +74 -0
- package/dist/parquet-source-loader-types.js.map +1 -0
- package/dist/parquet-source-loader.cjs +25975 -0
- package/dist/parquet-source-loader.cjs.map +6 -0
- package/dist/parquet-source-loader.d.ts +120 -0
- package/dist/parquet-source-loader.d.ts.map +1 -0
- package/dist/parquet-source-loader.js +812 -0
- package/dist/parquet-source-loader.js.map +1 -0
- package/dist/parquet-source-types.d.ts +384 -0
- package/dist/parquet-source-types.d.ts.map +1 -0
- package/dist/parquet-source-types.js +5 -0
- package/dist/parquet-source-types.js.map +1 -0
- package/dist/parquet-source-worker-url.d.ts +3 -0
- package/dist/parquet-source-worker-url.d.ts.map +1 -0
- package/dist/parquet-source-worker-url.js +22 -0
- package/dist/parquet-source-worker-url.js.map +1 -0
- package/dist/parquet-source-worker.js +38491 -0
- package/dist/parquet-source-worker.js.map +7 -0
- package/dist/parquet-worker-node.cjs +54779 -0
- package/dist/parquet-worker-node.cjs.map +7 -0
- package/dist/parquet-worker-url.d.ts +3 -0
- package/dist/parquet-worker-url.d.ts.map +1 -0
- package/dist/parquet-worker-url.js +15 -0
- package/dist/parquet-worker-url.js.map +1 -0
- package/dist/parquet-worker.js +19920 -0
- package/dist/parquet-worker.js.map +7 -0
- package/dist/parquet-writer.d.ts +10 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +27 -0
- package/dist/parquet-writer.js.map +1 -0
- package/dist/parquet_wasm_bg.wasm +0 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts +9 -0
- package/dist/parquetjs/codecs/byte-stream-split.d.ts.map +1 -0
- package/dist/parquetjs/codecs/byte-stream-split.js +159 -0
- package/dist/parquetjs/codecs/byte-stream-split.js.map +1 -0
- package/dist/parquetjs/codecs/declare.d.ts +20 -3
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +7 -1
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts +17 -0
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -0
- package/dist/parquetjs/codecs/delta.js +439 -0
- package/dist/parquetjs/codecs/delta.js.map +1 -0
- package/dist/parquetjs/codecs/dictionary.d.ts +5 -2
- package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +16 -4
- package/dist/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +18 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +3 -3
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +88 -76
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +3 -3
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +196 -68
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +7 -8
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +95 -141
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts +17 -0
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/dictionary-planner.js +64 -0
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -0
- package/dist/parquetjs/encoder/page-planner.d.ts +11 -0
- package/dist/parquetjs/encoder/page-planner.d.ts.map +1 -0
- package/dist/parquetjs/encoder/page-planner.js +60 -0
- package/dist/parquetjs/encoder/page-planner.js.map +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +11 -4
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +259 -52
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +6 -6
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +3 -3
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +2 -2
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts +8 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js +14 -0
- package/dist/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts +2 -1
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +2 -2
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js +27 -0
- package/dist/parquetjs/parquet-thrift/Float16Type.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts +14 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js +51 -0
- package/dist/parquetjs/parquet-thrift/GeographyType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js +41 -0
- package/dist/parquetjs/parquet-thrift/GeometryType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +17 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +97 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts +9 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js +27 -0
- package/dist/parquetjs/parquet-thrift/NanoSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +2 -2
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts +14 -10
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +38 -2
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +5 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +25 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts +11 -0
- package/dist/parquetjs/parquet-thrift/VariantType.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js +41 -0
- package/dist/parquetjs/parquet-thrift/VariantType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -2
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -2
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +14 -3
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +381 -117
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +35 -10
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +179 -59
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +62 -8
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts +3 -0
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +21 -3
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts +2 -0
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +71 -23
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +272 -51
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/utils/binary-utils.d.ts +41 -0
- package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/binary-utils.js +106 -0
- package/dist/parquetjs/utils/binary-utils.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.d.ts +1 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +3 -3
- package/dist/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts +14 -6
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +46 -39
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/parquetjs/utils/thrift-runtime.d.ts +106 -0
- package/dist/parquetjs/utils/thrift-runtime.d.ts.map +1 -0
- package/dist/parquetjs/utils/thrift-runtime.js +50 -0
- package/dist/parquetjs/utils/thrift-runtime.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +80 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
- package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
- package/dist/polyfills/process.d.ts +12 -0
- package/dist/polyfills/process.d.ts.map +1 -0
- package/dist/polyfills/process.js +17 -0
- package/dist/polyfills/process.js.map +1 -0
- package/dist/unbundled.d.ts +9 -0
- package/dist/unbundled.d.ts.map +1 -0
- package/dist/unbundled.js +10 -0
- package/dist/unbundled.js.map +1 -0
- package/dist/wasm.d.ts +3 -0
- package/dist/wasm.d.ts.map +1 -0
- package/dist/wasm.js +6 -0
- package/dist/wasm.js.map +1 -0
- package/dist/workers/parquet-source-worker.d.ts +2 -0
- package/dist/workers/parquet-source-worker.d.ts.map +1 -0
- package/dist/workers/parquet-source-worker.js +17 -0
- package/dist/workers/parquet-source-worker.js.map +1 -0
- package/dist/workers/parquet-worker-node.d.ts +2 -0
- package/dist/workers/parquet-worker-node.d.ts.map +1 -0
- package/dist/workers/parquet-worker-node.js +5 -0
- package/dist/workers/parquet-worker-node.js.map +1 -0
- package/dist/workers/parquet-worker.d.ts +2 -0
- package/dist/workers/parquet-worker.d.ts.map +1 -0
- package/dist/workers/parquet-worker.js +27 -0
- package/dist/workers/parquet-worker.js.map +1 -0
- package/package.json +63 -33
- package/src/bundled.ts +33 -0
- package/src/geoparquet-loader-with-parser.ts +95 -0
- package/src/geoparquet-loader.ts +54 -0
- package/src/index.ts +49 -23
- package/src/lib/arrow/convert-columns-to-row-group.ts +3 -0
- package/src/lib/arrow/convert-schema-from-parquet.ts +97 -30
- package/src/lib/arrow/convert-schema-to-parquet.ts +16 -7
- package/src/lib/arrow/create-nested-arrow-vector.ts +589 -0
- package/src/lib/constants.ts +0 -2
- package/src/lib/encoders/encode-arrow-to-parquet.ts +10 -16
- package/src/lib/encoders/encode-table-to-parquet-js.ts +261 -0
- package/src/lib/geo/geospatial-metadata.ts +457 -0
- package/src/lib/parquet-page-index.ts +581 -0
- package/src/lib/parquet-predicate.ts +374 -0
- package/src/lib/parquet-source-worker-client.ts +39 -0
- package/src/lib/parquet-source-worker-decoder.ts +156 -0
- package/src/lib/parquet-source-worker-file.ts +56 -0
- package/src/lib/parquet-source-worker-types.ts +107 -0
- package/src/lib/parquet-worker-transport.ts +57 -0
- package/src/lib/parsers/convert-parquet-tables.ts +39 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
- package/src/lib/parsers/parse-parquet-tables.ts +82 -0
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +829 -0
- package/src/lib/parsers/parse-parquet-to-arrow.ts +63 -36
- package/src/lib/parsers/parse-parquet-to-columns.ts +6 -7
- package/src/lib/parsers/parse-parquet-to-json.ts +101 -33
- package/src/lib/sources/parquet-range-file.ts +359 -0
- package/src/lib/sources/single-batch-queue.ts +95 -0
- package/src/lib/utils/load-wasm-node.ts +25 -0
- package/src/lib/utils/load-wasm.ts +36 -13
- package/src/lib/utils/make-stream-iterator.ts +126 -14
- package/src/lib/utils/normalize-parquet-options.ts +14 -0
- package/src/parquet-dataset-source.ts +488 -0
- package/src/parquet-format.ts +2 -0
- package/src/parquet-js-loader-types.ts +52 -0
- package/src/parquet-js-loader.ts +83 -0
- package/src/parquet-js-writer.ts +68 -0
- package/src/parquet-loader-base.ts +34 -0
- package/src/parquet-loader-options.ts +51 -0
- package/src/parquet-loader-types.ts +35 -0
- package/src/parquet-loader.ts +115 -0
- package/src/parquet-source-capabilities.ts +56 -0
- package/src/parquet-source-loader-types.ts +87 -0
- package/src/parquet-source-loader.ts +1265 -0
- package/src/parquet-source-types.ts +458 -0
- package/src/parquet-source-worker-url.ts +23 -0
- package/src/parquet-worker-url.ts +18 -0
- package/src/parquet-writer.ts +42 -0
- package/src/parquetjs/LICENSE +23 -1
- package/src/parquetjs/codecs/byte-stream-split.ts +221 -0
- package/src/parquetjs/codecs/declare.ts +35 -3
- package/src/parquetjs/codecs/delta.ts +587 -0
- package/src/parquetjs/codecs/dictionary.ts +22 -4
- package/src/parquetjs/codecs/index.ts +18 -0
- package/src/parquetjs/codecs/plain.ts +149 -87
- package/src/parquetjs/codecs/rle.ts +268 -82
- package/src/parquetjs/compression.ts +110 -155
- package/src/parquetjs/encoder/dictionary-planner.ts +99 -0
- package/src/parquetjs/encoder/page-planner.ts +83 -0
- package/src/parquetjs/encoder/parquet-encoder.ts +378 -77
- package/src/parquetjs/parquet-thrift/BsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +2 -2
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +13 -13
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +2 -2
- package/src/parquetjs/parquet-thrift/ColumnOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/DataPageHeaderV2.ts +1 -1
- package/src/parquetjs/parquet-thrift/DateType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DecimalType.ts +1 -1
- package/src/parquetjs/parquet-thrift/DictionaryPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/EdgeInterpolationAlgorithm.ts +13 -0
- package/src/parquetjs/parquet-thrift/Encoding.ts +2 -1
- package/src/parquetjs/parquet-thrift/EnumType.ts +1 -1
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +2 -2
- package/src/parquetjs/parquet-thrift/Float16Type.ts +28 -0
- package/src/parquetjs/parquet-thrift/GeographyType.ts +54 -0
- package/src/parquetjs/parquet-thrift/GeometryType.ts +43 -0
- package/src/parquetjs/parquet-thrift/IndexPageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/IntType.ts +1 -1
- package/src/parquetjs/parquet-thrift/JsonType.ts +1 -1
- package/src/parquetjs/parquet-thrift/KeyValue.ts +1 -1
- package/src/parquetjs/parquet-thrift/ListType.ts +1 -1
- package/src/parquetjs/parquet-thrift/LogicalType.ts +97 -1
- package/src/parquetjs/parquet-thrift/MapType.ts +1 -1
- package/src/parquetjs/parquet-thrift/MicroSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/MilliSeconds.ts +1 -1
- package/src/parquetjs/parquet-thrift/NanoSeconds.ts +28 -0
- package/src/parquetjs/parquet-thrift/NullType.ts +1 -1
- package/src/parquetjs/parquet-thrift/OffsetIndex.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageEncodingStats.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageHeader.ts +1 -1
- package/src/parquetjs/parquet-thrift/PageLocation.ts +2 -2
- package/src/parquetjs/parquet-thrift/RowGroup.ts +2 -2
- package/src/parquetjs/parquet-thrift/SchemaElement.ts +1 -1
- package/src/parquetjs/parquet-thrift/SortingColumn.ts +1 -1
- package/src/parquetjs/parquet-thrift/Statistics.ts +50 -14
- package/src/parquetjs/parquet-thrift/StringType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TimeUnit.ts +25 -1
- package/src/parquetjs/parquet-thrift/TimestampType.ts +1 -1
- package/src/parquetjs/parquet-thrift/TypeDefinedOrder.ts +1 -1
- package/src/parquetjs/parquet-thrift/UUIDType.ts +1 -1
- package/src/parquetjs/parquet-thrift/VariantType.ts +43 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -4
- package/src/parquetjs/parser/decoders.ts +487 -133
- package/src/parquetjs/parser/parquet-reader.ts +297 -71
- package/src/parquetjs/schema/declare.ts +99 -7
- package/src/parquetjs/schema/schema.ts +23 -3
- package/src/parquetjs/schema/shred.ts +112 -35
- package/src/parquetjs/schema/types.ts +294 -61
- package/src/parquetjs/utils/binary-utils.ts +132 -0
- package/src/parquetjs/utils/file-utils.ts +4 -4
- package/src/parquetjs/utils/read-utils.ts +56 -55
- package/src/parquetjs/utils/thrift-runtime.ts +109 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
- package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
- package/src/polyfills/process.ts +17 -0
- package/src/unbundled.ts +33 -0
- package/src/wasm.ts +6 -0
- package/src/workers/parquet-source-worker.ts +21 -0
- package/src/workers/parquet-worker-node.ts +5 -0
- package/src/workers/parquet-worker.ts +31 -0
- package/dist/parquet-arrow-loader.d.ts +0 -43
- package/dist/parquet-arrow-loader.d.ts.map +0 -1
- package/dist/parquet-arrow-loader.js +0 -49
- package/dist/parquet-arrow-loader.js.map +0 -1
- package/dist/parquet-arrow-writer.d.ts +0 -9
- package/dist/parquet-arrow-writer.d.ts.map +0 -1
- package/dist/parquet-arrow-writer.js +0 -23
- package/dist/parquet-arrow-writer.js.map +0 -1
- package/dist/parquet-json-loader.d.ts +0 -148
- package/dist/parquet-json-loader.d.ts.map +0 -1
- package/dist/parquet-json-loader.js +0 -106
- package/dist/parquet-json-loader.js.map +0 -1
- package/dist/parquet-json-writer.d.ts +0 -19
- package/dist/parquet-json-writer.d.ts.map +0 -1
- package/dist/parquet-json-writer.js +0 -20
- package/dist/parquet-json-writer.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.d.ts +0 -222
- package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer.js +0 -1868
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.d.ts +0 -4
- package/dist/polyfills/buffer/index.d.ts.map +0 -1
- package/dist/polyfills/buffer/index.js +0 -9
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/workers/parquet-json-worker.d.ts +0 -2
- package/dist/workers/parquet-json-worker.d.ts.map +0 -1
- package/dist/workers/parquet-json-worker.js +0 -7
- package/dist/workers/parquet-json-worker.js.map +0 -1
- package/src/parquet-arrow-loader.ts +0 -74
- package/src/parquet-arrow-writer.ts +0 -33
- package/src/parquet-json-loader.ts +0 -160
- package/src/parquet-json-writer.ts +0 -27
- package/src/parquetjs/modules.d.ts +0 -25
- package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
- package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
- package/src/polyfills/buffer/buffer.ts +0 -2205
- package/src/polyfills/buffer/index.ts +0 -11
- package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
- package/src/workers/parquet-json-worker.ts +0 -8
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
import { RangeRequestScheduler } from '@loaders.gl/loader-utils';
|
|
5
|
+
/** Strict HTTP range-backed file with Parquet object-version validation. */
|
|
6
|
+
export class ParquetRangeFile {
|
|
7
|
+
/** URL used as the file handle. */
|
|
8
|
+
handle;
|
|
9
|
+
/** URL of the remote Parquet object. */
|
|
10
|
+
url;
|
|
11
|
+
/** Fetch implementation inherited from source loader options. */
|
|
12
|
+
fetch;
|
|
13
|
+
/** Headers inherited from source loader options. */
|
|
14
|
+
headers;
|
|
15
|
+
/** Source telemetry callback invoked after transport and cache operations. */
|
|
16
|
+
onTelemetry;
|
|
17
|
+
/** Scheduler used to coalesce later row-group reads. */
|
|
18
|
+
scheduler;
|
|
19
|
+
/** Aborts all active requests when the file is closed. */
|
|
20
|
+
closeController = new AbortController();
|
|
21
|
+
/** Exact ranges already loaded during initialization. */
|
|
22
|
+
cache = new Map();
|
|
23
|
+
/** Remote object byte length discovered from `Content-Range`. */
|
|
24
|
+
fileByteLength = 0;
|
|
25
|
+
/** Object validators captured by the opening range request. */
|
|
26
|
+
version = {};
|
|
27
|
+
/** Whether `close()` has been called. */
|
|
28
|
+
closed = false;
|
|
29
|
+
/** Creates an unopened remote Parquet file. Use `open()` before reading it. */
|
|
30
|
+
constructor(url, options) {
|
|
31
|
+
this.handle = url;
|
|
32
|
+
this.url = url;
|
|
33
|
+
this.fetch = options.fetch;
|
|
34
|
+
this.headers = options.headers;
|
|
35
|
+
this.onTelemetry = options.onTelemetry;
|
|
36
|
+
this.scheduler =
|
|
37
|
+
options.rangeRequests?.scheduler ||
|
|
38
|
+
new RangeRequestScheduler({
|
|
39
|
+
batchDelayMs: options.rangeRequests?.batchDelayMs ?? 0,
|
|
40
|
+
maxGapBytes: options.rangeRequests?.maxGapBytes,
|
|
41
|
+
rangeExpansionBytes: options.rangeRequests?.rangeExpansionBytes,
|
|
42
|
+
maxMergedBytes: options.rangeRequests?.maxMergedBytes,
|
|
43
|
+
stats: options.rangeRequests?.stats,
|
|
44
|
+
onEvent: options.rangeRequests?.onEvent
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
/** Total remote object byte length after `open()`. */
|
|
48
|
+
get size() {
|
|
49
|
+
return this.fileByteLength;
|
|
50
|
+
}
|
|
51
|
+
/** Total remote object byte length as a bigint after `open()`. */
|
|
52
|
+
get bigsize() {
|
|
53
|
+
return BigInt(this.fileByteLength);
|
|
54
|
+
}
|
|
55
|
+
/** Immutable copy of the HTTP validators captured when the object was opened. */
|
|
56
|
+
get objectVersion() {
|
|
57
|
+
return { ...this.version };
|
|
58
|
+
}
|
|
59
|
+
/** Opens the object with a four-byte range probe and caches the Parquet header. */
|
|
60
|
+
async open(signal) {
|
|
61
|
+
this.assertOpen();
|
|
62
|
+
const result = await this.fetchExactRange(0, 4, signal, false);
|
|
63
|
+
this.fileByteLength = result.contentRange.total;
|
|
64
|
+
this.version = getObjectVersion(result.response.headers);
|
|
65
|
+
this.cache.set(getRangeKey(0, 4), result.arrayBuffer);
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
/** Reads an exact byte range without downloading the complete object. */
|
|
69
|
+
async read(start = 0, length = this.size - Number(start), signal) {
|
|
70
|
+
this.assertOpen();
|
|
71
|
+
const offset = Number(start);
|
|
72
|
+
assertValidRange(offset, length, this.size);
|
|
73
|
+
if (length === 0) {
|
|
74
|
+
return new ArrayBuffer(0);
|
|
75
|
+
}
|
|
76
|
+
const cached = this.cache.get(getRangeKey(offset, length));
|
|
77
|
+
if (cached) {
|
|
78
|
+
this.onTelemetry?.({ type: 'cache-hit', cacheHits: 1 });
|
|
79
|
+
return cached.slice(0);
|
|
80
|
+
}
|
|
81
|
+
const abortContext = createCombinedAbortSignal(signal, this.closeController.signal);
|
|
82
|
+
try {
|
|
83
|
+
return await this.scheduler.scheduleRequest({
|
|
84
|
+
sourceId: getVersionedSourceId(this.url, this.version),
|
|
85
|
+
offset,
|
|
86
|
+
length,
|
|
87
|
+
signal: abortContext.signal,
|
|
88
|
+
fetchRange: async (transportOffset, transportLength, transportSignal) => {
|
|
89
|
+
const result = await this.fetchExactRange(transportOffset, transportLength, transportSignal, true);
|
|
90
|
+
return {
|
|
91
|
+
arrayBuffer: result.arrayBuffer,
|
|
92
|
+
status: result.response.status,
|
|
93
|
+
transportBytes: result.arrayBuffer.byteLength
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
finally {
|
|
99
|
+
abortContext.dispose();
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/** Returns file length information without another HTTP request. */
|
|
103
|
+
async stat() {
|
|
104
|
+
this.assertOpen();
|
|
105
|
+
return { size: this.size, bigsize: this.bigsize, isDirectory: false };
|
|
106
|
+
}
|
|
107
|
+
/** Aborts active requests and prevents future reads. */
|
|
108
|
+
async close() {
|
|
109
|
+
if (!this.closed) {
|
|
110
|
+
this.closed = true;
|
|
111
|
+
this.closeController.abort();
|
|
112
|
+
this.cache.clear();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/** Fetches and validates one exact HTTP byte range. */
|
|
116
|
+
async fetchExactRange(offset, length, signal, validateVersion) {
|
|
117
|
+
const abortContext = createCombinedAbortSignal(signal, this.closeController.signal);
|
|
118
|
+
const startTime = getCurrentTime();
|
|
119
|
+
try {
|
|
120
|
+
const response = await this.fetch(this.url, {
|
|
121
|
+
headers: createRangeHeaders(this.headers, offset, length, validateVersion && this.version),
|
|
122
|
+
signal: abortContext.signal
|
|
123
|
+
});
|
|
124
|
+
if (response.status === 412) {
|
|
125
|
+
throw new Error('Parquet object changed while it was being read');
|
|
126
|
+
}
|
|
127
|
+
if (response.status !== 206) {
|
|
128
|
+
await response.body?.cancel().catch(() => { });
|
|
129
|
+
throw new Error(`ParquetSource requires HTTP byte ranges (expected 206, received ${response.status})`);
|
|
130
|
+
}
|
|
131
|
+
const contentRange = parseContentRange(response.headers.get('Content-Range'));
|
|
132
|
+
if (contentRange.start !== offset ||
|
|
133
|
+
contentRange.end !== offset + length - 1 ||
|
|
134
|
+
(this.fileByteLength > 0 && contentRange.total !== this.fileByteLength)) {
|
|
135
|
+
await response.body?.cancel().catch(() => { });
|
|
136
|
+
throw new Error('Parquet range response does not match the requested object range');
|
|
137
|
+
}
|
|
138
|
+
if (validateVersion) {
|
|
139
|
+
assertObjectVersion(this.version, getObjectVersion(response.headers));
|
|
140
|
+
}
|
|
141
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
142
|
+
if (arrayBuffer.byteLength !== length) {
|
|
143
|
+
throw new Error(`Parquet range response returned ${arrayBuffer.byteLength} bytes; expected ${length}`);
|
|
144
|
+
}
|
|
145
|
+
this.onTelemetry?.({
|
|
146
|
+
type: 'range-request',
|
|
147
|
+
rangeRequestCount: 1,
|
|
148
|
+
requestedBytes: length,
|
|
149
|
+
downloadedBytes: arrayBuffer.byteLength,
|
|
150
|
+
networkDurationMs: getCurrentTime() - startTime
|
|
151
|
+
});
|
|
152
|
+
return { response, arrayBuffer, contentRange };
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
this.onTelemetry?.({
|
|
156
|
+
type: 'range-request',
|
|
157
|
+
rangeRequestCount: 1,
|
|
158
|
+
requestedBytes: length,
|
|
159
|
+
networkDurationMs: getCurrentTime() - startTime,
|
|
160
|
+
failedRangeRequestCount: 1,
|
|
161
|
+
abortedRangeRequestCount: abortContext.signal.aborted || isAbortError(error) ? 1 : undefined,
|
|
162
|
+
error
|
|
163
|
+
});
|
|
164
|
+
throw error;
|
|
165
|
+
}
|
|
166
|
+
finally {
|
|
167
|
+
abortContext.dispose();
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/** Throws when the file has already been closed. */
|
|
171
|
+
assertOpen() {
|
|
172
|
+
if (this.closed) {
|
|
173
|
+
throw new Error('Parquet source is closed');
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/** Returns a monotonic timestamp when available and falls back to wall-clock time. */
|
|
178
|
+
function getCurrentTime() {
|
|
179
|
+
return globalThis.performance?.now() ?? Date.now();
|
|
180
|
+
}
|
|
181
|
+
/** Returns true for DOM and cross-runtime abort errors. */
|
|
182
|
+
function isAbortError(error) {
|
|
183
|
+
return error instanceof Error && error.name === 'AbortError';
|
|
184
|
+
}
|
|
185
|
+
/** Validates a requested range against the known object byte length. */
|
|
186
|
+
function assertValidRange(offset, length, fileByteLength) {
|
|
187
|
+
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || offset < 0 || length < 0) {
|
|
188
|
+
throw new Error('Parquet byte range must use non-negative safe integers');
|
|
189
|
+
}
|
|
190
|
+
if (offset + length > fileByteLength) {
|
|
191
|
+
throw new Error('Parquet byte range exceeds the object length');
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/** Creates request headers for one byte range and optional object precondition. */
|
|
195
|
+
function createRangeHeaders(defaultHeaders, offset, length, version) {
|
|
196
|
+
const headers = new Headers(defaultHeaders);
|
|
197
|
+
headers.set('Range', `bytes=${offset}-${offset + length - 1}`);
|
|
198
|
+
if (version && version.etag && !version.etag.startsWith('W/')) {
|
|
199
|
+
headers.set('If-Match', version.etag);
|
|
200
|
+
}
|
|
201
|
+
else if (version && version.lastModified) {
|
|
202
|
+
headers.set('If-Unmodified-Since', version.lastModified);
|
|
203
|
+
}
|
|
204
|
+
return headers;
|
|
205
|
+
}
|
|
206
|
+
/** Parses a satisfied single-range `Content-Range` header. */
|
|
207
|
+
function parseContentRange(value) {
|
|
208
|
+
const match = value?.match(/^bytes (\d+)-(\d+)\/(\d+)$/);
|
|
209
|
+
if (!match) {
|
|
210
|
+
throw new Error(`Invalid Parquet Content-Range header: ${value}`);
|
|
211
|
+
}
|
|
212
|
+
return { start: Number(match[1]), end: Number(match[2]), total: Number(match[3]) };
|
|
213
|
+
}
|
|
214
|
+
/** Reads supported object validators from response headers. */
|
|
215
|
+
function getObjectVersion(headers) {
|
|
216
|
+
const etag = headers.get('ETag') || undefined;
|
|
217
|
+
const lastModified = headers.get('Last-Modified') || undefined;
|
|
218
|
+
return { etag, lastModified };
|
|
219
|
+
}
|
|
220
|
+
/** Rejects a response that identifies a different remote object version. */
|
|
221
|
+
function assertObjectVersion(expected, actual) {
|
|
222
|
+
if (expected.etag && actual.etag && expected.etag !== actual.etag) {
|
|
223
|
+
throw new Error('Parquet object ETag changed while it was being read');
|
|
224
|
+
}
|
|
225
|
+
if (!expected.etag &&
|
|
226
|
+
expected.lastModified &&
|
|
227
|
+
actual.lastModified &&
|
|
228
|
+
expected.lastModified !== actual.lastModified) {
|
|
229
|
+
throw new Error('Parquet object Last-Modified changed while it was being read');
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/** Creates a cache key for one exact range. */
|
|
233
|
+
function getRangeKey(offset, length) {
|
|
234
|
+
return `${offset}:${length}`;
|
|
235
|
+
}
|
|
236
|
+
/** Creates a scheduler identity that isolates object versions. */
|
|
237
|
+
function getVersionedSourceId(url, version) {
|
|
238
|
+
return `${url}#${version.etag || version.lastModified || 'unversioned'}`;
|
|
239
|
+
}
|
|
240
|
+
/** Combines two abort signals and returns cleanup for installed listeners. */
|
|
241
|
+
function createCombinedAbortSignal(firstSignal, secondSignal) {
|
|
242
|
+
const controller = new AbortController();
|
|
243
|
+
const abort = () => controller.abort();
|
|
244
|
+
const signals = [firstSignal, secondSignal].filter(Boolean);
|
|
245
|
+
for (const signal of signals) {
|
|
246
|
+
if (signal.aborted) {
|
|
247
|
+
controller.abort();
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
signal: controller.signal,
|
|
254
|
+
dispose: () => {
|
|
255
|
+
for (const signal of signals) {
|
|
256
|
+
signal.removeEventListener('abort', abort);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
//# sourceMappingURL=parquet-range-file.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet-range-file.js","sourceRoot":"","sources":["../../../src/lib/sources/parquet-range-file.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AA4C/D,4EAA4E;AAC5E,MAAM,OAAO,gBAAgB;IAC3B,mCAAmC;IAC1B,MAAM,CAAS;IACxB,wCAAwC;IAC/B,GAAG,CAAS;IAErB,iEAAiE;IAChD,KAAK,CAAY;IAClC,oDAAoD;IACnC,OAAO,CAAe;IACvC,8EAA8E;IAC7D,WAAW,CAA+C;IAC3E,wDAAwD;IACvC,SAAS,CAAwB;IAClD,0DAA0D;IACzC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACzD,yDAAyD;IACxC,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IACxD,iEAAiE;IACzD,cAAc,GAAG,CAAC,CAAC;IAC3B,+DAA+D;IACvD,OAAO,GAAyB,EAAE,CAAC;IAC3C,yCAAyC;IACjC,MAAM,GAAG,KAAK,CAAC;IAEvB,+EAA+E;IAC/E,YAAY,GAAW,EAAE,OAAgC;QACvD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,SAAS;YACZ,OAAO,CAAC,aAAa,EAAE,SAAS;gBAChC,IAAI,qBAAqB,CAAC;oBACxB,YAAY,EAAE,OAAO,CAAC,aAAa,EAAE,YAAY,IAAI,CAAC;oBACtD,WAAW,EAAE,OAAO,CAAC,aAAa,EAAE,WAAW;oBAC/C,mBAAmB,EAAE,OAAO,CAAC,aAAa,EAAE,mBAAmB;oBAC/D,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE,cAAc;oBACrD,KAAK,EAAE,OAAO,CAAC,aAAa,EAAE,KAAK;oBACnC,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,OAAO;iBACxC,CAAC,CAAC;IACP,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,kEAAkE;IAClE,IAAI,OAAO;QACT,OAAO,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACrC,CAAC;IAED,iFAAiF;IACjF,IAAI,aAAa;QACf,OAAO,EAAC,GAAG,IAAI,CAAC,OAAO,EAAC,CAAC;IAC3B,CAAC;IAED,mFAAmF;IACnF,KAAK,CAAC,IAAI,CAAC,MAAoB;QAC7B,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC/D,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,IAAI,CACR,QAAyB,CAAC,EAC1B,SAAiB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAC1C,MAAoB;QAEpB,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAC3D,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,EAAE,CAAC,EAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAC,CAAC,CAAC;YACtD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC;gBAC1C,QAAQ,EAAE,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;gBACtD,MAAM;gBACN,MAAM;gBACN,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,EAAE;oBACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CACvC,eAAe,EACf,eAAe,EACf,eAAe,EACf,IAAI,CACL,CAAC;oBACF,OAAO;wBACL,WAAW,EAAE,MAAM,CAAC,WAAW;wBAC/B,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;wBAC9B,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU;qBAC9C,CAAC;gBACJ,CAAC;aACF,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,OAAO,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAC,CAAC;IACtE,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,uDAAuD;IAC/C,KAAK,CAAC,eAAe,CAC3B,MAAc,EACd,MAAc,EACd,MAA+B,EAC/B,eAAwB;QAExB,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACpF,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC1C,OAAO,EAAE,kBAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,IAAI,IAAI,CAAC,OAAO,CAAC;gBAC1F,MAAM,EAAE,YAAY,CAAC,MAAM;aAC5B,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,KAAK,CACb,mEAAmE,QAAQ,CAAC,MAAM,GAAG,CACtF,CAAC;YACJ,CAAC;YAED,MAAM,YAAY,GAAG,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC;YAC9E,IACE,YAAY,CAAC,KAAK,KAAK,MAAM;gBAC7B,YAAY,CAAC,GAAG,KAAK,MAAM,GAAG,MAAM,GAAG,CAAC;gBACxC,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,YAAY,CAAC,KAAK,KAAK,IAAI,CAAC,cAAc,CAAC,EACvE,CAAC;gBACD,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;YACtF,CAAC;YAED,IAAI,eAAe,EAAE,CAAC;gBACpB,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YACxE,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACjD,IAAI,WAAW,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CACb,mCAAmC,WAAW,CAAC,UAAU,oBAAoB,MAAM,EAAE,CACtF,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,IAAI,EAAE,eAAe;gBACrB,iBAAiB,EAAE,CAAC;gBACpB,cAAc,EAAE,MAAM;gBACtB,eAAe,EAAE,WAAW,CAAC,UAAU;gBACvC,iBAAiB,EAAE,cAAc,EAAE,GAAG,SAAS;aAChD,CAAC,CAAC;YACH,OAAO,EAAC,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,IAAI,EAAE,eAAe;gBACrB,iBAAiB,EAAE,CAAC;gBACpB,cAAc,EAAE,MAAM;gBACtB,iBAAiB,EAAE,cAAc,EAAE,GAAG,SAAS;gBAC/C,uBAAuB,EAAE,CAAC;gBAC1B,wBAAwB,EACtB,YAAY,CAAC,MAAM,CAAC,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;gBACpE,KAAK;aACN,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,oDAAoD;IAC5C,UAAU;QAChB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;CACF;AAED,sFAAsF;AACtF,SAAS,cAAc;IACrB,OAAO,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AACrD,CAAC;AAED,2DAA2D;AAC3D,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;AAC/D,CAAC;AAED,wEAAwE;AACxE,SAAS,gBAAgB,CAAC,MAAc,EAAE,MAAc,EAAE,cAAsB;IAC9E,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/F,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,MAAM,GAAG,MAAM,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,mFAAmF;AACnF,SAAS,kBAAkB,CACzB,cAAuC,EACvC,MAAc,EACd,MAAc,EACd,OAAqC;IAErC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,MAAM,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;SAAM,IAAI,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,8DAA8D;AAC9D,SAAS,iBAAiB,CAAC,KAAoB;IAC7C,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACzD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,EAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAC,CAAC;AACnF,CAAC;AAED,+DAA+D;AAC/D,SAAS,gBAAgB,CAAC,OAAgB;IACxC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,SAAS,CAAC;IAC/D,OAAO,EAAC,IAAI,EAAE,YAAY,EAAC,CAAC;AAC9B,CAAC;AAED,4EAA4E;AAC5E,SAAS,mBAAmB,CAAC,QAA8B,EAAE,MAA4B;IACvF,IAAI,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,IACE,CAAC,QAAQ,CAAC,IAAI;QACd,QAAQ,CAAC,YAAY;QACrB,MAAM,CAAC,YAAY;QACnB,QAAQ,CAAC,YAAY,KAAK,MAAM,CAAC,YAAY,EAC7C,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,+CAA+C;AAC/C,SAAS,WAAW,CAAC,MAAc,EAAE,MAAc;IACjD,OAAO,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;AAC/B,CAAC;AAED,kEAAkE;AAClE,SAAS,oBAAoB,CAAC,GAAW,EAAE,OAA6B;IACtE,OAAO,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,YAAY,IAAI,aAAa,EAAE,CAAC;AAC3E,CAAC;AAED,8EAA8E;AAC9E,SAAS,yBAAyB,CAChC,WAAyB,EACzB,YAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,CAAkB,CAAC;IAC7E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM;QACR,CAAC;QACD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,OAAO,EAAE,GAAG,EAAE;YACZ,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** One-slot asynchronous queue that applies backpressure between a producer and ordered output. */
|
|
2
|
+
export declare class SingleBatchQueue<Value> implements AsyncIterable<Value> {
|
|
3
|
+
/** Buffered batch, limited to one value. */
|
|
4
|
+
private value;
|
|
5
|
+
/** Whether the producer completed successfully. */
|
|
6
|
+
private finished;
|
|
7
|
+
/** Producer failure rethrown by the consumer. */
|
|
8
|
+
private error;
|
|
9
|
+
/** Wakes a consumer waiting for a value. */
|
|
10
|
+
private wakeConsumer;
|
|
11
|
+
/** Wakes a producer after its buffered value is consumed. */
|
|
12
|
+
private wakeProducer;
|
|
13
|
+
/** Buffers one value and waits until it has been consumed. */
|
|
14
|
+
push(value: Value, signal: AbortSignal): Promise<void>;
|
|
15
|
+
/** Marks the queue complete and wakes its consumer. */
|
|
16
|
+
finish(): void;
|
|
17
|
+
/** Stores a producer error and wakes its consumer. */
|
|
18
|
+
fail(error: unknown): void;
|
|
19
|
+
/** Returns the ordered asynchronous batch iterator. */
|
|
20
|
+
[Symbol.asyncIterator](): AsyncIterator<Value>;
|
|
21
|
+
/** Waits for a producer or consumer queue transition and supports read cancellation. */
|
|
22
|
+
private waitForWakeup;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=single-batch-queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"single-batch-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/sources/single-batch-queue.ts"],"names":[],"mappings":"AAIA,mGAAmG;AACnG,qBAAa,gBAAgB,CAAC,KAAK,CAAE,YAAW,aAAa,CAAC,KAAK,CAAC;IAClE,4CAA4C;IAC5C,OAAO,CAAC,KAAK,CAAoB;IACjC,mDAAmD;IACnD,OAAO,CAAC,QAAQ,CAAS;IACzB,iDAAiD;IACjD,OAAO,CAAC,KAAK,CAAU;IACvB,4CAA4C;IAC5C,OAAO,CAAC,YAAY,CAA6B;IACjD,6DAA6D;IAC7D,OAAO,CAAC,YAAY,CAA6B;IAEjD,8DAA8D;IACxD,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D,uDAAuD;IACvD,MAAM,IAAI,IAAI;IAMd,sDAAsD;IACtD,IAAI,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAO1B,uDAAuD;IAChD,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC;IAoBrD,wFAAwF;IACxF,OAAO,CAAC,aAAa;CAmBtB"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
/** One-slot asynchronous queue that applies backpressure between a producer and ordered output. */
|
|
5
|
+
export class SingleBatchQueue {
|
|
6
|
+
/** Buffered batch, limited to one value. */
|
|
7
|
+
value;
|
|
8
|
+
/** Whether the producer completed successfully. */
|
|
9
|
+
finished = false;
|
|
10
|
+
/** Producer failure rethrown by the consumer. */
|
|
11
|
+
error;
|
|
12
|
+
/** Wakes a consumer waiting for a value. */
|
|
13
|
+
wakeConsumer = null;
|
|
14
|
+
/** Wakes a producer after its buffered value is consumed. */
|
|
15
|
+
wakeProducer = null;
|
|
16
|
+
/** Buffers one value and waits until it has been consumed. */
|
|
17
|
+
async push(value, signal) {
|
|
18
|
+
throwIfAborted(signal);
|
|
19
|
+
while (this.value !== undefined) {
|
|
20
|
+
await this.waitForWakeup('producer', signal);
|
|
21
|
+
}
|
|
22
|
+
this.value = value;
|
|
23
|
+
this.wakeConsumer?.();
|
|
24
|
+
this.wakeConsumer = null;
|
|
25
|
+
while (this.value !== undefined) {
|
|
26
|
+
await this.waitForWakeup('producer', signal);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** Marks the queue complete and wakes its consumer. */
|
|
30
|
+
finish() {
|
|
31
|
+
this.finished = true;
|
|
32
|
+
this.wakeConsumer?.();
|
|
33
|
+
this.wakeConsumer = null;
|
|
34
|
+
}
|
|
35
|
+
/** Stores a producer error and wakes its consumer. */
|
|
36
|
+
fail(error) {
|
|
37
|
+
this.error = error;
|
|
38
|
+
this.finished = true;
|
|
39
|
+
this.wakeConsumer?.();
|
|
40
|
+
this.wakeConsumer = null;
|
|
41
|
+
}
|
|
42
|
+
/** Returns the ordered asynchronous batch iterator. */
|
|
43
|
+
async *[Symbol.asyncIterator]() {
|
|
44
|
+
while (true) {
|
|
45
|
+
if (this.value !== undefined) {
|
|
46
|
+
const value = this.value;
|
|
47
|
+
this.value = undefined;
|
|
48
|
+
this.wakeProducer?.();
|
|
49
|
+
this.wakeProducer = null;
|
|
50
|
+
yield value;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (this.error !== undefined) {
|
|
54
|
+
throw this.error;
|
|
55
|
+
}
|
|
56
|
+
if (this.finished) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
await this.waitForWakeup('consumer');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Waits for a producer or consumer queue transition and supports read cancellation. */
|
|
63
|
+
waitForWakeup(type, signal) {
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
const abort = () => reject(signal?.reason || new DOMException('Aborted', 'AbortError'));
|
|
66
|
+
const wakeup = () => {
|
|
67
|
+
signal?.removeEventListener('abort', abort);
|
|
68
|
+
resolve();
|
|
69
|
+
};
|
|
70
|
+
if (type === 'producer') {
|
|
71
|
+
this.wakeProducer = wakeup;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
this.wakeConsumer = wakeup;
|
|
75
|
+
}
|
|
76
|
+
if (signal?.aborted) {
|
|
77
|
+
abort();
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** Throws a standard abort failure when a queued operation has been cancelled. */
|
|
86
|
+
function throwIfAborted(signal) {
|
|
87
|
+
if (signal?.aborted) {
|
|
88
|
+
throw signal.reason || new DOMException('Aborted', 'AbortError');
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=single-batch-queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"single-batch-queue.js","sourceRoot":"","sources":["../../../src/lib/sources/single-batch-queue.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,mGAAmG;AACnG,MAAM,OAAO,gBAAgB;IAC3B,4CAA4C;IACpC,KAAK,CAAoB;IACjC,mDAAmD;IAC3C,QAAQ,GAAG,KAAK,CAAC;IACzB,iDAAiD;IACzC,KAAK,CAAU;IACvB,4CAA4C;IACpC,YAAY,GAAwB,IAAI,CAAC;IACjD,6DAA6D;IACrD,YAAY,GAAwB,IAAI,CAAC;IAEjD,8DAA8D;IAC9D,KAAK,CAAC,IAAI,CAAC,KAAY,EAAE,MAAmB;QAC1C,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,OAAO,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,MAAM;QACJ,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,sDAAsD;IACtD,IAAI,CAAC,KAAc;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBACzB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;gBACvB,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;gBACzB,MAAM,KAAK,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,KAAK,CAAC;YACnB,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,wFAAwF;IAChF,aAAa,CAAC,IAA6B,EAAE,MAAoB;QACvE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;YAC9F,MAAM,MAAM,GAAG,GAAS,EAAE;gBACxB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC5C,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;gBACxB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;YAC7B,CAAC;YACD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpB,KAAK,EAAE,CAAC;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;YACzD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,kFAAkF;AAClF,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACnE,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-wasm-node.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm-node.ts"],"names":[],"mappings":"AAIA,gGAAgG;AAChG,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAczE"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
/** Loads the installed parquet-wasm binary in Node.js or defers to browser asset resolution. */
|
|
5
|
+
export async function loadNodeWasmInput() {
|
|
6
|
+
if (!isNodeRuntime()) {
|
|
7
|
+
return undefined;
|
|
8
|
+
}
|
|
9
|
+
// Keep these specifiers dynamic so browser bundles do not include Node.js built-ins.
|
|
10
|
+
const nodeProtocol = 'node:';
|
|
11
|
+
const { readFile } = await import(/* @vite-ignore */ `${nodeProtocol}fs/promises`);
|
|
12
|
+
const { createRequire } = await import(/* @vite-ignore */ `${nodeProtocol}module`);
|
|
13
|
+
const { pathToFileURL } = await import(/* @vite-ignore */ `${nodeProtocol}url`);
|
|
14
|
+
const moduleUrl = typeof __filename === 'string' ? pathToFileURL(__filename).toString() : import.meta.url;
|
|
15
|
+
const require = createRequire(moduleUrl);
|
|
16
|
+
const wasmPath = require.resolve('parquet-wasm/esm/parquet_wasm_bg.wasm');
|
|
17
|
+
return await readFile(wasmPath);
|
|
18
|
+
}
|
|
19
|
+
/** Returns true when Node.js built-ins are available in the current runtime. */
|
|
20
|
+
function isNodeRuntime() {
|
|
21
|
+
return Boolean(globalThis.process?.versions?.node);
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=load-wasm-node.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-wasm-node.js","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm-node.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;QACrB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,qFAAqF;IACrF,MAAM,YAAY,GAAG,OAAO,CAAC;IAC7B,MAAM,EAAC,QAAQ,EAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,GAAG,YAAY,aAAa,CAAC,CAAC;IACjF,MAAM,EAAC,aAAa,EAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,GAAG,YAAY,QAAQ,CAAC,CAAC;IACjF,MAAM,EAAC,aAAa,EAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC;IAC9E,MAAM,SAAS,GACb,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;IAC1F,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;IAC1E,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,gFAAgF;AAChF,SAAS,aAAa;IACpB,OAAO,OAAO,CAAE,UAAuD,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;AACnG,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import * as
|
|
2
|
-
export declare function loadWasm(wasmUrl?:
|
|
1
|
+
import type * as ParquetWasm from 'parquet-wasm/esm/parquet_wasm.js';
|
|
2
|
+
export declare function loadWasm(wasmUrl?: ParquetWasm.InitInput | Promise<ParquetWasm.InitInput>): Promise<typeof ParquetWasm>;
|
|
3
3
|
//# sourceMappingURL=load-wasm.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"load-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"load-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,WAAW,MAAM,kCAAkC,CAAC;AAKrE,wBAAsB,QAAQ,CAC5B,OAAO,CAAC,EAAE,WAAW,CAAC,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,GAC/D,OAAO,CAAC,OAAO,WAAW,CAAC,CAa7B"}
|
|
@@ -1,20 +1,36 @@
|
|
|
1
1
|
// loaders.gl
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
|
-
|
|
5
|
-
import initWasm from 'parquet-wasm';
|
|
6
|
-
import * as parquetWasm from 'parquet-wasm';
|
|
7
|
-
import { PARQUET_WASM_URL } from "../constants.js";
|
|
4
|
+
import { loadNodeWasmInput } from "./load-wasm-node.js";
|
|
8
5
|
let initializePromise;
|
|
9
|
-
export async function loadWasm(wasmUrl
|
|
10
|
-
if (!initializePromise
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
6
|
+
export async function loadWasm(wasmUrl) {
|
|
7
|
+
if (!initializePromise) {
|
|
8
|
+
const nextInitializePromise = loadAndInitializeWasm(wasmUrl);
|
|
9
|
+
const cachedInitializePromise = nextInitializePromise.catch(error => {
|
|
10
|
+
if (initializePromise === cachedInitializePromise) {
|
|
11
|
+
initializePromise = undefined;
|
|
12
|
+
}
|
|
13
|
+
throw error;
|
|
14
|
+
});
|
|
15
|
+
initializePromise = cachedInitializePromise;
|
|
16
|
+
}
|
|
17
|
+
return await initializePromise;
|
|
18
|
+
}
|
|
19
|
+
async function loadAndInitializeWasm(wasmUrl) {
|
|
20
|
+
const parquetWasm = await import('parquet-wasm/esm/parquet_wasm.js');
|
|
21
|
+
const moduleOrPath = wasmUrl ? await wasmUrl : await loadDefaultWasmInput();
|
|
22
|
+
if (moduleOrPath) {
|
|
23
|
+
await parquetWasm.default({ module_or_path: moduleOrPath });
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
// parquet-wasm resolves its sibling asset with new URL(..., import.meta.url), which allows
|
|
27
|
+
// browser bundlers to copy and fingerprint the WASM file without a network CDN default.
|
|
28
|
+
await parquetWasm.default();
|
|
16
29
|
}
|
|
17
|
-
await initializePromise;
|
|
18
30
|
return parquetWasm;
|
|
19
31
|
}
|
|
32
|
+
/** Loads a local Node asset or defers to parquet-wasm's browser import.meta resolver. */
|
|
33
|
+
async function loadDefaultWasmInput() {
|
|
34
|
+
return await loadNodeWasmInput();
|
|
35
|
+
}
|
|
20
36
|
//# sourceMappingURL=load-wasm.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"load-wasm.js","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;
|
|
1
|
+
{"version":3,"file":"load-wasm.js","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,iBAAiB,EAAC,4BAAyB;AAEnD,IAAI,iBAA0D,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,OAAgE;IAEhE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAC7D,MAAM,uBAAuB,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAClE,IAAI,iBAAiB,KAAK,uBAAuB,EAAE,CAAC;gBAClD,iBAAiB,GAAG,SAAS,CAAC;YAChC,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,iBAAiB,GAAG,uBAAuB,CAAC;IAC9C,CAAC;IAED,OAAO,MAAM,iBAAiB,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,OAAgE;IAEhE,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAC;IACrE,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;IAC5E,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,WAAW,CAAC,OAAO,CAAC,EAAC,cAAc,EAAE,YAAY,EAAC,CAAC,CAAC;IAC5D,CAAC;SAAM,CAAC;QACN,2FAA2F;QAC3F,wFAAwF;QACxF,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,yFAAyF;AACzF,KAAK,UAAU,oBAAoB;IACjC,OAAO,MAAM,iBAAiB,EAAE,CAAC;AACnC,CAAC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { Readable } from 'stream';
|
|
2
2
|
export type StreamIteratorOptions = {
|
|
3
3
|
_streamReadAhead?: boolean;
|
|
4
|
+
/** Cancels the active stream read and rejects iteration with the signal reason. */
|
|
5
|
+
signal?: AbortSignal;
|
|
4
6
|
};
|
|
5
7
|
/**
|
|
6
8
|
* Returns an async iterable that reads from a stream (works in both Node.js and browsers)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"make-stream-iterator.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/make-stream-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAGrC,MAAM,MAAM,qBAAqB,GAAG;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"make-stream-iterator.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/make-stream-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAGrC,MAAM,MAAM,qBAAqB,GAAG;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mFAAmF;IACnF,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,QAAQ,EACpC,OAAO,CAAC,EAAE,qBAAqB,GAC9B,aAAa,CAAC,CAAC,CAAC,CAIlB"}
|