@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/bundled.d.ts +1 -1
- package/dist/bundled.d.ts.map +1 -1
- package/dist/bundled.js.map +1 -1
- package/dist/delta-format.d.ts +9 -0
- package/dist/delta-format.d.ts.map +1 -0
- package/dist/delta-format.js +12 -0
- package/dist/delta-format.js.map +1 -0
- package/dist/delta-source-loader-types.d.ts +27 -0
- package/dist/delta-source-loader-types.d.ts.map +1 -0
- package/dist/delta-source-loader-types.js +28 -0
- package/dist/delta-source-loader-types.js.map +1 -0
- package/dist/delta-source.d.ts +48 -0
- package/dist/delta-source.d.ts.map +1 -0
- package/dist/delta-source.js +222 -0
- package/dist/delta-source.js.map +1 -0
- package/dist/delta-types.d.ts +38 -0
- package/dist/delta-types.d.ts.map +1 -0
- package/dist/delta-types.js +5 -0
- package/dist/delta-types.js.map +1 -0
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.d.ts +49 -0
- package/dist/iceberg-rest-catalog.d.ts.map +1 -0
- package/dist/iceberg-rest-catalog.js +78 -0
- package/dist/iceberg-rest-catalog.js.map +1 -0
- package/dist/iceberg-table-source.d.ts +63 -0
- package/dist/iceberg-table-source.d.ts.map +1 -0
- package/dist/iceberg-table-source.js +797 -0
- package/dist/iceberg-table-source.js.map +1 -0
- package/dist/iceberg-types.d.ts +150 -0
- package/dist/iceberg-types.d.ts.map +1 -0
- package/dist/iceberg-types.js +5 -0
- package/dist/iceberg-types.js.map +1 -0
- package/dist/index.cjs +15005 -3143
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -1
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +94 -19
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -1
- package/dist/lib/arrow/create-nested-arrow-vector.js +89 -0
- package/dist/lib/arrow/create-nested-arrow-vector.js.map +1 -1
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.js +202 -7
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -1
- package/dist/lib/geo/geoparquet-covering.d.ts +19 -0
- package/dist/lib/geo/geoparquet-covering.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-covering.js +137 -0
- package/dist/lib/geo/geoparquet-covering.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -1
- package/dist/lib/geo/geospatial-metadata.js +39 -13
- package/dist/lib/geo/geospatial-metadata.js.map +1 -1
- package/dist/lib/parquet-bloom-filter-planner.d.ts +16 -0
- package/dist/lib/parquet-bloom-filter-planner.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter-planner.js +57 -0
- package/dist/lib/parquet-bloom-filter-planner.js.map +1 -0
- package/dist/lib/parquet-bloom-filter.d.ts +30 -0
- package/dist/lib/parquet-bloom-filter.d.ts.map +1 -0
- package/dist/lib/parquet-bloom-filter.js +289 -0
- package/dist/lib/parquet-bloom-filter.js.map +1 -0
- package/dist/lib/parquet-encryption.d.ts +80 -0
- package/dist/lib/parquet-encryption.d.ts.map +1 -0
- package/dist/lib/parquet-encryption.js +190 -0
- package/dist/lib/parquet-encryption.js.map +1 -0
- package/dist/lib/parquet-page-index.d.ts +34 -5
- package/dist/lib/parquet-page-index.d.ts.map +1 -1
- package/dist/lib/parquet-page-index.js +114 -28
- package/dist/lib/parquet-page-index.js.map +1 -1
- package/dist/lib/parquet-predicate.d.ts +5 -1
- package/dist/lib/parquet-predicate.d.ts.map +1 -1
- package/dist/lib/parquet-predicate.js +42 -14
- package/dist/lib/parquet-predicate.js.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts +4 -1
- package/dist/lib/parquet-source-worker-decoder.d.ts.map +1 -1
- package/dist/lib/parquet-source-worker-decoder.js +83 -17
- package/dist/lib/parquet-source-worker-decoder.js.map +1 -1
- package/dist/lib/parquet-source-worker-types.d.ts +24 -0
- package/dist/lib/parquet-source-worker-types.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +327 -18
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +5 -5
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +3 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +18 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/slice-parquet-batch.d.ts +9 -0
- package/dist/lib/slice-parquet-batch.d.ts.map +1 -0
- package/dist/lib/slice-parquet-batch.js +28 -0
- package/dist/lib/slice-parquet-batch.js.map +1 -0
- package/dist/lib/sources/parquet-range-file.d.ts +1 -1
- package/dist/lib/sources/parquet-range-file.d.ts.map +1 -1
- package/dist/lib/sources/parquet-range-file.js +28 -22
- package/dist/lib/sources/parquet-range-file.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.js +15 -1
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -1
- package/dist/parquet-dataset-source.cjs +15766 -4896
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-dataset-source.d.ts +31 -4
- package/dist/parquet-dataset-source.d.ts.map +1 -1
- package/dist/parquet-dataset-source.js +296 -74
- package/dist/parquet-dataset-source.js.map +1 -1
- package/dist/parquet-js-loader-types.d.ts +2 -0
- package/dist/parquet-js-loader-types.d.ts.map +1 -1
- package/dist/parquet-js-loader-types.js +2 -1
- package/dist/parquet-js-loader-types.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +1 -0
- package/dist/parquet-js-loader.d.ts.map +1 -1
- package/dist/parquet-js-loader.js +2 -1
- package/dist/parquet-js-loader.js.map +1 -1
- package/dist/parquet-js-writer.d.ts +23 -2
- package/dist/parquet-js-writer.d.ts.map +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-js-writer.js.map +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-loader-options.d.ts +11 -0
- package/dist/parquet-loader-options.d.ts.map +1 -1
- package/dist/parquet-loader-options.js.map +1 -1
- package/dist/parquet-source-capabilities.d.ts +8 -1
- package/dist/parquet-source-capabilities.d.ts.map +1 -1
- package/dist/parquet-source-capabilities.js +10 -0
- package/dist/parquet-source-capabilities.js.map +1 -1
- package/dist/parquet-source-loader-types.d.ts +10 -0
- package/dist/parquet-source-loader-types.d.ts.map +1 -1
- package/dist/parquet-source-loader-types.js +11 -1
- package/dist/parquet-source-loader-types.js.map +1 -1
- package/dist/parquet-source-loader.cjs +14911 -4316
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-loader.d.ts +42 -4
- package/dist/parquet-source-loader.d.ts.map +1 -1
- package/dist/parquet-source-loader.js +585 -38
- package/dist/parquet-source-loader.js.map +1 -1
- package/dist/parquet-source-types.d.ts +180 -30
- package/dist/parquet-source-types.d.ts.map +1 -1
- package/dist/parquet-source-worker.js +7055 -6989
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +8505 -1041
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker-url.js +5 -1
- package/dist/parquet-worker-url.js.map +1 -1
- package/dist/parquet-worker.js +289 -218
- package/dist/parquet-worker.js.map +4 -4
- package/dist/parquetjs/codecs/declare.d.ts +15 -0
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +10 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -1
- package/dist/parquetjs/codecs/delta.d.ts.map +1 -1
- package/dist/parquetjs/codecs/delta.js +356 -7
- package/dist/parquetjs/codecs/delta.js.map +1 -1
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +4 -0
- package/dist/parquetjs/codecs/index.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +153 -12
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +4 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +406 -3
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +9 -0
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +86 -67
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.d.ts.map +1 -1
- package/dist/parquetjs/encoder/dictionary-planner.js +5 -2
- package/dist/parquetjs/encoder/dictionary-planner.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +43 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +691 -34
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts +29 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js +85 -0
- package/dist/parquetjs/parquet-thrift/BoundingBox.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts +35 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js +135 -0
- package/dist/parquetjs/parquet-thrift/ColumnCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +10 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +73 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts +22 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js +111 -0
- package/dist/parquetjs/parquet-thrift/EncryptionAlgorithm.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts +16 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js +57 -0
- package/dist/parquetjs/parquet-thrift/FileCryptoMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +35 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts +18 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js +62 -0
- package/dist/parquetjs/parquet-thrift/GeospatialStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts +26 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.d.ts.map +1 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js +85 -0
- package/dist/parquetjs/parquet-thrift/SizeStatistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts +6 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +10 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +318 -14
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +78 -12
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +605 -66
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +22 -3
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js.map +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +10 -0
- package/dist/parquetjs/schema/schema.js.map +1 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +219 -11
- package/dist/parquetjs/schema/shred.js.map +1 -1
- package/dist/parquetjs/schema/types.js +19 -21
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/schema/variant.d.ts +3 -0
- package/dist/parquetjs/schema/variant.d.ts.map +1 -0
- package/dist/parquetjs/schema/variant.js +248 -0
- package/dist/parquetjs/schema/variant.js.map +1 -0
- package/dist/parquetjs/utils/crc32.d.ts +3 -0
- package/dist/parquetjs/utils/crc32.d.ts.map +1 -0
- package/dist/parquetjs/utils/crc32.js +15 -0
- package/dist/parquetjs/utils/crc32.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.d.ts +11 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +21 -1
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/unbundled.d.ts +2 -1
- package/dist/unbundled.d.ts.map +1 -1
- package/dist/unbundled.js +1 -0
- package/dist/unbundled.js.map +1 -1
- package/package.json +36 -11
- package/src/bundled.ts +3 -0
- package/src/delta-format.ts +12 -0
- package/src/delta-source-loader-types.ts +40 -0
- package/src/delta-source.ts +305 -0
- package/src/delta-types.ts +40 -0
- package/src/iceberg-rest-catalog.ts +122 -0
- package/src/iceberg-table-source.ts +1037 -0
- package/src/iceberg-types.ts +165 -0
- package/src/index.ts +50 -1
- package/src/lib/arrow/convert-schema-from-parquet.ts +113 -19
- package/src/lib/arrow/create-nested-arrow-vector.ts +116 -0
- package/src/lib/encoders/encode-table-to-parquet-js.ts +268 -7
- package/src/lib/geo/geoparquet-covering.ts +201 -0
- package/src/lib/geo/geospatial-metadata.ts +42 -15
- package/src/lib/parquet-bloom-filter-planner.ts +86 -0
- package/src/lib/parquet-bloom-filter.ts +345 -0
- package/src/lib/parquet-encryption.ts +344 -0
- package/src/lib/parquet-page-index.ts +180 -34
- package/src/lib/parquet-predicate.ts +52 -16
- package/src/lib/parquet-source-worker-decoder.ts +116 -20
- package/src/lib/parquet-source-worker-types.ts +25 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +397 -17
- package/src/lib/parsers/parse-parquet-to-arrow.ts +4 -5
- package/src/lib/parsers/parse-parquet-to-columns.ts +3 -1
- package/src/lib/parsers/parse-parquet-to-json.ts +18 -2
- package/src/lib/slice-parquet-batch.ts +32 -0
- package/src/lib/sources/parquet-range-file.ts +35 -29
- package/src/lib/utils/normalize-parquet-options.ts +17 -1
- package/src/parquet-dataset-source.ts +397 -85
- package/src/parquet-js-loader-types.ts +1 -0
- package/src/parquet-js-loader.ts +2 -1
- package/src/parquet-js-writer.ts +30 -0
- package/src/parquet-loader-options.ts +11 -0
- package/src/parquet-source-capabilities.ts +18 -1
- package/src/parquet-source-loader-types.ts +10 -0
- package/src/parquet-source-loader.ts +929 -52
- package/src/parquet-source-types.ts +197 -28
- package/src/parquet-worker-url.ts +6 -1
- package/src/parquetjs/codecs/declare.ts +27 -0
- package/src/parquetjs/codecs/delta.ts +466 -6
- package/src/parquetjs/codecs/index.ts +4 -0
- package/src/parquetjs/codecs/plain.ts +177 -11
- package/src/parquetjs/codecs/rle.ts +573 -3
- package/src/parquetjs/compression.ts +102 -76
- package/src/parquetjs/encoder/dictionary-planner.ts +7 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +1016 -38
- package/src/parquetjs/parquet-thrift/BoundingBox.ts +108 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +35 -0
- package/src/parquetjs/parquet-thrift/ColumnCryptoMetaData.ts +139 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +73 -0
- package/src/parquetjs/parquet-thrift/EncryptionAlgorithm.ts +127 -0
- package/src/parquetjs/parquet-thrift/FileCryptoMetaData.ts +60 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +35 -0
- package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts +68 -0
- package/src/parquetjs/parquet-thrift/SizeStatistics.ts +98 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +451 -26
- package/src/parquetjs/parser/parquet-reader.ts +935 -87
- package/src/parquetjs/schema/declare.ts +22 -2
- package/src/parquetjs/schema/schema.ts +10 -0
- package/src/parquetjs/schema/shred.ts +238 -11
- package/src/parquetjs/schema/types.ts +16 -25
- package/src/parquetjs/schema/variant.ts +313 -0
- package/src/parquetjs/utils/crc32.ts +15 -0
- package/src/parquetjs/utils/read-utils.ts +26 -1
- package/src/unbundled.ts +14 -0
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import type {DataType, Field, ObjectRowTable, Schema, Table} from '@loaders.gl/schema';
|
|
6
|
+
import {
|
|
7
|
+
getGeoMetadata,
|
|
8
|
+
type GeoArrowEdgeType,
|
|
9
|
+
type GeoArrowMetadata,
|
|
10
|
+
type GeoColumnMetadata
|
|
11
|
+
} from '@loaders.gl/gis';
|
|
6
12
|
import {concatUint8Arrays} from '../../parquetjs/utils/binary-utils';
|
|
7
13
|
import {ParquetEncoder} from '../../parquetjs/encoder/parquet-encoder';
|
|
8
14
|
import {ParquetSchema} from '../../parquetjs/schema/schema';
|
|
@@ -62,7 +68,17 @@ function convertSchemaToParquetSchema(
|
|
|
62
68
|
const parquetFields: SchemaDefinition = {};
|
|
63
69
|
const columnEncodings = options.parquet?.columnEncodings || {};
|
|
64
70
|
const columnDictionaries = options.parquet?.columnDictionaries || {};
|
|
71
|
+
const bloomFilterColumns =
|
|
72
|
+
options.parquet?.bloomFilter && typeof options.parquet.bloomFilter === 'object'
|
|
73
|
+
? options.parquet.bloomFilter
|
|
74
|
+
: {};
|
|
75
|
+
const pageIndexColumns =
|
|
76
|
+
options.parquet?.pageIndex && typeof options.parquet.pageIndex === 'object'
|
|
77
|
+
? options.parquet.pageIndex
|
|
78
|
+
: {};
|
|
79
|
+
const columnKeyMetadata = options.parquet?.encryption?.columnKeyMetadata || {};
|
|
65
80
|
const fieldNames = new Set(schema.fields.map(field => field.name));
|
|
81
|
+
const geoMetadata = getGeoMetadata(schema.metadata);
|
|
66
82
|
for (const columnName of Object.keys(columnEncodings)) {
|
|
67
83
|
if (!fieldNames.has(columnName)) {
|
|
68
84
|
throw new Error(`ParquetJSWriter: Unknown column encoding override "${columnName}"`);
|
|
@@ -73,10 +89,32 @@ function convertSchemaToParquetSchema(
|
|
|
73
89
|
throw new Error(`ParquetJSWriter: Unknown column dictionary override "${columnName}"`);
|
|
74
90
|
}
|
|
75
91
|
}
|
|
92
|
+
for (const columnName of Object.keys(bloomFilterColumns)) {
|
|
93
|
+
if (!fieldNames.has(columnName)) {
|
|
94
|
+
throw new Error(`ParquetJSWriter: Unknown Bloom-filter column "${columnName}"`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
for (const columnName of Object.keys(pageIndexColumns)) {
|
|
98
|
+
if (!fieldNames.has(columnName)) {
|
|
99
|
+
throw new Error(`ParquetJSWriter: Unknown page-index column "${columnName}"`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
for (const columnName of Object.keys(columnKeyMetadata)) {
|
|
103
|
+
if (!fieldNames.has(columnName)) {
|
|
104
|
+
throw new Error(`ParquetJSWriter: Unknown encryption column key "${columnName}"`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
76
107
|
|
|
77
108
|
for (const field of schema.fields) {
|
|
78
109
|
parquetFields[field.name] = {
|
|
79
|
-
...convertFieldToParquetFieldDefinition(
|
|
110
|
+
...convertFieldToParquetFieldDefinition(
|
|
111
|
+
field,
|
|
112
|
+
objectRowTable.data,
|
|
113
|
+
geoMetadata?.version,
|
|
114
|
+
geoMetadata?.columns?.[field.name],
|
|
115
|
+
undefined,
|
|
116
|
+
options.parquet?.int96AsTimestamp
|
|
117
|
+
),
|
|
80
118
|
encoding: columnEncodings[field.name]
|
|
81
119
|
};
|
|
82
120
|
}
|
|
@@ -92,12 +130,116 @@ function convertSchemaToParquetSchema(
|
|
|
92
130
|
*/
|
|
93
131
|
function convertFieldToParquetFieldDefinition(
|
|
94
132
|
field: Field,
|
|
95
|
-
rows: Array<Record<string, unknown
|
|
133
|
+
rows: Array<Record<string, unknown>>,
|
|
134
|
+
geoParquetVersion?: string,
|
|
135
|
+
geoColumnMetadata?: GeoColumnMetadata,
|
|
136
|
+
fieldValues?: readonly unknown[],
|
|
137
|
+
int96AsTimestamp = false
|
|
96
138
|
): FieldDefinition {
|
|
97
|
-
const
|
|
139
|
+
const values = fieldValues || rows.map(row => row[field.name]);
|
|
140
|
+
const sampleValue = getFirstDefinedValue(values);
|
|
98
141
|
const nullable = field.nullable ?? sampleValue === undefined;
|
|
99
142
|
const dataType = field.type === 'null' ? getDataTypeFromValue(sampleValue) : field.type;
|
|
100
143
|
|
|
144
|
+
if (typeof dataType === 'object') {
|
|
145
|
+
switch (dataType.type) {
|
|
146
|
+
case 'struct':
|
|
147
|
+
return {
|
|
148
|
+
optional: nullable,
|
|
149
|
+
fields: Object.fromEntries(
|
|
150
|
+
dataType.children.map(child => [
|
|
151
|
+
child.name,
|
|
152
|
+
convertFieldToParquetFieldDefinition(
|
|
153
|
+
child,
|
|
154
|
+
[],
|
|
155
|
+
undefined,
|
|
156
|
+
undefined,
|
|
157
|
+
getStructChildValues(values, child.name),
|
|
158
|
+
int96AsTimestamp
|
|
159
|
+
)
|
|
160
|
+
])
|
|
161
|
+
)
|
|
162
|
+
};
|
|
163
|
+
case 'list': {
|
|
164
|
+
const child = dataType.children[0];
|
|
165
|
+
if (!child) {
|
|
166
|
+
throw new Error(`ParquetJSWriter: List field "${field.name}" has no value child`);
|
|
167
|
+
}
|
|
168
|
+
return {
|
|
169
|
+
optional: nullable,
|
|
170
|
+
logicalType: {type: 'LIST'},
|
|
171
|
+
fields: {
|
|
172
|
+
list: {
|
|
173
|
+
repeated: true,
|
|
174
|
+
fields: {
|
|
175
|
+
element: convertFieldToParquetFieldDefinition(
|
|
176
|
+
child,
|
|
177
|
+
[],
|
|
178
|
+
undefined,
|
|
179
|
+
undefined,
|
|
180
|
+
flattenListValues(values),
|
|
181
|
+
int96AsTimestamp
|
|
182
|
+
)
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
case 'map': {
|
|
189
|
+
const mapEntryField = dataType.children.length === 1 ? dataType.children[0] : undefined;
|
|
190
|
+
const mapEntryChildren =
|
|
191
|
+
mapEntryField &&
|
|
192
|
+
typeof mapEntryField.type === 'object' &&
|
|
193
|
+
mapEntryField.type.type === 'struct'
|
|
194
|
+
? mapEntryField.type.children
|
|
195
|
+
: dataType.children;
|
|
196
|
+
const [keyField, valueField] = mapEntryChildren;
|
|
197
|
+
if (!keyField || !valueField) {
|
|
198
|
+
throw new Error(
|
|
199
|
+
`ParquetJSWriter: Map field "${field.name}" needs key and value children`
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
if (keyField.nullable) {
|
|
203
|
+
throw new Error(`ParquetJSWriter: Map keys must be non-nullable ("${field.name}")`);
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
optional: nullable,
|
|
207
|
+
logicalType: {type: 'MAP'},
|
|
208
|
+
fields: {
|
|
209
|
+
key_value: {
|
|
210
|
+
repeated: true,
|
|
211
|
+
fields: {
|
|
212
|
+
key: {
|
|
213
|
+
...convertFieldToParquetFieldDefinition(
|
|
214
|
+
{...keyField, name: 'key', nullable: false},
|
|
215
|
+
[],
|
|
216
|
+
undefined,
|
|
217
|
+
undefined,
|
|
218
|
+
getMapChildValues(values, 'key'),
|
|
219
|
+
int96AsTimestamp
|
|
220
|
+
),
|
|
221
|
+
optional: false
|
|
222
|
+
},
|
|
223
|
+
value: {
|
|
224
|
+
...convertFieldToParquetFieldDefinition(
|
|
225
|
+
{...valueField, name: 'value'},
|
|
226
|
+
[],
|
|
227
|
+
undefined,
|
|
228
|
+
undefined,
|
|
229
|
+
getMapChildValues(values, 'value'),
|
|
230
|
+
int96AsTimestamp
|
|
231
|
+
)
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
default:
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
101
243
|
switch (dataType) {
|
|
102
244
|
case 'bool':
|
|
103
245
|
return {type: 'BOOLEAN', optional: nullable};
|
|
@@ -141,6 +283,26 @@ function convertFieldToParquetFieldDefinition(
|
|
|
141
283
|
return {type: 'UTF8', optional: nullable};
|
|
142
284
|
|
|
143
285
|
case 'binary':
|
|
286
|
+
if (
|
|
287
|
+
geoParquetVersion?.startsWith('2.') &&
|
|
288
|
+
String(geoColumnMetadata?.encoding).toLowerCase() === 'wkb'
|
|
289
|
+
) {
|
|
290
|
+
return getNativeGeospatialFieldDefinition(
|
|
291
|
+
nullable,
|
|
292
|
+
getNativeGeospatialCRS(geoColumnMetadata?.crs),
|
|
293
|
+
geoColumnMetadata?.edges
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
if (!geoParquetVersion) {
|
|
297
|
+
const geoArrowMetadata = getWKBGeoArrowMetadata(field);
|
|
298
|
+
if (geoArrowMetadata) {
|
|
299
|
+
return getNativeGeospatialFieldDefinition(
|
|
300
|
+
nullable,
|
|
301
|
+
getGeoArrowNativeCRS(geoArrowMetadata),
|
|
302
|
+
geoArrowMetadata.edges
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
144
306
|
return {type: 'BYTE_ARRAY', optional: nullable};
|
|
145
307
|
|
|
146
308
|
case 'date-day':
|
|
@@ -155,7 +317,10 @@ function convertFieldToParquetFieldDefinition(
|
|
|
155
317
|
return {type: 'TIMESTAMP_MICROS', optional: nullable};
|
|
156
318
|
|
|
157
319
|
case 'timestamp-nanosecond':
|
|
158
|
-
return {
|
|
320
|
+
return {
|
|
321
|
+
type: int96AsTimestamp ? 'INT96' : 'TIMESTAMP_NANOS',
|
|
322
|
+
optional: nullable
|
|
323
|
+
};
|
|
159
324
|
|
|
160
325
|
case 'time-second':
|
|
161
326
|
case 'time-millisecond':
|
|
@@ -200,6 +365,70 @@ function convertFieldToParquetFieldDefinition(
|
|
|
200
365
|
}
|
|
201
366
|
}
|
|
202
367
|
|
|
368
|
+
/** Serializes GeoParquet CRS metadata into the native Parquet logical-type string. */
|
|
369
|
+
function getNativeGeospatialCRS(crs: object | null | undefined): string | undefined {
|
|
370
|
+
if (crs === null) return 'srid:0';
|
|
371
|
+
return crs === undefined ? undefined : JSON.stringify(crs);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** Creates a native GEOMETRY or GEOGRAPHY field from normalized spatial semantics. */
|
|
375
|
+
function getNativeGeospatialFieldDefinition(
|
|
376
|
+
nullable: boolean,
|
|
377
|
+
crs: string | undefined,
|
|
378
|
+
edges: 'planar' | GeoArrowEdgeType | undefined
|
|
379
|
+
): FieldDefinition {
|
|
380
|
+
return edges && edges !== 'planar'
|
|
381
|
+
? {
|
|
382
|
+
type: 'GEOGRAPHY',
|
|
383
|
+
optional: nullable,
|
|
384
|
+
logicalType: {type: 'GEOGRAPHY', crs, algorithm: edges.toUpperCase()}
|
|
385
|
+
}
|
|
386
|
+
: {
|
|
387
|
+
type: 'GEOMETRY',
|
|
388
|
+
optional: nullable,
|
|
389
|
+
logicalType: {type: 'GEOMETRY', crs}
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** Reads validated GeoArrow WKB extension metadata from a field. */
|
|
394
|
+
function getWKBGeoArrowMetadata(field: Field): GeoArrowMetadata | undefined {
|
|
395
|
+
if (field.metadata?.['ARROW:extension:name']?.toLowerCase() !== 'geoarrow.wkb') return undefined;
|
|
396
|
+
const serializedMetadata = field.metadata['ARROW:extension:metadata'];
|
|
397
|
+
if (!serializedMetadata) return {};
|
|
398
|
+
let metadata: unknown;
|
|
399
|
+
try {
|
|
400
|
+
metadata = JSON.parse(serializedMetadata);
|
|
401
|
+
} catch (error) {
|
|
402
|
+
throw new Error(`ParquetJSWriter: Invalid GeoArrow metadata on "${field.name}"`, {
|
|
403
|
+
cause: error
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
if (!metadata || typeof metadata !== 'object') {
|
|
407
|
+
throw new Error(`ParquetJSWriter: Invalid GeoArrow metadata on "${field.name}"`);
|
|
408
|
+
}
|
|
409
|
+
const candidate = metadata as GeoArrowMetadata;
|
|
410
|
+
if (
|
|
411
|
+
candidate.crs !== undefined &&
|
|
412
|
+
typeof candidate.crs !== 'string' &&
|
|
413
|
+
(typeof candidate.crs !== 'object' || candidate.crs === null)
|
|
414
|
+
) {
|
|
415
|
+
throw new Error(`ParquetJSWriter: Invalid GeoArrow CRS on "${field.name}"`);
|
|
416
|
+
}
|
|
417
|
+
if (
|
|
418
|
+
candidate.edges !== undefined &&
|
|
419
|
+
!['spherical', 'vincenty', 'thomas', 'andoyer', 'karney'].includes(candidate.edges)
|
|
420
|
+
) {
|
|
421
|
+
throw new Error(`ParquetJSWriter: Invalid GeoArrow edges on "${field.name}"`);
|
|
422
|
+
}
|
|
423
|
+
return candidate;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** Maps GeoArrow CRS semantics onto the native Parquet logical-type property. */
|
|
427
|
+
function getGeoArrowNativeCRS(metadata: GeoArrowMetadata): string {
|
|
428
|
+
if (metadata.crs === undefined) return 'srid:0';
|
|
429
|
+
return typeof metadata.crs === 'string' ? metadata.crs : JSON.stringify(metadata.crs);
|
|
430
|
+
}
|
|
431
|
+
|
|
203
432
|
/** Returns the minimum fixed byte width required by one decimal precision. */
|
|
204
433
|
function getDecimalByteWidth(precision: number): number {
|
|
205
434
|
for (let byteWidth = 1; byteWidth <= 32; byteWidth++) {
|
|
@@ -215,9 +444,8 @@ function getDecimalByteWidth(precision: number): number {
|
|
|
215
444
|
* @param columnName column name
|
|
216
445
|
* @returns first defined value, if any
|
|
217
446
|
*/
|
|
218
|
-
function getFirstDefinedValue(
|
|
219
|
-
for (const
|
|
220
|
-
const value = row[columnName];
|
|
447
|
+
function getFirstDefinedValue(values: readonly unknown[]): unknown {
|
|
448
|
+
for (const value of values) {
|
|
221
449
|
if (value !== null && value !== undefined) {
|
|
222
450
|
return value;
|
|
223
451
|
}
|
|
@@ -226,6 +454,39 @@ function getFirstDefinedValue(rows: Array<Record<string, unknown>>, columnName:
|
|
|
226
454
|
return undefined;
|
|
227
455
|
}
|
|
228
456
|
|
|
457
|
+
/** Extracts one named child from struct-like values for recursive schema conversion. */
|
|
458
|
+
function getStructChildValues(values: readonly unknown[], childName: string): unknown[] {
|
|
459
|
+
return values.flatMap(value => {
|
|
460
|
+
if (!value || typeof value !== 'object' || ArrayBuffer.isView(value)) return [];
|
|
461
|
+
return [Reflect.get(value, childName)];
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** Flattens list values while retaining nested arrays as child values. */
|
|
466
|
+
function flattenListValues(values: readonly unknown[]): unknown[] {
|
|
467
|
+
return values.flatMap(value => (Array.isArray(value) ? value : []));
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/** Extracts key/value members from Map, object, tuple, or Arrow object-row map values. */
|
|
471
|
+
function getMapChildValues(values: readonly unknown[], member: 'key' | 'value'): unknown[] {
|
|
472
|
+
const output: unknown[] = [];
|
|
473
|
+
for (const value of values) {
|
|
474
|
+
if (value instanceof Map) {
|
|
475
|
+
for (const [key, mapValue] of value) output.push(member === 'key' ? key : mapValue);
|
|
476
|
+
} else if (Array.isArray(value)) {
|
|
477
|
+
for (const entry of value) {
|
|
478
|
+
if (Array.isArray(entry) && entry.length >= 2) output.push(entry[member === 'key' ? 0 : 1]);
|
|
479
|
+
else if (entry && typeof entry === 'object') output.push(Reflect.get(entry, member));
|
|
480
|
+
}
|
|
481
|
+
} else if (value && typeof value === 'object') {
|
|
482
|
+
for (const [key, mapValue] of Object.entries(value)) {
|
|
483
|
+
output.push(member === 'key' ? key : mapValue);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
return output;
|
|
488
|
+
}
|
|
489
|
+
|
|
229
490
|
/**
|
|
230
491
|
* Infers a loaders.gl data type from a JavaScript value.
|
|
231
492
|
* @param value sample value
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {getGeoMetadata} from '@loaders.gl/gis';
|
|
6
|
+
|
|
7
|
+
import type {
|
|
8
|
+
ParquetBoundingBox,
|
|
9
|
+
ParquetPredicate,
|
|
10
|
+
ParquetRowGroupMetadata,
|
|
11
|
+
ParquetSourceMetadata
|
|
12
|
+
} from '../../parquet-source-types';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Conservatively tests a row group using native Parquet geospatial statistics.
|
|
16
|
+
*
|
|
17
|
+
* Missing or malformed statistics retain the row group. Longitude intervals support the
|
|
18
|
+
* Parquet GEOGRAPHY antimeridian representation where `xmin` is greater than `xmax`.
|
|
19
|
+
*/
|
|
20
|
+
export function canGeoParquetRowGroupMatch(
|
|
21
|
+
metadata: ParquetSourceMetadata,
|
|
22
|
+
rowGroup: ParquetRowGroupMetadata,
|
|
23
|
+
bbox: ParquetBoundingBox,
|
|
24
|
+
geometryColumn?: string
|
|
25
|
+
): boolean {
|
|
26
|
+
const bounds = getBoundingBoxCoordinates(bbox, true);
|
|
27
|
+
if (!bounds) return true;
|
|
28
|
+
const geoMetadata = getGeoMetadata(metadata.schema.metadata);
|
|
29
|
+
const selectedGeometryColumn = geometryColumn || geoMetadata?.primary_column;
|
|
30
|
+
if (!selectedGeometryColumn) return true;
|
|
31
|
+
const statistics = rowGroup.columns.find(
|
|
32
|
+
column => column.path.length === 1 && column.path[0] === selectedGeometryColumn
|
|
33
|
+
)?.geospatialStatistics?.bbox;
|
|
34
|
+
if (!statistics) return true;
|
|
35
|
+
if (
|
|
36
|
+
![statistics.xmin, statistics.xmax, statistics.ymin, statistics.ymax].every(Number.isFinite)
|
|
37
|
+
) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
return (
|
|
41
|
+
statistics.ymin <= bounds.north &&
|
|
42
|
+
statistics.ymax >= bounds.south &&
|
|
43
|
+
doLongitudeIntervalsIntersect(statistics.xmin, statistics.xmax, bounds.west, bounds.east)
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
type GeoParquetBoundingBoxCovering = {
|
|
48
|
+
/** Parquet schema path containing the per-row minimum x coordinate. */
|
|
49
|
+
xmin: readonly [string, 'xmin'];
|
|
50
|
+
/** Parquet schema path containing the per-row minimum y coordinate. */
|
|
51
|
+
ymin: readonly [string, 'ymin'];
|
|
52
|
+
/** Parquet schema path containing the per-row maximum x coordinate. */
|
|
53
|
+
xmax: readonly [string, 'xmax'];
|
|
54
|
+
/** Parquet schema path containing the per-row maximum y coordinate. */
|
|
55
|
+
ymax: readonly [string, 'ymax'];
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Creates an exact per-row bounding-box predicate from GeoParquet 1.1 covering metadata.
|
|
60
|
+
*
|
|
61
|
+
* Invalid, missing, unsupported, or antimeridian-crossing coverings return `undefined` so spatial
|
|
62
|
+
* filtering remains conservative. GeoParquet 1.1 explicitly excludes antimeridian-crossing
|
|
63
|
+
* geometries from its bounding-box covering technique.
|
|
64
|
+
*/
|
|
65
|
+
export function createGeoParquetBoundingBoxPredicate(
|
|
66
|
+
metadata: ParquetSourceMetadata,
|
|
67
|
+
bbox: ParquetBoundingBox,
|
|
68
|
+
geometryColumn?: string
|
|
69
|
+
): ParquetPredicate | undefined {
|
|
70
|
+
const bounds = getBoundingBoxCoordinates(bbox, false);
|
|
71
|
+
if (!bounds) {
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
const geoMetadata = getGeoMetadata(metadata.schema.metadata);
|
|
75
|
+
const selectedGeometryColumn = geometryColumn || geoMetadata?.primary_column;
|
|
76
|
+
const columnMetadata = selectedGeometryColumn
|
|
77
|
+
? geoMetadata?.columns?.[selectedGeometryColumn]
|
|
78
|
+
: undefined;
|
|
79
|
+
const covering = getBoundingBoxCovering(columnMetadata?.covering);
|
|
80
|
+
if (!covering || !hasCoveringColumns(metadata, covering)) {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
op: 'and',
|
|
86
|
+
args: [
|
|
87
|
+
{op: '<=', args: [{property: covering.xmin}, bounds.east]},
|
|
88
|
+
{op: '>=', args: [{property: covering.xmax}, bounds.west]},
|
|
89
|
+
{op: '<=', args: [{property: covering.ymin}, bounds.north]},
|
|
90
|
+
{op: '>=', args: [{property: covering.ymax}, bounds.south]}
|
|
91
|
+
]
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Combines an application predicate with an optional GeoParquet covering predicate. */
|
|
96
|
+
export function combineParquetPredicates(
|
|
97
|
+
predicate: ParquetPredicate | undefined,
|
|
98
|
+
spatialPredicate: ParquetPredicate | undefined
|
|
99
|
+
): ParquetPredicate | undefined {
|
|
100
|
+
if (!predicate) {
|
|
101
|
+
return spatialPredicate;
|
|
102
|
+
}
|
|
103
|
+
if (!spatialPredicate) {
|
|
104
|
+
return predicate;
|
|
105
|
+
}
|
|
106
|
+
return {op: 'and', args: [predicate, spatialPredicate]};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Validates the normative GeoParquet 1.1 two-level bbox covering representation. */
|
|
110
|
+
function getBoundingBoxCovering(value: unknown): GeoParquetBoundingBoxCovering | undefined {
|
|
111
|
+
if (!value || typeof value !== 'object') {
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
const bbox = (value as {bbox?: unknown}).bbox;
|
|
115
|
+
if (!bbox || typeof bbox !== 'object') {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
const candidate = bbox as Record<string, unknown>;
|
|
119
|
+
const xmin = getCoveringPath(candidate.xmin, 'xmin');
|
|
120
|
+
const ymin = getCoveringPath(candidate.ymin, 'ymin');
|
|
121
|
+
const xmax = getCoveringPath(candidate.xmax, 'xmax');
|
|
122
|
+
const ymax = getCoveringPath(candidate.ymax, 'ymax');
|
|
123
|
+
if (!xmin || !ymin || !xmax || !ymax) {
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
const rootColumn = xmin[0];
|
|
127
|
+
if (ymin[0] !== rootColumn || xmax[0] !== rootColumn || ymax[0] !== rootColumn) {
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
return {xmin, ymin, xmax, ymax};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Validates one GeoParquet 1.1 bbox leaf path. */
|
|
134
|
+
function getCoveringPath<T extends 'xmin' | 'ymin' | 'xmax' | 'ymax'>(
|
|
135
|
+
value: unknown,
|
|
136
|
+
expectedLeaf: T
|
|
137
|
+
): readonly [string, T] | undefined {
|
|
138
|
+
return Array.isArray(value) &&
|
|
139
|
+
value.length === 2 &&
|
|
140
|
+
typeof value[0] === 'string' &&
|
|
141
|
+
value[0].length > 0 &&
|
|
142
|
+
value[1] === expectedLeaf
|
|
143
|
+
? [value[0], expectedLeaf]
|
|
144
|
+
: undefined;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Ensures every covering path resolves to a physical column in the Parquet footer. */
|
|
148
|
+
function hasCoveringColumns(
|
|
149
|
+
metadata: ParquetSourceMetadata,
|
|
150
|
+
covering: GeoParquetBoundingBoxCovering
|
|
151
|
+
): boolean {
|
|
152
|
+
const availablePaths = new Set(
|
|
153
|
+
metadata.rowGroups.flatMap(rowGroup => rowGroup.columns.map(column => column.path.join('\0')))
|
|
154
|
+
);
|
|
155
|
+
return Object.values(covering).every(path => availablePaths.has(path.join('\0')));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Validates and normalizes the two horizontal dimensions of a query bounding box. */
|
|
159
|
+
function getBoundingBoxCoordinates(
|
|
160
|
+
bbox: ParquetBoundingBox,
|
|
161
|
+
allowAntimeridian: boolean
|
|
162
|
+
): {west: number; south: number; east: number; north: number} | undefined {
|
|
163
|
+
const west = bbox[0];
|
|
164
|
+
const south = bbox[1];
|
|
165
|
+
const east = bbox.length === 8 ? bbox[4] : bbox.length === 6 ? bbox[3] : bbox[2];
|
|
166
|
+
const north = bbox.length === 8 ? bbox[5] : bbox.length === 6 ? bbox[4] : bbox[3];
|
|
167
|
+
if (
|
|
168
|
+
![west, south, east, north].every(Number.isFinite) ||
|
|
169
|
+
(!allowAntimeridian && west > east) ||
|
|
170
|
+
south > north
|
|
171
|
+
) {
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
return {west, south, east, north};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Tests two longitude intervals, each of which may wrap across the antimeridian. */
|
|
178
|
+
function doLongitudeIntervalsIntersect(
|
|
179
|
+
firstWest: number,
|
|
180
|
+
firstEast: number,
|
|
181
|
+
secondWest: number,
|
|
182
|
+
secondEast: number
|
|
183
|
+
): boolean {
|
|
184
|
+
const firstIntervals =
|
|
185
|
+
firstWest <= firstEast
|
|
186
|
+
? [[firstWest, firstEast]]
|
|
187
|
+
: [
|
|
188
|
+
[firstWest, 180],
|
|
189
|
+
[-180, firstEast]
|
|
190
|
+
];
|
|
191
|
+
const secondIntervals =
|
|
192
|
+
secondWest <= secondEast
|
|
193
|
+
? [[secondWest, secondEast]]
|
|
194
|
+
: [
|
|
195
|
+
[secondWest, 180],
|
|
196
|
+
[-180, secondEast]
|
|
197
|
+
];
|
|
198
|
+
return firstIntervals.some(first =>
|
|
199
|
+
secondIntervals.some(second => first[0] <= second[1] && first[1] >= second[0])
|
|
200
|
+
);
|
|
201
|
+
}
|
|
@@ -104,7 +104,7 @@ function getGeometryMetadataForField(
|
|
|
104
104
|
const columnMetadata = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_METADATA_KEY);
|
|
105
105
|
if (columnMetadata) {
|
|
106
106
|
try {
|
|
107
|
-
const parsedMetadata =
|
|
107
|
+
const parsedMetadata = parseGeoArrowMetadata(JSON.parse(columnMetadata));
|
|
108
108
|
metadata = {
|
|
109
109
|
...(metadata || {}),
|
|
110
110
|
...parsedMetadata
|
|
@@ -117,6 +117,22 @@ function getGeometryMetadataForField(
|
|
|
117
117
|
return metadata;
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
+
/** Validates GeoArrow metadata, treating a mislabeled string CRS as opaque metadata. */
|
|
121
|
+
function parseGeoArrowMetadata(value: unknown): GeoArrowMetadata {
|
|
122
|
+
const result = GeoArrowMetadataSchema.safeParse(value);
|
|
123
|
+
if (result.success) {
|
|
124
|
+
return result.data;
|
|
125
|
+
}
|
|
126
|
+
if (value && typeof value === 'object' && typeof (value as {crs?: unknown}).crs === 'string') {
|
|
127
|
+
const {crs_type: _untrustedCRSType, ...opaqueMetadata} = value as Record<string, unknown>;
|
|
128
|
+
const opaqueResult = GeoArrowMetadataSchema.safeParse(opaqueMetadata);
|
|
129
|
+
if (opaqueResult.success) {
|
|
130
|
+
return opaqueResult.data;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
throw result.error;
|
|
134
|
+
}
|
|
135
|
+
|
|
120
136
|
/**
|
|
121
137
|
* Applies GeoParquet schema metadata to matching geometry fields as GeoArrow field metadata.
|
|
122
138
|
*
|
|
@@ -327,14 +343,15 @@ function getGeoArrowMetadataFromGeoParquetField(
|
|
|
327
343
|
}
|
|
328
344
|
|
|
329
345
|
const extensionMetadata: Record<string, unknown> = {};
|
|
330
|
-
if (columnMetadata.crs
|
|
346
|
+
if (columnMetadata.crs === undefined) {
|
|
347
|
+
extensionMetadata.crs = 'OGC:CRS84';
|
|
348
|
+
extensionMetadata.crs_type = 'authority_code';
|
|
349
|
+
} else if (columnMetadata.crs !== null) {
|
|
331
350
|
extensionMetadata.crs = columnMetadata.crs;
|
|
351
|
+
extensionMetadata.crs_type = 'projjson';
|
|
332
352
|
}
|
|
333
|
-
if (columnMetadata.
|
|
334
|
-
extensionMetadata.
|
|
335
|
-
}
|
|
336
|
-
if (columnMetadata.edges === 'spherical') {
|
|
337
|
-
extensionMetadata.edges = 'spherical';
|
|
353
|
+
if (columnMetadata.edges && columnMetadata.edges !== 'planar') {
|
|
354
|
+
extensionMetadata.edges = columnMetadata.edges;
|
|
338
355
|
}
|
|
339
356
|
|
|
340
357
|
return Object.keys(extensionMetadata).length > 0
|
|
@@ -368,19 +385,29 @@ function synthesizeGeoParquetColumnMetadata(
|
|
|
368
385
|
geometry_types: geometryTypes
|
|
369
386
|
};
|
|
370
387
|
|
|
371
|
-
if (geometryMetadata.crs
|
|
388
|
+
if (geometryMetadata.crs === undefined) {
|
|
389
|
+
columnMetadata.crs = null;
|
|
390
|
+
} else if (typeof geometryMetadata.crs === 'object') {
|
|
372
391
|
columnMetadata.crs = geometryMetadata.crs;
|
|
392
|
+
} else if (
|
|
393
|
+
typeof geometryMetadata.crs !== 'string' ||
|
|
394
|
+
!isDefaultCRS84Identifier(geometryMetadata.crs)
|
|
395
|
+
) {
|
|
396
|
+
return null;
|
|
373
397
|
}
|
|
374
|
-
if (geometryMetadata.
|
|
375
|
-
columnMetadata.
|
|
376
|
-
}
|
|
377
|
-
if (geometryMetadata.edges === 'spherical') {
|
|
378
|
-
columnMetadata.edges = 'spherical';
|
|
398
|
+
if (geometryMetadata.edges) {
|
|
399
|
+
columnMetadata.edges = geometryMetadata.edges;
|
|
379
400
|
}
|
|
380
401
|
|
|
381
402
|
return columnMetadata;
|
|
382
403
|
}
|
|
383
404
|
|
|
405
|
+
/** Returns whether a compact GeoArrow CRS identifier has GeoParquet's default CRS semantics. */
|
|
406
|
+
function isDefaultCRS84Identifier(crs: string): boolean {
|
|
407
|
+
const normalizedCRS = crs.toUpperCase();
|
|
408
|
+
return normalizedCRS === 'OGC:CRS84' || normalizedCRS === 'EPSG:4326';
|
|
409
|
+
}
|
|
410
|
+
|
|
384
411
|
function inferGeometryTypesFromGeoArrowEncoding(
|
|
385
412
|
encoding?: GeoArrowEncoding
|
|
386
413
|
): GeoParquetGeometryType[] {
|
|
@@ -420,8 +447,8 @@ function pickValidOptionalGeoParquetColumnMetadata(
|
|
|
420
447
|
|
|
421
448
|
function isValidBBox(
|
|
422
449
|
bbox: GeoColumnMetadata['bbox']
|
|
423
|
-
): bbox is [
|
|
424
|
-
if (!Array.isArray(bbox) ||
|
|
450
|
+
): bbox is NonNullable<GeoColumnMetadata['bbox']> {
|
|
451
|
+
if (!Array.isArray(bbox) || ![4, 6, 8].includes(bbox.length)) {
|
|
425
452
|
return false;
|
|
426
453
|
}
|
|
427
454
|
|