@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundled.d.ts +6 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +7 -0
- package/dist/bundled.js.map +1 -0
- package/dist/geoparquet-loader-with-parser.d.ts +34 -0
- package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
- package/dist/geoparquet-loader-with-parser.js +52 -0
- package/dist/geoparquet-loader-with-parser.js.map +1 -0
- package/dist/geoparquet-loader.d.ts +61 -0
- package/dist/geoparquet-loader.d.ts.map +1 -0
- package/dist/geoparquet-loader.js +30 -0
- package/dist/geoparquet-loader.js.map +1 -0
- package/dist/index.cjs +5879 -5643
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -11
- package/dist/index.js.map +1 -1
- package/dist/lib/constants.d.ts +1 -1
- package/dist/lib/constants.js +2 -2
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js +149 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
- package/dist/lib/geo/geospatial-metadata.js +357 -0
- package/dist/lib/geo/geospatial-metadata.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-tables.d.ts +50 -0
- package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-tables.js +101 -0
- package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +6 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +23 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -4
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +77 -26
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/utils/load-wasm.d.ts +2 -2
- package/dist/lib/utils/load-wasm.d.ts.map +1 -1
- package/dist/lib/utils/load-wasm.js +7 -7
- package/dist/lib/utils/load-wasm.js.map +1 -1
- package/dist/lib/utils/make-stream-iterator.js +1 -1
- package/dist/lib/utils/make-stream-iterator.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
- package/dist/lib/utils/normalize-parquet-options.js +11 -0
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
- package/dist/parquet-format.d.ts +2 -0
- package/dist/parquet-format.d.ts.map +1 -1
- package/dist/parquet-format.js +2 -0
- package/dist/parquet-format.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +57 -0
- package/dist/parquet-js-loader.d.ts.map +1 -0
- package/dist/parquet-js-loader.js +33 -0
- package/dist/parquet-js-loader.js.map +1 -0
- package/dist/parquet-js-writer.d.ts +34 -0
- package/dist/parquet-js-writer.d.ts.map +1 -0
- package/dist/parquet-js-writer.js +28 -0
- package/dist/parquet-js-writer.js.map +1 -0
- package/dist/parquet-loader-options.d.ts +56 -0
- package/dist/parquet-loader-options.d.ts.map +1 -0
- package/dist/parquet-loader-options.js +11 -0
- package/dist/parquet-loader-options.js.map +1 -0
- package/dist/parquet-loader-with-parser.d.ts +33 -0
- package/dist/parquet-loader-with-parser.d.ts.map +1 -0
- package/dist/parquet-loader-with-parser.js +40 -0
- package/dist/parquet-loader-with-parser.js.map +1 -0
- package/dist/parquet-loader.d.ts +62 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +53 -0
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-wasm-loader-with-parser.d.ts +40 -0
- package/dist/parquet-wasm-loader-with-parser.d.ts.map +1 -0
- package/dist/parquet-wasm-loader-with-parser.js +68 -0
- package/dist/parquet-wasm-loader-with-parser.js.map +1 -0
- package/dist/parquet-writer.d.ts +10 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +29 -0
- package/dist/parquet-writer.js.map +1 -0
- package/dist/parquet_wasm_bg.wasm +0 -0
- package/dist/parquetjs/codecs/declare.d.ts +2 -2
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +1 -1
- package/dist/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +36 -41
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +1 -1
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +16 -21
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +2 -3
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +4 -105
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +4 -4
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +38 -33
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.d.ts +0 -1
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +0 -1
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +1 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +4 -22
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +9 -9
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +1 -1
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +36 -35
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/utils/binary-utils.d.ts +39 -0
- package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/binary-utils.js +102 -0
- package/dist/parquetjs/utils/binary-utils.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.d.ts +1 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +3 -3
- package/dist/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts +4 -4
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +12 -17
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
- package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
- package/dist/unbundled.d.ts +6 -0
- package/dist/unbundled.d.ts.map +1 -0
- package/dist/unbundled.js +7 -0
- package/dist/unbundled.js.map +1 -0
- package/package.json +21 -25
- package/src/bundled.ts +9 -0
- package/src/geoparquet-loader-with-parser.ts +95 -0
- package/src/geoparquet-loader.ts +54 -0
- package/src/index.ts +7 -25
- package/src/lib/constants.ts +1 -1
- package/src/lib/encoders/encode-arrow-to-parquet.ts +9 -15
- package/src/lib/encoders/encode-table-to-parquet-js.ts +199 -0
- package/src/lib/geo/geospatial-metadata.ts +507 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
- package/src/lib/parsers/parse-parquet-tables.ts +137 -0
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +36 -0
- package/src/lib/parsers/parse-parquet-to-arrow.ts +63 -36
- package/src/lib/parsers/parse-parquet-to-columns.ts +6 -7
- package/src/lib/parsers/parse-parquet-to-json.ts +101 -33
- package/src/lib/utils/load-wasm.ts +16 -9
- package/src/lib/utils/make-stream-iterator.ts +1 -1
- package/src/lib/utils/normalize-parquet-options.ts +14 -0
- package/src/parquet-format.ts +2 -0
- package/src/parquet-js-loader.ts +43 -0
- package/src/parquet-js-writer.ts +52 -0
- package/src/parquet-loader-options.ts +63 -0
- package/src/parquet-loader-with-parser.ts +68 -0
- package/src/parquet-loader.ts +79 -0
- package/src/parquet-wasm-loader-with-parser.ts +115 -0
- package/src/parquet-writer.ts +44 -0
- package/src/parquetjs/codecs/declare.ts +2 -2
- package/src/parquetjs/codecs/dictionary.ts +1 -1
- package/src/parquetjs/codecs/plain.ts +67 -53
- package/src/parquetjs/codecs/rle.ts +20 -26
- package/src/parquetjs/compression.ts +7 -114
- package/src/parquetjs/encoder/parquet-encoder.ts +49 -43
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +1 -1
- package/src/parquetjs/parquet-thrift/index.ts +0 -2
- package/src/parquetjs/parser/decoders.ts +5 -23
- package/src/parquetjs/parser/parquet-reader.ts +10 -12
- package/src/parquetjs/schema/declare.ts +1 -1
- package/src/parquetjs/schema/types.ts +48 -40
- package/src/parquetjs/utils/binary-utils.ts +127 -0
- package/src/parquetjs/utils/file-utils.ts +4 -4
- package/src/parquetjs/utils/read-utils.ts +19 -34
- package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
- package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
- package/src/unbundled.ts +9 -0
- package/dist/parquet-arrow-loader.d.ts +0 -43
- package/dist/parquet-arrow-loader.d.ts.map +0 -1
- package/dist/parquet-arrow-loader.js +0 -49
- package/dist/parquet-arrow-loader.js.map +0 -1
- package/dist/parquet-arrow-writer.d.ts +0 -9
- package/dist/parquet-arrow-writer.d.ts.map +0 -1
- package/dist/parquet-arrow-writer.js +0 -23
- package/dist/parquet-arrow-writer.js.map +0 -1
- package/dist/parquet-json-loader.d.ts +0 -148
- package/dist/parquet-json-loader.d.ts.map +0 -1
- package/dist/parquet-json-loader.js +0 -106
- package/dist/parquet-json-loader.js.map +0 -1
- package/dist/parquet-json-writer.d.ts +0 -19
- package/dist/parquet-json-writer.d.ts.map +0 -1
- package/dist/parquet-json-writer.js +0 -20
- package/dist/parquet-json-writer.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.d.ts +0 -222
- package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer.js +0 -1868
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.d.ts +0 -4
- package/dist/polyfills/buffer/index.d.ts.map +0 -1
- package/dist/polyfills/buffer/index.js +0 -9
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/workers/parquet-json-worker.d.ts +0 -2
- package/dist/workers/parquet-json-worker.d.ts.map +0 -1
- package/dist/workers/parquet-json-worker.js +0 -7
- package/dist/workers/parquet-json-worker.js.map +0 -1
- package/src/parquet-arrow-loader.ts +0 -74
- package/src/parquet-arrow-writer.ts +0 -33
- package/src/parquet-json-loader.ts +0 -160
- package/src/parquet-json-writer.ts +0 -27
- package/src/parquetjs/modules.d.ts +0 -25
- package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
- package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
- package/src/polyfills/buffer/buffer.ts +0 -2205
- package/src/polyfills/buffer/index.ts +0 -11
- package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
- package/src/workers/parquet-json-worker.ts +0 -8
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {DataType, Field, ObjectRowTable, Schema, Table} from '@loaders.gl/schema';
|
|
6
|
+
import {concatUint8Arrays} from '../../parquetjs/utils/binary-utils';
|
|
7
|
+
import {ParquetEncoder} from '../../parquetjs/encoder/parquet-encoder';
|
|
8
|
+
import {ParquetSchema} from '../../parquetjs/schema/schema';
|
|
9
|
+
import type {FieldDefinition, SchemaDefinition} from '../../parquetjs/schema/declare';
|
|
10
|
+
import type {ParquetJSWriterOptions} from '../../parquet-js-writer';
|
|
11
|
+
|
|
12
|
+
/** Encode a plain loaders.gl table with the parquetjs implementation. */
|
|
13
|
+
export async function encodeTableToParquetJs(
|
|
14
|
+
table: Table,
|
|
15
|
+
objectRowTable: ObjectRowTable,
|
|
16
|
+
options: ParquetJSWriterOptions
|
|
17
|
+
): Promise<ArrayBuffer> {
|
|
18
|
+
const parquetSchema = new ParquetSchema(
|
|
19
|
+
convertSchemaToParquetSchema(table.schema!, objectRowTable)
|
|
20
|
+
);
|
|
21
|
+
const chunks: Uint8Array[] = [];
|
|
22
|
+
const outputStream = {
|
|
23
|
+
write(chunk: Uint8Array, callback: (error?: Error) => void) {
|
|
24
|
+
chunks.push(chunk.slice());
|
|
25
|
+
callback();
|
|
26
|
+
},
|
|
27
|
+
close(callback: (error?: Error) => void) {
|
|
28
|
+
callback();
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const encoder = await ParquetEncoder.openStream(
|
|
33
|
+
parquetSchema,
|
|
34
|
+
outputStream as any,
|
|
35
|
+
options.parquet
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
for (const [key, value] of Object.entries(table.schema?.metadata || {})) {
|
|
39
|
+
encoder.setMetadata(key, value);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
for (const row of objectRowTable.data) {
|
|
43
|
+
await encoder.appendRow(row);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
await encoder.close();
|
|
47
|
+
|
|
48
|
+
return concatUint8Arrays(chunks).slice().buffer;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Converts a loaders.gl schema into a parquetjs schema definition.
|
|
53
|
+
* @param schema table schema
|
|
54
|
+
* @param objectRowTable table data in object-row form
|
|
55
|
+
* @returns parquetjs schema definition
|
|
56
|
+
*/
|
|
57
|
+
function convertSchemaToParquetSchema(
|
|
58
|
+
schema: Schema,
|
|
59
|
+
objectRowTable: ObjectRowTable
|
|
60
|
+
): SchemaDefinition {
|
|
61
|
+
const parquetFields: SchemaDefinition = {};
|
|
62
|
+
|
|
63
|
+
for (const field of schema.fields) {
|
|
64
|
+
parquetFields[field.name] = convertFieldToParquetFieldDefinition(field, objectRowTable.data);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return parquetFields;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Converts one loaders.gl field to the closest parquetjs field definition.
|
|
72
|
+
* @param field table field metadata
|
|
73
|
+
* @param rows object rows used for nullability and fallback inference
|
|
74
|
+
* @returns parquetjs field definition
|
|
75
|
+
*/
|
|
76
|
+
function convertFieldToParquetFieldDefinition(
|
|
77
|
+
field: Field,
|
|
78
|
+
rows: Array<Record<string, unknown>>
|
|
79
|
+
): FieldDefinition {
|
|
80
|
+
const sampleValue = getFirstDefinedValue(rows, field.name);
|
|
81
|
+
const nullable = field.nullable ?? sampleValue === undefined;
|
|
82
|
+
const dataType = field.type === 'null' ? getDataTypeFromValue(sampleValue) : field.type;
|
|
83
|
+
|
|
84
|
+
switch (dataType) {
|
|
85
|
+
case 'bool':
|
|
86
|
+
return {type: 'BOOLEAN', optional: nullable};
|
|
87
|
+
|
|
88
|
+
case 'int':
|
|
89
|
+
case 'int8':
|
|
90
|
+
case 'int16':
|
|
91
|
+
case 'int32':
|
|
92
|
+
case 'uint8':
|
|
93
|
+
case 'uint16':
|
|
94
|
+
case 'uint32':
|
|
95
|
+
return {type: 'INT32', optional: nullable};
|
|
96
|
+
|
|
97
|
+
case 'int64':
|
|
98
|
+
case 'uint64':
|
|
99
|
+
return {type: 'INT64', optional: nullable};
|
|
100
|
+
|
|
101
|
+
case 'float':
|
|
102
|
+
case 'float16':
|
|
103
|
+
case 'float32':
|
|
104
|
+
return {type: 'FLOAT', optional: nullable};
|
|
105
|
+
|
|
106
|
+
case 'float64':
|
|
107
|
+
return {type: 'DOUBLE', optional: nullable};
|
|
108
|
+
|
|
109
|
+
case 'utf8':
|
|
110
|
+
return {type: 'UTF8', optional: nullable};
|
|
111
|
+
|
|
112
|
+
case 'binary':
|
|
113
|
+
return {type: 'BYTE_ARRAY', optional: nullable};
|
|
114
|
+
|
|
115
|
+
case 'date-day':
|
|
116
|
+
return {type: 'DATE', optional: nullable};
|
|
117
|
+
|
|
118
|
+
case 'date-millisecond':
|
|
119
|
+
case 'timestamp-second':
|
|
120
|
+
case 'timestamp-millisecond':
|
|
121
|
+
return {type: 'TIMESTAMP_MILLIS', optional: nullable};
|
|
122
|
+
|
|
123
|
+
case 'timestamp-microsecond':
|
|
124
|
+
case 'timestamp-nanosecond':
|
|
125
|
+
return {type: 'TIMESTAMP_MICROS', optional: nullable};
|
|
126
|
+
|
|
127
|
+
case 'time-second':
|
|
128
|
+
case 'time-millisecond':
|
|
129
|
+
return {type: 'TIME_MILLIS', optional: nullable};
|
|
130
|
+
|
|
131
|
+
case 'time-microsecond':
|
|
132
|
+
case 'time-nanosecond':
|
|
133
|
+
return {type: 'TIME_MICROS', optional: nullable};
|
|
134
|
+
|
|
135
|
+
default:
|
|
136
|
+
if (typeof dataType === 'object' && dataType.type === 'fixed-size-binary') {
|
|
137
|
+
return {
|
|
138
|
+
type: 'FIXED_LEN_BYTE_ARRAY',
|
|
139
|
+
typeLength: dataType.byteWidth,
|
|
140
|
+
optional: nullable
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
throw new Error(
|
|
145
|
+
`ParquetJSWriter: Unsupported field "${field.name}" with type ${formatDataType(dataType)}`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Finds the first non-nullish value in a column.
|
|
152
|
+
* @param rows table rows
|
|
153
|
+
* @param columnName column name
|
|
154
|
+
* @returns first defined value, if any
|
|
155
|
+
*/
|
|
156
|
+
function getFirstDefinedValue(rows: Array<Record<string, unknown>>, columnName: string): unknown {
|
|
157
|
+
for (const row of rows) {
|
|
158
|
+
const value = row[columnName];
|
|
159
|
+
if (value !== null && value !== undefined) {
|
|
160
|
+
return value;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Infers a loaders.gl data type from a JavaScript value.
|
|
169
|
+
* @param value sample value
|
|
170
|
+
* @returns inferred data type
|
|
171
|
+
*/
|
|
172
|
+
function getDataTypeFromValue(value: unknown): DataType {
|
|
173
|
+
if (typeof value === 'boolean') {
|
|
174
|
+
return 'bool';
|
|
175
|
+
}
|
|
176
|
+
if (typeof value === 'number') {
|
|
177
|
+
return Number.isInteger(value) ? 'int32' : 'float64';
|
|
178
|
+
}
|
|
179
|
+
if (typeof value === 'string') {
|
|
180
|
+
return 'utf8';
|
|
181
|
+
}
|
|
182
|
+
if (value instanceof Date) {
|
|
183
|
+
return 'timestamp-millisecond';
|
|
184
|
+
}
|
|
185
|
+
if (value instanceof Uint8Array) {
|
|
186
|
+
return 'binary';
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
throw new Error('ParquetJSWriter: Unable to infer a Parquet type from the provided row data');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Formats a loaders.gl data type for error messages.
|
|
194
|
+
* @param dataType input data type
|
|
195
|
+
* @returns human-readable type string
|
|
196
|
+
*/
|
|
197
|
+
function formatDataType(dataType: DataType): string {
|
|
198
|
+
return typeof dataType === 'string' ? dataType : dataType.type;
|
|
199
|
+
}
|
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import * as arrow from 'apache-arrow';
|
|
6
|
+
|
|
7
|
+
import type {Field, Schema} from '@loaders.gl/schema';
|
|
8
|
+
import {
|
|
9
|
+
getGeoMetadata,
|
|
10
|
+
getMetadataValue,
|
|
11
|
+
setGeoMetadata,
|
|
12
|
+
setMetadataValue,
|
|
13
|
+
type GeoColumnMetadata,
|
|
14
|
+
type GeoMetadata,
|
|
15
|
+
type GeoParquetGeometryType
|
|
16
|
+
} from '@loaders.gl/gis';
|
|
17
|
+
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
|
|
18
|
+
|
|
19
|
+
const GEOARROW_EXTENSION_NAME_KEY = 'ARROW:extension:name';
|
|
20
|
+
const GEOARROW_EXTENSION_METADATA_KEY = 'ARROW:extension:metadata';
|
|
21
|
+
const GEOPARQUET_VERSION = '1.1.0';
|
|
22
|
+
|
|
23
|
+
type GeoArrowEncoding =
|
|
24
|
+
| 'geoarrow.geometry'
|
|
25
|
+
| 'geoarrow.geometrycollection'
|
|
26
|
+
| 'geoarrow.multipolygon'
|
|
27
|
+
| 'geoarrow.polygon'
|
|
28
|
+
| 'geoarrow.multilinestring'
|
|
29
|
+
| 'geoarrow.linestring'
|
|
30
|
+
| 'geoarrow.multipoint'
|
|
31
|
+
| 'geoarrow.point'
|
|
32
|
+
| 'geoarrow.wkb'
|
|
33
|
+
| 'geoarrow.wkt';
|
|
34
|
+
|
|
35
|
+
type GeoArrowMetadata = {
|
|
36
|
+
encoding?: GeoArrowEncoding;
|
|
37
|
+
crs?: Record<string, unknown>;
|
|
38
|
+
edges?: 'spherical';
|
|
39
|
+
[key: string]: unknown;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const GEOARROW_ENCODINGS = [
|
|
43
|
+
'geoarrow.geometry',
|
|
44
|
+
'geoarrow.geometrycollection',
|
|
45
|
+
'geoarrow.multipolygon',
|
|
46
|
+
'geoarrow.polygon',
|
|
47
|
+
'geoarrow.multilinestring',
|
|
48
|
+
'geoarrow.linestring',
|
|
49
|
+
'geoarrow.multipoint',
|
|
50
|
+
'geoarrow.point',
|
|
51
|
+
'geoarrow.wkb',
|
|
52
|
+
'geoarrow.wkt'
|
|
53
|
+
] as const satisfies GeoArrowEncoding[];
|
|
54
|
+
|
|
55
|
+
const GEOPARQUET_TO_GEOARROW_ENCODINGS = {
|
|
56
|
+
wkb: 'geoarrow.wkb',
|
|
57
|
+
point: 'geoarrow.point',
|
|
58
|
+
linestring: 'geoarrow.linestring',
|
|
59
|
+
polygon: 'geoarrow.polygon',
|
|
60
|
+
multipoint: 'geoarrow.multipoint',
|
|
61
|
+
multilinestring: 'geoarrow.multilinestring',
|
|
62
|
+
multipolygon: 'geoarrow.multipolygon'
|
|
63
|
+
} as const satisfies Record<string, GeoArrowEncoding>;
|
|
64
|
+
|
|
65
|
+
const GEOARROW_TO_GEOPARQUET_ENCODINGS = {
|
|
66
|
+
'geoarrow.wkb': 'wkb',
|
|
67
|
+
'geoarrow.point': 'point',
|
|
68
|
+
'geoarrow.linestring': 'linestring',
|
|
69
|
+
'geoarrow.polygon': 'polygon',
|
|
70
|
+
'geoarrow.multipoint': 'multipoint',
|
|
71
|
+
'geoarrow.multilinestring': 'multilinestring',
|
|
72
|
+
'geoarrow.multipolygon': 'multipolygon'
|
|
73
|
+
} as const;
|
|
74
|
+
|
|
75
|
+
const GEOARROW_TO_GEOMETRY_TYPE = {
|
|
76
|
+
'geoarrow.point': 'Point',
|
|
77
|
+
'geoarrow.linestring': 'LineString',
|
|
78
|
+
'geoarrow.polygon': 'Polygon',
|
|
79
|
+
'geoarrow.multipoint': 'MultiPoint',
|
|
80
|
+
'geoarrow.multilinestring': 'MultiLineString',
|
|
81
|
+
'geoarrow.multipolygon': 'MultiPolygon'
|
|
82
|
+
} as const satisfies Record<string, GeoParquetGeometryType>;
|
|
83
|
+
|
|
84
|
+
/** Returns GeoArrow geometry field metadata keyed by column name. */
|
|
85
|
+
function getGeometryColumnsFromSchema(schema: Schema): Record<string, GeoArrowMetadata> {
|
|
86
|
+
const geometryColumns: Record<string, GeoArrowMetadata> = {};
|
|
87
|
+
for (const field of schema.fields || []) {
|
|
88
|
+
const metadata = getGeometryMetadataForField(field.metadata || {});
|
|
89
|
+
if (metadata) {
|
|
90
|
+
geometryColumns[field.name] = metadata;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return geometryColumns;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Extracts GeoArrow metadata from one schema field metadata object. */
|
|
97
|
+
function getGeometryMetadataForField(
|
|
98
|
+
fieldMetadata: Record<string, string>
|
|
99
|
+
): GeoArrowMetadata | null {
|
|
100
|
+
let metadata: GeoArrowMetadata | null = null;
|
|
101
|
+
let geoEncoding = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_NAME_KEY);
|
|
102
|
+
if (geoEncoding) {
|
|
103
|
+
geoEncoding = geoEncoding.toLowerCase();
|
|
104
|
+
if (geoEncoding === 'wkb') {
|
|
105
|
+
geoEncoding = 'geoarrow.wkb';
|
|
106
|
+
}
|
|
107
|
+
if (geoEncoding === 'wkt') {
|
|
108
|
+
geoEncoding = 'geoarrow.wkt';
|
|
109
|
+
}
|
|
110
|
+
if (GEOARROW_ENCODINGS.includes(geoEncoding as GeoArrowEncoding)) {
|
|
111
|
+
metadata ||= {};
|
|
112
|
+
metadata.encoding = geoEncoding as GeoArrowEncoding;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const columnMetadata = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_METADATA_KEY);
|
|
117
|
+
if (columnMetadata) {
|
|
118
|
+
try {
|
|
119
|
+
metadata = {
|
|
120
|
+
...(metadata || {}),
|
|
121
|
+
...(JSON.parse(columnMetadata) as GeoArrowMetadata)
|
|
122
|
+
};
|
|
123
|
+
} catch {
|
|
124
|
+
return metadata;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return metadata;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Applies GeoParquet schema metadata to matching geometry fields as GeoArrow field metadata.
|
|
133
|
+
*
|
|
134
|
+
* @param schema - Schema to annotate.
|
|
135
|
+
* @returns The same schema reference.
|
|
136
|
+
*/
|
|
137
|
+
export function applyGeoParquetToFieldMetadata(schema: Schema): Schema {
|
|
138
|
+
const geoMetadata = getGeoMetadata(schema.metadata);
|
|
139
|
+
if (!geoMetadata) {
|
|
140
|
+
return schema;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
for (const field of schema.fields) {
|
|
144
|
+
const columnMetadata = geoMetadata.columns?.[field.name];
|
|
145
|
+
if (!columnMetadata) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const fieldMetadata = getGeoArrowMetadataFromGeoParquetField(field, columnMetadata);
|
|
150
|
+
if (!fieldMetadata) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
field.metadata ||= {};
|
|
155
|
+
setMetadataValue(field.metadata, GEOARROW_EXTENSION_NAME_KEY, fieldMetadata.extensionName);
|
|
156
|
+
|
|
157
|
+
if (fieldMetadata.extensionMetadata) {
|
|
158
|
+
setMetadataValue(
|
|
159
|
+
field.metadata,
|
|
160
|
+
GEOARROW_EXTENSION_METADATA_KEY,
|
|
161
|
+
JSON.stringify(fieldMetadata.extensionMetadata)
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return schema;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Creates an Arrow JS table whose schema metadata matches a loaders.gl schema.
|
|
171
|
+
*
|
|
172
|
+
* @param table - Arrow JS table to rewrap.
|
|
173
|
+
* @param schema - loaders.gl schema containing desired field/schema metadata.
|
|
174
|
+
* @returns Table using the updated Arrow JS schema.
|
|
175
|
+
*/
|
|
176
|
+
export function applyLoadersSchemaToArrowTable(table: arrow.Table, schema: Schema): arrow.Table {
|
|
177
|
+
const arrowSchema = createArrowJsSchema(table.schema, schema);
|
|
178
|
+
return new arrow.Table(arrowSchema, table.batches);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Normalizes an Arrow wrapper so GeoParquet metadata is preserved and GeoArrow field metadata is attached.
|
|
183
|
+
*
|
|
184
|
+
* @param arrowTable - loaders.gl Arrow table wrapper.
|
|
185
|
+
* @param schemaMetadata - Optional schema metadata override from the Parquet footer.
|
|
186
|
+
* @returns Arrow table wrapper with synchronized schema metadata.
|
|
187
|
+
*/
|
|
188
|
+
export function normalizeArrowTableGeoMetadata(
|
|
189
|
+
arrowTable: {shape: 'arrow-table'; data: arrow.Table; schema?: Schema},
|
|
190
|
+
schemaMetadata?: Map<string, string> | Record<string, string>
|
|
191
|
+
): {shape: 'arrow-table'; data: arrow.Table; schema: Schema} {
|
|
192
|
+
const schema = convertArrowToSchema(arrowTable.data.schema);
|
|
193
|
+
mergeSchemaMetadata(schema, schemaMetadata);
|
|
194
|
+
applyGeoParquetToFieldMetadata(schema);
|
|
195
|
+
|
|
196
|
+
return {
|
|
197
|
+
shape: 'arrow-table',
|
|
198
|
+
data: applyLoadersSchemaToArrowTable(arrowTable.data, schema),
|
|
199
|
+
schema
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Synthesizes GeoParquet schema metadata from GeoArrow field metadata when needed.
|
|
205
|
+
*
|
|
206
|
+
* @param schema - Schema to inspect and update.
|
|
207
|
+
* @returns The same schema reference.
|
|
208
|
+
*/
|
|
209
|
+
export function ensureGeoParquetMetadata(schema: Schema): Schema {
|
|
210
|
+
schema.metadata ||= {};
|
|
211
|
+
const rawExistingGeoMetadata = getGeoMetadata(schema.metadata);
|
|
212
|
+
if (isValidGeoParquetMetadata(rawExistingGeoMetadata)) {
|
|
213
|
+
return schema;
|
|
214
|
+
}
|
|
215
|
+
const existingGeoMetadata: GeoMetadata | undefined = rawExistingGeoMetadata || undefined;
|
|
216
|
+
|
|
217
|
+
const geometryColumns = getGeometryColumnsFromSchema(schema);
|
|
218
|
+
const geometryColumnNames = Object.keys(geometryColumns);
|
|
219
|
+
if (geometryColumnNames.length === 0) {
|
|
220
|
+
return schema;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const geoMetadata: GeoMetadata = {
|
|
224
|
+
version: GEOPARQUET_VERSION,
|
|
225
|
+
primary_column: geometryColumnNames[0],
|
|
226
|
+
columns: {}
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
for (const geometryColumnName of geometryColumnNames) {
|
|
230
|
+
const geometryMetadata = geometryColumns[geometryColumnName];
|
|
231
|
+
const synthesizedColumnMetadata = synthesizeGeoParquetColumnMetadata(geometryMetadata);
|
|
232
|
+
if (!synthesizedColumnMetadata) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const existingColumnMetadata = existingGeoMetadata?.columns?.[geometryColumnName];
|
|
237
|
+
geoMetadata.columns[geometryColumnName] = {
|
|
238
|
+
...pickValidOptionalGeoParquetColumnMetadata(existingColumnMetadata),
|
|
239
|
+
...synthesizedColumnMetadata
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if (Object.keys(geoMetadata.columns).length === 0) {
|
|
244
|
+
return schema;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
setGeoMetadata(schema.metadata, geoMetadata);
|
|
248
|
+
return schema;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Ensures an Arrow table wrapper carries valid GeoParquet schema metadata when GeoArrow fields are present.
|
|
253
|
+
*
|
|
254
|
+
* @param arrowTable - Arrow table wrapper to inspect and update.
|
|
255
|
+
* @returns Updated Arrow table wrapper.
|
|
256
|
+
*/
|
|
257
|
+
export function ensureGeoParquetMetadataOnArrowTable(arrowTable: {
|
|
258
|
+
shape: 'arrow-table';
|
|
259
|
+
data: arrow.Table;
|
|
260
|
+
schema?: Schema;
|
|
261
|
+
}): {shape: 'arrow-table'; data: arrow.Table; schema: Schema} {
|
|
262
|
+
const schema = arrowTable.schema || convertArrowToSchema(arrowTable.data.schema);
|
|
263
|
+
ensureGeoParquetMetadata(schema);
|
|
264
|
+
|
|
265
|
+
return {
|
|
266
|
+
shape: 'arrow-table',
|
|
267
|
+
data: applyLoadersSchemaToArrowTable(arrowTable.data, schema),
|
|
268
|
+
schema
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Returns Parquet file key/value metadata as a plain map.
|
|
274
|
+
*
|
|
275
|
+
* @param parquetMetadata - parquet-wasm metadata object.
|
|
276
|
+
* @returns Metadata map.
|
|
277
|
+
*/
|
|
278
|
+
export function getParquetFileMetadataMap(parquetMetadata: {
|
|
279
|
+
fileMetadata(): {keyValueMetadata(): Map<string, string>};
|
|
280
|
+
}): Map<string, string> {
|
|
281
|
+
return parquetMetadata.fileMetadata().keyValueMetadata();
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function mergeSchemaMetadata(
|
|
285
|
+
schema: Schema,
|
|
286
|
+
schemaMetadata?: Map<string, string> | Record<string, string>
|
|
287
|
+
): void {
|
|
288
|
+
if (!schemaMetadata) {
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
schema.metadata ||= {};
|
|
293
|
+
for (const [metadataKey, metadataValue] of getMetadataEntries(schemaMetadata)) {
|
|
294
|
+
setMetadataValue(schema.metadata, metadataKey, metadataValue);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function createArrowJsSchema(baseSchema: arrow.Schema, schema: Schema): arrow.Schema {
|
|
299
|
+
const fields = baseSchema.fields.map(baseField => {
|
|
300
|
+
const nextField = schema.fields.find(field => field.name === baseField.name);
|
|
301
|
+
if (!nextField?.metadata) {
|
|
302
|
+
return baseField;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
return baseField.clone({
|
|
306
|
+
metadata: createMetadataMap(nextField.metadata)
|
|
307
|
+
});
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
return new arrow.Schema(
|
|
311
|
+
fields,
|
|
312
|
+
createMetadataMap(schema.metadata),
|
|
313
|
+
baseSchema.dictionaries,
|
|
314
|
+
baseSchema.metadataVersion
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function getGeoArrowMetadataFromGeoParquetField(
|
|
319
|
+
field: Field,
|
|
320
|
+
columnMetadata: GeoColumnMetadata
|
|
321
|
+
): {extensionName: GeoArrowEncoding; extensionMetadata?: Record<string, unknown>} | null {
|
|
322
|
+
const normalizedEncoding = String(columnMetadata.encoding || '').toLowerCase();
|
|
323
|
+
const extensionName =
|
|
324
|
+
GEOPARQUET_TO_GEOARROW_ENCODINGS[
|
|
325
|
+
normalizedEncoding as keyof typeof GEOPARQUET_TO_GEOARROW_ENCODINGS
|
|
326
|
+
];
|
|
327
|
+
|
|
328
|
+
if (!extensionName) {
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if (normalizedEncoding === 'wkb' && field.type !== 'binary') {
|
|
333
|
+
return null;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (normalizedEncoding !== 'wkb' && (field.type === 'binary' || field.type === 'utf8')) {
|
|
337
|
+
return null;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const extensionMetadata: Record<string, unknown> = {};
|
|
341
|
+
if (columnMetadata.crs !== undefined) {
|
|
342
|
+
extensionMetadata.crs = columnMetadata.crs;
|
|
343
|
+
}
|
|
344
|
+
if (columnMetadata.crs_type !== undefined) {
|
|
345
|
+
extensionMetadata.crs_type = columnMetadata.crs_type;
|
|
346
|
+
}
|
|
347
|
+
if (columnMetadata.edges === 'spherical') {
|
|
348
|
+
extensionMetadata.edges = 'spherical';
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return Object.keys(extensionMetadata).length > 0
|
|
352
|
+
? {extensionName, extensionMetadata}
|
|
353
|
+
: {extensionName};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function isValidGeoParquetMetadata(geoMetadata: GeoMetadata | null): boolean {
|
|
357
|
+
if (!geoMetadata || typeof geoMetadata !== 'object') {
|
|
358
|
+
return false;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
if (
|
|
362
|
+
typeof geoMetadata.version !== 'string' ||
|
|
363
|
+
typeof geoMetadata.primary_column !== 'string' ||
|
|
364
|
+
!geoMetadata.columns ||
|
|
365
|
+
typeof geoMetadata.columns !== 'object'
|
|
366
|
+
) {
|
|
367
|
+
return false;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const primaryColumnMetadata = geoMetadata.columns[geoMetadata.primary_column];
|
|
371
|
+
if (!primaryColumnMetadata) {
|
|
372
|
+
return false;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
return Object.values(geoMetadata.columns).every(isValidGeoParquetColumnMetadata);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function isValidGeoParquetColumnMetadata(columnMetadata: unknown): boolean {
|
|
379
|
+
if (!columnMetadata || typeof columnMetadata !== 'object') {
|
|
380
|
+
return false;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
const {encoding, geometry_types} = columnMetadata as GeoColumnMetadata;
|
|
384
|
+
return (
|
|
385
|
+
typeof encoding === 'string' &&
|
|
386
|
+
Boolean(
|
|
387
|
+
GEOPARQUET_TO_GEOARROW_ENCODINGS[
|
|
388
|
+
encoding.toLowerCase() as keyof typeof GEOPARQUET_TO_GEOARROW_ENCODINGS
|
|
389
|
+
]
|
|
390
|
+
) &&
|
|
391
|
+
Array.isArray(geometry_types)
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function synthesizeGeoParquetColumnMetadata(geometryMetadata: {
|
|
396
|
+
encoding?: GeoArrowEncoding;
|
|
397
|
+
crs?: Record<string, unknown>;
|
|
398
|
+
crs_type?: 'projjson' | 'wkt2:2019';
|
|
399
|
+
edges?: 'spherical';
|
|
400
|
+
geometry_types?: GeoParquetGeometryType[];
|
|
401
|
+
}): GeoColumnMetadata | null {
|
|
402
|
+
const encoding = geometryMetadata.encoding
|
|
403
|
+
? GEOARROW_TO_GEOPARQUET_ENCODINGS[
|
|
404
|
+
geometryMetadata.encoding as keyof typeof GEOARROW_TO_GEOPARQUET_ENCODINGS
|
|
405
|
+
]
|
|
406
|
+
: null;
|
|
407
|
+
|
|
408
|
+
if (!encoding) {
|
|
409
|
+
return null;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const geometryTypes = Array.isArray(geometryMetadata.geometry_types)
|
|
413
|
+
? geometryMetadata.geometry_types
|
|
414
|
+
: inferGeometryTypesFromGeoArrowEncoding(geometryMetadata.encoding);
|
|
415
|
+
|
|
416
|
+
const columnMetadata: GeoColumnMetadata = {
|
|
417
|
+
encoding,
|
|
418
|
+
geometry_types: geometryTypes
|
|
419
|
+
};
|
|
420
|
+
|
|
421
|
+
if (geometryMetadata.crs !== undefined) {
|
|
422
|
+
columnMetadata.crs = geometryMetadata.crs;
|
|
423
|
+
}
|
|
424
|
+
if (geometryMetadata.crs_type !== undefined) {
|
|
425
|
+
columnMetadata.crs_type = geometryMetadata.crs_type;
|
|
426
|
+
}
|
|
427
|
+
if (geometryMetadata.edges === 'spherical') {
|
|
428
|
+
columnMetadata.edges = 'spherical';
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return columnMetadata;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function inferGeometryTypesFromGeoArrowEncoding(
|
|
435
|
+
encoding?: GeoArrowEncoding
|
|
436
|
+
): GeoParquetGeometryType[] {
|
|
437
|
+
const geometryType = encoding
|
|
438
|
+
? GEOARROW_TO_GEOMETRY_TYPE[encoding as keyof typeof GEOARROW_TO_GEOMETRY_TYPE]
|
|
439
|
+
: null;
|
|
440
|
+
return geometryType ? [geometryType] : [];
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function pickValidOptionalGeoParquetColumnMetadata(
|
|
444
|
+
columnMetadata: GeoColumnMetadata | undefined
|
|
445
|
+
): Partial<GeoColumnMetadata> {
|
|
446
|
+
if (!columnMetadata) {
|
|
447
|
+
return {};
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const nextColumnMetadata: Partial<GeoColumnMetadata> = {};
|
|
451
|
+
|
|
452
|
+
if (columnMetadata.orientation === 'counterclockwise') {
|
|
453
|
+
nextColumnMetadata.orientation = columnMetadata.orientation;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
if (isValidBBox(columnMetadata.bbox)) {
|
|
457
|
+
nextColumnMetadata.bbox = columnMetadata.bbox;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
if (typeof columnMetadata.epoch === 'number' && Number.isFinite(columnMetadata.epoch)) {
|
|
461
|
+
nextColumnMetadata.epoch = columnMetadata.epoch;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
if (columnMetadata.covering && typeof columnMetadata.covering === 'object') {
|
|
465
|
+
nextColumnMetadata.covering = columnMetadata.covering;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
return nextColumnMetadata;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function isValidBBox(
|
|
472
|
+
bbox: GeoColumnMetadata['bbox']
|
|
473
|
+
): bbox is [number, number, number, number] | [number, number, number, number, number, number] {
|
|
474
|
+
if (!Array.isArray(bbox) || (bbox.length !== 4 && bbox.length !== 6)) {
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
return bbox.every(value => typeof value === 'number' && Number.isFinite(value));
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function getMetadataEntries(
|
|
482
|
+
metadata: Map<string, string> | Record<string, string>
|
|
483
|
+
): [string, string][] {
|
|
484
|
+
if (metadata instanceof Map) {
|
|
485
|
+
return [...metadata.entries()];
|
|
486
|
+
}
|
|
487
|
+
return Object.entries(metadata);
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
function createMetadataMap(
|
|
491
|
+
metadata: Map<string, string> | Record<string, string> | undefined
|
|
492
|
+
): Map<string, string> {
|
|
493
|
+
if (!metadata) {
|
|
494
|
+
return new Map();
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
if (metadata instanceof Map) {
|
|
498
|
+
return new Map(metadata);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
return new Map(
|
|
502
|
+
Object.entries(metadata).map(([metadataKey, metadataValue]) => [
|
|
503
|
+
metadataKey,
|
|
504
|
+
String(metadataValue)
|
|
505
|
+
])
|
|
506
|
+
);
|
|
507
|
+
}
|