@loaders.gl/parquet 4.2.0-alpha.4 → 4.2.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +385 -598
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +12 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -1
- package/dist/lib/arrow/convert-columns-to-row-group.js +1 -2
- package/dist/lib/arrow/convert-row-group-to-columns.js +8 -6
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +3 -3
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js +71 -82
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -1
- package/dist/lib/arrow/convert-schema-to-parquet.js +65 -35
- package/dist/lib/constants.js +15 -2
- package/dist/lib/parsers/get-parquet-schema.d.ts +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +9 -7
- package/dist/lib/parsers/parse-geoparquet.d.ts +1 -1
- package/dist/lib/parsers/parse-geoparquet.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet.js +47 -45
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +36 -25
- package/dist/lib/parsers/parse-parquet.d.ts +1 -1
- package/dist/lib/parsers/parse-parquet.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet.js +62 -50
- package/dist/lib/wasm/encode-parquet-wasm.d.ts +1 -1
- package/dist/lib/wasm/encode-parquet-wasm.d.ts.map +1 -1
- package/dist/lib/wasm/encode-parquet-wasm.js +49 -12
- package/dist/lib/wasm/load-wasm.js +13 -10
- package/dist/lib/wasm/parse-parquet-wasm.d.ts +1 -1
- package/dist/lib/wasm/parse-parquet-wasm.d.ts.map +1 -1
- package/dist/lib/wasm/parse-parquet-wasm.js +19 -15
- package/dist/parquet-loader.d.ts +1 -1
- package/dist/parquet-loader.d.ts.map +1 -1
- package/dist/parquet-loader.js +78 -65
- package/dist/parquet-wasm-loader.js +25 -28
- package/dist/parquet-wasm-writer.js +19 -23
- package/dist/parquet-writer.js +17 -13
- package/dist/parquetjs/codecs/declare.d.ts +1 -1
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +0 -1
- package/dist/parquetjs/codecs/dictionary.js +4 -8
- package/dist/parquetjs/codecs/index.d.ts +3 -3
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +20 -17
- package/dist/parquetjs/codecs/plain.d.ts +2 -2
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +166 -162
- package/dist/parquetjs/codecs/rle.d.ts +2 -2
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +124 -105
- package/dist/parquetjs/compression.d.ts +1 -1
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +157 -43
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +3 -3
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +420 -275
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +12 -7
- package/dist/parquetjs/parquet-thrift/BsonType.js +31 -27
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +182 -166
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +183 -166
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts +6 -6
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +372 -333
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +74 -68
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +17 -12
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +31 -26
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +141 -124
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +205 -181
- package/dist/parquetjs/parquet-thrift/DateType.js +31 -27
- package/dist/parquetjs/parquet-thrift/DecimalType.js +78 -69
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +96 -86
- package/dist/parquetjs/parquet-thrift/Encoding.js +17 -12
- package/dist/parquetjs/parquet-thrift/EnumType.js +31 -27
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +12 -7
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts +4 -4
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +224 -205
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +31 -27
- package/dist/parquetjs/parquet-thrift/IntType.js +78 -69
- package/dist/parquetjs/parquet-thrift/JsonType.js +31 -27
- package/dist/parquetjs/parquet-thrift/KeyValue.js +75 -67
- package/dist/parquetjs/parquet-thrift/ListType.js +31 -27
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts +13 -13
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +338 -344
- package/dist/parquetjs/parquet-thrift/MapType.js +31 -27
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +31 -27
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +31 -27
- package/dist/parquetjs/parquet-thrift/NullType.js +31 -27
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +67 -60
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +101 -88
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts +5 -5
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +193 -175
- package/dist/parquetjs/parquet-thrift/PageLocation.js +111 -96
- package/dist/parquetjs/parquet-thrift/PageType.js +13 -8
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +149 -133
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts +4 -4
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +221 -205
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +101 -88
- package/dist/parquetjs/parquet-thrift/Statistics.js +149 -137
- package/dist/parquetjs/parquet-thrift/StringType.js +31 -27
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +78 -69
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts +2 -2
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +96 -91
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +78 -69
- package/dist/parquetjs/parquet-thrift/Type.js +17 -12
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +31 -27
- package/dist/parquetjs/parquet-thrift/UUIDType.js +31 -27
- package/dist/parquetjs/parquet-thrift/index.d.ts +44 -44
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +6 -1
- package/dist/parquetjs/parser/decoders.d.ts +3 -3
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +306 -242
- package/dist/parquetjs/parser/parquet-reader.d.ts +3 -3
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +183 -158
- package/dist/parquetjs/schema/declare.d.ts +1 -1
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js +10 -9
- package/dist/parquetjs/schema/schema.d.ts +1 -1
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +142 -120
- package/dist/parquetjs/schema/shred.d.ts +2 -2
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +301 -204
- package/dist/parquetjs/schema/types.d.ts +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +329 -314
- package/dist/parquetjs/utils/file-utils.js +24 -23
- package/dist/parquetjs/utils/read-utils.d.ts +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +76 -69
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +8 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.js +13 -8
- package/dist/polyfills/buffer/buffer.js +1747 -1367
- package/dist/polyfills/buffer/index.d.ts +3 -3
- package/dist/polyfills/buffer/index.d.ts.map +1 -1
- package/dist/polyfills/buffer/index.js +5 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +28 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +1 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +2 -1
- package/dist/polyfills/util.js +4 -1
- package/dist/workers/parquet-worker.js +3 -1
- package/package.json +21 -19
- package/src/index.ts +1 -1
- package/src/lib/parsers/parse-parquet-to-columns.ts +1 -1
- package/src/lib/parsers/parse-parquet.ts +1 -1
- package/src/parquetjs/encoder/parquet-encoder.ts +2 -2
- package/src/parquetjs/parser/decoders.ts +2 -2
- package/src/parquetjs/parser/parquet-reader.ts +3 -2
- package/src/parquetjs/schema/schema.ts +1 -1
- package/src/parquetjs/schema/types.ts +1 -0
- package/src/parquetjs/utils/read-utils.ts +2 -2
- package/src/polyfills/buffer/buffer.ts +0 -3
- package/dist/index.js.map +0 -1
- package/dist/lib/arrow/convert-columns-to-row-group.js.map +0 -1
- package/dist/lib/arrow/convert-row-group-to-columns.js.map +0 -1
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +0 -1
- package/dist/lib/arrow/convert-schema-to-parquet.js.map +0 -1
- package/dist/lib/constants.js.map +0 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +0 -1
- package/dist/lib/parsers/parse-geoparquet.js.map +0 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +0 -1
- package/dist/lib/parsers/parse-parquet.js.map +0 -1
- package/dist/lib/wasm/encode-parquet-wasm.js.map +0 -1
- package/dist/lib/wasm/load-wasm.js.map +0 -1
- package/dist/lib/wasm/parse-parquet-wasm.js.map +0 -1
- package/dist/lib/wip/convert-schema-deep.java.disabled +0 -910
- package/dist/lib/wip/convert-schema-deep.rs.disabled +0 -976
- package/dist/parquet-loader.js.map +0 -1
- package/dist/parquet-wasm-loader.js.map +0 -1
- package/dist/parquet-wasm-writer.js.map +0 -1
- package/dist/parquet-writer.js.map +0 -1
- package/dist/parquetjs/LICENSE +0 -20
- package/dist/parquetjs/codecs/declare.js.map +0 -1
- package/dist/parquetjs/codecs/dictionary.js.map +0 -1
- package/dist/parquetjs/codecs/index.js.map +0 -1
- package/dist/parquetjs/codecs/plain.js.map +0 -1
- package/dist/parquetjs/codecs/rle.js.map +0 -1
- package/dist/parquetjs/compression.js.map +0 -1
- package/dist/parquetjs/encoder/parquet-encoder.js.map +0 -1
- package/dist/parquetjs/modules.d.ts +0 -21
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DateType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/IntType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/ListType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/MapType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/NullType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/PageType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/StringType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/Type.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +0 -1
- package/dist/parquetjs/parquet-thrift/index.js.map +0 -1
- package/dist/parquetjs/parser/decoders.js.map +0 -1
- package/dist/parquetjs/parser/parquet-reader.js.map +0 -1
- package/dist/parquetjs/schema/declare.js.map +0 -1
- package/dist/parquetjs/schema/schema.js.map +0 -1
- package/dist/parquetjs/schema/shred.js.map +0 -1
- package/dist/parquetjs/schema/types.js.map +0 -1
- package/dist/parquetjs/utils/file-utils.js.map +0 -1
- package/dist/parquetjs/utils/read-utils.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/polyfills/util.js.map +0 -1
- package/dist/workers/parquet-worker.js.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
export { Buffer } from
|
|
2
|
-
export { ParquetWorkerLoader, ParquetLoader, GeoParquetWorkerLoader, GeoParquetLoader, ParquetColumnarWorkerLoader, ParquetColumnarLoader } from
|
|
3
|
-
export { ParquetWriter as _ParquetWriter } from
|
|
4
|
-
export type { ParquetWasmLoaderOptions } from
|
|
5
|
-
export { ParquetWasmLoader, ParquetWasmWorkerLoader } from
|
|
6
|
-
export { ParquetWasmWriter } from
|
|
7
|
-
export { preloadCompressions } from
|
|
8
|
-
export { ParquetSchema } from
|
|
9
|
-
export { ParquetReader } from
|
|
10
|
-
export { ParquetEncoder } from
|
|
11
|
-
export { convertParquetSchema, convertParquetSchema as convertParquetToArrowSchema } from
|
|
12
|
-
export { BufferPolyfill, installBufferPolyfill } from
|
|
1
|
+
export { Buffer } from "./polyfills/buffer/install-buffer-polyfill.js";
|
|
2
|
+
export { ParquetWorkerLoader, ParquetLoader, GeoParquetWorkerLoader, GeoParquetLoader, ParquetColumnarWorkerLoader, ParquetColumnarLoader } from "./parquet-loader.js";
|
|
3
|
+
export { ParquetWriter as _ParquetWriter } from "./parquet-writer.js";
|
|
4
|
+
export type { ParquetWasmLoaderOptions } from "./parquet-wasm-loader.js";
|
|
5
|
+
export { ParquetWasmLoader, ParquetWasmWorkerLoader } from "./parquet-wasm-loader.js";
|
|
6
|
+
export { ParquetWasmWriter } from "./parquet-wasm-writer.js";
|
|
7
|
+
export { preloadCompressions } from "./parquetjs/compression.js";
|
|
8
|
+
export { ParquetSchema } from "./parquetjs/schema/schema.js";
|
|
9
|
+
export { ParquetReader } from "./parquetjs/parser/parquet-reader.js";
|
|
10
|
+
export { ParquetEncoder } from "./parquetjs/encoder/parquet-encoder.js";
|
|
11
|
+
export { convertParquetSchema, convertParquetSchema as convertParquetToArrowSchema } from "./lib/arrow/convert-schema-from-parquet.js";
|
|
12
|
+
export { BufferPolyfill, installBufferPolyfill } from "./polyfills/buffer/index.js";
|
|
13
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAC,sDAAmD;AAIlE,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,gBAAgB,EAChB,2BAA2B,EAC3B,qBAAqB,EACtB,4BAAyB;AAE1B,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,4BAAyB;AAIjE,YAAY,EAAC,wBAAwB,EAAC,iCAA8B;AACpE,OAAO,EAAC,iBAAiB,EAAE,uBAAuB,EAAC,iCAA8B;AACjF,OAAO,EAAC,iBAAiB,EAAC,iCAA8B;AAIxD,OAAO,EAAC,mBAAmB,EAAC,mCAAgC;AAE5D,OAAO,EAAC,aAAa,EAAC,qCAAkC;AACxD,OAAO,EAAC,aAAa,EAAC,6CAA0C;AAChE,OAAO,EAAC,cAAc,EAAC,+CAA4C;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,mDAAgD;AAGjD,OAAO,EAAC,cAAc,EAAE,qBAAqB,EAAC,oCAAiC"}
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
export { Buffer } from "./polyfills/buffer/install-buffer-polyfill.js";
|
|
5
|
+
// import {ArrowTable, ArrowTableBatch} from '@loaders.gl/arrow';
|
|
2
6
|
export { ParquetWorkerLoader, ParquetLoader, GeoParquetWorkerLoader, GeoParquetLoader, ParquetColumnarWorkerLoader, ParquetColumnarLoader } from "./parquet-loader.js";
|
|
3
7
|
export { ParquetWriter as _ParquetWriter } from "./parquet-writer.js";
|
|
4
8
|
export { ParquetWasmLoader, ParquetWasmWorkerLoader } from "./parquet-wasm-loader.js";
|
|
5
9
|
export { ParquetWasmWriter } from "./parquet-wasm-writer.js";
|
|
10
|
+
// EXPERIMENTAL - expose the internal parquetjs API
|
|
6
11
|
export { preloadCompressions } from "./parquetjs/compression.js";
|
|
7
12
|
export { ParquetSchema } from "./parquetjs/schema/schema.js";
|
|
8
13
|
export { ParquetReader } from "./parquetjs/parser/parquet-reader.js";
|
|
9
14
|
export { ParquetEncoder } from "./parquetjs/encoder/parquet-encoder.js";
|
|
10
15
|
export { convertParquetSchema, convertParquetSchema as convertParquetToArrowSchema } from "./lib/arrow/convert-schema-from-parquet.js";
|
|
16
|
+
// Experimental
|
|
11
17
|
export { BufferPolyfill, installBufferPolyfill } from "./polyfills/buffer/index.js";
|
|
12
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
//# sourceMappingURL=convert-columns-to-row-group.js.map
|
|
1
|
+
"use strict";
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
export function convertParquetRowGroupToColumns(schema, rowGroup) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
const columns = {};
|
|
6
|
+
for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
|
|
7
|
+
columns[columnName] = columns[columnName] || data.values;
|
|
8
|
+
}
|
|
9
|
+
return columns;
|
|
7
10
|
}
|
|
8
|
-
//# sourceMappingURL=convert-row-group-to-columns.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Schema, DataType } from '@loaders.gl/schema';
|
|
2
|
-
import type { ParquetSchema } from
|
|
3
|
-
import type { ParquetType } from
|
|
4
|
-
import { FileMetaData } from
|
|
2
|
+
import type { ParquetSchema } from "../../parquetjs/schema/schema.js";
|
|
3
|
+
import type { ParquetType } from "../../parquetjs/schema/declare.js";
|
|
4
|
+
import { FileMetaData } from "../../parquetjs/parquet-thrift.js";
|
|
5
5
|
export declare const PARQUET_TYPE_MAPPING: {
|
|
6
6
|
[type in ParquetType]: DataType;
|
|
7
7
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,
|
|
1
|
+
{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,yCAAsC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,0CAAuC;AAC/F,OAAO,EAAC,YAAY,EAAC,0CAAuC;AAE5D,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,YAAY,GAAG,IAAI,GACnC,MAAM,CAUR"}
|
|
@@ -1,95 +1,84 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
export const PARQUET_TYPE_MAPPING = {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
5
|
+
BOOLEAN: 'bool',
|
|
6
|
+
INT32: 'int32',
|
|
7
|
+
INT64: 'float64',
|
|
8
|
+
INT96: 'float64',
|
|
9
|
+
FLOAT: 'float32',
|
|
10
|
+
DOUBLE: 'float64',
|
|
11
|
+
BYTE_ARRAY: 'binary',
|
|
12
|
+
FIXED_LEN_BYTE_ARRAY: 'binary',
|
|
13
|
+
UTF8: 'utf8',
|
|
14
|
+
DATE: 'int32',
|
|
15
|
+
TIME_MILLIS: 'int64',
|
|
16
|
+
TIME_MICROS: 'int64',
|
|
17
|
+
TIMESTAMP_MILLIS: 'int64',
|
|
18
|
+
TIMESTAMP_MICROS: 'int64',
|
|
19
|
+
UINT_8: 'int32',
|
|
20
|
+
UINT_16: 'uint16',
|
|
21
|
+
UINT_32: 'uint32',
|
|
22
|
+
UINT_64: 'uint64',
|
|
23
|
+
INT_8: 'int8',
|
|
24
|
+
INT_16: 'int16',
|
|
25
|
+
INT_32: 'int32',
|
|
26
|
+
INT_64: 'int64',
|
|
27
|
+
JSON: 'binary',
|
|
28
|
+
BSON: 'binary',
|
|
29
|
+
// TODO check interal type
|
|
30
|
+
INTERVAL: 'binary',
|
|
31
|
+
DECIMAL_INT32: 'float32',
|
|
32
|
+
DECIMAL_INT64: 'float64',
|
|
33
|
+
DECIMAL_BYTE_ARRAY: 'float64',
|
|
34
|
+
DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
|
|
31
35
|
};
|
|
32
36
|
export function convertParquetSchema(parquetSchema, parquetMetadata) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
const fields = getFields(parquetSchema.schema);
|
|
38
|
+
const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
|
|
39
|
+
const schema = {
|
|
40
|
+
fields,
|
|
41
|
+
metadata: metadata || {}
|
|
42
|
+
};
|
|
43
|
+
return schema;
|
|
40
44
|
}
|
|
41
45
|
function getFields(schema) {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
} else {
|
|
56
|
-
const type = PARQUET_TYPE_MAPPING[field.type];
|
|
57
|
-
const metadata = getFieldMetadata(field);
|
|
58
|
-
const arrowField = {
|
|
59
|
-
name,
|
|
60
|
-
type,
|
|
61
|
-
nullable: field.optional,
|
|
62
|
-
metadata
|
|
63
|
-
};
|
|
64
|
-
fields.push(arrowField);
|
|
46
|
+
const fields = [];
|
|
47
|
+
for (const name in schema) {
|
|
48
|
+
const field = schema[name];
|
|
49
|
+
if (field.fields) {
|
|
50
|
+
const children = getFields(field.fields);
|
|
51
|
+
fields.push({ name, type: { type: 'struct', children }, nullable: field.optional });
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
const type = PARQUET_TYPE_MAPPING[field.type];
|
|
55
|
+
const metadata = getFieldMetadata(field);
|
|
56
|
+
const arrowField = { name, type, nullable: field.optional, metadata };
|
|
57
|
+
fields.push(arrowField);
|
|
58
|
+
}
|
|
65
59
|
}
|
|
66
|
-
|
|
67
|
-
return fields;
|
|
60
|
+
return fields;
|
|
68
61
|
}
|
|
69
62
|
function getFieldMetadata(field) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
63
|
+
let metadata;
|
|
64
|
+
for (const key in field) {
|
|
65
|
+
if (key !== 'name') {
|
|
66
|
+
let value = field[key] || '';
|
|
67
|
+
value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
68
|
+
metadata = metadata || {};
|
|
69
|
+
metadata[key] = value;
|
|
70
|
+
}
|
|
77
71
|
}
|
|
78
|
-
|
|
79
|
-
return metadata;
|
|
72
|
+
return metadata;
|
|
80
73
|
}
|
|
81
74
|
function getSchemaMetadata(parquetMetadata) {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
metadata = metadata || {};
|
|
90
|
-
metadata[key] = value;
|
|
75
|
+
let metadata;
|
|
76
|
+
const keyValueList = parquetMetadata.key_value_metadata || [];
|
|
77
|
+
for (const { key, value } of keyValueList) {
|
|
78
|
+
if (typeof value === 'string') {
|
|
79
|
+
metadata = metadata || {};
|
|
80
|
+
metadata[key] = value;
|
|
81
|
+
}
|
|
91
82
|
}
|
|
92
|
-
|
|
93
|
-
return metadata;
|
|
83
|
+
return metadata;
|
|
94
84
|
}
|
|
95
|
-
//# sourceMappingURL=convert-schema-from-parquet.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ParquetType } from
|
|
1
|
+
import type { ParquetType } from "../../parquetjs/schema/declare.js";
|
|
2
2
|
import { Schema, DataType } from '@loaders.gl/schema';
|
|
3
3
|
export declare const PARQUET_TYPE_MAPPING: {
|
|
4
4
|
[type in ParquetType]: DataType;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,WAAW,EACZ,
|
|
1
|
+
{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,WAAW,EACZ,0CAAuC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}
|
|
@@ -1,39 +1,69 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
export const PARQUET_TYPE_MAPPING = {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
5
|
+
BOOLEAN: 'bool',
|
|
6
|
+
INT32: 'int32',
|
|
7
|
+
INT64: 'float64',
|
|
8
|
+
INT96: 'float64',
|
|
9
|
+
FLOAT: 'float32',
|
|
10
|
+
DOUBLE: 'float64',
|
|
11
|
+
BYTE_ARRAY: 'binary',
|
|
12
|
+
FIXED_LEN_BYTE_ARRAY: 'binary',
|
|
13
|
+
UTF8: 'utf8',
|
|
14
|
+
DATE: 'int32',
|
|
15
|
+
TIME_MILLIS: 'int64',
|
|
16
|
+
TIME_MICROS: 'int64',
|
|
17
|
+
TIMESTAMP_MILLIS: 'int64',
|
|
18
|
+
TIMESTAMP_MICROS: 'int64',
|
|
19
|
+
UINT_8: 'int32',
|
|
20
|
+
UINT_16: 'uint16',
|
|
21
|
+
UINT_32: 'uint32',
|
|
22
|
+
UINT_64: 'uint64',
|
|
23
|
+
INT_8: 'int8',
|
|
24
|
+
INT_16: 'int16',
|
|
25
|
+
INT_32: 'int32',
|
|
26
|
+
INT_64: 'int64',
|
|
27
|
+
JSON: 'binary',
|
|
28
|
+
BSON: 'binary',
|
|
29
|
+
// TODO check interval type
|
|
30
|
+
INTERVAL: 'binary',
|
|
31
|
+
DECIMAL_INT32: 'float32',
|
|
32
|
+
DECIMAL_INT64: 'float64',
|
|
33
|
+
DECIMAL_BYTE_ARRAY: 'float64',
|
|
34
|
+
DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
|
|
31
35
|
};
|
|
32
36
|
export function convertToParquetSchema(schema) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
fields,
|
|
36
|
-
metadata: {}
|
|
37
|
-
};
|
|
37
|
+
const fields = []; // getFields(schema.fields);
|
|
38
|
+
// TODO add metadata if needed.
|
|
39
|
+
return { fields, metadata: {} };
|
|
38
40
|
}
|
|
39
|
-
|
|
41
|
+
// function getFields(schema: Field[]): Definition[] {
|
|
42
|
+
// const fields: Field[] = [];
|
|
43
|
+
// for (const name in schema) {
|
|
44
|
+
// const field = schema[name];
|
|
45
|
+
// // @ts-ignore
|
|
46
|
+
// const children = field.children as DataType[];
|
|
47
|
+
// if (children) {
|
|
48
|
+
// const childField = getFields(field.fields);
|
|
49
|
+
// const nestedField = new Field(name, new Struct(childField), field.optional);
|
|
50
|
+
// fields.push(nestedField);
|
|
51
|
+
// } else {
|
|
52
|
+
// const FieldType = PARQUET_TYPE_MAPPING[field.type];
|
|
53
|
+
// const metadata = getFieldMetadata(field);
|
|
54
|
+
// const arrowField = new Field(name, new FieldType(), field.optional, metadata);
|
|
55
|
+
// fields.push(arrowField);
|
|
56
|
+
// }
|
|
57
|
+
// }
|
|
58
|
+
// return fields;
|
|
59
|
+
// }
|
|
60
|
+
// function getFieldMetadata(field: ParquetField): Map<string, string> {
|
|
61
|
+
// const metadata = new Map();
|
|
62
|
+
// for (const key in field) {
|
|
63
|
+
// if (key !== 'name') {
|
|
64
|
+
// const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
65
|
+
// metadata.set(key, value);
|
|
66
|
+
// }
|
|
67
|
+
// }
|
|
68
|
+
// return metadata;
|
|
69
|
+
// }
|
package/dist/lib/constants.js
CHANGED
|
@@ -1,8 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
+
export const VERSION = typeof "4.2.0-alpha.5" !== 'undefined' ? "4.2.0-alpha.5" : 'latest';
|
|
2
7
|
export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';
|
|
8
|
+
/**
|
|
9
|
+
* Parquet File Magic String
|
|
10
|
+
*/
|
|
3
11
|
export const PARQUET_MAGIC = 'PAR1';
|
|
4
12
|
export const PARQUET_MAGIC_ENCRYPTED = 'PARE';
|
|
13
|
+
/**
|
|
14
|
+
* Parquet File Format Version
|
|
15
|
+
*/
|
|
5
16
|
export const PARQUET_VERSION = 1;
|
|
17
|
+
/**
|
|
18
|
+
* Internal type used for repetition/definition levels
|
|
19
|
+
*/
|
|
6
20
|
export const PARQUET_RDLVL_TYPE = 'INT32';
|
|
7
21
|
export const PARQUET_RDLVL_ENCODING = 'RLE';
|
|
8
|
-
//# sourceMappingURL=constants.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { Schema } from '@loaders.gl/schema';
|
|
2
|
-
import { ParquetReader } from
|
|
2
|
+
import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
|
|
3
3
|
export declare function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema>;
|
|
4
4
|
//# sourceMappingURL=get-parquet-schema.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-parquet-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,aAAa,EAAC,
|
|
1
|
+
{"version":3,"file":"get-parquet-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,aAAa,EAAC,iDAA8C;AAIpE,wBAAsB,0BAA0B,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAOvF"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { convertParquetSchema } from "../arrow/convert-schema-from-parquet.js";
|
|
2
5
|
import { unpackGeoMetadata, unpackJSONStringMetadata } from '@loaders.gl/gis';
|
|
3
6
|
export async function getSchemaFromParquetReader(reader) {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
const parquetSchema = await reader.getSchema();
|
|
8
|
+
const parquetMetadata = await reader.getFileMetadata();
|
|
9
|
+
const schema = convertParquetSchema(parquetSchema, parquetMetadata);
|
|
10
|
+
unpackGeoMetadata(schema);
|
|
11
|
+
unpackJSONStringMetadata(schema, 'pandas');
|
|
12
|
+
return schema;
|
|
10
13
|
}
|
|
11
|
-
//# sourceMappingURL=get-parquet-schema.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ReadableFile } from '@loaders.gl/loader-utils';
|
|
2
2
|
import type { GeoJSONTable, GeoJSONTableBatch, ObjectRowTable, ObjectRowTableBatch } from '@loaders.gl/schema';
|
|
3
|
-
import type { ParquetLoaderOptions } from
|
|
3
|
+
import type { ParquetLoaderOptions } from "../../parquet-loader.js";
|
|
4
4
|
export declare function parseGeoParquetFile(file: ReadableFile, options?: ParquetLoaderOptions): Promise<ObjectRowTable | GeoJSONTable>;
|
|
5
5
|
export declare function parseGeoParquetFileInBatches(file: ReadableFile, options?: ParquetLoaderOptions): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch>;
|
|
6
6
|
//# sourceMappingURL=parse-geoparquet.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-geoparquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-geoparquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAC,oBAAoB,EAAC,
|
|
1
|
+
{"version":3,"file":"parse-geoparquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-geoparquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAI/D,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,CAIxC;AAED,wBAAuB,4BAA4B,CACjD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,CAOxD"}
|
|
@@ -1,56 +1,58 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { convertWKBTableToGeoJSON } from '@loaders.gl/gis';
|
|
2
5
|
import { WKTLoader, WKBLoader } from '@loaders.gl/wkt';
|
|
3
6
|
import { parseParquetFile, parseParquetFileInBatches } from "./parse-parquet.js";
|
|
4
7
|
export async function parseGeoParquetFile(file, options) {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
shape: 'object-row-table'
|
|
9
|
-
});
|
|
10
|
-
const shape = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.shape;
|
|
11
|
-
return convertTable(table, shape);
|
|
8
|
+
const table = await parseParquetFile(file, { ...options, shape: 'object-row-table' });
|
|
9
|
+
const shape = options?.parquet?.shape;
|
|
10
|
+
return convertTable(table, shape);
|
|
12
11
|
}
|
|
13
12
|
export async function* parseGeoParquetFileInBatches(file, options) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
var _options$parquet2;
|
|
20
|
-
const shape = options === null || options === void 0 ? void 0 : (_options$parquet2 = options.parquet) === null || _options$parquet2 === void 0 ? void 0 : _options$parquet2.shape;
|
|
21
|
-
yield convertBatch(batch, shape);
|
|
22
|
-
}
|
|
13
|
+
const tableBatches = parseParquetFileInBatches(file, { ...options, shape: 'object-row-table' });
|
|
14
|
+
for await (const batch of tableBatches) {
|
|
15
|
+
const shape = options?.parquet?.shape;
|
|
16
|
+
yield convertBatch(batch, shape);
|
|
17
|
+
}
|
|
23
18
|
}
|
|
24
19
|
function convertTable(objectRowTable, shape) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
20
|
+
switch (shape) {
|
|
21
|
+
case 'object-row-table':
|
|
22
|
+
return objectRowTable;
|
|
23
|
+
case 'geojson-table':
|
|
24
|
+
try {
|
|
25
|
+
return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema, [
|
|
26
|
+
WKTLoader,
|
|
27
|
+
WKBLoader
|
|
28
|
+
]);
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
return objectRowTable;
|
|
32
|
+
}
|
|
33
|
+
default:
|
|
34
|
+
throw new Error(shape);
|
|
35
|
+
}
|
|
37
36
|
}
|
|
38
37
|
function convertBatch(objectRowBatch, shape) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
38
|
+
switch (shape) {
|
|
39
|
+
case 'object-row-table':
|
|
40
|
+
return objectRowBatch;
|
|
41
|
+
case 'geojson-table':
|
|
42
|
+
try {
|
|
43
|
+
const geojsonTable = convertWKBTableToGeoJSON(objectRowBatch, objectRowBatch.schema, [
|
|
44
|
+
WKTLoader,
|
|
45
|
+
WKBLoader
|
|
46
|
+
]);
|
|
47
|
+
return {
|
|
48
|
+
...objectRowBatch,
|
|
49
|
+
...geojsonTable
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
return objectRowBatch;
|
|
54
|
+
}
|
|
55
|
+
default:
|
|
56
|
+
throw new Error(shape);
|
|
57
|
+
}
|
|
55
58
|
}
|
|
56
|
-
//# sourceMappingURL=parse-geoparquet.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ColumnarTable, ColumnarTableBatch } from '@loaders.gl/schema';
|
|
2
2
|
import type { ReadableFile } from '@loaders.gl/loader-utils';
|
|
3
|
-
import type { ParquetLoaderOptions } from
|
|
3
|
+
import type { ParquetLoaderOptions } from "../../parquet-loader.js";
|
|
4
4
|
/**
|
|
5
5
|
* @deprecated
|
|
6
6
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,
|
|
1
|
+
{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAQ/D;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAUxB;AAED;;GAEG;AACH,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,kBAAkB,CAAC,CAanC"}
|