@loaders.gl/parquet 4.3.2 → 4.4.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +76 -286
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -4
- package/dist/lib/constants.js +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +7 -0
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -0
- package/dist/lib/encoders/{encode-parquet-wasm.js → encode-arrow-to-parquet.js} +1 -1
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -3
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +6 -0
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -0
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +15 -0
- package/dist/lib/parsers/{parse-parquet-wasm.d.ts → parse-parquet-to-arrow.d.ts} +3 -3
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -0
- package/dist/lib/parsers/{parse-parquet-wasm.js → parse-parquet-to-arrow.js} +8 -8
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +4 -4
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -1
- package/dist/lib/parsers/{parse-parquet.d.ts → parse-parquet-to-json.d.ts} +4 -4
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -0
- package/dist/lib/parsers/{parse-parquet.js → parse-parquet-to-json.js} +2 -16
- package/dist/{parquet-wasm-loader.d.ts → parquet-arrow-loader.d.ts} +4 -6
- package/dist/parquet-arrow-loader.d.ts.map +1 -0
- package/dist/{parquet-wasm-loader.js → parquet-arrow-loader.js} +10 -11
- package/dist/parquet-arrow-writer.d.ts +9 -0
- package/dist/parquet-arrow-writer.d.ts.map +1 -0
- package/dist/{parquet-wasm-writer.js → parquet-arrow-writer.js} +6 -8
- package/dist/parquet-format.d.ts +9 -0
- package/dist/parquet-format.d.ts.map +1 -0
- package/dist/parquet-format.js +11 -0
- package/dist/{parquet-loader.d.ts → parquet-json-loader.d.ts} +57 -87
- package/dist/parquet-json-loader.d.ts.map +1 -0
- package/dist/{parquet-loader.js → parquet-json-loader.js} +42 -47
- package/dist/parquet-json-writer.d.ts +19 -0
- package/dist/parquet-json-writer.d.ts.map +1 -0
- package/dist/{parquet-writer.js → parquet-json-writer.js} +4 -6
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/declare.js +5 -0
- package/dist/parquetjs/codecs/dictionary.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +5 -0
- package/dist/parquetjs/codecs/index.d.ts.map +1 -1
- package/dist/parquetjs/codecs/index.js +5 -0
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +5 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +5 -1
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +5 -2
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +5 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +5 -0
- package/dist/parquetjs/parquet-thrift/BsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/BsonType.js +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +5 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +5 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +5 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +5 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +5 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +5 -0
- package/dist/parquetjs/parquet-thrift/DateType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DateType.js +5 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DecimalType.js +5 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +5 -0
- package/dist/parquetjs/parquet-thrift/Encoding.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Encoding.js +5 -0
- package/dist/parquetjs/parquet-thrift/EnumType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/EnumType.js +5 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +5 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +5 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +5 -0
- package/dist/parquetjs/parquet-thrift/IntType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/IntType.js +5 -0
- package/dist/parquetjs/parquet-thrift/JsonType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/JsonType.js +5 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/KeyValue.js +5 -0
- package/dist/parquetjs/parquet-thrift/ListType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/ListType.js +5 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/LogicalType.js +5 -0
- package/dist/parquetjs/parquet-thrift/MapType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MapType.js +5 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +5 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +5 -0
- package/dist/parquetjs/parquet-thrift/NullType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/NullType.js +5 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +5 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +5 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageHeader.js +5 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageLocation.js +5 -0
- package/dist/parquetjs/parquet-thrift/PageType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/PageType.js +5 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/RowGroup.js +5 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +5 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +5 -0
- package/dist/parquetjs/parquet-thrift/Statistics.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Statistics.js +5 -0
- package/dist/parquetjs/parquet-thrift/StringType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/StringType.js +5 -0
- package/dist/parquetjs/parquet-thrift/TimeType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeType.js +5 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +5 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TimestampType.js +5 -0
- package/dist/parquetjs/parquet-thrift/Type.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/Type.js +5 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +5 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/UUIDType.js +5 -0
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +5 -0
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +5 -0
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +5 -0
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/declare.js +5 -0
- package/dist/parquetjs/schema/schema.d.ts.map +1 -1
- package/dist/parquetjs/schema/schema.js +5 -1
- package/dist/parquetjs/schema/shred.d.ts.map +1 -1
- package/dist/parquetjs/schema/shred.js +5 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +5 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +5 -0
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +5 -0
- package/dist/polyfills/buffer/buffer.d.ts.map +1 -1
- package/dist/polyfills/buffer/buffer.js +0 -1
- package/dist/polyfills/buffer/index.d.ts.map +1 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +1 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +3 -0
- package/dist/workers/parquet-json-worker.d.ts +2 -0
- package/dist/workers/parquet-json-worker.d.ts.map +1 -0
- package/dist/workers/{parquet-worker.js → parquet-json-worker.js} +2 -2
- package/package.json +13 -13
- package/src/index.ts +16 -9
- package/src/lib/encoders/{encode-parquet-wasm.ts → encode-arrow-to-parquet.ts} +4 -4
- package/src/lib/parsers/get-parquet-schema.ts +3 -3
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +36 -0
- package/src/lib/parsers/{parse-parquet-wasm.ts → parse-parquet-to-arrow.ts} +9 -9
- package/src/lib/parsers/parse-parquet-to-columns.ts +4 -4
- package/src/lib/parsers/{parse-parquet.ts → parse-parquet-to-json.ts} +5 -25
- package/src/{parquet-wasm-loader.ts → parquet-arrow-loader.ts} +18 -20
- package/src/parquet-arrow-writer.ts +33 -0
- package/src/parquet-format.ts +14 -0
- package/src/{parquet-loader.ts → parquet-json-loader.ts} +42 -48
- package/src/{parquet-writer.ts → parquet-json-writer.ts} +6 -8
- package/src/parquetjs/codecs/declare.ts +6 -1
- package/src/parquetjs/codecs/dictionary.ts +6 -0
- package/src/parquetjs/codecs/index.ts +6 -1
- package/src/parquetjs/codecs/plain.ts +6 -1
- package/src/parquetjs/codecs/rle.ts +5 -1
- package/src/parquetjs/compression.ts +5 -2
- package/src/parquetjs/encoder/parquet-encoder.ts +6 -1
- package/src/parquetjs/modules.d.ts +5 -1
- package/src/parquetjs/parquet-thrift/BoundaryOrder.ts +6 -0
- package/src/parquetjs/parquet-thrift/BsonType.ts +6 -0
- package/src/parquetjs/parquet-thrift/ColumnChunk.ts +6 -0
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +6 -0
- package/src/parquetjs/parquet-thrift/ColumnMetaData.ts +6 -0
- package/src/parquetjs/parquet-thrift/ColumnOrder.ts +6 -0
- package/src/parquetjs/parquet-thrift/CompressionCodec.ts +6 -0
- package/src/parquetjs/parquet-thrift/ConvertedType.ts +6 -0
- package/src/parquetjs/parquet-thrift/DataPageHeader.ts +6 -0
- package/src/parquetjs/parquet-thrift/DataPageHeaderV2.ts +6 -0
- package/src/parquetjs/parquet-thrift/DateType.ts +6 -0
- package/src/parquetjs/parquet-thrift/DecimalType.ts +6 -0
- package/src/parquetjs/parquet-thrift/DictionaryPageHeader.ts +6 -0
- package/src/parquetjs/parquet-thrift/Encoding.ts +6 -0
- package/src/parquetjs/parquet-thrift/EnumType.ts +6 -0
- package/src/parquetjs/parquet-thrift/FieldRepetitionType.ts +6 -0
- package/src/parquetjs/parquet-thrift/FileMetaData.ts +6 -0
- package/src/parquetjs/parquet-thrift/IndexPageHeader.ts +6 -0
- package/src/parquetjs/parquet-thrift/IntType.ts +6 -0
- package/src/parquetjs/parquet-thrift/JsonType.ts +6 -0
- package/src/parquetjs/parquet-thrift/KeyValue.ts +6 -0
- package/src/parquetjs/parquet-thrift/ListType.ts +6 -0
- package/src/parquetjs/parquet-thrift/LogicalType.ts +6 -0
- package/src/parquetjs/parquet-thrift/MapType.ts +6 -0
- package/src/parquetjs/parquet-thrift/MicroSeconds.ts +6 -0
- package/src/parquetjs/parquet-thrift/MilliSeconds.ts +6 -0
- package/src/parquetjs/parquet-thrift/NullType.ts +6 -0
- package/src/parquetjs/parquet-thrift/OffsetIndex.ts +6 -0
- package/src/parquetjs/parquet-thrift/PageEncodingStats.ts +6 -0
- package/src/parquetjs/parquet-thrift/PageHeader.ts +6 -0
- package/src/parquetjs/parquet-thrift/PageLocation.ts +6 -0
- package/src/parquetjs/parquet-thrift/PageType.ts +6 -0
- package/src/parquetjs/parquet-thrift/RowGroup.ts +6 -0
- package/src/parquetjs/parquet-thrift/SchemaElement.ts +6 -0
- package/src/parquetjs/parquet-thrift/SortingColumn.ts +6 -0
- package/src/parquetjs/parquet-thrift/Statistics.ts +6 -0
- package/src/parquetjs/parquet-thrift/StringType.ts +6 -0
- package/src/parquetjs/parquet-thrift/TimeType.ts +6 -0
- package/src/parquetjs/parquet-thrift/TimeUnit.ts +6 -0
- package/src/parquetjs/parquet-thrift/TimestampType.ts +6 -0
- package/src/parquetjs/parquet-thrift/Type.ts +6 -0
- package/src/parquetjs/parquet-thrift/TypeDefinedOrder.ts +6 -0
- package/src/parquetjs/parquet-thrift/UUIDType.ts +6 -0
- package/src/parquetjs/parquet-thrift/index.ts +6 -0
- package/src/parquetjs/parser/decoders.ts +6 -1
- package/src/parquetjs/parser/parquet-reader.ts +6 -1
- package/src/parquetjs/schema/declare.ts +6 -1
- package/src/parquetjs/schema/schema.ts +5 -1
- package/src/parquetjs/schema/shred.ts +5 -1
- package/src/parquetjs/schema/types.ts +6 -1
- package/src/parquetjs/utils/file-utils.ts +6 -1
- package/src/parquetjs/utils/read-utils.ts +6 -0
- package/src/polyfills/buffer/buffer.ts +0 -1
- package/src/polyfills/buffer/index.ts +1 -0
- package/src/polyfills/buffer/install-buffer-polyfill.ts +4 -0
- package/src/workers/{parquet-worker.ts → parquet-json-worker.ts} +2 -2
- package/dist/lib/encoders/encode-parquet-wasm.d.ts +0 -7
- package/dist/lib/encoders/encode-parquet-wasm.d.ts.map +0 -1
- package/dist/lib/parsers/parse-geoparquet.d.ts +0 -6
- package/dist/lib/parsers/parse-geoparquet.d.ts.map +0 -1
- package/dist/lib/parsers/parse-geoparquet.js +0 -58
- package/dist/lib/parsers/parse-parquet-wasm.d.ts.map +0 -1
- package/dist/lib/parsers/parse-parquet.d.ts.map +0 -1
- package/dist/parquet-loader.d.ts.map +0 -1
- package/dist/parquet-wasm-loader.d.ts.map +0 -1
- package/dist/parquet-wasm-writer.d.ts +0 -9
- package/dist/parquet-wasm-writer.d.ts.map +0 -1
- package/dist/parquet-writer.d.ts +0 -17
- package/dist/parquet-writer.d.ts.map +0 -1
- package/dist/workers/parquet-worker.d.ts +0 -2
- package/dist/workers/parquet-worker.d.ts.map +0 -1
- package/src/lib/parsers/parse-geoparquet.ts +0 -88
- package/src/parquet-wasm-writer.ts +0 -35
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
6
|
+
|
|
2
7
|
import {
|
|
3
8
|
ParquetCodec,
|
|
4
9
|
ParquetColumnChunk,
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
6
|
+
|
|
2
7
|
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
3
8
|
|
|
4
9
|
import {ParquetSchema} from '../schema/schema';
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
6
|
+
|
|
2
7
|
import Int64 from 'node-int64';
|
|
3
8
|
import type {PageHeader} from '../parquet-thrift';
|
|
4
9
|
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
2
6
|
|
|
3
7
|
import {PARQUET_CODECS} from '../codecs/index';
|
|
4
8
|
import {PARQUET_COMPRESSION_METHODS} from '../compression';
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
2
6
|
|
|
3
7
|
import {ArrayType} from '@loaders.gl/schema';
|
|
4
8
|
import {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
6
|
+
|
|
2
7
|
/* eslint-disable camelcase */
|
|
3
8
|
import {BSONLoader, BSONWriter} from '@loaders.gl/bson';
|
|
4
9
|
import {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
//
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright (c) 2017 ironSource Ltd.
|
|
5
|
+
// Forked from https://github.com/kbajalc/parquets under MIT license
|
|
6
|
+
|
|
2
7
|
import {stream} from '@loaders.gl/loader-utils';
|
|
3
8
|
import * as fs from 'fs';
|
|
4
9
|
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
// This file is forked from https://github.com/feross/buffer under MIT license
|
|
5
|
-
// The code has primarily been converted to TypeScript.
|
|
6
5
|
|
|
7
6
|
/* !
|
|
8
7
|
* The buffer module from node.js, for the browser.
|
|
@@ -3,6 +3,6 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import {createLoaderWorker} from '@loaders.gl/loader-utils';
|
|
6
|
-
import {
|
|
6
|
+
import {ParquetJSONLoader} from '../parquet-json-loader';
|
|
7
7
|
|
|
8
|
-
createLoaderWorker(
|
|
8
|
+
createLoaderWorker(ParquetJSONLoader);
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { ArrowTable } from '@loaders.gl/arrow';
|
|
2
|
-
import type { ParquetWriterOptions } from "../../parquet-wasm-writer.js";
|
|
3
|
-
/**
|
|
4
|
-
* Encode Arrow arrow.Table to Parquet buffer
|
|
5
|
-
*/
|
|
6
|
-
export declare function encode(table: ArrowTable, options: ParquetWriterOptions): Promise<ArrayBuffer>;
|
|
7
|
-
//# sourceMappingURL=encode-parquet-wasm.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"encode-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/encode-parquet-wasm.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAKlD,OAAO,KAAK,EAAC,oBAAoB,EAAC,qCAAkC;AAEpE;;GAEG;AACH,wBAAsB,MAAM,CAC1B,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAsBtB"}
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { ReadableFile } from '@loaders.gl/loader-utils';
|
|
2
|
-
import type { GeoJSONTable, GeoJSONTableBatch, ObjectRowTable, ObjectRowTableBatch } from '@loaders.gl/schema';
|
|
3
|
-
import type { ParquetLoaderOptions } from "../../parquet-loader.js";
|
|
4
|
-
export declare function parseGeoParquetFile(file: ReadableFile, options?: ParquetLoaderOptions): Promise<ObjectRowTable | GeoJSONTable>;
|
|
5
|
-
export declare function parseGeoParquetFileInBatches(file: ReadableFile, options?: ParquetLoaderOptions): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch>;
|
|
6
|
-
//# sourceMappingURL=parse-geoparquet.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parse-geoparquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-geoparquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAI/D,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,CAIxC;AAED,wBAAuB,4BAA4B,CACjD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,CAOxD"}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
// loaders.gl
|
|
2
|
-
// SPDX-License-Identifier: MIT
|
|
3
|
-
// Copyright (c) vis.gl contributors
|
|
4
|
-
import { convertWKBTableToGeoJSON } from '@loaders.gl/gis';
|
|
5
|
-
import { WKTLoader, WKBLoader } from '@loaders.gl/wkt';
|
|
6
|
-
import { parseParquetFile, parseParquetFileInBatches } from "./parse-parquet.js";
|
|
7
|
-
export async function parseGeoParquetFile(file, options) {
|
|
8
|
-
const table = await parseParquetFile(file, { ...options, shape: 'object-row-table' });
|
|
9
|
-
const shape = options?.parquet?.shape;
|
|
10
|
-
return convertTable(table, shape);
|
|
11
|
-
}
|
|
12
|
-
export async function* parseGeoParquetFileInBatches(file, options) {
|
|
13
|
-
const tableBatches = parseParquetFileInBatches(file, { ...options, shape: 'object-row-table' });
|
|
14
|
-
for await (const batch of tableBatches) {
|
|
15
|
-
const shape = options?.parquet?.shape;
|
|
16
|
-
yield convertBatch(batch, shape);
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
function convertTable(objectRowTable, shape) {
|
|
20
|
-
switch (shape) {
|
|
21
|
-
case 'object-row-table':
|
|
22
|
-
return objectRowTable;
|
|
23
|
-
case 'geojson-table':
|
|
24
|
-
try {
|
|
25
|
-
return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema, [
|
|
26
|
-
WKTLoader,
|
|
27
|
-
WKBLoader
|
|
28
|
-
]);
|
|
29
|
-
}
|
|
30
|
-
catch (error) {
|
|
31
|
-
return objectRowTable;
|
|
32
|
-
}
|
|
33
|
-
default:
|
|
34
|
-
throw new Error(shape);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
function convertBatch(objectRowBatch, shape) {
|
|
38
|
-
switch (shape) {
|
|
39
|
-
case 'object-row-table':
|
|
40
|
-
return objectRowBatch;
|
|
41
|
-
case 'geojson-table':
|
|
42
|
-
try {
|
|
43
|
-
const geojsonTable = convertWKBTableToGeoJSON(objectRowBatch, objectRowBatch.schema, [
|
|
44
|
-
WKTLoader,
|
|
45
|
-
WKBLoader
|
|
46
|
-
]);
|
|
47
|
-
return {
|
|
48
|
-
...objectRowBatch,
|
|
49
|
-
...geojsonTable
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
catch (error) {
|
|
53
|
-
return objectRowBatch;
|
|
54
|
-
}
|
|
55
|
-
default:
|
|
56
|
-
throw new Error(shape);
|
|
57
|
-
}
|
|
58
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parse-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-wasm.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAS,MAAM,oBAAoB,CAAC;AAG5E,OAAO,KAAK,KAAK,WAAW,MAAM,cAAc,CAAC;AAMjD,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACtD,OAAO,CAAC,UAAU,CAAC,CAsBrB;AAED,wBAAuB,6BAA6B,CAClD,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACrD,aAAa,CAAC,eAAe,CAAC,CA0BhC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAO/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CAoCzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CA0BpC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EACnB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAC,gBAAgB,EAAE,yBAAyB,EAAC,uCAAoC;AACxF,OAAO,EAAC,mBAAmB,EAAE,4BAA4B,EAAC,0CAAuC;AACjG,OAAO,EACL,yBAAyB,EACzB,iCAAiC,EAClC,kDAA+C;AAKhD,OAAO,EAAC,MAAM,EAAC,sDAAmD;AAClE,OAAO,EAAC,MAAM,EAAC,CAAC;AAMhB,qCAAqC;AACrC,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,qCAAqC;IACrC,OAAO,CAAC,EAAE;QACR,qCAAqC;QACrC,KAAK,CAAC,EAAE,kBAAkB,GAAG,eAAe,CAAC;QAC7C,0FAA0F;QAC1F,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;QACnC,0DAA0D;QAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,qFAAqF;QACrF,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,gEAAgE;QAChE,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB;uBACD,cAAc;wBACb,mBAAmB;;;;;;;;;;;;;;;;;;;;CAqBmC,CAAC;AAEvF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;uBAGK,cAAc,GAAG,YAAY;wBAC5B,mBAAmB,GAAG,iBAAiB;kCAEhD,WAAW,YAAY,oBAAoB;;;;;;;;;;;;;;;;;;;;;;CASjE,CAAC;AAMF,eAAO,MAAM,sBAAsB;uBACJ,YAAY;wBACX,iBAAiB;;;;;;;;;;;;;;;;;;;;CAqBiC,CAAC;AAEnF,6BAA6B;AAC7B,eAAO,MAAM,gBAAgB;kCAGR,WAAW,YAAY,oBAAoB;;;uBA5BjC,YAAY;wBACX,iBAAiB;;;;;;;;;;;;;;;;;;;;CAoChD,CAAC;AAEF,yEAAyE;AACzE,eAAO,MAAM,2BAA2B;uBACb,aAAa;wBACZ,kBAAkB;;;;;;;;;;;;;;;;;;;;CAasC,CAAC;AAErF,yEAAyE;AACzE,eAAO,MAAM,qBAAqB;kCAEb,WAAW,YAAY,oBAAoB;;;uBAnBrC,aAAa;wBACZ,kBAAkB;;;;;;;;;;;;;;;;;;;;CAuBgD,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-wasm-loader.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAStF,kCAAkC;AAClC,MAAM,MAAM,wBAAwB,GAAG,aAAa,GAAG;IACrD,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,aAAa,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,uBAAuB;uBACL,UAAU;wBACT,eAAe;;;;;;;;;;;;;;;;;;;;;;;CAwBmC,CAAC;AAEnF,gCAAgC;AAChC,eAAO,MAAM,iBAAiB,KAiB8D,CAAC"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { WriterOptions } from '@loaders.gl/loader-utils';
|
|
2
|
-
export type ParquetWriterOptions = WriterOptions & {
|
|
3
|
-
parquet?: {
|
|
4
|
-
wasmUrl?: string;
|
|
5
|
-
};
|
|
6
|
-
};
|
|
7
|
-
/** Parquet WASM writer */
|
|
8
|
-
export declare const ParquetWasmWriter: any;
|
|
9
|
-
//# sourceMappingURL=parquet-wasm-writer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-wasm-writer.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-writer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAoB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAM/E,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,0BAA0B;AAC1B,eAAO,MAAM,iBAAiB,KAiBiD,CAAC"}
|
package/dist/parquet-writer.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { Table } from '@loaders.gl/schema';
|
|
2
|
-
export type ParquetWriterOptions = {};
|
|
3
|
-
export declare const ParquetWriter: {
|
|
4
|
-
readonly name: "Apache Parquet";
|
|
5
|
-
readonly id: "parquet";
|
|
6
|
-
readonly module: "parquet";
|
|
7
|
-
readonly version: any;
|
|
8
|
-
readonly extensions: ["parquet"];
|
|
9
|
-
readonly mimeTypes: ["application/octet-stream"];
|
|
10
|
-
readonly binary: true;
|
|
11
|
-
readonly options: {};
|
|
12
|
-
readonly encode: (data: Table, options: ParquetWriterOptions | undefined) => Promise<ArrayBuffer>;
|
|
13
|
-
readonly encodeSync: typeof encodeSync;
|
|
14
|
-
};
|
|
15
|
-
declare function encodeSync(data: any, options?: ParquetWriterOptions): ArrayBuffer;
|
|
16
|
-
export {};
|
|
17
|
-
//# sourceMappingURL=parquet-writer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,KAAK,EAAa,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEtC,eAAO,MAAM,aAAa;;;;;;;;;;;CAWqD,CAAC;AAEhF,iBAAS,UAAU,CAAC,IAAI,KAAA,EAAE,OAAO,CAAC,EAAE,oBAAoB,eAEvD"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-worker.d.ts","sourceRoot":"","sources":["../../src/workers/parquet-worker.ts"],"names":[],"mappings":""}
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
// loaders.gl
|
|
2
|
-
// SPDX-License-Identifier: MIT
|
|
3
|
-
// Copyright (c) vis.gl contributors
|
|
4
|
-
|
|
5
|
-
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
6
|
-
import type {
|
|
7
|
-
GeoJSONTable,
|
|
8
|
-
GeoJSONTableBatch,
|
|
9
|
-
ObjectRowTable,
|
|
10
|
-
ObjectRowTableBatch
|
|
11
|
-
} from '@loaders.gl/schema';
|
|
12
|
-
import {convertWKBTableToGeoJSON} from '@loaders.gl/gis';
|
|
13
|
-
import {WKTLoader, WKBLoader} from '@loaders.gl/wkt';
|
|
14
|
-
|
|
15
|
-
import type {ParquetLoaderOptions} from '../../parquet-loader';
|
|
16
|
-
|
|
17
|
-
import {parseParquetFile, parseParquetFileInBatches} from './parse-parquet';
|
|
18
|
-
|
|
19
|
-
export async function parseGeoParquetFile(
|
|
20
|
-
file: ReadableFile,
|
|
21
|
-
options?: ParquetLoaderOptions
|
|
22
|
-
): Promise<ObjectRowTable | GeoJSONTable> {
|
|
23
|
-
const table = await parseParquetFile(file, {...options, shape: 'object-row-table'});
|
|
24
|
-
const shape = options?.parquet?.shape;
|
|
25
|
-
return convertTable(table, shape);
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export async function* parseGeoParquetFileInBatches(
|
|
29
|
-
file: ReadableFile,
|
|
30
|
-
options?: ParquetLoaderOptions
|
|
31
|
-
): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {
|
|
32
|
-
const tableBatches = parseParquetFileInBatches(file, {...options, shape: 'object-row-table'});
|
|
33
|
-
|
|
34
|
-
for await (const batch of tableBatches) {
|
|
35
|
-
const shape = options?.parquet?.shape;
|
|
36
|
-
yield convertBatch(batch, shape);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function convertTable(
|
|
41
|
-
objectRowTable: ObjectRowTable,
|
|
42
|
-
shape?: 'object-row-table' | 'geojson-table'
|
|
43
|
-
): ObjectRowTable | GeoJSONTable {
|
|
44
|
-
switch (shape) {
|
|
45
|
-
case 'object-row-table':
|
|
46
|
-
return objectRowTable;
|
|
47
|
-
|
|
48
|
-
case 'geojson-table':
|
|
49
|
-
try {
|
|
50
|
-
return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!, [
|
|
51
|
-
WKTLoader,
|
|
52
|
-
WKBLoader
|
|
53
|
-
]);
|
|
54
|
-
} catch (error) {
|
|
55
|
-
return objectRowTable;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
default:
|
|
59
|
-
throw new Error(shape);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function convertBatch(
|
|
64
|
-
objectRowBatch: ObjectRowTableBatch,
|
|
65
|
-
shape?: 'object-row-table' | 'geojson-table'
|
|
66
|
-
): ObjectRowTableBatch | GeoJSONTableBatch {
|
|
67
|
-
switch (shape) {
|
|
68
|
-
case 'object-row-table':
|
|
69
|
-
return objectRowBatch;
|
|
70
|
-
|
|
71
|
-
case 'geojson-table':
|
|
72
|
-
try {
|
|
73
|
-
const geojsonTable = convertWKBTableToGeoJSON(objectRowBatch, objectRowBatch.schema!, [
|
|
74
|
-
WKTLoader,
|
|
75
|
-
WKBLoader
|
|
76
|
-
]);
|
|
77
|
-
return {
|
|
78
|
-
...objectRowBatch,
|
|
79
|
-
...geojsonTable
|
|
80
|
-
};
|
|
81
|
-
} catch (error) {
|
|
82
|
-
return objectRowBatch;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
default:
|
|
86
|
-
throw new Error(shape);
|
|
87
|
-
}
|
|
88
|
-
}
|