@loaders.gl/parquet 3.3.2 → 3.4.0-alpha.2

package/dist/lib/geo/geoparquet-schema.js

@@ -0,0 +1,69 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable camelcase */
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+exports.default = {
+    $schema: 'http://json-schema.org/draft-07/schema#',
+    title: 'GeoParquet',
+    description: 'Parquet metadata included in the geo field.',
+    type: 'object',
+    required: ['version', 'primary_column', 'columns'],
+    properties: {
+        version: { type: 'string', const: '1.0.0-beta.1' },
+        primary_column: { type: 'string', minLength: 1 },
+        columns: {
+            type: 'object',
+            minProperties: 1,
+            patternProperties: {
+                '.+': {
+                    type: 'object',
+                    required: ['encoding', 'geometry_types'],
+                    properties: {
+                        encoding: { type: 'string', const: 'WKB' },
+                        geometry_types: {
+                            type: 'array',
+                            uniqueItems: true,
+                            items: {
+                                type: 'string',
+                                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+                            }
+                        },
+                        crs: {
+                            oneOf: [
+                                {
+                                    $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+                                },
+                                { type: 'null' }
+                            ]
+                        },
+                        edges: { type: 'string', enum: ['planar', 'spherical'] },
+                        orientation: { type: 'string', const: 'counterclockwise' },
+                        bbox: {
+                            type: 'array',
+                            items: { type: 'number' },
+                            oneOf: [
+                                {
+                                    description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                                    minItems: 4,
+                                    maxItems: 4
+                                },
+                                {
+                                    description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                                    minItems: 6,
+                                    maxItems: 6
+                                }
+                            ]
+                        },
+                        epoch: { type: 'number' }
+                    }
+                }
+            },
+            additionalProperties: false
+        }
+    }
+};

package/dist/lib/parsers/parse-parquet-to-columns.d.ts

@@ -0,0 +1,5 @@
+import { ColumnarTableBatch } from '@loaders.gl/schema';
+import type { ParquetLoaderOptions } from '../../parquet-loader';
+export declare function parseParquetInColumns(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<ColumnarTableBatch | null>;
+export declare function parseParquetFileInColumnarBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncIterable<ColumnarTableBatch>;
+//# sourceMappingURL=parse-parquet-to-columns.d.ts.map

package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAO/D,wBAAsB,qBAAqB,CACzC,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,oBAAoB,sCAO/B;AAED,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,kBAAkB,CAAC,CAWnC"}

package/dist/lib/parsers/parse-parquet-to-columns.js

@@ -0,0 +1,40 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseParquetFileInColumnarBatches = exports.parseParquetInColumns = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
+const convert_schema_from_parquet_1 = require("../arrow/convert-schema-from-parquet");
+const convert_row_group_to_columns_1 = require("../arrow/convert-row-group-to-columns");
+const decode_geo_metadata_1 = require("../geo/decode-geo-metadata");
+async function parseParquetInColumns(arrayBuffer, options) {
+    const blob = new Blob([arrayBuffer]);
+    for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+        return batch;
+    }
+    return null;
+}
+exports.parseParquetInColumns = parseParquetInColumns;
+async function* parseParquetFileInColumnarBatches(blob, options) {
+    const file = (0, loader_utils_1.makeReadableFile)(blob);
+    const reader = new parquet_reader_1.ParquetReader(file);
+    const parquetSchema = await reader.getSchema();
+    const parquetMetadata = await reader.getFileMetadata();
+    const schema = (0, convert_schema_from_parquet_1.convertSchemaFromParquet)(parquetSchema, parquetMetadata);
+    (0, decode_geo_metadata_1.unpackGeoMetadata)(schema);
+    const rowGroups = reader.rowGroupIterator(options?.parquet);
+    for await (const rowGroup of rowGroups) {
+        yield convertRowGroupToTableBatch(schema, rowGroup);
+    }
+}
+exports.parseParquetFileInColumnarBatches = parseParquetFileInColumnarBatches;
+function convertRowGroupToTableBatch(schema, rowGroup) {
+    const data = (0, convert_row_group_to_columns_1.convertParquetRowGroupToColumns)(schema, rowGroup);
+    return {
+        shape: 'columnar-table',
+        batchType: 'data',
+        schema,
+        data,
+        length: rowGroup.rowCount
+    };
+}

package/dist/lib/parsers/parse-parquet-to-rows.d.ts

@@ -0,0 +1,4 @@
+import type { ParquetLoaderOptions } from '../../parquet-loader';
+export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<import("../../parquetjs/schema/declare").ParquetRecord[] | null>;
+export declare function parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncGenerator<import("../../parquetjs/schema/declare").ParquetRecord[], void, unknown>;
+//# sourceMappingURL=parse-parquet-to-rows.d.ts.map

package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-rows.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAG/D,wBAAsB,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,oBAAoB,4EAM1F;AAED,wBAAuB,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,oBAAoB,2FAO1F"}

package/dist/lib/{parse-parquet.js → parsers/parse-parquet-to-rows.js}

@@ -4,7 +4,7 @@ exports.parseParquetFileInBatches = exports.parseParquet = void 0;
 // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
 // import {ColumnarTableBatch} from '@loaders.gl/schema';
 const loader_utils_1 = require("@loaders.gl/loader-utils");
-const parquet_reader_1 = require("../parquetjs/parser/parquet-reader");
+const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
 async function parseParquet(arrayBuffer, options) {
     const blob = new Blob([arrayBuffer]);
     for await (const batch of parseParquetFileInBatches(blob, options)) {

package/dist/parquet-loader.d.ts

@@ -4,6 +4,7 @@ export type ParquetLoaderOptions = LoaderOptions & {
         type?: 'object-row-table';
         url?: string;
         columnList?: string[] | string[][];
+        geoparquet?: boolean;
     };
 };
 /** ParquetJS table loader */

package/dist/parquet-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,kBAAkB,CAAC;QAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;KACpC,CAAC;CACH,CAAC;AAUF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYzB,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAsB,CAAC"}
+{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,kBAAkB,CAAC;QAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;QACnC,UAAU,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;CACH,CAAC;AAWF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYzB,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAsB,CAAC"}

package/dist/parquet-loader.js

@@ -8,7 +8,8 @@ const DEFAULT_PARQUET_LOADER_OPTIONS = {
     parquet: {
         type: 'object-row-table',
         url: undefined,
-        columnList: []
+        columnList: [],
+        geoparquet: true
     }
 };
 /** ParquetJS table loader */