@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/index.js

@@ -1,15 +1,58 @@
-import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
-import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
-export { ParquetWorkerLoader };
-export const ParquetLoader = { ...ParquetWorkerLoader,
-  parse: parseParquet,
-  parseFileInBatches: parseParquetFileInBatches
-};
-export { ParquetWriter as _ParquetWriter } from './parquet-writer';
-export { preloadCompressions } from './parquetjs/compression';
-export { ParquetSchema } from './parquetjs/schema/schema';
-export { ParquetReader } from './parquetjs/parser/parquet-reader';
-export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
-export { convertParquetToArrowSchema } from './lib/convert-schema';
-export const _typecheckParquetLoader = ParquetLoader;
-//# sourceMappingURL=index.js.map
+"use strict";
+// loaders.gl, MIT license
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = exports.geoJSONSchema = exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.convertParquetSchema = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetColumnarLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
+// ParquetLoader
+const parquet_loader_1 = require("./parquet-loader");
+Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
+const parse_parquet_to_rows_1 = require("./lib/parsers/parse-parquet-to-rows");
+const parse_parquet_to_columns_1 = require("./lib/parsers/parse-parquet-to-columns");
+const parse_parquet_wasm_1 = require("./lib/wasm/parse-parquet-wasm");
+const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
+Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
+/** ParquetJS table loader */
+exports.ParquetLoader = {
+    ...parquet_loader_1.ParquetLoader,
+    parse: parse_parquet_to_rows_1.parseParquet,
+    parseFileInBatches: parse_parquet_to_rows_1.parseParquetFileInBatches
+};
+/** ParquetJS table loader */
+// @ts-expect-error
+exports.ParquetColumnarLoader = {
+    ...parquet_loader_1.ParquetLoader,
+    parse: parse_parquet_to_columns_1.parseParquetInColumns,
+    parseFileInBatches: parse_parquet_to_columns_1.parseParquetFileInColumnarBatches
+};
+exports.ParquetWasmLoader = {
+    ...parquet_wasm_loader_1.ParquetWasmLoader,
+    parse: parse_parquet_wasm_1.parseParquetWasm
+};
+// ParquetWriter
+var parquet_writer_1 = require("./parquet-writer");
+Object.defineProperty(exports, "_ParquetWriter", { enumerable: true, get: function () { return parquet_writer_1.ParquetWriter; } });
+var parquet_wasm_writer_1 = require("./parquet-wasm-writer");
+Object.defineProperty(exports, "ParquetWasmWriter", { enumerable: true, get: function () { return parquet_wasm_writer_1.ParquetWasmWriter; } });
+// EXPERIMENTAL - expose the internal parquetjs API
+var compression_1 = require("./parquetjs/compression");
+Object.defineProperty(exports, "preloadCompressions", { enumerable: true, get: function () { return compression_1.preloadCompressions; } });
+var schema_1 = require("./parquetjs/schema/schema");
+Object.defineProperty(exports, "ParquetSchema", { enumerable: true, get: function () { return schema_1.ParquetSchema; } });
+var parquet_reader_1 = require("./parquetjs/parser/parquet-reader");
+Object.defineProperty(exports, "ParquetReader", { enumerable: true, get: function () { return parquet_reader_1.ParquetReader; } });
+var parquet_encoder_1 = require("./parquetjs/encoder/parquet-encoder");
+Object.defineProperty(exports, "ParquetEncoder", { enumerable: true, get: function () { return parquet_encoder_1.ParquetEncoder; } });
+var convert_schema_from_parquet_1 = require("./lib/arrow/convert-schema-from-parquet");
+Object.defineProperty(exports, "convertParquetSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
+Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
+// TESTS
+exports._typecheckParquetLoader = exports.ParquetLoader;
+// Geo Metadata
+var geoparquet_schema_1 = require("./lib/geo/geoparquet-schema");
+Object.defineProperty(exports, "geoJSONSchema", { enumerable: true, get: function () { return __importDefault(geoparquet_schema_1).default; } });
+var decode_geo_metadata_1 = require("./lib/geo/decode-geo-metadata");
+Object.defineProperty(exports, "getGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.getGeoMetadata; } });
+Object.defineProperty(exports, "setGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.setGeoMetadata; } });
+Object.defineProperty(exports, "unpackGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.unpackGeoMetadata; } });

package/dist/lib/arrow/convert-columns-to-row-group.d.ts

@@ -0,0 +1 @@
+//# sourceMappingURL=convert-columns-to-row-group.d.ts.map

package/dist/lib/arrow/convert-columns-to-row-group.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-columns-to-row-group.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-columns-to-row-group.ts"],"names":[],"mappings":""}

package/dist/lib/arrow/convert-columns-to-row-group.js

@@ -0,0 +1 @@
+"use strict";

package/dist/lib/arrow/convert-row-group-to-columns.d.ts

@@ -0,0 +1,4 @@
+import { Schema } from '@loaders.gl/schema';
+import { ParquetRowGroup } from '@loaders.gl/parquet/parquetjs/schema/declare';
+export declare function convertParquetRowGroupToColumns(schema: Schema, rowGroup: ParquetRowGroup): Record<string, any[]>;
+//# sourceMappingURL=convert-row-group-to-columns.d.ts.map

package/dist/lib/arrow/convert-row-group-to-columns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-row-group-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-row-group-to-columns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,eAAe,EAAC,MAAM,8CAA8C,CAAC;AAE7E,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAMvB"}

package/dist/lib/arrow/convert-row-group-to-columns.js

@@ -0,0 +1,12 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertParquetRowGroupToColumns = void 0;
+function convertParquetRowGroupToColumns(schema, rowGroup) {
+    const columns = {};
+    for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
+        columns[columnName] = columns[columnName] || data.values;
+    }
+    return columns;
+}
+exports.convertParquetRowGroupToColumns = convertParquetRowGroupToColumns;

package/dist/lib/arrow/convert-schema-from-parquet.d.ts

@@ -0,0 +1,9 @@
+import { Schema, DataType } from '@loaders.gl/schema';
+import type { ParquetSchema } from '../../parquetjs/schema/schema';
+import type { ParquetType } from '../../parquetjs/schema/declare';
+import { FileMetaData } from '../../parquetjs/parquet-thrift';
+export declare const PARQUET_TYPE_MAPPING: {
+    [type in ParquetType]: DataType;
+};
+export declare function convertParquetSchema(parquetSchema: ParquetSchema, parquetMetadata: FileMetaData | null): Schema;
+//# sourceMappingURL=convert-schema-from-parquet.d.ts.map

package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAC,YAAY,EAAC,MAAM,gCAAgC,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,YAAY,GAAG,IAAI,GACnC,MAAM,CAUR"}

package/dist/lib/arrow/convert-schema-from-parquet.js

@@ -0,0 +1,86 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertParquetSchema = exports.PARQUET_TYPE_MAPPING = void 0;
+exports.PARQUET_TYPE_MAPPING = {
+    BOOLEAN: 'bool',
+    INT32: 'int32',
+    INT64: 'float64',
+    INT96: 'float64',
+    FLOAT: 'float32',
+    DOUBLE: 'float64',
+    BYTE_ARRAY: 'binary',
+    FIXED_LEN_BYTE_ARRAY: 'binary',
+    UTF8: 'utf8',
+    DATE: 'int32',
+    TIME_MILLIS: 'int64',
+    TIME_MICROS: 'int64',
+    TIMESTAMP_MILLIS: 'int64',
+    TIMESTAMP_MICROS: 'int64',
+    UINT_8: 'int32',
+    UINT_16: 'uint16',
+    UINT_32: 'uint32',
+    UINT_64: 'uint64',
+    INT_8: 'int8',
+    INT_16: 'int16',
+    INT_32: 'int32',
+    INT_64: 'int64',
+    JSON: 'binary',
+    BSON: 'binary',
+    // TODO check interal type
+    INTERVAL: 'binary',
+    DECIMAL_INT32: 'float32',
+    DECIMAL_INT64: 'float64',
+    DECIMAL_BYTE_ARRAY: 'float64',
+    DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+};
+function convertParquetSchema(parquetSchema, parquetMetadata) {
+    const fields = getFields(parquetSchema.schema);
+    const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
+    const schema = {
+        fields,
+        metadata: metadata || {}
+    };
+    return schema;
+}
+exports.convertParquetSchema = convertParquetSchema;
+function getFields(schema) {
+    const fields = [];
+    for (const name in schema) {
+        const field = schema[name];
+        if (field.fields) {
+            const children = getFields(field.fields);
+            fields.push({ name, type: { type: 'struct', children }, nullable: field.optional });
+        }
+        else {
+            const type = exports.PARQUET_TYPE_MAPPING[field.type];
+            const metadata = getFieldMetadata(field);
+            const arrowField = { name, type, nullable: field.optional, metadata };
+            fields.push(arrowField);
+        }
+    }
+    return fields;
+}
+function getFieldMetadata(field) {
+    let metadata;
+    for (const key in field) {
+        if (key !== 'name') {
+            let value = field[key] || '';
+            value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+            metadata = metadata || {};
+            metadata[key] = value;
+        }
+    }
+    return metadata;
+}
+function getSchemaMetadata(parquetMetadata) {
+    let metadata;
+    const keyValueList = parquetMetadata.key_value_metadata || [];
+    for (const { key, value } of keyValueList) {
+        if (typeof value === 'string') {
+            metadata = metadata || {};
+            metadata[key] = value;
+        }
+    }
+    return metadata;
+}

package/dist/lib/arrow/convert-schema-to-parquet.d.ts

@@ -0,0 +1,7 @@
+import type { ParquetType } from '../../parquetjs/schema/declare';
+import { Schema, DataType } from '@loaders.gl/schema';
+export declare const PARQUET_TYPE_MAPPING: {
+    [type in ParquetType]: DataType;
+};
+export declare function convertToParquetSchema(schema: Schema): Schema;
+//# sourceMappingURL=convert-schema-to-parquet.d.ts.map

package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}

package/dist/lib/arrow/convert-schema-to-parquet.js

@@ -0,0 +1,71 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToParquetSchema = exports.PARQUET_TYPE_MAPPING = void 0;
+exports.PARQUET_TYPE_MAPPING = {
+    BOOLEAN: 'bool',
+    INT32: 'int32',
+    INT64: 'float64',
+    INT96: 'float64',
+    FLOAT: 'float32',
+    DOUBLE: 'float64',
+    BYTE_ARRAY: 'binary',
+    FIXED_LEN_BYTE_ARRAY: 'binary',
+    UTF8: 'utf8',
+    DATE: 'int32',
+    TIME_MILLIS: 'int64',
+    TIME_MICROS: 'int64',
+    TIMESTAMP_MILLIS: 'int64',
+    TIMESTAMP_MICROS: 'int64',
+    UINT_8: 'int32',
+    UINT_16: 'uint16',
+    UINT_32: 'uint32',
+    UINT_64: 'uint64',
+    INT_8: 'int8',
+    INT_16: 'int16',
+    INT_32: 'int32',
+    INT_64: 'int64',
+    JSON: 'binary',
+    BSON: 'binary',
+    // TODO check interval type
+    INTERVAL: 'binary',
+    DECIMAL_INT32: 'float32',
+    DECIMAL_INT64: 'float64',
+    DECIMAL_BYTE_ARRAY: 'float64',
+    DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+};
+function convertToParquetSchema(schema) {
+    const fields = []; // getFields(schema.fields);
+    // TODO add metadata if needed.
+    return { fields, metadata: {} };
+}
+exports.convertToParquetSchema = convertToParquetSchema;
+// function getFields(schema: Field[]): Definition[] {
+//   const fields: Field[] = [];
+//   for (const name in schema) {
+//     const field = schema[name];
+//     // @ts-ignore
+//     const children = field.children as DataType[];
+//     if (children) {
+//       const childField = getFields(field.fields);
+//       const nestedField = new Field(name, new Struct(childField), field.optional);
+//       fields.push(nestedField);
+//     } else {
+//       const FieldType = PARQUET_TYPE_MAPPING[field.type];
+//       const metadata = getFieldMetadata(field);
+//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+//       fields.push(arrowField);
+//     }
+//   }
+//   return fields;
+// }
+// function getFieldMetadata(field: ParquetField): Map<string, string> {
+//   const metadata = new Map();
+//   for (const key in field) {
+//     if (key !== 'name') {
+//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+//       metadata.set(key, value);
+//     }
+//   }
+//   return metadata;
+// }

package/dist/lib/geo/decode-geo-metadata.d.ts

@@ -0,0 +1,31 @@
+import { Schema } from '@loaders.gl/schema';
+/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */
+export type GeoMetadata = {
+    version?: string;
+    primary_column?: string;
+    columns: Record<string, GeoColumnMetadata>;
+    [key: string]: unknown;
+};
+/** A geoarrow / geoparquet geo metadata for one geometry column  */
+export type GeoColumnMetadata = {
+    bounding_box?: [number, number, number, number] | [number, number, number, number, number, number];
+    crs?: string;
+    geometry_type?: string[];
+    edges?: string;
+    [key: string]: unknown;
+};
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
+export declare function getGeoMetadata(schema: Schema): GeoMetadata | null;
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+export declare function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void;
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export declare function unpackGeoMetadata(schema: Schema): void;
+//# sourceMappingURL=decode-geo-metadata.d.ts.map

package/dist/lib/geo/decode-geo-metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decode-geo-metadata.d.ts","sourceRoot":"","sources":["../../../src/lib/geo/decode-geo-metadata.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAQ,MAAM,oBAAoB,CAAC;AAIjD,mHAAmH;AACnH,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAC3C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,YAAY,CAAC,EACT,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAChC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF;;yGAEyG;AACzG,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAYjE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,GAAG,IAAI,CAG7E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CA6BtD"}

package/dist/lib/geo/decode-geo-metadata.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = void 0;
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
+function getGeoMetadata(schema) {
+    const stringifiedGeoMetadata = schema.metadata.geo;
+    if (!stringifiedGeoMetadata) {
+        return null;
+    }
+    try {
+        const geoMetadata = JSON.parse(stringifiedGeoMetadata);
+        return geoMetadata;
+    }
+    catch {
+        return null;
+    }
+}
+exports.getGeoMetadata = getGeoMetadata;
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+function setGeoMetadata(schema, geoMetadata) {
+    const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
+    schema.metadata.geo = stringifiedGeoMetadata;
+}
+exports.setGeoMetadata = setGeoMetadata;
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+function unpackGeoMetadata(schema) {
+    const geoMetadata = getGeoMetadata(schema);
+    if (!geoMetadata) {
+        return;
+    }
+    // Store Parquet Schema Level Metadata
+    const { version, primary_column, columns } = geoMetadata;
+    if (version) {
+        schema.metadata['geo.version'] = version;
+    }
+    if (primary_column) {
+        schema.metadata['geo.primary_column'] = primary_column;
+    }
+    // store column names as comma separated list
+    schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
+    for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
+        const field = schema.fields.find((field) => field.name === columnName);
+        if (field) {
+            if (field.name === primary_column) {
+                setFieldMetadata(field, 'geo.primary_field', 'true');
+            }
+            unpackGeoFieldMetadata(field, columnMetadata);
+        }
+    }
+}
+exports.unpackGeoMetadata = unpackGeoMetadata;
+function unpackGeoFieldMetadata(field, columnMetadata) {
+    for (const [key, value] of Object.entries(columnMetadata || {})) {
+        switch (key) {
+            case 'geometry_type':
+                setFieldMetadata(field, `geo.${key}`, value.join(','));
+                break;
+            case 'bbox':
+            case 'crs':
+            case 'edges':
+            default:
+                setFieldMetadata(field, `geo.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
+        }
+    }
+}
+function setFieldMetadata(field, key, value) {
+    field.metadata = field.metadata || {};
+    field.metadata[key] = value;
+}

package/dist/lib/geo/geoparquet-schema.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+declare const _default: {
+    $schema: string;
+    title: string;
+    description: string;
+    type: string;
+    required: string[];
+    properties: {
+        version: {
+            type: string;
+            const: string;
+        };
+        primary_column: {
+            type: string;
+            minLength: number;
+        };
+        columns: {
+            type: string;
+            minProperties: number;
+            patternProperties: {
+                '.+': {
+                    type: string;
+                    required: string[];
+                    properties: {
+                        encoding: {
+                            type: string;
+                            const: string;
+                        };
+                        geometry_types: {
+                            type: string;
+                            uniqueItems: boolean;
+                            items: {
+                                type: string;
+                                pattern: string;
+                            };
+                        };
+                        crs: {
+                            oneOf: ({
+                                $ref: string;
+                                type?: undefined;
+                            } | {
+                                type: string;
+                                $ref?: undefined;
+                            })[];
+                        };
+                        edges: {
+                            type: string;
+                            enum: string[];
+                        };
+                        orientation: {
+                            type: string;
+                            const: string;
+                        };
+                        bbox: {
+                            type: string;
+                            items: {
+                                type: string;
+                            };
+                            oneOf: {
+                                description: string;
+                                minItems: number;
+                                maxItems: number;
+                            }[];
+                        };
+                        epoch: {
+                            type: string;
+                        };
+                    };
+                };
+            };
+            additionalProperties: boolean;
+        };
+    };
+};
+export default _default;
+//# sourceMappingURL=geoparquet-schema.d.ts.map

package/dist/lib/geo/geoparquet-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"geoparquet-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/geo/geoparquet-schema.ts"],"names":[],"mappings":"AAIA;;;;GAIG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACH,wBA2DE"}

package/dist/lib/geo/geoparquet-schema.js

@@ -0,0 +1,69 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable camelcase */
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+exports.default = {
+    $schema: 'http://json-schema.org/draft-07/schema#',
+    title: 'GeoParquet',
+    description: 'Parquet metadata included in the geo field.',
+    type: 'object',
+    required: ['version', 'primary_column', 'columns'],
+    properties: {
+        version: { type: 'string', const: '1.0.0-beta.1' },
+        primary_column: { type: 'string', minLength: 1 },
+        columns: {
+            type: 'object',
+            minProperties: 1,
+            patternProperties: {
+                '.+': {
+                    type: 'object',
+                    required: ['encoding', 'geometry_types'],
+                    properties: {
+                        encoding: { type: 'string', const: 'WKB' },
+                        geometry_types: {
+                            type: 'array',
+                            uniqueItems: true,
+                            items: {
+                                type: 'string',
+                                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+                            }
+                        },
+                        crs: {
+                            oneOf: [
+                                {
+                                    $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+                                },
+                                { type: 'null' }
+                            ]
+                        },
+                        edges: { type: 'string', enum: ['planar', 'spherical'] },
+                        orientation: { type: 'string', const: 'counterclockwise' },
+                        bbox: {
+                            type: 'array',
+                            items: { type: 'number' },
+                            oneOf: [
+                                {
+                                    description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                                    minItems: 4,
+                                    maxItems: 4
+                                },
+                                {
+                                    description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                                    minItems: 6,
+                                    maxItems: 6
+                                }
+                            ]
+                        },
+                        epoch: { type: 'number' }
+                    }
+                }
+            },
+            additionalProperties: false
+        }
+    }
+};

package/dist/lib/parsers/parse-parquet-to-columns.d.ts

@@ -0,0 +1,5 @@
+import { ColumnarTable, ColumnarTableBatch } from '@loaders.gl/schema';
+import type { ParquetLoaderOptions } from '../../parquet-loader';
+export declare function parseParquetInColumns(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<ColumnarTable>;
+export declare function parseParquetFileInColumnarBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncIterable<ColumnarTableBatch>;
+//# sourceMappingURL=parse-parquet-to-columns.d.ts.map

package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAE7E,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAS/D,wBAAsB,qBAAqB,CACzC,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAUxB;AAED,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,kBAAkB,CAAC,CAWnC"}

package/dist/lib/parsers/parse-parquet-to-columns.js

@@ -0,0 +1,46 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseParquetFileInColumnarBatches = exports.parseParquetInColumns = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
+const convert_schema_from_parquet_1 = require("../arrow/convert-schema-from-parquet");
+const shred_1 = require("../../parquetjs/schema/shred");
+// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns.js';
+const decode_geo_metadata_1 = require("../geo/decode-geo-metadata");
+async function parseParquetInColumns(arrayBuffer, options) {
+    const blob = new Blob([arrayBuffer]);
+    for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+        return {
+            shape: 'columnar-table',
+            schema: batch.schema,
+            data: batch.data
+        };
+    }
+    throw new Error('empty table');
+}
+exports.parseParquetInColumns = parseParquetInColumns;
+async function* parseParquetFileInColumnarBatches(blob, options) {
+    const file = (0, loader_utils_1.makeReadableFile)(blob);
+    const reader = new parquet_reader_1.ParquetReader(file);
+    const parquetSchema = await reader.getSchema();
+    const parquetMetadata = await reader.getFileMetadata();
+    const schema = (0, convert_schema_from_parquet_1.convertParquetSchema)(parquetSchema, parquetMetadata);
+    (0, decode_geo_metadata_1.unpackGeoMetadata)(schema);
+    const rowGroups = reader.rowGroupIterator(options?.parquet);
+    for await (const rowGroup of rowGroups) {
+        yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
+    }
+}
+exports.parseParquetFileInColumnarBatches = parseParquetFileInColumnarBatches;
+function convertRowGroupToTableBatch(parquetSchema, rowGroup, schema) {
+    // const data = convertParquetRowGroupToColumns(schema, rowGroup);
+    const data = (0, shred_1.materializeColumns)(parquetSchema, rowGroup);
+    return {
+        shape: 'columnar-table',
+        batchType: 'data',
+        schema,
+        data,
+        length: rowGroup.rowCount
+    };
+}

package/dist/lib/parsers/parse-parquet-to-rows.d.ts

@@ -0,0 +1,5 @@
+import { ObjectRowTable, ObjectRowTableBatch } from '@loaders.gl/schema';
+import type { ParquetLoaderOptions } from '../../parquet-loader';
+export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<ObjectRowTable>;
+export declare function parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncIterable<ObjectRowTableBatch>;
+//# sourceMappingURL=parse-parquet-to-rows.d.ts.map

package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-rows.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AACvE,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAI/D,wBAAsB,YAAY,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CAgBzB;AAED,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CAYpC"}