@loaders.gl/parquet 4.0.0-alpha.22 → 4.0.0-alpha.24

package/dist/parquet-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,oBAAoB,CAUzE,CAAC"}
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEtC,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,oBAAoB,CAUzE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.0.0-alpha.22",
+  "version": "4.0.0-alpha.24",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -38,13 +38,15 @@
     "tls": false,
     "stream": false,
     "fs": false,
-    "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
+    "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts",
+    "./dist/es5/lib/wasm/load-wasm/load-wasm-node.js": "./dist/es5/lib/wasm/load-wasm/load-wasm-browser.js",
+    "./dist/esm/lib/wasm/load-wasm/load-wasm-node.js": "./dist/esm/lib/wasm/load-wasm/load-wasm-browser.js"
   },
   "dependencies": {
-    "@loaders.gl/bson": "4.0.0-alpha.22",
-    "@loaders.gl/compression": "4.0.0-alpha.22",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.22",
-    "@loaders.gl/schema": "4.0.0-alpha.22",
+    "@loaders.gl/bson": "4.0.0-alpha.24",
+    "@loaders.gl/compression": "4.0.0-alpha.24",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.24",
+    "@loaders.gl/schema": "4.0.0-alpha.24",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "int53": "^0.2.4",
@@ -58,14 +60,14 @@
     "zstd-codec": "^0.1"
   },
   "peerDependencies": {
-    "apache-arrow": "^9.0.0"
+    "apache-arrow": "^13.0.0"
   },
   "devDependencies": {
     "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0",
-    "apache-arrow": "^9.0.0"
+    "apache-arrow": "^13.0.0"
   },
-  "gitHead": "0da838c506d1275383f2fd3d244d9c72b25397d2"
+  "gitHead": "97a8990595c132fb14e3445a8768d9f4cb98ff05"
 }

package/src/index.ts

@@ -73,9 +73,6 @@ export {
   convertParquetSchema as convertParquetToArrowSchema
 } from './lib/arrow/convert-schema-from-parquet';
 
-// TESTS
-export const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;
-
 // Geo Metadata
 export {default as geoJSONSchema} from './lib/geo/geoparquet-schema';
 

package/src/parquet-writer.ts

@@ -9,8 +9,6 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type ParquetWriterOptions = {};
 
-const DEFAULT_PARQUET_LOADER_OPTIONS = {};
-
 export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
@@ -20,7 +18,7 @@ export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
   mimeTypes: ['application/octet-stream'],
   encodeSync,
   binary: true,
-  options: DEFAULT_PARQUET_LOADER_OPTIONS
+  options: {}
 };
 
 function encodeSync(data, options?: ParquetWriterOptions) {

package/dist/bundle.js

@@ -1,5 +0,0 @@
-"use strict";
-// @ts-nocheck
-const moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);

package/dist/constants.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PARQUET_RDLVL_ENCODING = exports.PARQUET_RDLVL_TYPE = exports.PARQUET_VERSION = exports.PARQUET_MAGIC_ENCRYPTED = exports.PARQUET_MAGIC = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-/**
- * Parquet File Magic String
- */
-exports.PARQUET_MAGIC = 'PAR1';
-exports.PARQUET_MAGIC_ENCRYPTED = 'PARE';
-/**
- * Parquet File Format Version
- */
-exports.PARQUET_VERSION = 1;
-/**
- * Internal type used for repetition/definition levels
- */
-exports.PARQUET_RDLVL_TYPE = 'INT32';
-exports.PARQUET_RDLVL_ENCODING = 'RLE';

package/dist/index.js

@@ -1,58 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = exports.geoJSONSchema = exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.convertParquetSchema = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetColumnarLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
-// ParquetLoader
-const parquet_loader_1 = require("./parquet-loader");
-Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
-const parse_parquet_to_rows_1 = require("./lib/parsers/parse-parquet-to-rows");
-const parse_parquet_to_columns_1 = require("./lib/parsers/parse-parquet-to-columns");
-const parse_parquet_wasm_1 = require("./lib/wasm/parse-parquet-wasm");
-const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
-Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
-/** ParquetJS table loader */
-exports.ParquetLoader = {
-    ...parquet_loader_1.ParquetLoader,
-    parse: parse_parquet_to_rows_1.parseParquet,
-    parseFileInBatches: parse_parquet_to_rows_1.parseParquetFileInBatches
-};
-/** ParquetJS table loader */
-// @ts-expect-error
-exports.ParquetColumnarLoader = {
-    ...parquet_loader_1.ParquetLoader,
-    parse: parse_parquet_to_columns_1.parseParquetInColumns,
-    parseFileInBatches: parse_parquet_to_columns_1.parseParquetFileInColumnarBatches
-};
-exports.ParquetWasmLoader = {
-    ...parquet_wasm_loader_1.ParquetWasmLoader,
-    parse: parse_parquet_wasm_1.parseParquetWasm
-};
-// ParquetWriter
-var parquet_writer_1 = require("./parquet-writer");
-Object.defineProperty(exports, "_ParquetWriter", { enumerable: true, get: function () { return parquet_writer_1.ParquetWriter; } });
-var parquet_wasm_writer_1 = require("./parquet-wasm-writer");
-Object.defineProperty(exports, "ParquetWasmWriter", { enumerable: true, get: function () { return parquet_wasm_writer_1.ParquetWasmWriter; } });
-// EXPERIMENTAL - expose the internal parquetjs API
-var compression_1 = require("./parquetjs/compression");
-Object.defineProperty(exports, "preloadCompressions", { enumerable: true, get: function () { return compression_1.preloadCompressions; } });
-var schema_1 = require("./parquetjs/schema/schema");
-Object.defineProperty(exports, "ParquetSchema", { enumerable: true, get: function () { return schema_1.ParquetSchema; } });
-var parquet_reader_1 = require("./parquetjs/parser/parquet-reader");
-Object.defineProperty(exports, "ParquetReader", { enumerable: true, get: function () { return parquet_reader_1.ParquetReader; } });
-var parquet_encoder_1 = require("./parquetjs/encoder/parquet-encoder");
-Object.defineProperty(exports, "ParquetEncoder", { enumerable: true, get: function () { return parquet_encoder_1.ParquetEncoder; } });
-var convert_schema_from_parquet_1 = require("./lib/arrow/convert-schema-from-parquet");
-Object.defineProperty(exports, "convertParquetSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
-Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
-// TESTS
-exports._typecheckParquetLoader = exports.ParquetLoader;
-// Geo Metadata
-var geoparquet_schema_1 = require("./lib/geo/geoparquet-schema");
-Object.defineProperty(exports, "geoJSONSchema", { enumerable: true, get: function () { return __importDefault(geoparquet_schema_1).default; } });
-var decode_geo_metadata_1 = require("./lib/geo/decode-geo-metadata");
-Object.defineProperty(exports, "getGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.getGeoMetadata; } });
-Object.defineProperty(exports, "setGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.setGeoMetadata; } });
-Object.defineProperty(exports, "unpackGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.unpackGeoMetadata; } });

package/dist/lib/arrow/convert-columns-to-row-group.js

@@ -1 +0,0 @@
-"use strict";

package/dist/lib/arrow/convert-row-group-to-columns.js

@@ -1,12 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertParquetRowGroupToColumns = void 0;
-function convertParquetRowGroupToColumns(schema, rowGroup) {
-    const columns = {};
-    for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
-        columns[columnName] = columns[columnName] || data.values;
-    }
-    return columns;
-}
-exports.convertParquetRowGroupToColumns = convertParquetRowGroupToColumns;

package/dist/lib/arrow/convert-schema-from-parquet.js

@@ -1,86 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertParquetSchema = exports.PARQUET_TYPE_MAPPING = void 0;
-exports.PARQUET_TYPE_MAPPING = {
-    BOOLEAN: 'bool',
-    INT32: 'int32',
-    INT64: 'float64',
-    INT96: 'float64',
-    FLOAT: 'float32',
-    DOUBLE: 'float64',
-    BYTE_ARRAY: 'binary',
-    FIXED_LEN_BYTE_ARRAY: 'binary',
-    UTF8: 'utf8',
-    DATE: 'int32',
-    TIME_MILLIS: 'int64',
-    TIME_MICROS: 'int64',
-    TIMESTAMP_MILLIS: 'int64',
-    TIMESTAMP_MICROS: 'int64',
-    UINT_8: 'int32',
-    UINT_16: 'uint16',
-    UINT_32: 'uint32',
-    UINT_64: 'uint64',
-    INT_8: 'int8',
-    INT_16: 'int16',
-    INT_32: 'int32',
-    INT_64: 'int64',
-    JSON: 'binary',
-    BSON: 'binary',
-    // TODO check interal type
-    INTERVAL: 'binary',
-    DECIMAL_INT32: 'float32',
-    DECIMAL_INT64: 'float64',
-    DECIMAL_BYTE_ARRAY: 'float64',
-    DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
-};
-function convertParquetSchema(parquetSchema, parquetMetadata) {
-    const fields = getFields(parquetSchema.schema);
-    const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
-    const schema = {
-        fields,
-        metadata: metadata || {}
-    };
-    return schema;
-}
-exports.convertParquetSchema = convertParquetSchema;
-function getFields(schema) {
-    const fields = [];
-    for (const name in schema) {
-        const field = schema[name];
-        if (field.fields) {
-            const children = getFields(field.fields);
-            fields.push({ name, type: { type: 'struct', children }, nullable: field.optional });
-        }
-        else {
-            const type = exports.PARQUET_TYPE_MAPPING[field.type];
-            const metadata = getFieldMetadata(field);
-            const arrowField = { name, type, nullable: field.optional, metadata };
-            fields.push(arrowField);
-        }
-    }
-    return fields;
-}
-function getFieldMetadata(field) {
-    let metadata;
-    for (const key in field) {
-        if (key !== 'name') {
-            let value = field[key] || '';
-            value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-            metadata = metadata || {};
-            metadata[key] = value;
-        }
-    }
-    return metadata;
-}
-function getSchemaMetadata(parquetMetadata) {
-    let metadata;
-    const keyValueList = parquetMetadata.key_value_metadata || [];
-    for (const { key, value } of keyValueList) {
-        if (typeof value === 'string') {
-            metadata = metadata || {};
-            metadata[key] = value;
-        }
-    }
-    return metadata;
-}

package/dist/lib/arrow/convert-schema-to-parquet.js

@@ -1,71 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertToParquetSchema = exports.PARQUET_TYPE_MAPPING = void 0;
-exports.PARQUET_TYPE_MAPPING = {
-    BOOLEAN: 'bool',
-    INT32: 'int32',
-    INT64: 'float64',
-    INT96: 'float64',
-    FLOAT: 'float32',
-    DOUBLE: 'float64',
-    BYTE_ARRAY: 'binary',
-    FIXED_LEN_BYTE_ARRAY: 'binary',
-    UTF8: 'utf8',
-    DATE: 'int32',
-    TIME_MILLIS: 'int64',
-    TIME_MICROS: 'int64',
-    TIMESTAMP_MILLIS: 'int64',
-    TIMESTAMP_MICROS: 'int64',
-    UINT_8: 'int32',
-    UINT_16: 'uint16',
-    UINT_32: 'uint32',
-    UINT_64: 'uint64',
-    INT_8: 'int8',
-    INT_16: 'int16',
-    INT_32: 'int32',
-    INT_64: 'int64',
-    JSON: 'binary',
-    BSON: 'binary',
-    // TODO check interval type
-    INTERVAL: 'binary',
-    DECIMAL_INT32: 'float32',
-    DECIMAL_INT64: 'float64',
-    DECIMAL_BYTE_ARRAY: 'float64',
-    DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
-};
-function convertToParquetSchema(schema) {
-    const fields = []; // getFields(schema.fields);
-    // TODO add metadata if needed.
-    return { fields, metadata: {} };
-}
-exports.convertToParquetSchema = convertToParquetSchema;
-// function getFields(schema: Field[]): Definition[] {
-//   const fields: Field[] = [];
-//   for (const name in schema) {
-//     const field = schema[name];
-//     // @ts-ignore
-//     const children = field.children as DataType[];
-//     if (children) {
-//       const childField = getFields(field.fields);
-//       const nestedField = new Field(name, new Struct(childField), field.optional);
-//       fields.push(nestedField);
-//     } else {
-//       const FieldType = PARQUET_TYPE_MAPPING[field.type];
-//       const metadata = getFieldMetadata(field);
-//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);
-//       fields.push(arrowField);
-//     }
-//   }
-//   return fields;
-// }
-// function getFieldMetadata(field: ParquetField): Map<string, string> {
-//   const metadata = new Map();
-//   for (const key in field) {
-//     if (key !== 'name') {
-//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-//       metadata.set(key, value);
-//     }
-//   }
-//   return metadata;
-// }

package/dist/lib/geo/decode-geo-metadata.js

@@ -1,77 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = void 0;
-/**
- * Reads the GeoMetadata object from the metadata
- * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
-function getGeoMetadata(schema) {
-    const stringifiedGeoMetadata = schema.metadata.geo;
-    if (!stringifiedGeoMetadata) {
-        return null;
-    }
-    try {
-        const geoMetadata = JSON.parse(stringifiedGeoMetadata);
-        return geoMetadata;
-    }
-    catch {
-        return null;
-    }
-}
-exports.getGeoMetadata = getGeoMetadata;
-/**
- * Stores a geoarrow / geoparquet geo metadata object in the schema
- * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
- */
-function setGeoMetadata(schema, geoMetadata) {
-    const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
-    schema.metadata.geo = stringifiedGeoMetadata;
-}
-exports.setGeoMetadata = setGeoMetadata;
-/**
- * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
- * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
- */
-function unpackGeoMetadata(schema) {
-    const geoMetadata = getGeoMetadata(schema);
-    if (!geoMetadata) {
-        return;
-    }
-    // Store Parquet Schema Level Metadata
-    const { version, primary_column, columns } = geoMetadata;
-    if (version) {
-        schema.metadata['geo.version'] = version;
-    }
-    if (primary_column) {
-        schema.metadata['geo.primary_column'] = primary_column;
-    }
-    // store column names as comma separated list
-    schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
-    for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
-        const field = schema.fields.find((field) => field.name === columnName);
-        if (field) {
-            if (field.name === primary_column) {
-                setFieldMetadata(field, 'geo.primary_field', 'true');
-            }
-            unpackGeoFieldMetadata(field, columnMetadata);
-        }
-    }
-}
-exports.unpackGeoMetadata = unpackGeoMetadata;
-function unpackGeoFieldMetadata(field, columnMetadata) {
-    for (const [key, value] of Object.entries(columnMetadata || {})) {
-        switch (key) {
-            case 'geometry_type':
-                setFieldMetadata(field, `geo.${key}`, value.join(','));
-                break;
-            case 'bbox':
-            case 'crs':
-            case 'edges':
-            default:
-                setFieldMetadata(field, `geo.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
-        }
-    }
-}
-function setFieldMetadata(field, key, value) {
-    field.metadata = field.metadata || {};
-    field.metadata[key] = value;
-}

package/dist/lib/geo/geoparquet-schema.js

@@ -1,69 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-/* eslint-disable camelcase */
-/**
- * Geoparquet JSON schema for geo metadata
- * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
- * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
- */
-exports.default = {
-    $schema: 'http://json-schema.org/draft-07/schema#',
-    title: 'GeoParquet',
-    description: 'Parquet metadata included in the geo field.',
-    type: 'object',
-    required: ['version', 'primary_column', 'columns'],
-    properties: {
-        version: { type: 'string', const: '1.0.0-beta.1' },
-        primary_column: { type: 'string', minLength: 1 },
-        columns: {
-            type: 'object',
-            minProperties: 1,
-            patternProperties: {
-                '.+': {
-                    type: 'object',
-                    required: ['encoding', 'geometry_types'],
-                    properties: {
-                        encoding: { type: 'string', const: 'WKB' },
-                        geometry_types: {
-                            type: 'array',
-                            uniqueItems: true,
-                            items: {
-                                type: 'string',
-                                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
-                            }
-                        },
-                        crs: {
-                            oneOf: [
-                                {
-                                    $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
-                                },
-                                { type: 'null' }
-                            ]
-                        },
-                        edges: { type: 'string', enum: ['planar', 'spherical'] },
-                        orientation: { type: 'string', const: 'counterclockwise' },
-                        bbox: {
-                            type: 'array',
-                            items: { type: 'number' },
-                            oneOf: [
-                                {
-                                    description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
-                                    minItems: 4,
-                                    maxItems: 4
-                                },
-                                {
-                                    description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
-                                    minItems: 6,
-                                    maxItems: 6
-                                }
-                            ]
-                        },
-                        epoch: { type: 'number' }
-                    }
-                }
-            },
-            additionalProperties: false
-        }
-    }
-};

package/dist/lib/parsers/parse-parquet-to-columns.js

@@ -1,46 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseParquetFileInColumnarBatches = exports.parseParquetInColumns = void 0;
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
-const convert_schema_from_parquet_1 = require("../arrow/convert-schema-from-parquet");
-const shred_1 = require("../../parquetjs/schema/shred");
-// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
-const decode_geo_metadata_1 = require("../geo/decode-geo-metadata");
-async function parseParquetInColumns(arrayBuffer, options) {
-    const blob = new Blob([arrayBuffer]);
-    for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
-        return {
-            shape: 'columnar-table',
-            schema: batch.schema,
-            data: batch.data
-        };
-    }
-    throw new Error('empty table');
-}
-exports.parseParquetInColumns = parseParquetInColumns;
-async function* parseParquetFileInColumnarBatches(blob, options) {
-    const file = (0, loader_utils_1.makeReadableFile)(blob);
-    const reader = new parquet_reader_1.ParquetReader(file);
-    const parquetSchema = await reader.getSchema();
-    const parquetMetadata = await reader.getFileMetadata();
-    const schema = (0, convert_schema_from_parquet_1.convertParquetSchema)(parquetSchema, parquetMetadata);
-    (0, decode_geo_metadata_1.unpackGeoMetadata)(schema);
-    const rowGroups = reader.rowGroupIterator(options?.parquet);
-    for await (const rowGroup of rowGroups) {
-        yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
-    }
-}
-exports.parseParquetFileInColumnarBatches = parseParquetFileInColumnarBatches;
-function convertRowGroupToTableBatch(parquetSchema, rowGroup, schema) {
-    // const data = convertParquetRowGroupToColumns(schema, rowGroup);
-    const data = (0, shred_1.materializeColumns)(parquetSchema, rowGroup);
-    return {
-        shape: 'columnar-table',
-        batchType: 'data',
-        schema,
-        data,
-        length: rowGroup.rowCount
-    };
-}

package/dist/lib/parsers/parse-parquet-to-rows.js

@@ -1,37 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseParquetFileInBatches = exports.parseParquet = void 0;
-// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
-// import {ColumnarTableBatch} from '@loaders.gl/schema';
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
-async function parseParquet(arrayBuffer, options) {
-    const blob = new Blob([arrayBuffer]);
-    const rows = [];
-    for await (const batch of parseParquetFileInBatches(blob, options)) {
-        // we have only one input batch so return
-        for (const row of batch.data) {
-            rows.push(row);
-        }
-    }
-    return {
-        shape: 'object-row-table',
-        // TODO - spread can fail for very large number of batches
-        data: rows
-    };
-}
-exports.parseParquet = parseParquet;
-async function* parseParquetFileInBatches(blob, options) {
-    const file = (0, loader_utils_1.makeReadableFile)(blob);
-    const reader = new parquet_reader_1.ParquetReader(file);
-    const rowBatches = reader.rowBatchIterator(options?.parquet);
-    for await (const rows of rowBatches) {
-        yield {
-            shape: 'object-row-table',
-            data: rows,
-            batchType: 'data',
-            length: rows.length
-        };
-    }
-}
-exports.parseParquetFileInBatches = parseParquetFileInBatches;

package/dist/lib/wasm/encode-parquet-wasm.js

@@ -1,30 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.tableToIPC = exports.encode = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const load_wasm_1 = require("./load-wasm");
-/**
- * Encode Arrow Table to Parquet buffer
- */
-async function encode(table, options) {
-    const wasmUrl = options?.parquet?.wasmUrl;
-    const wasm = await (0, load_wasm_1.loadWasm)(wasmUrl);
-    const arrowIPCBytes = tableToIPC(table);
-    // TODO: provide options for how to write table.
-    const writerProperties = new wasm.WriterPropertiesBuilder().build();
-    const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
-    return parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset);
-}
-exports.encode = encode;
-/**
- * Serialize a {@link Table} to the IPC format. This function is a convenience
- * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.
- * Opposite of {@link tableFromIPC}.
- *
- * @param table The Table to serialize.
- * @param type Whether to serialize the Table as a file or a stream.
- */
-function tableToIPC(table) {
-    return apache_arrow_1.RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
-}
-exports.tableToIPC = tableToIPC;

package/dist/lib/wasm/load-wasm/index.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadWasm = void 0;
-var load_wasm_node_1 = require("./load-wasm-node");
-Object.defineProperty(exports, "loadWasm", { enumerable: true, get: function () { return load_wasm_node_1.loadWasm; } });

package/dist/lib/wasm/load-wasm/load-wasm-browser.js

@@ -1,38 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadWasm = void 0;
-const wasmEsm = __importStar(require("parquet-wasm/esm2/arrow1"));
-let cached = null;
-async function loadWasm(wasmUrl) {
-    if (cached !== null) {
-        return cached;
-    }
-    // For ESM bundles, need to await the default export, which loads the WASM
-    await wasmEsm.default(wasmUrl);
-    cached = wasmEsm;
-    return wasmEsm;
-}
-exports.loadWasm = loadWasm;