@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.2

package/dist/parquetjs/schema/shred.js

@@ -155,75 +155,193 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
  */
 function materializeRecords(schema, buffer) {
     const records = [];
-    for (let i = 0; i < buffer.rowCount; i++)
+    for (let i = 0; i < buffer.rowCount; i++) {
         records.push({});
+    }
     for (const key in buffer.columnData) {
-        materializeColumn(schema, buffer, key, records);
+        const columnData = buffer.columnData[key];
+        if (columnData.count) {
+            materializeColumn(schema, columnData, key, records);
+        }
     }
     return records;
 }
 exports.materializeRecords = materializeRecords;
 // eslint-disable-next-line max-statements, complexity
-function materializeColumn(schema, buffer, key, records) {
-    const data = buffer.columnData[key];
-    if (!data.count)
-        return;
+function materializeColumn(schema, columnData, key, records) {
     const field = schema.findField(key);
     const branch = schema.findFieldBranch(key);
     // tslint:disable-next-line:prefer-array-literal
     const rLevels = new Array(field.rLevelMax + 1).fill(0);
     let vIndex = 0;
-    for (let i = 0; i < data.count; i++) {
-        const dLevel = data.dlevels[i];
-        const rLevel = data.rlevels[i];
+    for (let i = 0; i < columnData.count; i++) {
+        const dLevel = columnData.dlevels[i];
+        const rLevel = columnData.rlevels[i];
         rLevels[rLevel]++;
         rLevels.fill(0, rLevel + 1);
         let rIndex = 0;
         let record = records[rLevels[rIndex++] - 1];
-        // Internal nodes
+        // Internal nodes - Build a nested row object
         for (const step of branch) {
-            if (step === field)
+            if (step === field || dLevel < step.dLevelMax) {
                 break;
-            if (dLevel < step.dLevelMax)
-                break;
-            if (step.repetitionType === 'REPEATED') {
-                if (!(step.name in record)) {
-                    // eslint-disable max-depth
-                    record[step.name] = [];
-                }
-                const ix = rLevels[rIndex++];
-                while (record[step.name].length <= ix) {
-                    // eslint-disable max-depth
-                    record[step.name].push({});
-                }
-                record = record[step.name][ix];
             }
-            else {
-                record[step.name] = record[step.name] || {};
-                record = record[step.name];
+            switch (step.repetitionType) {
+                case 'REPEATED':
+                    if (!(step.name in record)) {
+                        // eslint-disable max-depth
+                        record[step.name] = [];
+                    }
+                    const ix = rLevels[rIndex++];
+                    while (record[step.name].length <= ix) {
+                        // eslint-disable max-depth
+                        record[step.name].push({});
+                    }
+                    record = record[step.name][ix];
+                    break;
+                default:
+                    record[step.name] = record[step.name] || {};
+                    record = record[step.name];
             }
         }
-        // Leaf node
+        // Leaf node - Add the value
         if (dLevel === field.dLevelMax) {
             const value = Types.fromPrimitive(
             // @ts-ignore
-            field.originalType || field.primitiveType, data.values[vIndex], field);
+            field.originalType || field.primitiveType, columnData.values[vIndex], field);
             vIndex++;
-            if (field.repetitionType === 'REPEATED') {
-                if (!(field.name in record)) {
-                    // eslint-disable max-depth
-                    record[field.name] = [];
-                }
-                const ix = rLevels[rIndex];
-                while (record[field.name].length <= ix) {
-                    // eslint-disable max-depth
-                    record[field.name].push(null);
-                }
-                record[field.name][ix] = value;
-            }
-            else {
-                record[field.name] = value;
+            switch (field.repetitionType) {
+                case 'REPEATED':
+                    if (!(field.name in record)) {
+                        // eslint-disable max-depth
+                        record[field.name] = [];
+                    }
+                    const ix = rLevels[rIndex];
+                    while (record[field.name].length <= ix) {
+                        // eslint-disable max-depth
+                        record[field.name].push(null);
+                    }
+                    record[field.name][ix] = value;
+                    break;
+                default:
+                    record[field.name] = value;
             }
         }
     }
 }
+// Columnar export
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The buffer argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ *
+export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
+  const columns: ParquetRecord = {};
+  for (const key in buffer.columnData) {
+    const columnData = buffer.columnData[key];
+    if (columnData.count) {
+      extractColumn(schema, columnData, key, columns);
+    }
+  }
+  return columns;
+}
+
+// eslint-disable-next-line max-statements, complexity
+function extractColumn(
+  schema: ParquetSchema,
+  columnData: ParquetData,
+  key: string,
+  columns: Record<string, unknown>
+) {
+  if (columnData.count <= 0) {
+    return;
+  }
+
+  const record = columns;
+
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+
+  // tslint:disable-next-line:prefer-array-literal
+  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+
+  let i = 0;
+  const dLevel = columnData.dlevels[i];
+  const rLevel = columnData.rlevels[i];
+  rLevels[rLevel]++;
+  rLevels.fill(0, rLevel + 1);
+
+  let rIndex = 0;
+  let record = records[rLevels[rIndex++] - 1];
+
+  // Internal nodes
+  for (const step of branch) {
+    if (step === field || dLevel < step.dLevelMax) {
+      break;
+    }
+
+    switch (step.repetitionType) {
+      case 'REPEATED':
+        if (!(step.name in record)) {
+          // eslint-disable max-depth
+          record[step.name] = [];
+        }
+        const ix = rLevels[rIndex++];
+        while (record[step.name].length <= ix) {
+          // eslint-disable max-depth
+          record[step.name].push({});
+        }
+        record = record[step.name][ix];
+        break;
+
+      default:
+        record[step.name] = record[step.name] || {};
+        record = record[step.name];
+    }
+  }
+
+  // Leaf node
+  if (dLevel === field.dLevelMax) {
+    const value = Types.fromPrimitive(
+      // @ts-ignore
+      field.originalType || field.primitiveType,
+      columnData.values[vIndex],
+      field
+    );
+    vIndex++;
+
+    switch (field.repetitionType) {
+      case 'REPEATED':
+        if (!(field.name in record)) {
+          // eslint-disable max-depth
+          record[field.name] = [];
+        }
+        const ix = rLevels[rIndex];
+        while (record[field.name].length <= ix) {
+          // eslint-disable max-depth
+          record[field.name].push(null);
+        }
+        record[field.name][ix] = value;
+        break;
+
+      default:
+        record[field.name] = value;
+    }
+  }
+}
+*/

package/dist/parquetjs/schema/types.d.ts

@@ -11,10 +11,10 @@ export declare const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit>;
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export declare function toPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+export declare function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown;
 /**
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export declare function fromPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+export declare function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField): any;
 //# sourceMappingURL=types.d.ts.map

package/dist/parquetjs/schema/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAM9E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAUhF"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,GAAG,OAAO,CAM5F;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,OAUpF"}

package/dist/parquetjs/utils/file-utils.d.ts

@@ -1,8 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import fs from 'fs';
-import { Writable } from 'stream';
+import { fs, stream } from '@loaders.gl/loader-utils';
 export declare function load(name: string): any;
 export interface WriteStreamOptions {
     flags?: string;
@@ -12,7 +11,7 @@ export interface WriteStreamOptions {
     autoClose?: boolean;
     start?: number;
 }
-export declare function oswrite(os: Writable, buf: Buffer): Promise<void>;
-export declare function osclose(os: Writable): Promise<void>;
+export declare function oswrite(os: stream.Writable, buf: Buffer): Promise<void>;
+export declare function osclose(os: stream.Writable): Promise<void>;
 export declare function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream>;
 //# sourceMappingURL=file-utils.d.ts.map

package/dist/parquetjs/utils/file-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAEhC,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAC,EAAE,EAAE,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAEpD,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAU1D;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}

package/dist/parquetjs/utils/file-utils.js

@@ -1,11 +1,8 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.osopen = exports.osclose = exports.oswrite = exports.load = void 0;
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const fs_1 = __importDefault(require("fs"));
+const loader_utils_1 = require("@loaders.gl/loader-utils");
 function load(name) {
     return (module || global).require(name);
 }
@@ -38,7 +35,7 @@ function osclose(os) {
 exports.osclose = osclose;
 function osopen(path, opts) {
     return new Promise((resolve, reject) => {
-        const outputStream = fs_1.default.createWriteStream(path, opts);
+        const outputStream = loader_utils_1.fs.createWriteStream(path, opts);
         outputStream.once('open', (fd) => resolve(outputStream));
         outputStream.once('error', (err) => reject(err));
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.4.0-alpha.1",
+  "version": "3.4.0-alpha.2",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -37,12 +37,14 @@
     "net": false,
     "tls": false,
     "lzo": false,
+    "stream": false,
+    "fs": false,
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.4.0-alpha.1",
-    "@loaders.gl/loader-utils": "3.4.0-alpha.1",
-    "@loaders.gl/schema": "3.4.0-alpha.1",
+    "@loaders.gl/compression": "3.4.0-alpha.2",
+    "@loaders.gl/loader-utils": "3.4.0-alpha.2",
+    "@loaders.gl/schema": "3.4.0-alpha.2",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -68,5 +70,5 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "4085b0323050e4361614471319a1fb4729547bbf"
+  "gitHead": "f1c00c124d8d0c41a138ff40afb0d1a00711bf2e"
 }

package/src/index.ts

@@ -4,7 +4,11 @@ import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 
 import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
 import {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';
-import {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';
+import {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';
+import {
+  parseParquetInColumns,
+  parseParquetFileInColumnarBatches
+} from './lib/parsers/parse-parquet-to-columns';
 import {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
 
 export {ParquetWorkerLoader, ParquetWasmWorkerLoader};
@@ -16,6 +20,13 @@ export const ParquetLoader = {
   parseFileInBatches: parseParquetFileInBatches
 };
 
+/** ParquetJS table loader */
+export const ParquetColumnarLoader = {
+  ...ParquetWorkerLoader,
+  parse: parseParquetInColumns,
+  parseFileInBatches: parseParquetFileInColumnarBatches
+};
+
 export const ParquetWasmLoader = {
   ...ParquetWasmWorkerLoader,
   parse: parseParquetWasm
@@ -32,9 +43,18 @@ export {preloadCompressions} from './parquetjs/compression';
 
 export {ParquetSchema} from './parquetjs/schema/schema';
 export {ParquetReader} from './parquetjs/parser/parquet-reader';
-export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
-// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
-export {convertParquetToArrowSchema} from './lib/convert-schema';
+export {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';
+
+export {
+  convertSchemaFromParquet,
+  convertSchemaFromParquet as convertParquetToArrowSchema
+} from './lib/arrow/convert-schema-from-parquet';
 
 // TESTS
 export const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;
+
+// Geo Metadata
+export {default as geoJSONSchema} from './lib/geo/geoparquet-schema';
+
+export type {GeoMetadata} from './lib/geo/decode-geo-metadata';
+export {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';

package/src/lib/arrow/convert-row-group-to-columns.ts

@@ -0,0 +1,15 @@
+// loaders.gl, MIT license
+
+import {Schema} from '@loaders.gl/schema';
+import {ParquetBuffer} from '@loaders.gl/parquet/parquetjs/schema/declare';
+
+export function convertParquetRowGroupToColumns(
+  schema: Schema,
+  rowGroup: ParquetBuffer
+): Record<string, any[]> {
+  const columns: Record<string, any[]> = {};
+  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
+    columns[columnName] = columns[columnName] || data.values;
+  }
+  return columns;
+}

package/src/lib/{convert-schema.ts → arrow/convert-schema-from-parquet.ts}

@@ -1,5 +1,8 @@
-import type {ParquetSchema} from '../parquetjs/schema/schema';
-import type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';
+// loaders.gl, MIT license
+
+import type {ParquetSchema} from '../../parquetjs/schema/schema';
+import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';
+import {FileMetaData} from '@loaders.gl/parquet/parquetjs/parquet-thrift';
 
 import {
   Schema,
@@ -45,7 +48,7 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
   INT_64: Int64,
   JSON: Binary,
   BSON: Binary,
-  // TODO check interal type
+  // TODO check interval type
   INTERVAL: Binary,
   DECIMAL_INT32: Float32,
   DECIMAL_INT64: Float64,
@@ -53,24 +56,13 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
   DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
 };
 
-export function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {
+export function convertSchemaFromParquet(
+  parquetSchema: ParquetSchema,
+  parquetMetadata?: FileMetaData
+): Schema {
   const fields = getFields(parquetSchema.schema);
-
-  // TODO add metadata if needed.
-  return new Schema(fields);
-}
-
-function getFieldMetadata(field: ParquetField): Map<string, string> {
-  const metadata = new Map();
-
-  for (const key in field) {
-    if (key !== 'name') {
-      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-      metadata.set(key, value);
-    }
-  }
-
-  return metadata;
+  const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
+  return new Schema(fields, metadata);
 }
 
 function getFields(schema: FieldDefinition): Field[] {
@@ -80,8 +72,8 @@ function getFields(schema: FieldDefinition): Field[] {
     const field = schema[name];
 
     if (field.fields) {
-      const childField = getFields(field.fields);
-      const nestedField = new Field(name, new Struct(childField), field.optional);
+      const childFields = getFields(field.fields);
+      const nestedField = new Field(name, new Struct(childFields), field.optional);
       fields.push(nestedField);
     } else {
       const FieldType = PARQUET_TYPE_MAPPING[field.type];
@@ -93,3 +85,30 @@ function getFields(schema: FieldDefinition): Field[] {
 
   return fields;
 }
+
+function getFieldMetadata(field: ParquetField): Map<string, string> {
+  const metadata = new Map();
+
+  for (const key in field) {
+    if (key !== 'name') {
+      let value = field[key] || '';
+      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+      metadata.set(key, value);
+    }
+  }
+
+  return metadata;
+}
+
+function getSchemaMetadata(parquetMetadata: FileMetaData): Map<string, string> {
+  const metadata = new Map();
+
+  const keyValueList = parquetMetadata.key_value_metadata || [];
+  for (const {key, value} of keyValueList) {
+    if (typeof value === 'string') {
+      metadata.set(key, value);
+    }
+  }
+
+  return metadata;
+}

package/src/lib/arrow/convert-schema-to-parquet.ts

@@ -0,0 +1,102 @@
+// loaders.gl, MIT license
+
+// import type {ParquetSchema} from '../../parquetjs/schema/schema';
+import type {
+  // FieldDefinition, ParquetField,
+  ParquetType
+} from '../../parquetjs/schema/declare';
+
+import {
+  Schema,
+  // Struct,
+  // Field,
+  DataType,
+  Bool,
+  Float64,
+  Int32,
+  Float32,
+  Binary,
+  Utf8,
+  Int64,
+  Uint16,
+  Uint32,
+  Uint64,
+  Int8,
+  Int16
+} from '@loaders.gl/schema';
+
+export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
+  BOOLEAN: Bool,
+  INT32: Int32,
+  INT64: Float64,
+  INT96: Float64,
+  FLOAT: Float32,
+  DOUBLE: Float64,
+  BYTE_ARRAY: Binary,
+  FIXED_LEN_BYTE_ARRAY: Binary,
+  UTF8: Utf8,
+  DATE: Int32,
+  TIME_MILLIS: Int64,
+  TIME_MICROS: Int64,
+  TIMESTAMP_MILLIS: Int64,
+  TIMESTAMP_MICROS: Int64,
+  UINT_8: Int32,
+  UINT_16: Uint16,
+  UINT_32: Uint32,
+  UINT_64: Uint64,
+  INT_8: Int8,
+  INT_16: Int16,
+  INT_32: Int32,
+  INT_64: Int64,
+  JSON: Binary,
+  BSON: Binary,
+  // TODO check interval type
+  INTERVAL: Binary,
+  DECIMAL_INT32: Float32,
+  DECIMAL_INT64: Float64,
+  DECIMAL_BYTE_ARRAY: Float64,
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
+};
+
+export function convertToParquetSchema(schema: Schema): Schema {
+  const fields = []; // getFields(schema.fields);
+
+  // TODO add metadata if needed.
+  return new Schema(fields);
+}
+
+// function getFields(schema: Field[]): Definition[] {
+//   const fields: Field[] = [];
+
+//   for (const name in schema) {
+//     const field = schema[name];
+
+//     // @ts-ignore
+//     const children = field.children as DataType[];
+//     if (children) {
+//       const childField = getFields(field.fields);
+//       const nestedField = new Field(name, new Struct(childField), field.optional);
+//       fields.push(nestedField);
+//     } else {
+//       const FieldType = PARQUET_TYPE_MAPPING[field.type];
+//       const metadata = getFieldMetadata(field);
+//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+//       fields.push(arrowField);
+//     }
+//   }
+
+//   return fields;
+// }
+
+// function getFieldMetadata(field: ParquetField): Map<string, string> {
+//   const metadata = new Map();
+
+//   for (const key in field) {
+//     if (key !== 'name') {
+//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+//       metadata.set(key, value);
+//     }
+//   }
+
+//   return metadata;
+// }