@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/parquetjs/utils/file-utils.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
-import fs from 'fs';
-import { Writable } from 'stream';
+/// <reference types="node" />
+/// <reference types="node" />
+import { fs, stream } from '@loaders.gl/loader-utils';
 export declare function load(name: string): any;
 export interface WriteStreamOptions {
     flags?: string;
@@ -10,7 +11,7 @@ export interface WriteStreamOptions {
     autoClose?: boolean;
     start?: number;
 }
-export declare function oswrite(os: Writable, buf: Buffer): Promise<void>;
-export declare function osclose(os: Writable): Promise<void>;
+export declare function oswrite(os: stream.Writable, buf: Buffer): Promise<void>;
+export declare function osclose(os: stream.Writable): Promise<void>;
 export declare function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream>;
 //# sourceMappingURL=file-utils.d.ts.map

package/dist/parquetjs/utils/file-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAEhC,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAC,EAAE,EAAE,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAEpD,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAU1D;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}

package/dist/parquetjs/utils/file-utils.js

@@ -1,34 +1,43 @@
-import fs from 'fs';
-export function load(name) {
-  return (module || global).require(name);
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.osopen = exports.osclose = exports.oswrite = exports.load = void 0;
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+function load(name) {
+    return (module || global).require(name);
 }
-export function oswrite(os, buf) {
-  return new Promise((resolve, reject) => {
-    os.write(buf, err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
+exports.load = load;
+function oswrite(os, buf) {
+    return new Promise((resolve, reject) => {
+        os.write(buf, (err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
     });
-  });
 }
-export function osclose(os) {
-  return new Promise((resolve, reject) => {
-    os.close(err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
+exports.oswrite = oswrite;
+function osclose(os) {
+    return new Promise((resolve, reject) => {
+        os.close((err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
     });
-  });
 }
-export function osopen(path, opts) {
-  return new Promise((resolve, reject) => {
-    const outputStream = fs.createWriteStream(path, opts);
-    outputStream.once('open', fd => resolve(outputStream));
-    outputStream.once('error', err => reject(err));
-  });
+exports.osclose = osclose;
+function osopen(path, opts) {
+    return new Promise((resolve, reject) => {
+        const outputStream = loader_utils_1.fs.createWriteStream(path, opts);
+        outputStream.once('open', (fd) => resolve(outputStream));
+        outputStream.once('error', (err) => reject(err));
+    });
 }
-//# sourceMappingURL=file-utils.js.map
+exports.osopen = osopen;

package/dist/parquetjs/utils/read-utils.js

@@ -1,105 +1,109 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { TBufferedTransport, TCompactProtocol, TFramedTransport } from 'thrift';
-import { FileMetaData, PageHeader } from '../parquet-thrift';
-
-class UFramedTransport extends TFramedTransport {
-  constructor(...args) {
-    super(...args);
-
-    _defineProperty(this, "readPos", 0);
-  }
-
-}
-
-export function serializeThrift(obj) {
-  const output = [];
-  const transport = new TBufferedTransport(undefined, buf => {
-    output.push(buf);
-  });
-  const protocol = new TCompactProtocol(transport);
-  obj.write(protocol);
-  transport.flush();
-  return Buffer.concat(output);
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fieldIndexOf = exports.getBitWidth = exports.decodePageHeader = exports.decodeFileMetadata = exports.getThriftEnum = exports.decodeThrift = exports.serializeThrift = void 0;
+const thrift_1 = require("thrift");
+const parquet_thrift_1 = require("../parquet-thrift");
+class UFramedTransport extends thrift_1.TFramedTransport {
+    constructor() {
+        super(...arguments);
+        this.readPos = 0;
+    }
 }
-export function decodeThrift(obj, buf, offset) {
-  if (!offset) {
-    offset = 0;
-  }
-
-  const transport = new UFramedTransport(buf);
-  transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  obj.read(protocol);
-  return transport.readPos - offset;
+/**
+ * Helper function that serializes a thrift object into a buffer
+ */
+function serializeThrift(obj) {
+    const output = [];
+    const transport = new thrift_1.TBufferedTransport(undefined, (buf) => {
+        output.push(buf);
+    });
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    obj.write(protocol);
+    transport.flush();
+    return Buffer.concat(output);
 }
-export function getThriftEnum(klass, value) {
-  for (const k in klass) {
-    if (klass[k] === value) {
-      return k;
+exports.serializeThrift = serializeThrift;
+function decodeThrift(obj, buf, offset) {
+    if (!offset) {
+        // tslint:disable-next-line:no-parameter-reassignment
+        offset = 0;
     }
-  }
-
-  throw new Error('Invalid ENUM value');
+    const transport = new UFramedTransport(buf);
+    transport.readPos = offset;
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    obj.read(protocol);
+    return transport.readPos - offset;
 }
-export function decodeFileMetadata(buf, offset) {
-  if (!offset) {
-    offset = 0;
-  }
-
-  const transport = new UFramedTransport(buf);
-  transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  const metadata = FileMetaData.read(protocol);
-  return {
-    length: transport.readPos - offset,
-    metadata
-  };
+exports.decodeThrift = decodeThrift;
+/**
+ * FIXME not ideal that this is linear
+ */
+function getThriftEnum(klass, value) {
+    for (const k in klass) {
+        if (klass[k] === value) {
+            return k;
+        }
+    }
+    throw new Error('Invalid ENUM value');
 }
-export function decodePageHeader(buf, offset) {
-  if (!offset) {
-    offset = 0;
-  }
-
-  const transport = new UFramedTransport(buf);
-  transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  const pageHeader = PageHeader.read(protocol);
-  return {
-    length: transport.readPos - offset,
-    pageHeader
-  };
+exports.getThriftEnum = getThriftEnum;
+function decodeFileMetadata(buf, offset) {
+    if (!offset) {
+        // tslint:disable-next-line:no-parameter-reassignment
+        offset = 0;
+    }
+    const transport = new UFramedTransport(buf);
+    transport.readPos = offset;
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    const metadata = parquet_thrift_1.FileMetaData.read(protocol);
+    return { length: transport.readPos - offset, metadata };
 }
-export function getBitWidth(val) {
-  if (val === 0) {
-    return 0;
-  }
-
-  return Math.ceil(Math.log2(val + 1));
+exports.decodeFileMetadata = decodeFileMetadata;
+function decodePageHeader(buf, offset) {
+    if (!offset) {
+        // tslint:disable-next-line:no-parameter-reassignment
+        offset = 0;
+    }
+    const transport = new UFramedTransport(buf);
+    transport.readPos = offset;
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    const pageHeader = parquet_thrift_1.PageHeader.read(protocol);
+    return { length: transport.readPos - offset, pageHeader };
 }
-export function fieldIndexOf(arr, elem) {
-  for (let j = 0; j < arr.length; j++) {
-    if (arr[j].length > elem.length) {
-      continue;
+exports.decodePageHeader = decodePageHeader;
+/**
+ * Get the number of bits required to store a given value
+ */
+function getBitWidth(val) {
+    if (val === 0) {
+        return 0;
+        // tslint:disable-next-line:no-else-after-return
     }
-
-    let m = true;
-
-    for (let i = 0; i < elem.length; i++) {
-      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
-        continue;
-      }
-
-      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
-        continue;
-      }
-
-      m = false;
-      break;
+    return Math.ceil(Math.log2(val + 1));
+}
+exports.getBitWidth = getBitWidth;
+// Supports MQTT path wildcards
+// + all immediate children
+// # all descendents
+function fieldIndexOf(arr, elem) {
+    for (let j = 0; j < arr.length; j++) {
+        if (arr[j].length > elem.length) {
+            continue; // eslint-disable-line no-continue
+        }
+        let m = true;
+        for (let i = 0; i < elem.length; i++) {
+            if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
+                continue; // eslint-disable-line no-continue
+            }
+            if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
+                continue; // eslint-disable-line no-continue
+            }
+            m = false;
+            break;
+        }
+        if (m)
+            return j;
     }
-
-    if (m) return j;
-  }
-
-  return -1;
+    return -1;
 }
-//# sourceMappingURL=read-utils.js.map
+exports.fieldIndexOf = fieldIndexOf;

package/dist/workers/parquet-worker.js

@@ -1,4 +1,5 @@
-import { createLoaderWorker } from '@loaders.gl/loader-utils';
-import { ParquetLoader } from '../index';
-createLoaderWorker(ParquetLoader);
-//# sourceMappingURL=parquet-worker.js.map
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const index_1 = require("../index");
+(0, loader_utils_1.createLoaderWorker)(index_1.ParquetLoader);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.0.0-alpha.5",
+  "version": "4.0.0-alpha.6",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -19,8 +19,8 @@
     "Apache Parquet"
   ],
   "types": "dist/index.d.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "files": [
     "src",
@@ -36,31 +36,36 @@
     "child_process": false,
     "net": false,
     "tls": false,
-    "lzo": false
+    "stream": false,
+    "fs": false,
+    "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "4.0.0-alpha.5",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.5",
-    "@loaders.gl/schema": "4.0.0-alpha.5",
+    "@loaders.gl/bson": "4.0.0-alpha.6",
+    "@loaders.gl/compression": "4.0.0-alpha.6",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.6",
+    "@loaders.gl/schema": "4.0.0-alpha.6",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
-    "bson": "^1.0.4",
     "int53": "^0.2.4",
     "lz4js": "^0.2.0",
-    "lzo": "^0.4.11",
     "node-int64": "^0.4.0",
     "object-stream": "0.0.1",
+    "parquet-wasm": "^0.3.1",
     "snappyjs": "^0.6.0",
     "thrift": "^0.14.2",
     "varint": "^5.0.0",
     "zstd-codec": "^0.1"
   },
+  "peerDependencies": {
+    "apache-arrow": "^9.0.0"
+  },
   "devDependencies": {
-    "@types/bson": "^4.0.0",
     "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
     "@types/thrift": "^0.10.8",
-    "@types/varint": "^5.0.0"
+    "@types/varint": "^5.0.0",
+    "apache-arrow": "^9.0.0"
   },
-  "gitHead": "7a71a54bdf1ddf985cc3af3db90b82e7fa97d025"
+  "gitHead": "acc1985050dfaa0f1f0c066f8da5bce7454a046c"
 }

package/src/index.ts

@@ -1,22 +1,64 @@
+// loaders.gl, MIT license
+
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ColumnarTable,
+  ColumnarTableBatch
+} from '@loaders.gl/schema';
+import type {Table as ArrowTable} from 'apache-arrow';
 
 // ParquetLoader
 
-import {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';
-import {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';
+import {
+  ParquetLoader as ParquetWorkerLoader,
+  ParquetLoader as ParquetColumnarWorkerLoader,
+  ParquetLoaderOptions
+} from './parquet-loader';
+import {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';
+import {
+  parseParquetInColumns,
+  parseParquetFileInColumnarBatches
+} from './lib/parsers/parse-parquet-to-columns';
+
+import {parseParquetWasm, ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';
+import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
 
-export {ParquetWorkerLoader};
+export {ParquetWorkerLoader, ParquetWasmWorkerLoader};
 
 /** ParquetJS table loader */
-export const ParquetLoader = {
+export const ParquetLoader: LoaderWithParser<
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ParquetLoaderOptions
+> = {
   ...ParquetWorkerLoader,
   parse: parseParquet,
   parseFileInBatches: parseParquetFileInBatches
 };
 
+/** ParquetJS table loader */
+// @ts-expect-error
+export const ParquetColumnarLoader: LoaderWithParser<
+  ColumnarTable,
+  ColumnarTableBatch,
+  ParquetLoaderOptions
+> = {
+  ...ParquetColumnarWorkerLoader,
+  parse: parseParquetInColumns,
+  parseFileInBatches: parseParquetFileInColumnarBatches
+};
+
+export const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {
+  ...ParquetWasmWorkerLoader,
+  parse: parseParquetWasm
+};
+
 // ParquetWriter
 
 export {ParquetWriter as _ParquetWriter} from './parquet-writer';
+export {ParquetWasmWriter} from './parquet-wasm-writer';
 
 // EXPERIMENTAL - expose the internal parquetjs API
 
@@ -24,9 +66,18 @@ export {preloadCompressions} from './parquetjs/compression';
 
 export {ParquetSchema} from './parquetjs/schema/schema';
 export {ParquetReader} from './parquetjs/parser/parquet-reader';
-export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
-// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
-export {convertParquetToArrowSchema} from './lib/convert-schema';
+export {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';
+
+export {
+  convertParquetSchema,
+  convertParquetSchema as convertParquetToArrowSchema
+} from './lib/arrow/convert-schema-from-parquet';
 
 // TESTS
 export const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;
+
+// Geo Metadata
+export {default as geoJSONSchema} from './lib/geo/geoparquet-schema';
+
+export type {GeoMetadata} from './lib/geo/decode-geo-metadata';
+export {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';

package/src/lib/arrow/convert-row-group-to-columns.ts

@@ -0,0 +1,15 @@
+// loaders.gl, MIT license
+
+import {Schema} from '@loaders.gl/schema';
+import {ParquetRowGroup} from '@loaders.gl/parquet/parquetjs/schema/declare';
+
+export function convertParquetRowGroupToColumns(
+  schema: Schema,
+  rowGroup: ParquetRowGroup
+): Record<string, any[]> {
+  const columns: Record<string, any[]> = {};
+  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
+    columns[columnName] = columns[columnName] || data.values;
+  }
+  return columns;
+}

package/src/lib/arrow/convert-schema-from-parquet.ts

@@ -0,0 +1,104 @@
+// loaders.gl, MIT license
+
+import {Schema, Field, DataType} from '@loaders.gl/schema';
+
+import type {ParquetSchema} from '../../parquetjs/schema/schema';
+import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';
+import {FileMetaData} from '../../parquetjs/parquet-thrift';
+
+export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
+  // TODO check interal type
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+};
+
+export function convertParquetSchema(
+  parquetSchema: ParquetSchema,
+  parquetMetadata: FileMetaData | null
+): Schema {
+  const fields = getFields(parquetSchema.schema);
+  const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
+
+  const schema: Schema = {
+    fields,
+    metadata: metadata || {}
+  };
+
+  return schema;
+}
+
+function getFields(schema: FieldDefinition): Field[] {
+  const fields: Field[] = [];
+
+  for (const name in schema) {
+    const field = schema[name];
+
+    if (field.fields) {
+      const children = getFields(field.fields);
+      fields.push({name, type: {type: 'struct', children}, nullable: field.optional});
+    } else {
+      const type = PARQUET_TYPE_MAPPING[field.type];
+      const metadata = getFieldMetadata(field);
+      const arrowField = {name, type, nullable: field.optional, metadata};
+      fields.push(arrowField);
+    }
+  }
+
+  return fields;
+}
+
+function getFieldMetadata(field: ParquetField): Record<string, string> | undefined {
+  let metadata: Record<string, string> | undefined;
+
+  for (const key in field) {
+    if (key !== 'name') {
+      let value = field[key] || '';
+      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+      metadata = metadata || {};
+      metadata[key] = value;
+    }
+  }
+
+  return metadata;
+}
+
+function getSchemaMetadata(parquetMetadata: FileMetaData): Record<string, string> | undefined {
+  let metadata: Record<string, string> | undefined;
+
+  const keyValueList = parquetMetadata.key_value_metadata || [];
+  for (const {key, value} of keyValueList) {
+    if (typeof value === 'string') {
+      metadata = metadata || {};
+      metadata[key] = value;
+    }
+  }
+
+  return metadata;
+}