@loaders.gl/parquet 3.3.0-alpha.7 → 3.3.0

package/dist/parquetjs/schema/shred.js

@@ -155,75 +155,193 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
  */
 function materializeRecords(schema, buffer) {
     const records = [];
-    for (let i = 0; i < buffer.rowCount; i++)
+    for (let i = 0; i < buffer.rowCount; i++) {
         records.push({});
+    }
     for (const key in buffer.columnData) {
-        materializeColumn(schema, buffer, key, records);
+        const columnData = buffer.columnData[key];
+        if (columnData.count) {
+            materializeColumn(schema, columnData, key, records);
+        }
     }
     return records;
 }
 exports.materializeRecords = materializeRecords;
 // eslint-disable-next-line max-statements, complexity
-function materializeColumn(schema, buffer, key, records) {
-    const data = buffer.columnData[key];
-    if (!data.count)
-        return;
+function materializeColumn(schema, columnData, key, records) {
     const field = schema.findField(key);
     const branch = schema.findFieldBranch(key);
     // tslint:disable-next-line:prefer-array-literal
     const rLevels = new Array(field.rLevelMax + 1).fill(0);
     let vIndex = 0;
-    for (let i = 0; i < data.count; i++) {
-        const dLevel = data.dlevels[i];
-        const rLevel = data.rlevels[i];
+    for (let i = 0; i < columnData.count; i++) {
+        const dLevel = columnData.dlevels[i];
+        const rLevel = columnData.rlevels[i];
         rLevels[rLevel]++;
         rLevels.fill(0, rLevel + 1);
         let rIndex = 0;
         let record = records[rLevels[rIndex++] - 1];
-        // Internal nodes
+        // Internal nodes - Build a nested row object
         for (const step of branch) {
-            if (step === field)
+            if (step === field || dLevel < step.dLevelMax) {
                 break;
-            if (dLevel < step.dLevelMax)
-                break;
-            if (step.repetitionType === 'REPEATED') {
-                if (!(step.name in record)) {
-                    // eslint-disable max-depth
-                    record[step.name] = [];
-                }
-                const ix = rLevels[rIndex++];
-                while (record[step.name].length <= ix) {
-                    // eslint-disable max-depth
-                    record[step.name].push({});
-                }
-                record = record[step.name][ix];
             }
-            else {
-                record[step.name] = record[step.name] || {};
-                record = record[step.name];
+            switch (step.repetitionType) {
+                case 'REPEATED':
+                    if (!(step.name in record)) {
+                        // eslint-disable max-depth
+                        record[step.name] = [];
+                    }
+                    const ix = rLevels[rIndex++];
+                    while (record[step.name].length <= ix) {
+                        // eslint-disable max-depth
+                        record[step.name].push({});
+                    }
+                    record = record[step.name][ix];
+                    break;
+                default:
+                    record[step.name] = record[step.name] || {};
+                    record = record[step.name];
             }
         }
-        // Leaf node
+        // Leaf node - Add the value
         if (dLevel === field.dLevelMax) {
             const value = Types.fromPrimitive(
             // @ts-ignore
-            field.originalType || field.primitiveType, data.values[vIndex], field);
+            field.originalType || field.primitiveType, columnData.values[vIndex], field);
             vIndex++;
-            if (field.repetitionType === 'REPEATED') {
-                if (!(field.name in record)) {
-                    // eslint-disable max-depth
-                    record[field.name] = [];
-                }
-                const ix = rLevels[rIndex];
-                while (record[field.name].length <= ix) {
-                    // eslint-disable max-depth
-                    record[field.name].push(null);
-                }
-                record[field.name][ix] = value;
-            }
-            else {
-                record[field.name] = value;
+            switch (field.repetitionType) {
+                case 'REPEATED':
+                    if (!(field.name in record)) {
+                        // eslint-disable max-depth
+                        record[field.name] = [];
+                    }
+                    const ix = rLevels[rIndex];
+                    while (record[field.name].length <= ix) {
+                        // eslint-disable max-depth
+                        record[field.name].push(null);
+                    }
+                    record[field.name][ix] = value;
+                    break;
+                default:
+                    record[field.name] = value;
             }
         }
     }
 }
+// Columnar export
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The buffer argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ *
+export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
+  const columns: ParquetRecord = {};
+  for (const key in buffer.columnData) {
+    const columnData = buffer.columnData[key];
+    if (columnData.count) {
+      extractColumn(schema, columnData, key, columns);
+    }
+  }
+  return columns;
+}
+
+// eslint-disable-next-line max-statements, complexity
+function extractColumn(
+  schema: ParquetSchema,
+  columnData: ParquetData,
+  key: string,
+  columns: Record<string, unknown>
+) {
+  if (columnData.count <= 0) {
+    return;
+  }
+
+  const record = columns;
+
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+
+  // tslint:disable-next-line:prefer-array-literal
+  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+
+  let i = 0;
+  const dLevel = columnData.dlevels[i];
+  const rLevel = columnData.rlevels[i];
+  rLevels[rLevel]++;
+  rLevels.fill(0, rLevel + 1);
+
+  let rIndex = 0;
+  let record = records[rLevels[rIndex++] - 1];
+
+  // Internal nodes
+  for (const step of branch) {
+    if (step === field || dLevel < step.dLevelMax) {
+      break;
+    }
+
+    switch (step.repetitionType) {
+      case 'REPEATED':
+        if (!(step.name in record)) {
+          // eslint-disable max-depth
+          record[step.name] = [];
+        }
+        const ix = rLevels[rIndex++];
+        while (record[step.name].length <= ix) {
+          // eslint-disable max-depth
+          record[step.name].push({});
+        }
+        record = record[step.name][ix];
+        break;
+
+      default:
+        record[step.name] = record[step.name] || {};
+        record = record[step.name];
+    }
+  }
+
+  // Leaf node
+  if (dLevel === field.dLevelMax) {
+    const value = Types.fromPrimitive(
+      // @ts-ignore
+      field.originalType || field.primitiveType,
+      columnData.values[vIndex],
+      field
+    );
+    vIndex++;
+
+    switch (field.repetitionType) {
+      case 'REPEATED':
+        if (!(field.name in record)) {
+          // eslint-disable max-depth
+          record[field.name] = [];
+        }
+        const ix = rLevels[rIndex];
+        while (record[field.name].length <= ix) {
+          // eslint-disable max-depth
+          record[field.name].push(null);
+        }
+        record[field.name][ix] = value;
+        break;
+
+      default:
+        record[field.name] = value;
+    }
+  }
+}
+*/

package/dist/parquetjs/schema/types.d.ts

@@ -11,10 +11,10 @@ export declare const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit>;
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export declare function toPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+export declare function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown;
 /**
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export declare function fromPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+export declare function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField): any;
 //# sourceMappingURL=types.d.ts.map

package/dist/parquetjs/schema/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAM9E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAUhF"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,GAAG,OAAO,CAM5F;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,OAUpF"}

package/dist/parquetjs/utils/file-utils.d.ts

@@ -1,8 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import fs from 'fs';
-import { Writable } from 'stream';
+import { fs, stream } from '@loaders.gl/loader-utils';
 export declare function load(name: string): any;
 export interface WriteStreamOptions {
     flags?: string;
@@ -12,7 +11,7 @@ export interface WriteStreamOptions {
     autoClose?: boolean;
     start?: number;
 }
-export declare function oswrite(os: Writable, buf: Buffer): Promise<void>;
-export declare function osclose(os: Writable): Promise<void>;
+export declare function oswrite(os: stream.Writable, buf: Buffer): Promise<void>;
+export declare function osclose(os: stream.Writable): Promise<void>;
 export declare function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream>;
 //# sourceMappingURL=file-utils.d.ts.map

package/dist/parquetjs/utils/file-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAEhC,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAC,EAAE,EAAE,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAEpD,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAU1D;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}

package/dist/parquetjs/utils/file-utils.js

@@ -1,11 +1,8 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.osopen = exports.osclose = exports.oswrite = exports.load = void 0;
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const fs_1 = __importDefault(require("fs"));
+const loader_utils_1 = require("@loaders.gl/loader-utils");
 function load(name) {
     return (module || global).require(name);
 }
@@ -38,7 +35,7 @@ function osclose(os) {
 exports.osclose = osclose;
 function osopen(path, opts) {
     return new Promise((resolve, reject) => {
-        const outputStream = fs_1.default.createWriteStream(path, opts);
+        const outputStream = loader_utils_1.fs.createWriteStream(path, opts);
         outputStream.once('open', (fd) => resolve(outputStream));
         outputStream.once('error', (err) => reject(err));
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.3.0-alpha.7",
+  "version": "3.3.0",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -37,12 +37,14 @@
     "net": false,
     "tls": false,
     "lzo": false,
+    "stream": false,
+    "fs": false,
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.3.0-alpha.7",
-    "@loaders.gl/loader-utils": "3.3.0-alpha.7",
-    "@loaders.gl/schema": "3.3.0-alpha.7",
+    "@loaders.gl/compression": "3.3.0",
+    "@loaders.gl/loader-utils": "3.3.0",
+    "@loaders.gl/schema": "3.3.0",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -68,5 +70,5 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "29b08f3519c50984e84bf4234e607cab7c7d1c3e"
+  "gitHead": "566d194f175a03e042d3e588cdd514048f428cc1"
 }

package/src/index.ts

@@ -32,8 +32,8 @@ export {preloadCompressions} from './parquetjs/compression';
 
 export {ParquetSchema} from './parquetjs/schema/schema';
 export {ParquetReader} from './parquetjs/parser/parquet-reader';
-export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
-// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
+export {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';
+
 export {convertParquetToArrowSchema} from './lib/convert-schema';
 
 // TESTS