@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/parquetjs/schema/schema.js

@@ -1,176 +1,162 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { PARQUET_CODECS } from '../codecs';
-import { PARQUET_COMPRESSION_METHODS } from '../compression';
-import { materializeRecords, shredBuffer, shredRecord } from './shred';
-import { PARQUET_LOGICAL_TYPES } from './types';
-export class ParquetSchema {
-  constructor(schema) {
-    _defineProperty(this, "schema", void 0);
-
-    _defineProperty(this, "fields", void 0);
-
-    _defineProperty(this, "fieldList", void 0);
-
-    this.schema = schema;
-    this.fields = buildFields(schema, 0, 0, []);
-    this.fieldList = listFields(this.fields);
-  }
-
-  findField(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
-    } else {
-      path = path.slice(0);
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetSchema = void 0;
+const codecs_1 = require("../codecs");
+const compression_1 = require("../compression");
+const shred_1 = require("./shred");
+const types_1 = require("./types");
+/**
+ * A parquet file schema
+ */
+class ParquetSchema {
+    /**
+     * Create a new schema from a JSON schema definition
+     */
+    constructor(schema) {
+        this.schema = schema;
+        this.fields = buildFields(schema, 0, 0, []);
+        this.fieldList = listFields(this.fields);
     }
-
-    let n = this.fields;
-
-    for (; path.length > 1; path.shift()) {
-      n = n[path[0]].fields;
+    /**
+     * Retrieve a field definition
+     */
+    findField(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        else {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.slice(0); // clone array
+        }
+        let n = this.fields;
+        for (; path.length > 1; path.shift()) {
+            n = n[path[0]].fields;
+        }
+        return n[path[0]];
     }
-
-    return n[path[0]];
-  }
-
-  findFieldBranch(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
+    /**
+     * Retrieve a field definition and all the field's ancestors
+     */
+    findFieldBranch(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        const branch = [];
+        let n = this.fields;
+        for (; path.length > 0; path.shift()) {
+            branch.push(n[path[0]]);
+            if (path.length > 1) {
+                n = n[path[0]].fields;
+            }
+        }
+        return branch;
     }
-
-    const branch = [];
-    let n = this.fields;
-
-    for (; path.length > 0; path.shift()) {
-      branch.push(n[path[0]]);
-
-      if (path.length > 1) {
-        n = n[path[0]].fields;
-      }
+    shredRecord(row, rowGroup) {
+        (0, shred_1.shredRecord)(this, row, rowGroup);
+    }
+    materializeRows(rowGroup) {
+        return (0, shred_1.materializeRows)(this, rowGroup);
+    }
+    compress(type) {
+        setCompress(this.schema, type);
+        setCompress(this.fields, type);
+        return this;
+    }
+    rowGroup() {
+        return (0, shred_1.shredBuffer)(this);
     }
-
-    return branch;
-  }
-
-  shredRecord(record, buffer) {
-    shredRecord(this, record, buffer);
-  }
-
-  materializeRecords(buffer) {
-    return materializeRecords(this, buffer);
-  }
-
-  compress(type) {
-    setCompress(this.schema, type);
-    setCompress(this.fields, type);
-    return this;
-  }
-
-  buffer() {
-    return shredBuffer(this);
-  }
-
 }
-
+exports.ParquetSchema = ParquetSchema;
 function setCompress(schema, type) {
-  for (const name in schema) {
-    const node = schema[name];
-
-    if (node.fields) {
-      setCompress(node.fields, type);
-    } else {
-      node.compression = type;
+    for (const name in schema) {
+        const node = schema[name];
+        if (node.fields) {
+            setCompress(node.fields, type);
+        }
+        else {
+            node.compression = type;
+        }
     }
-  }
 }
-
+// eslint-disable-next-line max-statements, complexity
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
-  const fieldList = {};
-
-  for (const name in schema) {
-    const opts = schema[name];
-    const required = !opts.optional;
-    const repeated = Boolean(opts.repeated);
-    let rLevelMax = rLevelParentMax;
-    let dLevelMax = dLevelParentMax;
-    let repetitionType = 'REQUIRED';
-
-    if (!required) {
-      repetitionType = 'OPTIONAL';
-      dLevelMax++;
-    }
-
-    if (repeated) {
-      repetitionType = 'REPEATED';
-      rLevelMax++;
-      if (required) dLevelMax++;
-    }
-
-    if (opts.fields) {
-      const cpath = path.concat([name]);
-      fieldList[name] = {
-        name,
-        path: cpath,
-        key: cpath.join(),
-        repetitionType,
-        rLevelMax,
-        dLevelMax,
-        isNested: true,
-        fieldCount: Object.keys(opts.fields).length,
-        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
-      };
-      continue;
-    }
-
-    const typeDef = PARQUET_LOGICAL_TYPES[opts.type];
-
-    if (!typeDef) {
-      throw new Error("invalid parquet type: ".concat(opts.type));
-    }
-
-    opts.encoding = opts.encoding || 'PLAIN';
-
-    if (!(opts.encoding in PARQUET_CODECS)) {
-      throw new Error("unsupported parquet encoding: ".concat(opts.encoding));
-    }
-
-    opts.compression = opts.compression || 'UNCOMPRESSED';
-
-    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
-      throw new Error("unsupported compression method: ".concat(opts.compression));
+    const fieldList = {};
+    for (const name in schema) {
+        const opts = schema[name];
+        /* field repetition type */
+        const required = !opts.optional;
+        const repeated = Boolean(opts.repeated);
+        let rLevelMax = rLevelParentMax;
+        let dLevelMax = dLevelParentMax;
+        let repetitionType = 'REQUIRED';
+        if (!required) {
+            repetitionType = 'OPTIONAL';
+            dLevelMax++;
+        }
+        if (repeated) {
+            repetitionType = 'REPEATED';
+            rLevelMax++;
+            if (required)
+                dLevelMax++;
+        }
+        /* nested field */
+        if (opts.fields) {
+            const cpath = path.concat([name]);
+            fieldList[name] = {
+                name,
+                path: cpath,
+                key: cpath.join(),
+                repetitionType,
+                rLevelMax,
+                dLevelMax,
+                isNested: true,
+                fieldCount: Object.keys(opts.fields).length,
+                fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+            };
+            continue; // eslint-disable-line no-continue
+        }
+        const typeDef = types_1.PARQUET_LOGICAL_TYPES[opts.type];
+        if (!typeDef) {
+            throw new Error(`invalid parquet type: ${opts.type}`);
+        }
+        opts.encoding = opts.encoding || 'PLAIN';
+        if (!(opts.encoding in codecs_1.PARQUET_CODECS)) {
+            throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
+        }
+        opts.compression = opts.compression || 'UNCOMPRESSED';
+        if (!(opts.compression in compression_1.PARQUET_COMPRESSION_METHODS)) {
+            throw new Error(`unsupported compression method: ${opts.compression}`);
+        }
+        /* add to schema */
+        const cpath = path.concat([name]);
+        fieldList[name] = {
+            name,
+            primitiveType: typeDef.primitiveType,
+            originalType: typeDef.originalType,
+            path: cpath,
+            key: cpath.join(),
+            repetitionType,
+            encoding: opts.encoding,
+            compression: opts.compression,
+            typeLength: opts.typeLength || typeDef.typeLength,
+            presision: opts.presision,
+            scale: opts.scale,
+            rLevelMax,
+            dLevelMax
+        };
     }
-
-    const cpath = path.concat([name]);
-    fieldList[name] = {
-      name,
-      primitiveType: typeDef.primitiveType,
-      originalType: typeDef.originalType,
-      path: cpath,
-      key: cpath.join(),
-      repetitionType,
-      encoding: opts.encoding,
-      compression: opts.compression,
-      typeLength: opts.typeLength || typeDef.typeLength,
-      presision: opts.presision,
-      scale: opts.scale,
-      rLevelMax,
-      dLevelMax
-    };
-  }
-
-  return fieldList;
+    return fieldList;
 }
-
 function listFields(fields) {
-  let list = [];
-
-  for (const k in fields) {
-    list.push(fields[k]);
-
-    if (fields[k].isNested) {
-      list = list.concat(listFields(fields[k].fields));
+    let list = [];
+    for (const k in fields) {
+        list.push(fields[k]);
+        if (fields[k].isNested) {
+            list = list.concat(listFields(fields[k].fields));
+        }
     }
-  }
-
-  return list;
+    return list;
 }
-//# sourceMappingURL=schema.js.map

package/dist/parquetjs/schema/shred.d.ts

@@ -1,19 +1,20 @@
-import { ParquetBuffer, ParquetRecord } from './declare';
+import { ArrayType } from '@loaders.gl/schema';
+import { ParquetRowGroup, ParquetRow } from './declare';
 import { ParquetSchema } from './schema';
-export { ParquetBuffer };
-export declare function shredBuffer(schema: ParquetSchema): ParquetBuffer;
+export { ParquetRowGroup };
+export declare function shredBuffer(schema: ParquetSchema): ParquetRowGroup;
 /**
  * 'Shred' a record into a list of <value, repetition_level, definition_level>
  * tuples per column using the Google Dremel Algorithm..
  *
- * The buffer argument must point to an object into which the shredded record
- * will be returned. You may re-use the buffer for repeated calls to this function
- * to append to an existing buffer, as long as the schema is unchanged.
+ * The rowGroup argument must point to an object into which the shredded record
+ * will be returned. You may re-use the rowGroup for repeated calls to this function
+ * to append to an existing rowGroup, as long as the schema is unchanged.
  *
- * The format in which the shredded records will be stored in the buffer is as
+ * The format in which the shredded records will be stored in the rowGroup is as
  * follows:
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -24,16 +25,16 @@ export declare function shredBuffer(schema: ParquetSchema): ParquetBuffer;
  *      rowCount: X,
  *   }
  */
-export declare function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void;
+export declare function shredRecord(schema: ParquetSchema, record: ParquetRow, rowGroup: ParquetRowGroup): void;
 /**
  * 'Materialize' a list of <value, repetition_level, definition_level>
  * tuples back to nested records (objects/arrays) using the Google Dremel
  * Algorithm..
  *
- * The buffer argument must point to an object with the following structure (i.e.
+ * The rowGroup argument must point to an object with the following structure (i.e.
  * the same structure that is returned by shredRecords):
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -44,5 +45,25 @@ export declare function shredRecord(schema: ParquetSchema, record: any, buffer:
  *      rowCount: X,
  *   }
  */
-export declare function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[];
+export declare function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[];
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The rowGroup argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   rowGroup = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ */
+export declare function materializeColumns(schema: ParquetSchema, rowGroup: ParquetRowGroup): Record<string, ArrayType>;
 //# sourceMappingURL=shred.d.ts.map

package/dist/parquetjs/schema/shred.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAOhG"}
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAoC,UAAU,EAAC,MAAM,WAAW,CAAC;AACxF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,eAAe,GACxB,IAAI,CAmBN;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE,CAa9F;AAoFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAS3B"}