@loaders.gl/parquet 4.2.0-alpha.3 → 4.2.0-alpha.5

package/dist/parquetjs/schema/schema.js

@@ -1,139 +1,158 @@
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 import { PARQUET_CODECS } from "../codecs/index.js";
 import { PARQUET_COMPRESSION_METHODS } from "../compression.js";
 import { materializeRows, shredBuffer, shredRecord } from "./shred.js";
 import { PARQUET_LOGICAL_TYPES } from "./types.js";
+/**
+ * A parquet file schema
+ */
 export class ParquetSchema {
-  constructor(schema) {
-    this.schema = void 0;
-    this.fields = void 0;
-    this.fieldList = void 0;
-    this.schema = schema;
-    this.fields = buildFields(schema, 0, 0, []);
-    this.fieldList = listFields(this.fields);
-  }
-  findField(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
-    } else {
-      path = path.slice(0);
+    /**
+     * Create a new schema from a JSON schema definition
+     */
+    constructor(schema) {
+        this.schema = schema;
+        this.fields = buildFields(schema, 0, 0, []);
+        this.fieldList = listFields(this.fields);
     }
-    let n = this.fields;
-    for (; path.length > 1; path.shift()) {
-      n = n[path[0]].fields;
+    /**
+     * Retrieve a field definition
+     */
+    findField(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        else {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.slice(0); // clone array
+        }
+        let n = this.fields;
+        for (; path.length > 1; path.shift()) {
+            n = n[path[0]].fields;
+        }
+        return n[path[0]];
     }
-    return n[path[0]];
-  }
-  findFieldBranch(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
+    /**
+     * Retrieve a field definition and all the field's ancestors
+     */
+    findFieldBranch(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        const branch = [];
+        let n = this.fields;
+        for (; path.length > 0; path.shift()) {
+            branch.push(n[path[0]]);
+            if (path.length > 1) {
+                n = n[path[0]].fields;
+            }
+        }
+        return branch;
     }
-    const branch = [];
-    let n = this.fields;
-    for (; path.length > 0; path.shift()) {
-      branch.push(n[path[0]]);
-      if (path.length > 1) {
-        n = n[path[0]].fields;
-      }
+    shredRecord(row, rowGroup) {
+        shredRecord(this, row, rowGroup);
+    }
+    materializeRows(rowGroup) {
+        return materializeRows(this, rowGroup);
+    }
+    compress(type) {
+        setCompress(this.schema, type);
+        setCompress(this.fields, type);
+        return this;
+    }
+    rowGroup() {
+        return shredBuffer(this);
     }
-    return branch;
-  }
-  shredRecord(row, rowGroup) {
-    shredRecord(this, row, rowGroup);
-  }
-  materializeRows(rowGroup) {
-    return materializeRows(this, rowGroup);
-  }
-  compress(type) {
-    setCompress(this.schema, type);
-    setCompress(this.fields, type);
-    return this;
-  }
-  rowGroup() {
-    return shredBuffer(this);
-  }
 }
 function setCompress(schema, type) {
-  for (const name in schema) {
-    const node = schema[name];
-    if (node.fields) {
-      setCompress(node.fields, type);
-    } else {
-      node.compression = type;
+    for (const name in schema) {
+        const node = schema[name];
+        if (node.fields) {
+            setCompress(node.fields, type);
+        }
+        else {
+            node.compression = type;
+        }
     }
-  }
 }
+// eslint-disable-next-line max-statements, complexity
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
-  const fieldList = {};
-  for (const name in schema) {
-    const opts = schema[name];
-    const required = !opts.optional;
-    const repeated = Boolean(opts.repeated);
-    let rLevelMax = rLevelParentMax;
-    let dLevelMax = dLevelParentMax;
-    let repetitionType = 'REQUIRED';
-    if (!required) {
-      repetitionType = 'OPTIONAL';
-      dLevelMax++;
-    }
-    if (repeated) {
-      repetitionType = 'REPEATED';
-      rLevelMax++;
-      if (required) dLevelMax++;
-    }
-    if (opts.fields) {
-      const cpath = path.concat([name]);
-      fieldList[name] = {
-        name,
-        path: cpath,
-        key: cpath.join(),
-        repetitionType,
-        rLevelMax,
-        dLevelMax,
-        isNested: true,
-        fieldCount: Object.keys(opts.fields).length,
-        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
-      };
-      continue;
-    }
-    const typeDef = PARQUET_LOGICAL_TYPES[opts.type];
-    if (!typeDef) {
-      throw new Error(`invalid parquet type: ${opts.type}`);
-    }
-    opts.encoding = opts.encoding || 'PLAIN';
-    if (!(opts.encoding in PARQUET_CODECS)) {
-      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
-    }
-    opts.compression = opts.compression || 'UNCOMPRESSED';
-    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
-      throw new Error(`unsupported compression method: ${opts.compression}`);
+    const fieldList = {};
+    for (const name in schema) {
+        const opts = schema[name];
+        /* field repetition type */
+        const required = !opts.optional;
+        const repeated = Boolean(opts.repeated);
+        let rLevelMax = rLevelParentMax;
+        let dLevelMax = dLevelParentMax;
+        let repetitionType = 'REQUIRED';
+        if (!required) {
+            repetitionType = 'OPTIONAL';
+            dLevelMax++;
+        }
+        if (repeated) {
+            repetitionType = 'REPEATED';
+            rLevelMax++;
+            if (required)
+                dLevelMax++;
+        }
+        /* nested field */
+        if (opts.fields) {
+            const cpath = path.concat([name]);
+            fieldList[name] = {
+                name,
+                path: cpath,
+                key: cpath.join(),
+                repetitionType,
+                rLevelMax,
+                dLevelMax,
+                isNested: true,
+                fieldCount: Object.keys(opts.fields).length,
+                fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+            };
+            continue; // eslint-disable-line no-continue
+        }
+        const typeDef = PARQUET_LOGICAL_TYPES[opts.type];
+        if (!typeDef) {
+            throw new Error(`invalid parquet type: ${opts.type}`);
+        }
+        opts.encoding = opts.encoding || 'PLAIN';
+        if (!(opts.encoding in PARQUET_CODECS)) {
+            throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
+        }
+        opts.compression = opts.compression || 'UNCOMPRESSED';
+        if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
+            throw new Error(`unsupported compression method: ${opts.compression}`);
+        }
+        /* add to schema */
+        const cpath = path.concat([name]);
+        fieldList[name] = {
+            name,
+            primitiveType: typeDef.primitiveType,
+            originalType: typeDef.originalType,
+            path: cpath,
+            key: cpath.join(),
+            repetitionType,
+            encoding: opts.encoding,
+            compression: opts.compression,
+            typeLength: opts.typeLength || typeDef.typeLength,
+            presision: opts.presision,
+            scale: opts.scale,
+            rLevelMax,
+            dLevelMax
+        };
     }
-    const cpath = path.concat([name]);
-    fieldList[name] = {
-      name,
-      primitiveType: typeDef.primitiveType,
-      originalType: typeDef.originalType,
-      path: cpath,
-      key: cpath.join(),
-      repetitionType,
-      encoding: opts.encoding,
-      compression: opts.compression,
-      typeLength: opts.typeLength || typeDef.typeLength,
-      presision: opts.presision,
-      scale: opts.scale,
-      rLevelMax,
-      dLevelMax
-    };
-  }
-  return fieldList;
+    return fieldList;
 }
 function listFields(fields) {
-  let list = [];
-  for (const k in fields) {
-    list.push(fields[k]);
-    if (fields[k].isNested) {
-      list = list.concat(listFields(fields[k].fields));
+    let list = [];
+    for (const k in fields) {
+        list.push(fields[k]);
+        if (fields[k].isNested) {
+            list = list.concat(listFields(fields[k].fields));
+        }
     }
-  }
-  return list;
+    return list;
 }
-//# sourceMappingURL=schema.js.map

package/dist/parquetjs/schema/shred.d.ts

@@ -1,6 +1,6 @@
 import { ArrayType } from '@loaders.gl/schema';
-import { ParquetRowGroup, ParquetRow } from './declare';
-import { ParquetSchema } from './schema';
+import { ParquetRowGroup, ParquetRow } from "./declare.js";
+import { ParquetSchema } from "./schema.js";
 export { ParquetRowGroup };
 export declare function shredBuffer(schema: ParquetSchema): ParquetRowGroup;
 /**

package/dist/parquetjs/schema/shred.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAoC,UAAU,EAAC,MAAM,WAAW,CAAC;AACxF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,eAAe,GACxB,IAAI,CAmBN;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE,CAa9F;AAoFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAS3B"}
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAoC,UAAU,EAAC,qBAAkB;AACxF,OAAO,EAAC,aAAa,EAAC,oBAAiB;AAGvC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,eAAe,GACxB,IAAI,CAmBN;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE,CAa9F;AAoFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAS3B"}