@loaders.gl/parquet 3.1.0-beta.3 → 3.1.1

package/dist/es5/parquetjs/schema/schema.js

@@ -7,6 +7,10 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ParquetSchema = void 0;
 
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
 var _codecs = require("../codecs");
@@ -17,8 +21,9 @@ var _shred = require("./shred");
 
 var _types = require("./types");
 
-class ParquetSchema {
-  constructor(schema) {
+var ParquetSchema = function () {
+  function ParquetSchema(schema) {
+    (0, _classCallCheck2.default)(this, ParquetSchema);
     (0, _defineProperty2.default)(this, "schema", void 0);
     (0, _defineProperty2.default)(this, "fields", void 0);
     (0, _defineProperty2.default)(this, "fieldList", void 0);
@@ -27,66 +32,74 @@ class ParquetSchema {
     this.fieldList = listFields(this.fields);
   }
 
-  findField(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
-    } else {
-      path = path.slice(0);
-    }
-
-    let n = this.fields;
-
-    for (; path.length > 1; path.shift()) {
-      n = n[path[0]].fields;
-    }
-
-    return n[path[0]];
-  }
-
-  findFieldBranch(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
-    }
-
-    const branch = [];
-    let n = this.fields;
+  (0, _createClass2.default)(ParquetSchema, [{
+    key: "findField",
+    value: function findField(path) {
+      if (typeof path === 'string') {
+        path = path.split(',');
+      } else {
+        path = path.slice(0);
+      }
 
-    for (; path.length > 0; path.shift()) {
-      branch.push(n[path[0]]);
+      var n = this.fields;
 
-      if (path.length > 1) {
+      for (; path.length > 1; path.shift()) {
         n = n[path[0]].fields;
       }
-    }
-
-    return branch;
-  }
 
-  shredRecord(record, buffer) {
-    (0, _shred.shredRecord)(this, record, buffer);
-  }
+      return n[path[0]];
+    }
+  }, {
+    key: "findFieldBranch",
+    value: function findFieldBranch(path) {
+      if (typeof path === 'string') {
+        path = path.split(',');
+      }
 
-  materializeRecords(buffer) {
-    return (0, _shred.materializeRecords)(this, buffer);
-  }
+      var branch = [];
+      var n = this.fields;
 
-  compress(type) {
-    setCompress(this.schema, type);
-    setCompress(this.fields, type);
-    return this;
-  }
+      for (; path.length > 0; path.shift()) {
+        branch.push(n[path[0]]);
 
-  buffer() {
-    return (0, _shred.shredBuffer)(this);
-  }
+        if (path.length > 1) {
+          n = n[path[0]].fields;
+        }
+      }
 
-}
+      return branch;
+    }
+  }, {
+    key: "shredRecord",
+    value: function shredRecord(record, buffer) {
+      (0, _shred.shredRecord)(this, record, buffer);
+    }
+  }, {
+    key: "materializeRecords",
+    value: function materializeRecords(buffer) {
+      return (0, _shred.materializeRecords)(this, buffer);
+    }
+  }, {
+    key: "compress",
+    value: function compress(type) {
+      setCompress(this.schema, type);
+      setCompress(this.fields, type);
+      return this;
+    }
+  }, {
+    key: "buffer",
+    value: function buffer() {
+      return (0, _shred.shredBuffer)(this);
+    }
+  }]);
+  return ParquetSchema;
+}();
 
 exports.ParquetSchema = ParquetSchema;
 
 function setCompress(schema, type) {
-  for (const name in schema) {
-    const node = schema[name];
+  for (var name in schema) {
+    var node = schema[name];
 
     if (node.fields) {
       setCompress(node.fields, type);
@@ -97,15 +110,15 @@ function setCompress(schema, type) {
 }
 
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
-  const fieldList = {};
+  var fieldList = {};
 
-  for (const name in schema) {
-    const opts = schema[name];
-    const required = !opts.optional;
-    const repeated = Boolean(opts.repeated);
-    let rLevelMax = rLevelParentMax;
-    let dLevelMax = dLevelParentMax;
-    let repetitionType = 'REQUIRED';
+  for (var name in schema) {
+    var opts = schema[name];
+    var required = !opts.optional;
+    var repeated = Boolean(opts.repeated);
+    var rLevelMax = rLevelParentMax;
+    var dLevelMax = dLevelParentMax;
+    var repetitionType = 'REQUIRED';
 
     if (!required) {
       repetitionType = 'OPTIONAL';
@@ -119,22 +132,23 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
     }
 
     if (opts.fields) {
-      const cpath = path.concat([name]);
+      var _cpath = path.concat([name]);
+
       fieldList[name] = {
-        name,
-        path: cpath,
-        key: cpath.join(),
-        repetitionType,
-        rLevelMax,
-        dLevelMax,
+        name: name,
+        path: _cpath,
+        key: _cpath.join(),
+        repetitionType: repetitionType,
+        rLevelMax: rLevelMax,
+        dLevelMax: dLevelMax,
         isNested: true,
         fieldCount: Object.keys(opts.fields).length,
-        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+        fields: buildFields(opts.fields, rLevelMax, dLevelMax, _cpath)
       };
       continue;
     }
 
-    const typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
+    var typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
 
     if (!typeDef) {
       throw new Error("invalid parquet type: ".concat(opts.type));
@@ -152,21 +166,21 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       throw new Error("unsupported compression method: ".concat(opts.compression));
     }
 
-    const cpath = path.concat([name]);
+    var cpath = path.concat([name]);
     fieldList[name] = {
-      name,
+      name: name,
       primitiveType: typeDef.primitiveType,
       originalType: typeDef.originalType,
       path: cpath,
       key: cpath.join(),
-      repetitionType,
+      repetitionType: repetitionType,
       encoding: opts.encoding,
       compression: opts.compression,
       typeLength: opts.typeLength || typeDef.typeLength,
       presision: opts.presision,
       scale: opts.scale,
-      rLevelMax,
-      dLevelMax
+      rLevelMax: rLevelMax,
+      dLevelMax: dLevelMax
     };
   }
 
@@ -174,9 +188,9 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
 }
 
 function listFields(fields) {
-  let list = [];
+  var list = [];
 
-  for (const k in fields) {
+  for (var k in fields) {
     list.push(fields[k]);
 
     if (fields[k].isNested) {

package/dist/es5/parquetjs/schema/schema.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/schema/schema.ts"],"names":["ParquetSchema","constructor","schema","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","record","buffer","materializeRecords","compress","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"mappings":";;;;;;;;;;;AAEA;;AACA;;AAUA;;AACA;;AAKO,MAAMA,aAAN,CAAoB;AAQzBC,EAAAA,WAAW,CAACC,MAAD,EAA2B;AAAA;AAAA;AAAA;AACpC,SAAKA,MAAL,GAAcA,MAAd;AACA,SAAKC,MAAL,GAAcC,WAAW,CAACF,MAAD,EAAS,CAAT,EAAY,CAAZ,EAAe,EAAf,CAAzB;AACA,SAAKG,SAAL,GAAiBC,UAAU,CAAC,KAAKH,MAAN,CAA3B;AACD;;AAKDI,EAAAA,SAAS,CAACC,IAAD,EAAwC;AAC/C,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,MAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD,KAHD,MAGO;AAELD,MAAAA,IAAI,GAAGA,IAAI,CAACE,KAAL,CAAW,CAAX,CAAP;AACD;;AAED,QAAIC,CAAC,GAAG,KAAKR,MAAb;;AACA,WAAOK,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCF,MAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWL,MAAf;AACD;;AAED,WAAOQ,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAR;AACD;;AAKDM,EAAAA,eAAe,CAACN,IAAD,EAA0C;AACvD,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,MAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD;;AACD,UAAMM,MAAsB,GAAG,EAA/B;AACA,QAAIJ,CAAC,GAAG,KAAKR,MAAb;;AACA,WAAOK,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCE,MAAAA,MAAM,CAACC,IAAP,CAAYL,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAb;;AACA,UAAIA,IAAI,CAACI,MAAL,GAAc,CAAlB,EAAqB;AACnBD,QAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWL,MAAf;AACD;AACF;;AACD,WAAOY,MAAP;AACD;;AAEDE,EAAAA,WAAW,CAACC,MAAD,EAAwBC,MAAxB,EAAqD;AAC9D,4BAAY,IAAZ,EAAkBD,MAAlB,EAA0BC,MAA1B;AACD;;AAEDC,EAAAA,kBAAkB,CAACD,MAAD,EAAyC;AACzD,WAAO,+BAAmB,IAAnB,EAAyBA,MAAzB,CAAP;AACD;;AAEDE,EAAAA,QAAQ,CAACC,IAAD,EAAiC;AACvCC,IAAAA,WAAW,CAAC,KAAKrB,MAAN,EAAcoB,IAAd,CAAX;AACAC,IAAAA,WAAW,CAAC,KAAKpB,MAAN,EAAcmB,IAAd,CAAX;AACA,WAAO,IAAP;AACD;;AAEDH,EAAAA,MAAM,GAAkB;AACtB,WAAO,wBAAY,IAAZ,CAAP;AACD;;AArEwB;;;;AAwE3B,SAASI,WAAT,CAAqBrB,MAArB,EAAkCoB,IAAlC,EAA4D;AAC1D,OAAK,MAAME,IAAX,IAAmBtB,MAAnB,EAA2B;AACzB,UAAMuB,IAAI,GAAGvB,MAAM,CAACsB,IAAD,CAAnB;;AACA,QAAIC,IAAI,CAACtB,MAAT,EAAiB;AACfoB,MAAAA,WAAW,CAACE,IAAI,CAACtB,MAAN,EAAcmB,IAAd,CAAX;AACD,KAFD,MAEO;AACLG,MAAAA,IAAI,CAACC,WAAL,GAAmBJ,IAAnB;AACD;AACF;AACF;;AAGD,SAASlB,WAAT,CACEF,MADF,EAEEyB,eAFF,EAGEC,eAHF,EAIEpB,IAJF,EAKgC;AAC9B,QAAMH,SAAuC,GAAG,EAAhD;;AAEA,OAAK,MAAMmB,IAAX,IAAmBtB,MAAnB,EAA2B;AACzB,UAAM2B,IAAI,GAAG3B,MAAM,CAACsB,IAAD,CAAnB;AAGA,UAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAvB;AACA,UAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAN,CAAxB;AACA,QAAIE,SAAS,GAAGP,eAAhB;AACA,QAAIQ,SAAS,GAAGP,eAAhB;AAEA,QAAIQ,cAA8B,GAAG,UAArC;;AACA,QAAI,CAACN,QAAL,EAAe;AACbM,MAAAA,cAAc,GAAG,UAAjB;AACAD,MAAAA,SAAS;AACV;;AACD,QAAIH,QAAJ,EAAc;AACZI,MAAAA,cAAc,GAAG,UAAjB;AACAF,MAAAA,SAAS;AACT,UAAIJ,QAAJ,EAAcK,SAAS;AACxB;;AAGD,QAAIN,IAAI,CAAC1B,MAAT,EAAiB;AACf,YAAMkC,KAAK,GAAG7B,IAAI,CAAC8B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAnB,MAAAA,SAAS,CAACmB,IAAD,CAAT,GAAkB;AAChBA,QAAAA,IADgB;AAEhBhB,QAAAA,IAAI,EAAE6B,KAFU;AAGhBE,QAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EAHW;AAIhBJ,QAAAA,cAJgB;AAKhBF,QAAAA,SALgB;AAMhBC,QAAAA,SANgB;AAOhBM,QAAAA,QAAQ,EAAE,IAPM;AAQhBC,QAAAA,UAAU,EAAEC,MAAM,CAACC,IAAP,CAAYf,IAAI,CAAC1B,MAAjB,EAAyBS,MARrB;AAShBT,QAAAA,MAAM,EAAEC,WAAW,CAACyB,IAAI,CAAC1B,MAAN,EAAc+B,SAAd,EAAyBC,SAAzB,EAAoCE,KAApC;AATH,OAAlB;AAWA;AACD;;AAED,UAAMQ,OAAY,GAAGC,6BAAsBjB,IAAI,CAACP,IAA3B,CAArB;;AACA,QAAI,CAACuB,OAAL,EAAc;AACZ,YAAM,IAAIE,KAAJ,iCAAmClB,IAAI,CAACP,IAAxC,EAAN;AACD;;AAEDO,IAAAA,IAAI,CAACmB,QAAL,GAAgBnB,IAAI,CAACmB,QAAL,IAAiB,OAAjC;;AACA,QAAI,EAAEnB,IAAI,CAACmB,QAAL,IAAiBC,sBAAnB,CAAJ,EAAwC;AACtC,YAAM,IAAIF,KAAJ,yCAA2ClB,IAAI,CAACmB,QAAhD,EAAN;AACD;;AAEDnB,IAAAA,IAAI,CAACH,WAAL,GAAmBG,IAAI,CAACH,WAAL,IAAoB,cAAvC;;AACA,QAAI,EAAEG,IAAI,CAACH,WAAL,IAAoBwB,wCAAtB,CAAJ,EAAwD;AACtD,YAAM,IAAIH,KAAJ,2CAA6ClB,IAAI,CAACH,WAAlD,EAAN;AACD;;AAGD,UAAMW,KAAK,GAAG7B,IAAI,CAAC8B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAnB,IAAAA,SAAS,CAACmB,IAAD,CAAT,GAAkB;AAChBA,MAAAA,IADgB;AAEhB2B,MAAAA,aAAa,EAAEN,OAAO,CAACM,aAFP;AAGhBC,MAAAA,YAAY,EAAEP,OAAO,CAACO,YAHN;AAIhB5C,MAAAA,IAAI,EAAE6B,KAJU;AAKhBE,MAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EALW;AAMhBJ,MAAAA,cANgB;AAOhBY,MAAAA,QAAQ,EAAEnB,IAAI,CAACmB,QAPC;AAQhBtB,MAAAA,WAAW,EAAEG,IAAI,CAACH,WARF;AAShB2B,MAAAA,UAAU,EAAExB,IAAI,CAACwB,UAAL,IAAmBR,OAAO,CAACQ,UATvB;AAUhBC,MAAAA,SAAS,EAAEzB,IAAI,CAACyB,SAVA;AAWhBC,MAAAA,KAAK,EAAE1B,IAAI,CAAC0B,KAXI;AAYhBrB,MAAAA,SAZgB;AAahBC,MAAAA;AAbgB,KAAlB;AAeD;;AACD,SAAO9B,SAAP;AACD;;AAED,SAASC,UAAT,CAAoBH,MAApB,EAA0E;AACxE,MAAIqD,IAAoB,GAAG,EAA3B;;AACA,OAAK,MAAMC,CAAX,IAAgBtD,MAAhB,EAAwB;AACtBqD,IAAAA,IAAI,CAACxC,IAAL,CAAUb,MAAM,CAACsD,CAAD,CAAhB;;AACA,QAAItD,MAAM,CAACsD,CAAD,CAAN,CAAUhB,QAAd,EAAwB;AACtBe,MAAAA,IAAI,GAAGA,IAAI,CAAClB,MAAL,CAAYhC,UAAU,CAACH,MAAM,CAACsD,CAAD,CAAN,CAAUtD,MAAX,CAAtB,CAAP;AACD;AACF;;AACD,SAAOqD,IAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"file":"schema.js"}
+{"version":3,"sources":["../../../../src/parquetjs/schema/schema.ts"],"names":["ParquetSchema","schema","fields","buildFields","fieldList","listFields","path","split","slice","n","length","shift","branch","push","record","buffer","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"mappings":";;;;;;;;;;;;;;;AAEA;;AACA;;AAUA;;AACA;;IAKaA,a;AAQX,yBAAYC,MAAZ,EAAsC;AAAA;AAAA;AAAA;AAAA;AACpC,SAAKA,MAAL,GAAcA,MAAd;AACA,SAAKC,MAAL,GAAcC,WAAW,CAACF,MAAD,EAAS,CAAT,EAAY,CAAZ,EAAe,EAAf,CAAzB;AACA,SAAKG,SAAL,GAAiBC,UAAU,CAAC,KAAKH,MAAN,CAA3B;AACD;;;;WAKD,mBAAUI,IAAV,EAAiD;AAC/C,UAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,QAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD,OAHD,MAGO;AAELD,QAAAA,IAAI,GAAGA,IAAI,CAACE,KAAL,CAAW,CAAX,CAAP;AACD;;AAED,UAAIC,CAAC,GAAG,KAAKP,MAAb;;AACA,aAAOI,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCF,QAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWJ,MAAf;AACD;;AAED,aAAOO,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAR;AACD;;;WAKD,yBAAgBA,IAAhB,EAAyD;AACvD,UAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,QAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD;;AACD,UAAMK,MAAsB,GAAG,EAA/B;AACA,UAAIH,CAAC,GAAG,KAAKP,MAAb;;AACA,aAAOI,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCC,QAAAA,MAAM,CAACC,IAAP,CAAYJ,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAb;;AACA,YAAIA,IAAI,CAACI,MAAL,GAAc,CAAlB,EAAqB;AACnBD,UAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWJ,MAAf;AACD;AACF;;AACD,aAAOU,MAAP;AACD;;;WAED,qBAAYE,MAAZ,EAAmCC,MAAnC,EAAgE;AAC9D,8BAAY,IAAZ,EAAkBD,MAAlB,EAA0BC,MAA1B;AACD;;;WAED,4BAAmBA,MAAnB,EAA2D;AACzD,aAAO,+BAAmB,IAAnB,EAAyBA,MAAzB,CAAP;AACD;;;WAED,kBAASC,IAAT,EAAyC;AACvCC,MAAAA,WAAW,CAAC,KAAKhB,MAAN,EAAce,IAAd,CAAX;AACAC,MAAAA,WAAW,CAAC,KAAKf,MAAN,EAAcc,IAAd,CAAX;AACA,aAAO,IAAP;AACD;;;WAED,kBAAwB;AACtB,aAAO,wBAAY,IAAZ,CAAP;AACD;;;;;;;AAGH,SAASC,WAAT,CAAqBhB,MAArB,EAAkCe,IAAlC,EAA4D;AAC1D,OAAK,IAAME,IAAX,IAAmBjB,MAAnB,EAA2B;AACzB,QAAMkB,IAAI,GAAGlB,MAAM,CAACiB,IAAD,CAAnB;;AACA,QAAIC,IAAI,CAACjB,MAAT,EAAiB;AACfe,MAAAA,WAAW,CAACE,IAAI,CAACjB,MAAN,EAAcc,IAAd,CAAX;AACD,KAFD,MAEO;AACLG,MAAAA,IAAI,CAACC,WAAL,GAAmBJ,IAAnB;AACD;AACF;AACF;;AAGD,SAASb,WAAT,CACEF,MADF,EAEEoB,eAFF,EAGEC,eAHF,EAIEhB,IAJF,EAKgC;AAC9B,MAAMF,SAAuC,GAAG,EAAhD;;AAEA,OAAK,IAAMc,IAAX,IAAmBjB,MAAnB,EAA2B;AACzB,QAAMsB,IAAI,GAAGtB,MAAM,CAACiB,IAAD,CAAnB;AAGA,QAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAvB;AACA,QAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAN,CAAxB;AACA,QAAIE,SAAS,GAAGP,eAAhB;AACA,QAAIQ,SAAS,GAAGP,eAAhB;AAEA,QAAIQ,cAA8B,GAAG,UAArC;;AACA,QAAI,CAACN,QAAL,EAAe;AACbM,MAAAA,cAAc,GAAG,UAAjB;AACAD,MAAAA,SAAS;AACV;;AACD,QAAIH,QAAJ,EAAc;AACZI,MAAAA,cAAc,GAAG,UAAjB;AACAF,MAAAA,SAAS;AACT,UAAIJ,QAAJ,EAAcK,SAAS;AACxB;;AAGD,QAAIN,IAAI,CAACrB,MAAT,EAAiB;AACf,UAAM6B,MAAK,GAAGzB,IAAI,CAAC0B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;;AACAd,MAAAA,SAAS,CAACc,IAAD,CAAT,GAAkB;AAChBA,QAAAA,IAAI,EAAJA,IADgB;AAEhBZ,QAAAA,IAAI,EAAEyB,MAFU;AAGhBE,QAAAA,GAAG,EAAEF,MAAK,CAACG,IAAN,EAHW;AAIhBJ,QAAAA,cAAc,EAAdA,cAJgB;AAKhBF,QAAAA,SAAS,EAATA,SALgB;AAMhBC,QAAAA,SAAS,EAATA,SANgB;AAOhBM,QAAAA,QAAQ,EAAE,IAPM;AAQhBC,QAAAA,UAAU,EAAEC,MAAM,CAACC,IAAP,CAAYf,IAAI,CAACrB,MAAjB,EAAyBQ,MARrB;AAShBR,QAAAA,MAAM,EAAEC,WAAW,CAACoB,IAAI,CAACrB,MAAN,EAAc0B,SAAd,EAAyBC,SAAzB,EAAoCE,MAApC;AATH,OAAlB;AAWA;AACD;;AAED,QAAMQ,OAAY,GAAGC,6BAAsBjB,IAAI,CAACP,IAA3B,CAArB;;AACA,QAAI,CAACuB,OAAL,EAAc;AACZ,YAAM,IAAIE,KAAJ,iCAAmClB,IAAI,CAACP,IAAxC,EAAN;AACD;;AAEDO,IAAAA,IAAI,CAACmB,QAAL,GAAgBnB,IAAI,CAACmB,QAAL,IAAiB,OAAjC;;AACA,QAAI,EAAEnB,IAAI,CAACmB,QAAL,IAAiBC,sBAAnB,CAAJ,EAAwC;AACtC,YAAM,IAAIF,KAAJ,yCAA2ClB,IAAI,CAACmB,QAAhD,EAAN;AACD;;AAEDnB,IAAAA,IAAI,CAACH,WAAL,GAAmBG,IAAI,CAACH,WAAL,IAAoB,cAAvC;;AACA,QAAI,EAAEG,IAAI,CAACH,WAAL,IAAoBwB,wCAAtB,CAAJ,EAAwD;AACtD,YAAM,IAAIH,KAAJ,2CAA6ClB,IAAI,CAACH,WAAlD,EAAN;AACD;;AAGD,QAAMW,KAAK,GAAGzB,IAAI,CAAC0B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAd,IAAAA,SAAS,CAACc,IAAD,CAAT,GAAkB;AAChBA,MAAAA,IAAI,EAAJA,IADgB;AAEhB2B,MAAAA,aAAa,EAAEN,OAAO,CAACM,aAFP;AAGhBC,MAAAA,YAAY,EAAEP,OAAO,CAACO,YAHN;AAIhBxC,MAAAA,IAAI,EAAEyB,KAJU;AAKhBE,MAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EALW;AAMhBJ,MAAAA,cAAc,EAAdA,cANgB;AAOhBY,MAAAA,QAAQ,EAAEnB,IAAI,CAACmB,QAPC;AAQhBtB,MAAAA,WAAW,EAAEG,IAAI,CAACH,WARF;AAShB2B,MAAAA,UAAU,EAAExB,IAAI,CAACwB,UAAL,IAAmBR,OAAO,CAACQ,UATvB;AAUhBC,MAAAA,SAAS,EAAEzB,IAAI,CAACyB,SAVA;AAWhBC,MAAAA,KAAK,EAAE1B,IAAI,CAAC0B,KAXI;AAYhBrB,MAAAA,SAAS,EAATA,SAZgB;AAahBC,MAAAA,SAAS,EAATA;AAbgB,KAAlB;AAeD;;AACD,SAAOzB,SAAP;AACD;;AAED,SAASC,UAAT,CAAoBH,MAApB,EAA0E;AACxE,MAAIgD,IAAoB,GAAG,EAA3B;;AACA,OAAK,IAAMC,CAAX,IAAgBjD,MAAhB,EAAwB;AACtBgD,IAAAA,IAAI,CAACrC,IAAL,CAAUX,MAAM,CAACiD,CAAD,CAAhB;;AACA,QAAIjD,MAAM,CAACiD,CAAD,CAAN,CAAUhB,QAAd,EAAwB;AACtBe,MAAAA,IAAI,GAAGA,IAAI,CAAClB,MAAL,CAAY3B,UAAU,CAACH,MAAM,CAACiD,CAAD,CAAN,CAAUjD,MAAX,CAAtB,CAAP;AACD;AACF;;AACD,SAAOgD,IAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"file":"schema.js"}

package/dist/es5/parquetjs/schema/shred.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var _typeof = require("@babel/runtime/helpers/typeof");
+
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
@@ -8,7 +10,7 @@ exports.shredRecord = shredRecord;
 exports.materializeRecords = materializeRecords;
 Object.defineProperty(exports, "ParquetBuffer", {
   enumerable: true,
-  get: function () {
+  get: function get() {
     return _declare.ParquetBuffer;
   }
 });
@@ -17,31 +19,47 @@ var _declare = require("./declare");
 
 var Types = _interopRequireWildcard(require("./types"));
 
-function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
 
-function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
 
 function shredBuffer(schema) {
-  const columnData = {};
-
-  for (const field of schema.fieldList) {
-    columnData[field.key] = {
-      dlevels: [],
-      rlevels: [],
-      values: [],
-      pageHeaders: [],
-      count: 0
-    };
+  var columnData = {};
+
+  var _iterator = _createForOfIteratorHelper(schema.fieldList),
+      _step;
+
+  try {
+    for (_iterator.s(); !(_step = _iterator.n()).done;) {
+      var field = _step.value;
+      columnData[field.key] = {
+        dlevels: [],
+        rlevels: [],
+        values: [],
+        pageHeaders: [],
+        count: 0
+      };
+    }
+  } catch (err) {
+    _iterator.e(err);
+  } finally {
+    _iterator.f();
   }
 
   return {
     rowCount: 0,
-    columnData
+    columnData: columnData
   };
 }
 
 function shredRecord(schema, record, buffer) {
-  const data = shredBuffer(schema).columnData;
+  var data = shredBuffer(schema).columnData;
   shredRecordFields(schema.fields, record, data, 0, 0);
 
   if (buffer.rowCount === 0) {
@@ -52,18 +70,28 @@ function shredRecord(schema, record, buffer) {
 
   buffer.rowCount += 1;
 
-  for (const field of schema.fieldList) {
-    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
-    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
-    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
-    buffer.columnData[field.key].count += data[field.key].count;
+  var _iterator2 = _createForOfIteratorHelper(schema.fieldList),
+      _step2;
+
+  try {
+    for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
+      var field = _step2.value;
+      Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
+      Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
+      Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
+      buffer.columnData[field.key].count += data[field.key].count;
+    }
+  } catch (err) {
+    _iterator2.e(err);
+  } finally {
+    _iterator2.f();
   }
 }
 
 function shredRecordFields(fields, record, data, rLevel, dLevel) {
-  for (const name in fields) {
-    const field = fields[name];
-    let values = [];
+  for (var name in fields) {
+    var field = fields[name];
+    var values = [];
 
     if (record && field.name in record && record[field.name] !== undefined && record[field.name] !== null) {
       if (record[field.name].constructor === Array) {
@@ -93,8 +121,8 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
       continue;
     }
 
-    for (let i = 0; i < values.length; i++) {
-      const rlvl = i === 0 ? rLevel : field.rLevelMax;
+    for (var i = 0; i < values.length; i++) {
+      var rlvl = i === 0 ? rLevel : field.rLevelMax;
 
       if (field.isNested) {
         shredRecordFields(field.fields, values[i], data, rlvl, field.dLevelMax);
@@ -109,11 +137,13 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
 }
 
 function materializeRecords(schema, buffer) {
-  const records = [];
+  var records = [];
 
-  for (let i = 0; i < buffer.rowCount; i++) records.push({});
+  for (var i = 0; i < buffer.rowCount; i++) {
+    records.push({});
+  }
 
-  for (const key in buffer.columnData) {
+  for (var key in buffer.columnData) {
     materializeColumn(schema, buffer, key, records);
   }
 
@@ -121,45 +151,55 @@ function materializeRecords(schema, buffer) {
 }
 
 function materializeColumn(schema, buffer, key, records) {
-  const data = buffer.columnData[key];
+  var data = buffer.columnData[key];
   if (!data.count) return;
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-  const rLevels = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-
-  for (let i = 0; i < data.count; i++) {
-    const dLevel = data.dlevels[i];
-    const rLevel = data.rlevels[i];
+  var field = schema.findField(key);
+  var branch = schema.findFieldBranch(key);
+  var rLevels = new Array(field.rLevelMax + 1).fill(0);
+  var vIndex = 0;
+
+  for (var i = 0; i < data.count; i++) {
+    var dLevel = data.dlevels[i];
+    var rLevel = data.rlevels[i];
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
-    let rIndex = 0;
-    let record = records[rLevels[rIndex++] - 1];
+    var rIndex = 0;
+    var record = records[rLevels[rIndex++] - 1];
 
-    for (const step of branch) {
-      if (step === field) break;
-      if (dLevel < step.dLevelMax) break;
+    var _iterator3 = _createForOfIteratorHelper(branch),
+        _step3;
 
-      if (step.repetitionType === 'REPEATED') {
-        if (!(step.name in record)) {
-          record[step.name] = [];
-        }
+    try {
+      for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
+        var step = _step3.value;
+        if (step === field) break;
+        if (dLevel < step.dLevelMax) break;
 
-        const ix = rLevels[rIndex++];
+        if (step.repetitionType === 'REPEATED') {
+          if (!(step.name in record)) {
+            record[step.name] = [];
+          }
 
-        while (record[step.name].length <= ix) {
-          record[step.name].push({});
-        }
+          var _ix = rLevels[rIndex++];
 
-        record = record[step.name][ix];
-      } else {
-        record[step.name] = record[step.name] || {};
-        record = record[step.name];
+          while (record[step.name].length <= _ix) {
+            record[step.name].push({});
+          }
+
+          record = record[step.name][_ix];
+        } else {
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
+        }
       }
+    } catch (err) {
+      _iterator3.e(err);
+    } finally {
+      _iterator3.f();
     }
 
     if (dLevel === field.dLevelMax) {
-      const value = Types.fromPrimitive(field.originalType || field.primitiveType, data.values[vIndex], field);
+      var value = Types.fromPrimitive(field.originalType || field.primitiveType, data.values[vIndex], field);
       vIndex++;
 
       if (field.repetitionType === 'REPEATED') {
@@ -167,7 +207,7 @@ function materializeColumn(schema, buffer, key, records) {
           record[field.name] = [];
         }
 
-        const ix = rLevels[rIndex];
+        var ix = rLevels[rIndex];
 
         while (record[field.name].length <= ix) {
           record[field.name].push(null);

package/dist/es5/parquetjs/schema/shred.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/schema/shred.ts"],"names":["shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","Types","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"mappings":";;;;;;;;;;;;;;;AAEA;;AAEA;;;;;;AAIO,SAASA,WAAT,CAAqBC,MAArB,EAA2D;AAChE,QAAMC,UAAuC,GAAG,EAAhD;;AACA,OAAK,MAAMC,KAAX,IAAoBF,MAAM,CAACG,SAA3B,EAAsC;AACpCF,IAAAA,UAAU,CAACC,KAAK,CAACE,GAAP,CAAV,GAAwB;AACtBC,MAAAA,OAAO,EAAE,EADa;AAEtBC,MAAAA,OAAO,EAAE,EAFa;AAGtBC,MAAAA,MAAM,EAAE,EAHc;AAItBC,MAAAA,WAAW,EAAE,EAJS;AAKtBC,MAAAA,KAAK,EAAE;AALe,KAAxB;AAOD;;AACD,SAAO;AAACC,IAAAA,QAAQ,EAAE,CAAX;AAAcT,IAAAA;AAAd,GAAP;AACD;;AAwBM,SAASU,WAAT,CAAqBX,MAArB,EAA4CY,MAA5C,EAAyDC,MAAzD,EAAsF;AAE3F,QAAMC,IAAI,GAAGf,WAAW,CAACC,MAAD,CAAX,CAAoBC,UAAjC;AAEAc,EAAAA,iBAAiB,CAACf,MAAM,CAACgB,MAAR,EAAgBJ,MAAhB,EAAwBE,IAAxB,EAA8B,CAA9B,EAAiC,CAAjC,CAAjB;;AAGA,MAAID,MAAM,CAACH,QAAP,KAAoB,CAAxB,EAA2B;AACzBG,IAAAA,MAAM,CAACH,QAAP,GAAkB,CAAlB;AACAG,IAAAA,MAAM,CAACZ,UAAP,GAAoBa,IAApB;AACA;AACD;;AACDD,EAAAA,MAAM,CAACH,QAAP,IAAmB,CAAnB;;AACA,OAAK,MAAMR,KAAX,IAAoBF,MAAM,CAACG,SAA3B,EAAsC;AACpCc,IAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BE,OAAxD,EAAiEQ,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBE,OAAjF;AACAW,IAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BC,OAAxD,EAAiES,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBC,OAAjF;AACAY,IAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BG,MAAxD,EAAgEO,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBG,MAAhF;AACAM,IAAAA,MAAM,CAACZ,UAAP,CAAkBC,KAAK,CAACE,GAAxB,EAA6BK,KAA7B,IAAsCK,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBK,KAAtD;AACD;AACF;;AAGD,SAASM,iBAAT,CACEC,MADF,EAEEJ,MAFF,EAGEE,IAHF,EAIEO,MAJF,EAKEC,MALF,EAME;AACA,OAAK,MAAMC,IAAX,IAAmBP,MAAnB,EAA2B;AACzB,UAAMd,KAAK,GAAGc,MAAM,CAACO,IAAD,CAApB;AAGA,QAAIhB,MAAa,GAAG,EAApB;;AACA,QACEK,MAAM,IACNV,KAAK,CAACqB,IAAN,IAAcX,MADd,IAEAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,KAAuBC,SAFvB,IAGAZ,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,KAAuB,IAJzB,EAKE;AACA,UAAIX,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmBE,WAAnB,KAAmCR,KAAvC,EAA8C;AAC5CV,QAAAA,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAf;AACD,OAFD,MAEO;AACLhB,QAAAA,MAAM,CAACY,IAAP,CAAYP,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAlB;AACD;AACF;;AAED,QAAIhB,MAAM,CAACmB,MAAP,KAAkB,CAAlB,IAAuBC,OAAO,CAACf,MAAD,CAA9B,IAA0CV,KAAK,CAAC0B,cAAN,KAAyB,UAAvE,EAAmF;AACjF,YAAM,IAAIC,KAAJ,mCAAqC3B,KAAK,CAACqB,IAA3C,EAAN;AACD;;AACD,QAAIhB,MAAM,CAACmB,MAAP,GAAgB,CAAhB,IAAqBxB,KAAK,CAAC0B,cAAN,KAAyB,UAAlD,EAA8D;AAC5D,YAAM,IAAIC,KAAJ,sCAAwC3B,KAAK,CAACqB,IAA9C,EAAN;AACD;;AAGD,QAAIhB,MAAM,CAACmB,MAAP,KAAkB,CAAtB,EAAyB;AACvB,UAAIxB,KAAK,CAAC4B,QAAV,EAAoB;AAClBf,QAAAA,iBAAiB,CAACb,KAAK,CAACc,MAAP,EAAgB,IAAhB,EAAsBF,IAAtB,EAA4BO,MAA5B,EAAoCC,MAApC,CAAjB;AACD,OAFD,MAEO;AACLR,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBK,KAAhB,IAAyB,CAAzB;AACAK,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBE,OAAhB,CAAwBa,IAAxB,CAA6BE,MAA7B;AACAP,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBC,OAAhB,CAAwBc,IAAxB,CAA6BG,MAA7B;AACD;;AACD;AACD;;AAGD,SAAK,IAAIS,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGxB,MAAM,CAACmB,MAA3B,EAAmCK,CAAC,EAApC,EAAwC;AACtC,YAAMC,IAAI,GAAGD,CAAC,KAAK,CAAN,GAAUV,MAAV,GAAmBnB,KAAK,CAAC+B,SAAtC;;AACA,UAAI/B,KAAK,CAAC4B,QAAV,EAAoB;AAClBf,QAAAA,iBAAiB,CAACb,KAAK,CAACc,MAAP,EAAgBT,MAAM,CAACwB,CAAD,CAAtB,EAA2BjB,IAA3B,EAAiCkB,IAAjC,EAAuC9B,KAAK,CAACgC,SAA7C,CAAjB;AACD,OAFD,MAEO;AACLpB,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBK,KAAhB,IAAyB,CAAzB;AACAK,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBE,OAAhB,CAAwBa,IAAxB,CAA6Ba,IAA7B;AACAlB,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBC,OAAhB,CAAwBc,IAAxB,CAA6BjB,KAAK,CAACgC,SAAnC;AACApB,QAAAA,IAAI,CAACZ,KAAK,CAACE,GAAP,CAAJ,CAAgBG,MAAhB,CAAuBY,IAAvB,CACEgB,KAAK,CAACC,WAAN,CAAmBlC,KAAK,CAACmC,YAAN,IAAsBnC,KAAK,CAACoC,aAA/C,EAAgE/B,MAAM,CAACwB,CAAD,CAAtE,CADF;AAGD;AACF;AACF;AACF;;AAqBM,SAASQ,kBAAT,CAA4BvC,MAA5B,EAAmDa,MAAnD,EAA2F;AAChG,QAAM2B,OAAwB,GAAG,EAAjC;;AACA,OAAK,IAAIT,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGlB,MAAM,CAACH,QAA3B,EAAqCqB,CAAC,EAAtC,EAA0CS,OAAO,CAACrB,IAAR,CAAa,EAAb;;AAC1C,OAAK,MAAMf,GAAX,IAAkBS,MAAM,CAACZ,UAAzB,EAAqC;AACnCwC,IAAAA,iBAAiB,CAACzC,MAAD,EAASa,MAAT,EAAiBT,GAAjB,EAAsBoC,OAAtB,CAAjB;AACD;;AACD,SAAOA,OAAP;AACD;;AAGD,SAASC,iBAAT,CACEzC,MADF,EAEEa,MAFF,EAGET,GAHF,EAIEoC,OAJF,EAKE;AACA,QAAM1B,IAAI,GAAGD,MAAM,CAACZ,UAAP,CAAkBG,GAAlB,CAAb;AACA,MAAI,CAACU,IAAI,CAACL,KAAV,EAAiB;AAEjB,QAAMP,KAAK,GAAGF,MAAM,CAAC0C,SAAP,CAAiBtC,GAAjB,CAAd;AACA,QAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAP,CAAuBxC,GAAvB,CAAf;AAGA,QAAMyC,OAAiB,GAAG,IAAI5B,KAAJ,CAAUf,KAAK,CAAC+B,SAAN,GAAkB,CAA5B,EAA+Ba,IAA/B,CAAoC,CAApC,CAA1B;AACA,MAAIC,MAAM,GAAG,CAAb;;AACA,OAAK,IAAIhB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGjB,IAAI,CAACL,KAAzB,EAAgCsB,CAAC,EAAjC,EAAqC;AACnC,UAAMT,MAAM,GAAGR,IAAI,CAACT,OAAL,CAAa0B,CAAb,CAAf;AACA,UAAMV,MAAM,GAAGP,IAAI,CAACR,OAAL,CAAayB,CAAb,CAAf;AACAc,IAAAA,OAAO,CAACxB,MAAD,CAAP;AACAwB,IAAAA,OAAO,CAACC,IAAR,CAAa,CAAb,EAAgBzB,MAAM,GAAG,CAAzB;AAEA,QAAI2B,MAAM,GAAG,CAAb;AACA,QAAIpC,MAAM,GAAG4B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAP,CAAP,GAAoB,CAArB,CAApB;;AAGA,SAAK,MAAMC,IAAX,IAAmBN,MAAnB,EAA2B;AACzB,UAAIM,IAAI,KAAK/C,KAAb,EAAoB;AACpB,UAAIoB,MAAM,GAAG2B,IAAI,CAACf,SAAlB,EAA6B;;AAC7B,UAAIe,IAAI,CAACrB,cAAL,KAAwB,UAA5B,EAAwC;AACtC,YAAI,EAAEqB,IAAI,CAAC1B,IAAL,IAAaX,MAAf,CAAJ,EAA4B;AAE1BA,UAAAA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,GAAoB,EAApB;AACD;;AACD,cAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAP,CAAlB;;AACA,eAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,CAAkBG,MAAlB,IAA4BwB,EAAnC,EAAuC;AAErCtC,UAAAA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,CAAkBJ,IAAlB,CAAuB,EAAvB;AACD;;AACDP,QAAAA,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,CAAkB2B,EAAlB,CAAT;AACD,OAXD,MAWO;AACLtC,QAAAA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,GAAoBX,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,IAAqB,EAAzC;AACAX,QAAAA,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAf;AACD;AACF;;AAGD,QAAID,MAAM,KAAKpB,KAAK,CAACgC,SAArB,EAAgC;AAC9B,YAAMiB,KAAK,GAAGhB,KAAK,CAACiB,aAAN,CAEZlD,KAAK,CAACmC,YAAN,IAAsBnC,KAAK,CAACoC,aAFhB,EAGZxB,IAAI,CAACP,MAAL,CAAYwC,MAAZ,CAHY,EAIZ7C,KAJY,CAAd;AAMA6C,MAAAA,MAAM;;AACN,UAAI7C,KAAK,CAAC0B,cAAN,KAAyB,UAA7B,EAAyC;AACvC,YAAI,EAAE1B,KAAK,CAACqB,IAAN,IAAcX,MAAhB,CAAJ,EAA6B;AAE3BA,UAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,GAAqB,EAArB;AACD;;AACD,cAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAD,CAAlB;;AACA,eAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmBG,MAAnB,IAA6BwB,EAApC,EAAwC;AAEtCtC,UAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmBJ,IAAnB,CAAwB,IAAxB;AACD;;AACDP,QAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,CAAmB2B,EAAnB,IAAyBC,KAAzB;AACD,OAXD,MAWO;AACLvC,QAAAA,MAAM,CAACV,KAAK,CAACqB,IAAP,CAAN,GAAqB4B,KAArB;AACD;AACF;AACF;AACF","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) records.push({});\n  for (const key in buffer.columnData) {\n    materializeColumn(schema, buffer, key, records);\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  buffer: ParquetBuffer,\n  key: string,\n  records: ParquetRecord[]\n) {\n  const data = buffer.columnData[key];\n  if (!data.count) return;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < data.count; i++) {\n    const dLevel = data.dlevels[i];\n    const rLevel = data.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes\n    for (const step of branch) {\n      if (step === field) break;\n      if (dLevel < step.dLevelMax) break;\n      if (step.repetitionType === 'REPEATED') {\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n      } else {\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n      }\n    }\n\n    // Leaf node\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        data.values[vIndex],\n        field\n      );\n      vIndex++;\n      if (field.repetitionType === 'REPEATED') {\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n      } else {\n        record[field.name] = value;\n      }\n    }\n  }\n}\n"],"file":"shred.js"}
+{"version":3,"sources":["../../../../src/parquetjs/schema/shred.ts"],"names":["shredBuffer","schema","columnData","fieldList","field","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","Types","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"mappings":";;;;;;;;;;;;;;;;;AAEA;;AAEA;;;;;;;;;;;;AAIO,SAASA,WAAT,CAAqBC,MAArB,EAA2D;AAChE,MAAMC,UAAuC,GAAG,EAAhD;;AADgE,6CAE5CD,MAAM,CAACE,SAFqC;AAAA;;AAAA;AAEhE,wDAAsC;AAAA,UAA3BC,KAA2B;AACpCF,MAAAA,UAAU,CAACE,KAAK,CAACC,GAAP,CAAV,GAAwB;AACtBC,QAAAA,OAAO,EAAE,EADa;AAEtBC,QAAAA,OAAO,EAAE,EAFa;AAGtBC,QAAAA,MAAM,EAAE,EAHc;AAItBC,QAAAA,WAAW,EAAE,EAJS;AAKtBC,QAAAA,KAAK,EAAE;AALe,OAAxB;AAOD;AAV+D;AAAA;AAAA;AAAA;AAAA;;AAWhE,SAAO;AAACC,IAAAA,QAAQ,EAAE,CAAX;AAAcT,IAAAA,UAAU,EAAVA;AAAd,GAAP;AACD;;AAwBM,SAASU,WAAT,CAAqBX,MAArB,EAA4CY,MAA5C,EAAyDC,MAAzD,EAAsF;AAE3F,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAD,CAAX,CAAoBC,UAAjC;AAEAc,EAAAA,iBAAiB,CAACf,MAAM,CAACgB,MAAR,EAAgBJ,MAAhB,EAAwBE,IAAxB,EAA8B,CAA9B,EAAiC,CAAjC,CAAjB;;AAGA,MAAID,MAAM,CAACH,QAAP,KAAoB,CAAxB,EAA2B;AACzBG,IAAAA,MAAM,CAACH,QAAP,GAAkB,CAAlB;AACAG,IAAAA,MAAM,CAACZ,UAAP,GAAoBa,IAApB;AACA;AACD;;AACDD,EAAAA,MAAM,CAACH,QAAP,IAAmB,CAAnB;;AAZ2F,8CAavEV,MAAM,CAACE,SAbgE;AAAA;;AAAA;AAa3F,2DAAsC;AAAA,UAA3BC,KAA2B;AACpCc,MAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBE,KAAK,CAACC,GAAxB,EAA6BE,OAAxD,EAAiEQ,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBE,OAAjF;AACAW,MAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBE,KAAK,CAACC,GAAxB,EAA6BC,OAAxD,EAAiES,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBC,OAAjF;AACAY,MAAAA,KAAK,CAACC,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2BP,MAAM,CAACZ,UAAP,CAAkBE,KAAK,CAACC,GAAxB,EAA6BG,MAAxD,EAAgEO,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBG,MAAhF;AACAM,MAAAA,MAAM,CAACZ,UAAP,CAAkBE,KAAK,CAACC,GAAxB,EAA6BK,KAA7B,IAAsCK,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBK,KAAtD;AACD;AAlB0F;AAAA;AAAA;AAAA;AAAA;AAmB5F;;AAGD,SAASM,iBAAT,CACEC,MADF,EAEEJ,MAFF,EAGEE,IAHF,EAIEO,MAJF,EAKEC,MALF,EAME;AACA,OAAK,IAAMC,IAAX,IAAmBP,MAAnB,EAA2B;AACzB,QAAMb,KAAK,GAAGa,MAAM,CAACO,IAAD,CAApB;AAGA,QAAIhB,MAAa,GAAG,EAApB;;AACA,QACEK,MAAM,IACNT,KAAK,CAACoB,IAAN,IAAcX,MADd,IAEAA,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,KAAuBC,SAFvB,IAGAZ,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,KAAuB,IAJzB,EAKE;AACA,UAAIX,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,CAAmBE,WAAnB,KAAmCR,KAAvC,EAA8C;AAC5CV,QAAAA,MAAM,GAAGK,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAf;AACD,OAFD,MAEO;AACLhB,QAAAA,MAAM,CAACY,IAAP,CAAYP,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAlB;AACD;AACF;;AAED,QAAIhB,MAAM,CAACmB,MAAP,KAAkB,CAAlB,IAAuBC,OAAO,CAACf,MAAD,CAA9B,IAA0CT,KAAK,CAACyB,cAAN,KAAyB,UAAvE,EAAmF;AACjF,YAAM,IAAIC,KAAJ,mCAAqC1B,KAAK,CAACoB,IAA3C,EAAN;AACD;;AACD,QAAIhB,MAAM,CAACmB,MAAP,GAAgB,CAAhB,IAAqBvB,KAAK,CAACyB,cAAN,KAAyB,UAAlD,EAA8D;AAC5D,YAAM,IAAIC,KAAJ,sCAAwC1B,KAAK,CAACoB,IAA9C,EAAN;AACD;;AAGD,QAAIhB,MAAM,CAACmB,MAAP,KAAkB,CAAtB,EAAyB;AACvB,UAAIvB,KAAK,CAAC2B,QAAV,EAAoB;AAClBf,QAAAA,iBAAiB,CAACZ,KAAK,CAACa,MAAP,EAAgB,IAAhB,EAAsBF,IAAtB,EAA4BO,MAA5B,EAAoCC,MAApC,CAAjB;AACD,OAFD,MAEO;AACLR,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBK,KAAhB,IAAyB,CAAzB;AACAK,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBE,OAAhB,CAAwBa,IAAxB,CAA6BE,MAA7B;AACAP,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBC,OAAhB,CAAwBc,IAAxB,CAA6BG,MAA7B;AACD;;AACD;AACD;;AAGD,SAAK,IAAIS,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGxB,MAAM,CAACmB,MAA3B,EAAmCK,CAAC,EAApC,EAAwC;AACtC,UAAMC,IAAI,GAAGD,CAAC,KAAK,CAAN,GAAUV,MAAV,GAAmBlB,KAAK,CAAC8B,SAAtC;;AACA,UAAI9B,KAAK,CAAC2B,QAAV,EAAoB;AAClBf,QAAAA,iBAAiB,CAACZ,KAAK,CAACa,MAAP,EAAgBT,MAAM,CAACwB,CAAD,CAAtB,EAA2BjB,IAA3B,EAAiCkB,IAAjC,EAAuC7B,KAAK,CAAC+B,SAA7C,CAAjB;AACD,OAFD,MAEO;AACLpB,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBK,KAAhB,IAAyB,CAAzB;AACAK,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBE,OAAhB,CAAwBa,IAAxB,CAA6Ba,IAA7B;AACAlB,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBC,OAAhB,CAAwBc,IAAxB,CAA6BhB,KAAK,CAAC+B,SAAnC;AACApB,QAAAA,IAAI,CAACX,KAAK,CAACC,GAAP,CAAJ,CAAgBG,MAAhB,CAAuBY,IAAvB,CACEgB,KAAK,CAACC,WAAN,CAAmBjC,KAAK,CAACkC,YAAN,IAAsBlC,KAAK,CAACmC,aAA/C,EAAgE/B,MAAM,CAACwB,CAAD,CAAtE,CADF;AAGD;AACF;AACF;AACF;;AAqBM,SAASQ,kBAAT,CAA4BvC,MAA5B,EAAmDa,MAAnD,EAA2F;AAChG,MAAM2B,OAAwB,GAAG,EAAjC;;AACA,OAAK,IAAIT,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGlB,MAAM,CAACH,QAA3B,EAAqCqB,CAAC,EAAtC;AAA0CS,IAAAA,OAAO,CAACrB,IAAR,CAAa,EAAb;AAA1C;;AACA,OAAK,IAAMf,GAAX,IAAkBS,MAAM,CAACZ,UAAzB,EAAqC;AACnCwC,IAAAA,iBAAiB,CAACzC,MAAD,EAASa,MAAT,EAAiBT,GAAjB,EAAsBoC,OAAtB,CAAjB;AACD;;AACD,SAAOA,OAAP;AACD;;AAGD,SAASC,iBAAT,CACEzC,MADF,EAEEa,MAFF,EAGET,GAHF,EAIEoC,OAJF,EAKE;AACA,MAAM1B,IAAI,GAAGD,MAAM,CAACZ,UAAP,CAAkBG,GAAlB,CAAb;AACA,MAAI,CAACU,IAAI,CAACL,KAAV,EAAiB;AAEjB,MAAMN,KAAK,GAAGH,MAAM,CAAC0C,SAAP,CAAiBtC,GAAjB,CAAd;AACA,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAP,CAAuBxC,GAAvB,CAAf;AAGA,MAAMyC,OAAiB,GAAG,IAAI5B,KAAJ,CAAUd,KAAK,CAAC8B,SAAN,GAAkB,CAA5B,EAA+Ba,IAA/B,CAAoC,CAApC,CAA1B;AACA,MAAIC,MAAM,GAAG,CAAb;;AACA,OAAK,IAAIhB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGjB,IAAI,CAACL,KAAzB,EAAgCsB,CAAC,EAAjC,EAAqC;AACnC,QAAMT,MAAM,GAAGR,IAAI,CAACT,OAAL,CAAa0B,CAAb,CAAf;AACA,QAAMV,MAAM,GAAGP,IAAI,CAACR,OAAL,CAAayB,CAAb,CAAf;AACAc,IAAAA,OAAO,CAACxB,MAAD,CAAP;AACAwB,IAAAA,OAAO,CAACC,IAAR,CAAa,CAAb,EAAgBzB,MAAM,GAAG,CAAzB;AAEA,QAAI2B,MAAM,GAAG,CAAb;AACA,QAAIpC,MAAM,GAAG4B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAP,CAAP,GAAoB,CAArB,CAApB;;AAPmC,gDAUhBL,MAVgB;AAAA;;AAAA;AAUnC,6DAA2B;AAAA,YAAhBM,IAAgB;AACzB,YAAIA,IAAI,KAAK9C,KAAb,EAAoB;AACpB,YAAImB,MAAM,GAAG2B,IAAI,CAACf,SAAlB,EAA6B;;AAC7B,YAAIe,IAAI,CAACrB,cAAL,KAAwB,UAA5B,EAAwC;AACtC,cAAI,EAAEqB,IAAI,CAAC1B,IAAL,IAAaX,MAAf,CAAJ,EAA4B;AAE1BA,YAAAA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,GAAoB,EAApB;AACD;;AACD,cAAM2B,GAAE,GAAGL,OAAO,CAACG,MAAM,EAAP,CAAlB;;AACA,iBAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,CAAkBG,MAAlB,IAA4BwB,GAAnC,EAAuC;AAErCtC,YAAAA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,CAAkBJ,IAAlB,CAAuB,EAAvB;AACD;;AACDP,UAAAA,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,CAAkB2B,GAAlB,CAAT;AACD,SAXD,MAWO;AACLtC,UAAAA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,GAAoBX,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAN,IAAqB,EAAzC;AACAX,UAAAA,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAN,CAAf;AACD;AACF;AA5BkC;AAAA;AAAA;AAAA;AAAA;;AA+BnC,QAAID,MAAM,KAAKnB,KAAK,CAAC+B,SAArB,EAAgC;AAC9B,UAAMiB,KAAK,GAAGhB,KAAK,CAACiB,aAAN,CAEZjD,KAAK,CAACkC,YAAN,IAAsBlC,KAAK,CAACmC,aAFhB,EAGZxB,IAAI,CAACP,MAAL,CAAYwC,MAAZ,CAHY,EAIZ5C,KAJY,CAAd;AAMA4C,MAAAA,MAAM;;AACN,UAAI5C,KAAK,CAACyB,cAAN,KAAyB,UAA7B,EAAyC;AACvC,YAAI,EAAEzB,KAAK,CAACoB,IAAN,IAAcX,MAAhB,CAAJ,EAA6B;AAE3BA,UAAAA,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,GAAqB,EAArB;AACD;;AACD,YAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAD,CAAlB;;AACA,eAAOpC,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,CAAmBG,MAAnB,IAA6BwB,EAApC,EAAwC;AAEtCtC,UAAAA,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,CAAmBJ,IAAnB,CAAwB,IAAxB;AACD;;AACDP,QAAAA,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,CAAmB2B,EAAnB,IAAyBC,KAAzB;AACD,OAXD,MAWO;AACLvC,QAAAA,MAAM,CAACT,KAAK,CAACoB,IAAP,CAAN,GAAqB4B,KAArB;AACD;AACF;AACF;AACF","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) records.push({});\n  for (const key in buffer.columnData) {\n    materializeColumn(schema, buffer, key, records);\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  buffer: ParquetBuffer,\n  key: string,\n  records: ParquetRecord[]\n) {\n  const data = buffer.columnData[key];\n  if (!data.count) return;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < data.count; i++) {\n    const dLevel = data.dlevels[i];\n    const rLevel = data.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes\n    for (const step of branch) {\n      if (step === field) break;\n      if (dLevel < step.dLevelMax) break;\n      if (step.repetitionType === 'REPEATED') {\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n      } else {\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n      }\n    }\n\n    // Leaf node\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        data.values[vIndex],\n        field\n      );\n      vIndex++;\n      if (field.repetitionType === 'REPEATED') {\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n      } else {\n        record[field.name] = value;\n      }\n    }\n  }\n}\n"],"file":"shred.js"}