@loaders.gl/parquet 3.3.0-alpha.5 → 3.3.0-alpha.7

package/dist/es5/parquetjs/parser/parquet-reader.js

@@ -1,60 +1,39 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetReader = void 0;
-
 var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
 var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
 var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
 var _parquetEnvelopeReader = require("./parquet-envelope-reader");
-
 var _schema = require("../schema/schema");
-
 var _parquetCursor = require("./parquet-cursor");
-
 var _constants = require("../../constants");
-
 var _decoders = require("./decoders");
-
 var _Symbol$asyncIterator;
-
 function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-
 function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
 function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
 _Symbol$asyncIterator = Symbol.asyncIterator;
-
 var ParquetReader = function () {
   function ParquetReader(metadata, envelopeReader) {
     (0, _classCallCheck2.default)(this, ParquetReader);
     (0, _defineProperty2.default)(this, "metadata", void 0);
     (0, _defineProperty2.default)(this, "envelopeReader", void 0);
     (0, _defineProperty2.default)(this, "schema", void 0);
-
     if (metadata.version !== _constants.PARQUET_VERSION) {
       throw new Error('invalid parquet version');
     }
-
     this.metadata = metadata;
     this.envelopeReader = envelopeReader;
     var root = this.metadata.schema[0];
-
     var _decodeSchema = (0, _decoders.decodeSchema)(this.metadata.schema, 1, root.num_children),
-        schema = _decodeSchema.schema;
-
+      schema = _decodeSchema.schema;
     this.schema = new _schema.ParquetSchema(schema);
   }
 
@@ -68,7 +47,6 @@ var ParquetReader = function () {
               case 0:
                 _context.next = 2;
                 return this.envelopeReader.close();
-
               case 2:
               case "end":
                 return _context.stop();
@@ -76,11 +54,9 @@ var ParquetReader = function () {
           }
         }, _callee, this);
       }));
-
       function close() {
         return _close.apply(this, arguments);
       }
-
       return close;
     }()
   }, {
@@ -95,24 +71,28 @@ var ParquetReader = function () {
       });
       return new _parquetCursor.ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
     }
+
   }, {
     key: "getRowCount",
-    value: function getRowCount() {
+    value:
+    function getRowCount() {
       return Number(this.metadata.num_rows);
     }
+
   }, {
     key: "getSchema",
-    value: function getSchema() {
+    value:
+    function getSchema() {
       return this.schema;
     }
+
   }, {
     key: "getMetadata",
-    value: function getMetadata() {
+    value:
+    function getMetadata() {
       var md = {};
-
       var _iterator = _createForOfIteratorHelper(this.metadata.key_value_metadata),
-          _step;
-
+        _step;
       try {
         for (_iterator.s(); !(_step = _iterator.n()).done;) {
           var kv = _step.value;
@@ -123,12 +103,13 @@ var ParquetReader = function () {
       } finally {
         _iterator.f();
       }
-
       return md;
     }
+
   }, {
     key: _Symbol$asyncIterator,
-    value: function value() {
+    value:
+    function value() {
       return this.getCursor()[Symbol.asyncIterator]();
     }
   }], [{
@@ -149,11 +130,9 @@ var ParquetReader = function () {
                           case 0:
                             _context2.next = 2;
                             return blob.slice(start, start + length).arrayBuffer();
-
                           case 2:
                             arrayBuffer = _context2.sent;
                             return _context2.abrupt("return", Buffer.from(arrayBuffer));
-
                           case 4:
                           case "end":
                             return _context2.stop();
@@ -161,12 +140,10 @@ var ParquetReader = function () {
                       }
                     }, _callee2);
                   }));
-
                   return function readFn(_x2, _x3) {
                     return _ref.apply(this, arguments);
                   };
                 }();
-
                 closeFn = function () {
                   var _ref2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3() {
                     return _regenerator.default.wrap(function _callee3$(_context3) {
@@ -179,35 +156,28 @@ var ParquetReader = function () {
                       }
                     }, _callee3);
                   }));
-
                   return function closeFn() {
                     return _ref2.apply(this, arguments);
                   };
                 }();
-
                 size = blob.size;
                 envelopeReader = new _parquetEnvelopeReader.ParquetEnvelopeReader(readFn, closeFn, size);
                 _context4.prev = 4;
                 _context4.next = 7;
                 return envelopeReader.readHeader();
-
               case 7:
                 _context4.next = 9;
                 return envelopeReader.readFooter();
-
               case 9:
                 metadata = _context4.sent;
                 return _context4.abrupt("return", new ParquetReader(metadata, envelopeReader));
-
               case 13:
                 _context4.prev = 13;
                 _context4.t0 = _context4["catch"](4);
                 _context4.next = 17;
                 return envelopeReader.close();
-
               case 17:
                 throw _context4.t0;
-
               case 18:
               case "end":
                 return _context4.stop();
@@ -215,11 +185,9 @@ var ParquetReader = function () {
           }
         }, _callee4, null, [[4, 13]]);
       }));
-
       function openBlob(_x) {
         return _openBlob.apply(this, arguments);
       }
-
       return openBlob;
     }()
   }, {
@@ -238,7 +206,6 @@ var ParquetReader = function () {
                         switch (_context5.prev = _context5.next) {
                           case 0:
                             return _context5.abrupt("return", Buffer.from(arrayBuffer, start, length));
-
                           case 1:
                           case "end":
                             return _context5.stop();
@@ -246,12 +213,10 @@ var ParquetReader = function () {
                       }
                     }, _callee5);
                   }));
-
                   return function readFn(_x5, _x6) {
                     return _ref3.apply(this, arguments);
                   };
                 }();
-
                 closeFn = function () {
                   var _ref4 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee6() {
                     return _regenerator.default.wrap(function _callee6$(_context6) {
@@ -264,35 +229,28 @@ var ParquetReader = function () {
                       }
                     }, _callee6);
                   }));
-
                   return function closeFn() {
                     return _ref4.apply(this, arguments);
                   };
                 }();
-
                 size = arrayBuffer.byteLength;
                 envelopeReader = new _parquetEnvelopeReader.ParquetEnvelopeReader(readFn, closeFn, size);
                 _context7.prev = 4;
                 _context7.next = 7;
                 return envelopeReader.readHeader();
-
               case 7:
                 _context7.next = 9;
                 return envelopeReader.readFooter();
-
               case 9:
                 metadata = _context7.sent;
                 return _context7.abrupt("return", new ParquetReader(metadata, envelopeReader));
-
               case 13:
                 _context7.prev = 13;
                 _context7.t0 = _context7["catch"](4);
                 _context7.next = 17;
                 return envelopeReader.close();
-
               case 17:
                 throw _context7.t0;
-
               case 18:
               case "end":
                 return _context7.stop();
@@ -300,11 +258,9 @@ var ParquetReader = function () {
           }
         }, _callee7, null, [[4, 13]]);
       }));
-
       function openArrayBuffer(_x4) {
         return _openArrayBuffer.apply(this, arguments);
       }
-
       return openArrayBuffer;
     }()
   }, {
@@ -318,30 +274,24 @@ var ParquetReader = function () {
               case 0:
                 _context8.next = 2;
                 return _parquetEnvelopeReader.ParquetEnvelopeReader.openBuffer(buffer);
-
               case 2:
                 envelopeReader = _context8.sent;
                 _context8.prev = 3;
                 _context8.next = 6;
                 return envelopeReader.readHeader();
-
               case 6:
                 _context8.next = 8;
                 return envelopeReader.readFooter();
-
               case 8:
                 metadata = _context8.sent;
                 return _context8.abrupt("return", new ParquetReader(metadata, envelopeReader));
-
               case 12:
                 _context8.prev = 12;
                 _context8.t0 = _context8["catch"](3);
                 _context8.next = 16;
                 return envelopeReader.close();
-
               case 16:
                 throw _context8.t0;
-
               case 17:
               case "end":
                 return _context8.stop();
@@ -349,16 +299,13 @@ var ParquetReader = function () {
           }
         }, _callee8, null, [[3, 12]]);
       }));
-
       function openBuffer(_x7) {
         return _openBuffer.apply(this, arguments);
       }
-
       return openBuffer;
     }()
   }]);
   return ParquetReader;
 }();
-
 exports.ParquetReader = ParquetReader;
 //# sourceMappingURL=parquet-reader.js.map

package/dist/es5/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"names":["Symbol","asyncIterator","ParquetReader","metadata","envelopeReader","version","PARQUET_VERSION","Error","root","schema","num_children","ParquetSchema","close","columnList","map","x","Array","isArray","ParquetCursor","Number","num_rows","md","key_value_metadata","kv","key","value","getCursor","blob","readFn","start","length","slice","arrayBuffer","Buffer","from","closeFn","size","ParquetEnvelopeReader","readHeader","readFooter","byteLength","buffer","openBuffer"],"mappings":";;;;;;;;;;;;;;;;;;;AACA;;AAEA;;AACA;;AACA;;AACA;;;;;;;;;;wBAyJGA,MAAM,CAACC,a;;IAhJGC,a;AA8DX,yBAAYC,QAAZ,EAAoCC,cAApC,EAA2E;AAAA;AAAA;AAAA;AAAA;;AACzE,QAAID,QAAQ,CAACE,OAAT,KAAqBC,0BAAzB,EAA0C;AACxC,YAAM,IAAIC,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAED,SAAKJ,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,QAAMI,IAAI,GAAG,KAAKL,QAAL,CAAcM,MAAd,CAAqB,CAArB,CAAb;;AACA,wBAAiB,4BAAa,KAAKN,QAAL,CAAcM,MAA3B,EAAmC,CAAnC,EAAsCD,IAAI,CAACE,YAA3C,CAAjB;AAAA,QAAOD,MAAP,iBAAOA,MAAP;;AACA,SAAKA,MAAL,GAAc,IAAIE,qBAAJ,CAAkBF,MAAlB,CAAd;AACD;;;;;6EAMD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACQ,KAAKL,cAAL,CAAoBQ,KAApB,EADR;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;WAmBA,mBAAUC,UAAV,EAAyE;AACvE,UAAI,CAACA,UAAL,EAAiB;AAEfA,QAAAA,UAAU,GAAG,EAAb;AACD;;AAGDA,MAAAA,UAAU,GAAGA,UAAU,CAACC,GAAX,CAAe,UAACC,CAAD;AAAA,eAAQC,KAAK,CAACC,OAAN,CAAcF,CAAd,IAAmBA,CAAnB,GAAuB,CAACA,CAAD,CAA/B;AAAA,OAAf,CAAb;AAEA,aAAO,IAAIG,4BAAJ,CACL,KAAKf,QADA,EAEL,KAAKC,cAFA,EAGL,KAAKK,MAHA,EAILI,UAJK,CAAP;AAMD;;;WAMD,uBAAsB;AACpB,aAAOM,MAAM,CAAC,KAAKhB,QAAL,CAAciB,QAAf,CAAb;AACD;;;WAKD,qBAA2B;AACzB,aAAO,KAAKX,MAAZ;AACD;;;WAKD,uBAAsC;AACpC,UAAMY,EAA0B,GAAG,EAAnC;;AADoC,iDAEnB,KAAKlB,QAAL,CAAcmB,kBAFK;AAAA;;AAAA;AAEpC,4DAAoD;AAAA,cAAzCC,EAAyC;AAClDF,UAAAA,EAAE,CAACE,EAAE,CAACC,GAAJ,CAAF,GAAaD,EAAE,CAACE,KAAhB;AACD;AAJmC;AAAA;AAAA;AAAA;AAAA;;AAKpC,aAAOJ,EAAP;AACD;;;WAMD,iBAA2C;AACzC,aAAO,KAAKK,SAAL,GAAiB1B,MAAM,CAACC,aAAxB,GAAP;AACD;;;;gFA9ID,kBAAyB0B,IAAzB;AAAA;AAAA;AAAA;AAAA;AAAA;AACQC,gBAAAA,MADR;AAAA,uFACiB,kBAAOC,KAAP,EAAsBC,MAAtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mCACaH,IAAI,CAACI,KAAL,CAAWF,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,EAAkCE,WAAlC,EADb;;AAAA;AACPA,4BAAAA,WADO;AAAA,8DAENC,MAAM,CAACC,IAAP,CAAYF,WAAZ,CAFM;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBADjB;;AAAA,kCACQJ,MADR;AAAA;AAAA;AAAA;;AAKQO,gBAAAA,OALR;AAAA,wFAKkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBALlB;;AAAA,kCAKQA,OALR;AAAA;AAAA;AAAA;;AAMQC,gBAAAA,IANR,GAMeT,IAAI,CAACS,IANpB;AAOQhC,gBAAAA,cAPR,GAOyB,IAAIiC,4CAAJ,CAA0BT,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAPzB;AAAA;AAAA;AAAA,uBASUhC,cAAc,CAACkC,UAAf,EATV;;AAAA;AAAA;AAAA,uBAU2BlC,cAAc,CAACmC,UAAf,EAV3B;;AAAA;AAUUpC,gBAAAA,QAVV;AAAA,kDAWW,IAAID,aAAJ,CAAkBC,QAAlB,EAA4BC,cAA5B,CAXX;;AAAA;AAAA;AAAA;AAAA;AAAA,uBAaUA,cAAc,CAACQ,KAAf,EAbV;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;uFAqBA,kBAAgCoB,WAAhC;AAAA;AAAA;AAAA;AAAA;AAAA;AACQJ,gBAAAA,MADR;AAAA,wFACiB,kBAAOC,KAAP,EAAsBC,MAAtB;AAAA;AAAA;AAAA;AAAA;AAAA,8DAAyCG,MAAM,CAACC,IAAP,CAAYF,WAAZ,EAAyBH,KAAzB,EAAgCC,MAAhC,CAAzC;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBADjB;;AAAA,kCACQF,MADR;AAAA;AAAA;AAAA;;AAEQO,gBAAAA,OAFR;AAAA,wFAEkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBAFlB;;AAAA,kCAEQA,OAFR;AAAA;AAAA;AAAA;;AAGQC,gBAAAA,IAHR,GAGeJ,WAAW,CAACQ,UAH3B;AAIQpC,gBAAAA,cAJR,GAIyB,IAAIiC,4CAAJ,CAA0BT,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAJzB;AAAA;AAAA;AAAA,uBAMUhC,cAAc,CAACkC,UAAf,EANV;;AAAA;AAAA;AAAA,uBAO2BlC,cAAc,CAACmC,UAAf,EAP3B;;AAAA;AAOUpC,gBAAAA,QAPV;AAAA,kDAQW,IAAID,aAAJ,CAAkBC,QAAlB,EAA4BC,cAA5B,CARX;;AAAA;AAAA;AAAA;AAAA;AAAA,uBAUUA,cAAc,CAACQ,KAAf,EAVV;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFAeA,kBAA2B6B,MAA3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAC+BJ,6CAAsBK,UAAtB,CAAiCD,MAAjC,CAD/B;;AAAA;AACQrC,gBAAAA,cADR;AAAA;AAAA;AAAA,uBAGUA,cAAc,CAACkC,UAAf,EAHV;;AAAA;AAAA;AAAA,uBAI2BlC,cAAc,CAACmC,UAAf,EAJ3B;;AAAA;AAIUpC,gBAAAA,QAJV;AAAA,kDAKW,IAAID,aAAJ,CAAqBC,QAArB,EAA+BC,cAA/B,CALX;;AAAA;AAAA;AAAA;AAAA;AAAA,uBAOUA,cAAc,CAACQ,KAAf,EAPV;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"file":"parquet-reader.js"}
+{"version":3,"file":"parquet-reader.js","names":["Symbol","asyncIterator","ParquetReader","metadata","envelopeReader","version","PARQUET_VERSION","Error","root","schema","decodeSchema","num_children","ParquetSchema","close","columnList","map","x","Array","isArray","ParquetCursor","Number","num_rows","md","key_value_metadata","kv","key","value","getCursor","blob","readFn","start","length","slice","arrayBuffer","Buffer","from","closeFn","size","ParquetEnvelopeReader","readHeader","readFooter","byteLength","buffer","openBuffer"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"mappings":";;;;;;;;;;;;AACA;AAEA;AACA;AACA;AACA;AAAwC;AAAA;AAAA;AAAA;AAAA,wBAyJrCA,MAAM,CAACC,aAAa;AAAA,IAhJVC,aAAa;EA8DxB,uBAAYC,QAAsB,EAAEC,cAAqC,EAAE;IAAA;IAAA;IAAA;IAAA;IACzE,IAAID,QAAQ,CAACE,OAAO,KAAKC,0BAAe,EAAE;MACxC,MAAM,IAAIC,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IAEA,IAAI,CAACJ,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAMI,IAAI,GAAG,IAAI,CAACL,QAAQ,CAACM,MAAM,CAAC,CAAC,CAAC;IACpC,oBAAiB,IAAAC,sBAAY,EAAC,IAAI,CAACP,QAAQ,CAACM,MAAM,EAAE,CAAC,EAAED,IAAI,CAACG,YAAY,CAAE;MAAnEF,MAAM,iBAANA,MAAM;IACb,IAAI,CAACA,MAAM,GAAG,IAAIG,qBAAa,CAACH,MAAM,CAAC;EACzC;;EAAC;IAAA;IAAA;MAAA,uEAMD;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACQ,IAAI,CAACL,cAAc,CAACS,KAAK,EAAE;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAGlC;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA,OAeD,mBAAUC,UAAkC,EAA6B;MACvE,IAAI,CAACA,UAAU,EAAE;QAEfA,UAAU,GAAG,EAAE;MACjB;;MAGAA,UAAU,GAAGA,UAAU,CAACC,GAAG,CAAC,UAACC,CAAC;QAAA,OAAMC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;MAAA,CAAC,CAAC;MAEhE,OAAO,IAAIG,4BAAa,CACtB,IAAI,CAAChB,QAAQ,EACb,IAAI,CAACC,cAAc,EACnB,IAAI,CAACK,MAAM,EACXK,UAAU,CACX;IACH;;EAAC;IAAA;IAAA;IAMD,uBAAsB;MACpB,OAAOM,MAAM,CAAC,IAAI,CAACjB,QAAQ,CAACkB,QAAQ,CAAC;IACvC;;EAAC;IAAA;IAAA;IAKD,qBAA2B;MACzB,OAAO,IAAI,CAACZ,MAAM;IACpB;;EAAC;IAAA;IAAA;IAKD,uBAAsC;MACpC,IAAMa,EAA0B,GAAG,CAAC,CAAC;MAAC,2CACrB,IAAI,CAACnB,QAAQ,CAACoB,kBAAkB;QAAA;MAAA;QAAjD,oDAAoD;UAAA,IAAzCC,EAAE;UACXF,EAAE,CAACE,EAAE,CAACC,GAAG,CAAC,GAAGD,EAAE,CAACE,KAAM;QACxB;MAAC;QAAA;MAAA;QAAA;MAAA;MACD,OAAOJ,EAAE;IACX;;EAAC;IAAA;IAAA;IAMD,iBAA2C;MACzC,OAAO,IAAI,CAACK,SAAS,EAAE,CAAC3B,MAAM,CAACC,aAAa,CAAC,EAAE;IACjD;EAAC;IAAA;IAAA;MAAA,0EA9ID,kBAAyB2B,IAAU;QAAA;QAAA;UAAA;YAAA;cAAA;gBAC3BC,MAAM;kBAAA,qEAAG,kBAAOC,KAAa,EAAEC,MAAc;oBAAA;oBAAA;sBAAA;wBAAA;0BAAA;4BAAA;4BAAA,OACvBH,IAAI,CAACI,KAAK,CAACF,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC,CAACE,WAAW,EAAE;0BAAA;4BAAnEA,WAAW;4BAAA,kCACVC,MAAM,CAACC,IAAI,CAACF,WAAW,CAAC;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA,CAChC;kBAAA,gBAHKJ,MAAM;oBAAA;kBAAA;gBAAA;gBAINO,OAAO;kBAAA,sEAAG;oBAAA;sBAAA;wBAAA;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA,CAAc;kBAAA,gBAAxBA,OAAO;oBAAA;kBAAA;gBAAA;gBACPC,IAAI,GAAGT,IAAI,CAACS,IAAI;gBAChBjC,cAAc,GAAG,IAAIkC,4CAAqB,CAACT,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;gBAAA;gBAAA;gBAAA,OAE/DjC,cAAc,CAACmC,UAAU,EAAE;cAAA;gBAAA;gBAAA,OACVnC,cAAc,CAACoC,UAAU,EAAE;cAAA;gBAA5CrC,QAAQ;gBAAA,kCACP,IAAID,aAAa,CAACC,QAAQ,EAAEC,cAAc,CAAC;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,OAE5CA,cAAc,CAACS,KAAK,EAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAG/B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,iFAKD,kBAAgCoB,WAAwB;QAAA;QAAA;UAAA;YAAA;cAAA;gBAChDJ,MAAM;kBAAA,sEAAG,kBAAOC,KAAa,EAAEC,MAAc;oBAAA;sBAAA;wBAAA;0BAAA;4BAAA,kCAAKG,MAAM,CAACC,IAAI,CAACF,WAAW,EAAEH,KAAK,EAAEC,MAAM,CAAC;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA;kBAAA,gBAAzFF,MAAM;oBAAA;kBAAA;gBAAA;gBACNO,OAAO;kBAAA,sEAAG;oBAAA;sBAAA;wBAAA;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA,CAAc;kBAAA,gBAAxBA,OAAO;oBAAA;kBAAA;gBAAA;gBACPC,IAAI,GAAGJ,WAAW,CAACQ,UAAU;gBAC7BrC,cAAc,GAAG,IAAIkC,4CAAqB,CAACT,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;gBAAA;gBAAA;gBAAA,OAE/DjC,cAAc,CAACmC,UAAU,EAAE;cAAA;gBAAA;gBAAA,OACVnC,cAAc,CAACoC,UAAU,EAAE;cAAA;gBAA5CrC,QAAQ;gBAAA,kCACP,IAAID,aAAa,CAACC,QAAQ,EAAEC,cAAc,CAAC;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,OAE5CA,cAAc,CAACS,KAAK,EAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAG/B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,4EAED,kBAA2B6B,MAAc;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACVJ,4CAAqB,CAACK,UAAU,CAACD,MAAM,CAAC;cAAA;gBAA/DtC,cAAc;gBAAA;gBAAA;gBAAA,OAEZA,cAAc,CAACmC,UAAU,EAAE;cAAA;gBAAA;gBAAA,OACVnC,cAAc,CAACoC,UAAU,EAAE;cAAA;gBAA5CrC,QAAQ;gBAAA,kCACP,IAAID,aAAa,CAAIC,QAAQ,EAAEC,cAAc,CAAC;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,OAE/CA,cAAc,CAACS,KAAK,EAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAG/B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;EAAA;AAAA;AAAA"}

package/dist/es5/parquetjs/schema/declare.js

@@ -1,17 +1,14 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetBuffer = void 0;
-
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
 var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var ParquetBuffer = function ParquetBuffer() {
+var ParquetBuffer = (0, _createClass2.default)(function ParquetBuffer() {
   var rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
   var columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
   (0, _classCallCheck2.default)(this, ParquetBuffer);
@@ -19,7 +16,6 @@ var ParquetBuffer = function ParquetBuffer() {
   (0, _defineProperty2.default)(this, "columnData", void 0);
   this.rowCount = rowCount;
   this.columnData = columnData;
-};
-
+});
 exports.ParquetBuffer = ParquetBuffer;
 //# sourceMappingURL=declare.js.map

package/dist/es5/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/schema/declare.ts"],"names":["ParquetBuffer","rowCount","columnData"],"mappings":";;;;;;;;;;;;;IAmIaA,a,GAGX,yBAAgF;AAAA,MAApEC,QAAoE,uEAAjD,CAAiD;AAAA,MAA9CC,UAA8C,uEAAJ,EAAI;AAAA;AAAA;AAAA;AAC9E,OAAKD,QAAL,GAAgBA,QAAhB;AACA,OAAKC,UAAL,GAAkBA,UAAlB;AACD,C","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  dictionary?: ParquetDictionary;\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"file":"declare.js"}
+{"version":3,"file":"declare.js","names":["ParquetBuffer","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  dictionary?: ParquetDictionary;\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"mappings":";;;;;;;;;;IAmIaA,aAAa,8BAGxB,yBAAgF;EAAA,IAApEC,QAAgB,uEAAG,CAAC;EAAA,IAAEC,UAAuC,uEAAG,CAAC,CAAC;EAAA;EAAA;EAAA;EAC5E,IAAI,CAACD,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACC,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAA"}

package/dist/es5/parquetjs/schema/schema.js

@@ -1,26 +1,17 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetSchema = void 0;
-
 var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
 var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
 var _codecs = require("../codecs");
-
 var _compression = require("../compression");
-
 var _shred = require("./shred");
-
 var _types = require("./types");
-
 var ParquetSchema = function () {
   function ParquetSchema(schema) {
     (0, _classCallCheck2.default)(this, ParquetSchema);
@@ -34,7 +25,8 @@ var ParquetSchema = function () {
 
   (0, _createClass2.default)(ParquetSchema, [{
     key: "findField",
-    value: function findField(path) {
+    value:
+    function findField(path) {
       if (typeof path === 'string') {
         path = path.split(',');
       } else {
@@ -42,31 +34,27 @@ var ParquetSchema = function () {
       }
 
       var n = this.fields;
-
       for (; path.length > 1; path.shift()) {
         n = n[path[0]].fields;
       }
-
       return n[path[0]];
     }
+
   }, {
     key: "findFieldBranch",
-    value: function findFieldBranch(path) {
+    value:
+    function findFieldBranch(path) {
       if (typeof path === 'string') {
         path = path.split(',');
       }
-
       var branch = [];
       var n = this.fields;
-
       for (; path.length > 0; path.shift()) {
         branch.push(n[path[0]]);
-
         if (path.length > 1) {
           n = n[path[0]].fields;
         }
       }
-
       return branch;
     }
   }, {
@@ -94,13 +82,10 @@ var ParquetSchema = function () {
   }]);
   return ParquetSchema;
 }();
-
 exports.ParquetSchema = ParquetSchema;
-
 function setCompress(schema, type) {
   for (var name in schema) {
     var node = schema[name];
-
     if (node.fields) {
       setCompress(node.fields, type);
     } else {
@@ -111,20 +96,18 @@ function setCompress(schema, type) {
 
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
   var fieldList = {};
-
   for (var name in schema) {
     var opts = schema[name];
+
     var required = !opts.optional;
     var repeated = Boolean(opts.repeated);
     var rLevelMax = rLevelParentMax;
     var dLevelMax = dLevelParentMax;
     var repetitionType = 'REQUIRED';
-
     if (!required) {
       repetitionType = 'OPTIONAL';
       dLevelMax++;
     }
-
     if (repeated) {
       repetitionType = 'REPEATED';
       rLevelMax++;
@@ -133,7 +116,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
 
     if (opts.fields) {
       var _cpath = path.concat([name]);
-
       fieldList[name] = {
         name: name,
         path: _cpath,
@@ -149,19 +131,14 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
     }
 
     var typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
-
     if (!typeDef) {
       throw new Error("invalid parquet type: ".concat(opts.type));
     }
-
     opts.encoding = opts.encoding || 'PLAIN';
-
     if (!(opts.encoding in _codecs.PARQUET_CODECS)) {
       throw new Error("unsupported parquet encoding: ".concat(opts.encoding));
     }
-
     opts.compression = opts.compression || 'UNCOMPRESSED';
-
     if (!(opts.compression in _compression.PARQUET_COMPRESSION_METHODS)) {
       throw new Error("unsupported compression method: ".concat(opts.compression));
     }
@@ -183,21 +160,16 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       dLevelMax: dLevelMax
     };
   }
-
   return fieldList;
 }
-
 function listFields(fields) {
   var list = [];
-
   for (var k in fields) {
     list.push(fields[k]);
-
     if (fields[k].isNested) {
       list = list.concat(listFields(fields[k].fields));
     }
   }
-
   return list;
 }
 //# sourceMappingURL=schema.js.map

package/dist/es5/parquetjs/schema/schema.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/schema/schema.ts"],"names":["ParquetSchema","schema","fields","buildFields","fieldList","listFields","path","split","slice","n","length","shift","branch","push","record","buffer","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"mappings":";;;;;;;;;;;;;;;AAEA;;AACA;;AAUA;;AACA;;IAKaA,a;AAQX,yBAAYC,MAAZ,EAAsC;AAAA;AAAA;AAAA;AAAA;AACpC,SAAKA,MAAL,GAAcA,MAAd;AACA,SAAKC,MAAL,GAAcC,WAAW,CAACF,MAAD,EAAS,CAAT,EAAY,CAAZ,EAAe,EAAf,CAAzB;AACA,SAAKG,SAAL,GAAiBC,UAAU,CAAC,KAAKH,MAAN,CAA3B;AACD;;;;WAKD,mBAAUI,IAAV,EAAiD;AAC/C,UAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,QAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD,OAHD,MAGO;AAELD,QAAAA,IAAI,GAAGA,IAAI,CAACE,KAAL,CAAW,CAAX,CAAP;AACD;;AAED,UAAIC,CAAC,GAAG,KAAKP,MAAb;;AACA,aAAOI,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCF,QAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWJ,MAAf;AACD;;AAED,aAAOO,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAR;AACD;;;WAKD,yBAAgBA,IAAhB,EAAyD;AACvD,UAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,QAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD;;AACD,UAAMK,MAAsB,GAAG,EAA/B;AACA,UAAIH,CAAC,GAAG,KAAKP,MAAb;;AACA,aAAOI,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCC,QAAAA,MAAM,CAACC,IAAP,CAAYJ,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAb;;AACA,YAAIA,IAAI,CAACI,MAAL,GAAc,CAAlB,EAAqB;AACnBD,UAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWJ,MAAf;AACD;AACF;;AACD,aAAOU,MAAP;AACD;;;WAED,qBAAYE,MAAZ,EAAmCC,MAAnC,EAAgE;AAC9D,8BAAY,IAAZ,EAAkBD,MAAlB,EAA0BC,MAA1B;AACD;;;WAED,4BAAmBA,MAAnB,EAA2D;AACzD,aAAO,+BAAmB,IAAnB,EAAyBA,MAAzB,CAAP;AACD;;;WAED,kBAASC,IAAT,EAAyC;AACvCC,MAAAA,WAAW,CAAC,KAAKhB,MAAN,EAAce,IAAd,CAAX;AACAC,MAAAA,WAAW,CAAC,KAAKf,MAAN,EAAcc,IAAd,CAAX;AACA,aAAO,IAAP;AACD;;;WAED,kBAAwB;AACtB,aAAO,wBAAY,IAAZ,CAAP;AACD;;;;;;;AAGH,SAASC,WAAT,CAAqBhB,MAArB,EAAkCe,IAAlC,EAA4D;AAC1D,OAAK,IAAME,IAAX,IAAmBjB,MAAnB,EAA2B;AACzB,QAAMkB,IAAI,GAAGlB,MAAM,CAACiB,IAAD,CAAnB;;AACA,QAAIC,IAAI,CAACjB,MAAT,EAAiB;AACfe,MAAAA,WAAW,CAACE,IAAI,CAACjB,MAAN,EAAcc,IAAd,CAAX;AACD,KAFD,MAEO;AACLG,MAAAA,IAAI,CAACC,WAAL,GAAmBJ,IAAnB;AACD;AACF;AACF;;AAGD,SAASb,WAAT,CACEF,MADF,EAEEoB,eAFF,EAGEC,eAHF,EAIEhB,IAJF,EAKgC;AAC9B,MAAMF,SAAuC,GAAG,EAAhD;;AAEA,OAAK,IAAMc,IAAX,IAAmBjB,MAAnB,EAA2B;AACzB,QAAMsB,IAAI,GAAGtB,MAAM,CAACiB,IAAD,CAAnB;AAGA,QAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAvB;AACA,QAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAN,CAAxB;AACA,QAAIE,SAAS,GAAGP,eAAhB;AACA,QAAIQ,SAAS,GAAGP,eAAhB;AAEA,QAAIQ,cAA8B,GAAG,UAArC;;AACA,QAAI,CAACN,QAAL,EAAe;AACbM,MAAAA,cAAc,GAAG,UAAjB;AACAD,MAAAA,SAAS;AACV;;AACD,QAAIH,QAAJ,EAAc;AACZI,MAAAA,cAAc,GAAG,UAAjB;AACAF,MAAAA,SAAS;AACT,UAAIJ,QAAJ,EAAcK,SAAS;AACxB;;AAGD,QAAIN,IAAI,CAACrB,MAAT,EAAiB;AACf,UAAM6B,MAAK,GAAGzB,IAAI,CAAC0B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;;AACAd,MAAAA,SAAS,CAACc,IAAD,CAAT,GAAkB;AAChBA,QAAAA,IAAI,EAAJA,IADgB;AAEhBZ,QAAAA,IAAI,EAAEyB,MAFU;AAGhBE,QAAAA,GAAG,EAAEF,MAAK,CAACG,IAAN,EAHW;AAIhBJ,QAAAA,cAAc,EAAdA,cAJgB;AAKhBF,QAAAA,SAAS,EAATA,SALgB;AAMhBC,QAAAA,SAAS,EAATA,SANgB;AAOhBM,QAAAA,QAAQ,EAAE,IAPM;AAQhBC,QAAAA,UAAU,EAAEC,MAAM,CAACC,IAAP,CAAYf,IAAI,CAACrB,MAAjB,EAAyBQ,MARrB;AAShBR,QAAAA,MAAM,EAAEC,WAAW,CAACoB,IAAI,CAACrB,MAAN,EAAc0B,SAAd,EAAyBC,SAAzB,EAAoCE,MAApC;AATH,OAAlB;AAWA;AACD;;AAED,QAAMQ,OAAY,GAAGC,6BAAsBjB,IAAI,CAACP,IAA3B,CAArB;;AACA,QAAI,CAACuB,OAAL,EAAc;AACZ,YAAM,IAAIE,KAAJ,iCAAmClB,IAAI,CAACP,IAAxC,EAAN;AACD;;AAEDO,IAAAA,IAAI,CAACmB,QAAL,GAAgBnB,IAAI,CAACmB,QAAL,IAAiB,OAAjC;;AACA,QAAI,EAAEnB,IAAI,CAACmB,QAAL,IAAiBC,sBAAnB,CAAJ,EAAwC;AACtC,YAAM,IAAIF,KAAJ,yCAA2ClB,IAAI,CAACmB,QAAhD,EAAN;AACD;;AAEDnB,IAAAA,IAAI,CAACH,WAAL,GAAmBG,IAAI,CAACH,WAAL,IAAoB,cAAvC;;AACA,QAAI,EAAEG,IAAI,CAACH,WAAL,IAAoBwB,wCAAtB,CAAJ,EAAwD;AACtD,YAAM,IAAIH,KAAJ,2CAA6ClB,IAAI,CAACH,WAAlD,EAAN;AACD;;AAGD,QAAMW,KAAK,GAAGzB,IAAI,CAAC0B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAd,IAAAA,SAAS,CAACc,IAAD,CAAT,GAAkB;AAChBA,MAAAA,IAAI,EAAJA,IADgB;AAEhB2B,MAAAA,aAAa,EAAEN,OAAO,CAACM,aAFP;AAGhBC,MAAAA,YAAY,EAAEP,OAAO,CAACO,YAHN;AAIhBxC,MAAAA,IAAI,EAAEyB,KAJU;AAKhBE,MAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EALW;AAMhBJ,MAAAA,cAAc,EAAdA,cANgB;AAOhBY,MAAAA,QAAQ,EAAEnB,IAAI,CAACmB,QAPC;AAQhBtB,MAAAA,WAAW,EAAEG,IAAI,CAACH,WARF;AAShB2B,MAAAA,UAAU,EAAExB,IAAI,CAACwB,UAAL,IAAmBR,OAAO,CAACQ,UATvB;AAUhBC,MAAAA,SAAS,EAAEzB,IAAI,CAACyB,SAVA;AAWhBC,MAAAA,KAAK,EAAE1B,IAAI,CAAC0B,KAXI;AAYhBrB,MAAAA,SAAS,EAATA,SAZgB;AAahBC,MAAAA,SAAS,EAATA;AAbgB,KAAlB;AAeD;;AACD,SAAOzB,SAAP;AACD;;AAED,SAASC,UAAT,CAAoBH,MAApB,EAA0E;AACxE,MAAIgD,IAAoB,GAAG,EAA3B;;AACA,OAAK,IAAMC,CAAX,IAAgBjD,MAAhB,EAAwB;AACtBgD,IAAAA,IAAI,CAACrC,IAAL,CAAUX,MAAM,CAACiD,CAAD,CAAhB;;AACA,QAAIjD,MAAM,CAACiD,CAAD,CAAN,CAAUhB,QAAd,EAAwB;AACtBe,MAAAA,IAAI,GAAGA,IAAI,CAAClB,MAAL,CAAY3B,UAAU,CAACH,MAAM,CAACiD,CAAD,CAAN,CAAUjD,MAAX,CAAtB,CAAP;AACD;AACF;;AACD,SAAOgD,IAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"file":"schema.js"}
+{"version":3,"file":"schema.js","names":["ParquetSchema","schema","fields","buildFields","fieldList","listFields","path","split","slice","n","length","shift","branch","push","record","buffer","shredRecord","materializeRecords","type","setCompress","shredBuffer","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";;;;;;;;;;AAEA;AACA;AAUA;AACA;AAA8C,IAKjCA,aAAa;EAQxB,uBAAYC,MAAwB,EAAE;IAAA;IAAA;IAAA;IAAA;IACpC,IAAI,CAACA,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,MAAM,GAAGC,WAAW,CAACF,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACG,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;;EAAC;IAAA;IAAA;IAKD,mBAAUI,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACP,MAAM;MACnB,OAAOI,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACJ,MAAsC;MACvD;MAEA,OAAOO,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;;EAAC;IAAA;IAAA;IAKD,yBAAgBA,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMK,MAAsB,GAAG,EAAE;MACjC,IAAIH,CAAC,GAAG,IAAI,CAACP,MAAM;MACnB,OAAOI,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;QACpCC,MAAM,CAACC,IAAI,CAACJ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACJ,MAAsC;QACvD;MACF;MACA,OAAOU,MAAM;IACf;EAAC;IAAA;IAAA,OAED,qBAAYE,MAAqB,EAAEC,MAAqB,EAAQ;MAC9D,IAAAC,kBAAW,EAAC,IAAI,EAAEF,MAAM,EAAEC,MAAM,CAAC;IACnC;EAAC;IAAA;IAAA,OAED,4BAAmBA,MAAqB,EAAmB;MACzD,OAAO,IAAAE,yBAAkB,EAAC,IAAI,EAAEF,MAAM,CAAC;IACzC;EAAC;IAAA;IAAA,OAED,kBAASG,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEiB,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACjB,MAAM,EAAEgB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAA;IAAA,OAED,kBAAwB;MACtB,OAAO,IAAAE,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA;AAAA;AAAA;AAGH,SAASD,WAAW,CAAClB,MAAW,EAAEiB,IAAwB,EAAE;EAC1D,KAAK,IAAMG,IAAI,IAAIpB,MAAM,EAAE;IACzB,IAAMqB,IAAI,GAAGrB,MAAM,CAACoB,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACpB,MAAM,EAAE;MACfiB,WAAW,CAACG,IAAI,CAACpB,MAAM,EAAEgB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLI,IAAI,CAACC,WAAW,GAAGL,IAAI;IACzB;EACF;AACF;;AAGA,SAASf,WAAW,CAClBF,MAAwB,EACxBuB,eAAuB,EACvBC,eAAuB,EACvBnB,IAAc,EACgB;EAC9B,IAAMF,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMiB,IAAI,IAAIpB,MAAM,EAAE;IACzB,IAAMyB,IAAI,GAAGzB,MAAM,CAACoB,IAAI,CAAC;;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;;IAGA,IAAIN,IAAI,CAACxB,MAAM,EAAE;MACf,IAAMgC,MAAK,GAAG5B,IAAI,CAAC6B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJf,IAAI,EAAE4B,MAAK;QACXE,GAAG,EAAEF,MAAK,CAACG,IAAI,EAAE;QACjBJ,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAACxB,MAAM,CAAC,CAACQ,MAAM;QAC3CR,MAAM,EAAEC,WAAW,CAACuB,IAAI,CAACxB,MAAM,EAAE6B,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;;IAEA,IAAMQ,OAAY,GAAGC,4BAAqB,CAACjB,IAAI,CAACR,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,iCAA0BlB,IAAI,CAACR,IAAI,EAAG;IACvD;IAEAQ,IAAI,CAACmB,QAAQ,GAAGnB,IAAI,CAACmB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAEnB,IAAI,CAACmB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,yCAAkClB,IAAI,CAACmB,QAAQ,EAAG;IACnE;IAEAnB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIwB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,2CAAoClB,IAAI,CAACH,WAAW,EAAG;IACxE;;IAGA,IAAMW,KAAK,GAAG5B,IAAI,CAAC6B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ2B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC3C,IAAI,EAAE4B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,EAAE;MACjBJ,cAAc,EAAdA,cAAc;MACdY,QAAQ,EAAEnB,IAAI,CAACmB,QAAQ;MACvBtB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B2B,UAAU,EAAExB,IAAI,CAACwB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAEzB,IAAI,CAACyB,SAAS;MACzBC,KAAK,EAAE1B,IAAI,CAAC0B,KAAK;MACjBrB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAO5B,SAAS;AAClB;AAEA,SAASC,UAAU,CAACH,MAAoC,EAAkB;EACxE,IAAImD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAIpD,MAAM,EAAE;IACtBmD,IAAI,CAACxC,IAAI,CAACX,MAAM,CAACoD,CAAC,CAAC,CAAC;IACpB,IAAIpD,MAAM,CAACoD,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAAClB,MAAM,CAAC9B,UAAU,CAACH,MAAM,CAACoD,CAAC,CAAC,CAACpD,MAAM,CAAE,CAAC;IACnD;EACF;EACA,OAAOmD,IAAI;AACb"}