@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/es5/parquetjs/parser/parquet-envelope-reader.js

@@ -7,41 +7,28 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ParquetEnvelopeReader = void 0;
 
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
 var _constants = require("../../constants");
 
 var _parquetThrift = require("../parquet-thrift");
 
-var _fileUtils = require("../utils/file-utils");
-
 var _readUtils = require("../utils/read-utils");
 
 var _decoders = require("./decoders");
 
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
+const DEFAULT_DICTIONARY_SIZE = 1e6;
 
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+class ParquetEnvelopeReader {
+  static async openBuffer(buffer) {
+    const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
 
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+    const closeFn = () => Promise.resolve();
 
-var DEFAULT_DICTIONARY_SIZE = 1e6;
+    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
+  }
 
-var ParquetEnvelopeReader = function () {
-  function ParquetEnvelopeReader(read, close, fileSize, options) {
-    (0, _classCallCheck2.default)(this, ParquetEnvelopeReader);
+  constructor(read, close, fileSize, options) {
     (0, _defineProperty2.default)(this, "read", void 0);
     (0, _defineProperty2.default)(this, "close", void 0);
     (0, _defineProperty2.default)(this, "fileSize", void 0);
@@ -52,401 +39,130 @@ var ParquetEnvelopeReader = function () {
     this.defaultDictionarySize = (options === null || options === void 0 ? void 0 : options.defaultDictionarySize) || DEFAULT_DICTIONARY_SIZE;
   }
 
-  (0, _createClass2.default)(ParquetEnvelopeReader, [{
-    key: "readHeader",
-    value: function () {
-      var _readHeader = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee() {
-        var buffer, magic;
-        return _regenerator.default.wrap(function _callee$(_context) {
-          while (1) {
-            switch (_context.prev = _context.next) {
-              case 0:
-                _context.next = 2;
-                return this.read(0, _constants.PARQUET_MAGIC.length);
-
-              case 2:
-                buffer = _context.sent;
-                magic = buffer.toString();
-                _context.t0 = magic;
-                _context.next = _context.t0 === _constants.PARQUET_MAGIC ? 7 : _context.t0 === _constants.PARQUET_MAGIC_ENCRYPTED ? 8 : 9;
-                break;
-
-              case 7:
-                return _context.abrupt("break", 10);
-
-              case 8:
-                throw new Error('Encrypted parquet file not supported');
-
-              case 9:
-                throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
-
-              case 10:
-              case "end":
-                return _context.stop();
-            }
-          }
-        }, _callee, this);
-      }));
-
-      function readHeader() {
-        return _readHeader.apply(this, arguments);
-      }
+  async readHeader() {
+    const buffer = await this.read(0, _constants.PARQUET_MAGIC.length);
+    const magic = buffer.toString();
 
-      return readHeader;
-    }()
-  }, {
-    key: "readRowGroup",
-    value: function () {
-      var _readRowGroup = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(schema, rowGroup, columnList) {
-        var buffer, _iterator, _step, colChunk, colMetadata, colKey;
-
-        return _regenerator.default.wrap(function _callee2$(_context2) {
-          while (1) {
-            switch (_context2.prev = _context2.next) {
-              case 0:
-                buffer = {
-                  rowCount: Number(rowGroup.num_rows),
-                  columnData: {}
-                };
-                _iterator = _createForOfIteratorHelper(rowGroup.columns);
-                _context2.prev = 2;
-
-                _iterator.s();
-
-              case 4:
-                if ((_step = _iterator.n()).done) {
-                  _context2.next = 15;
-                  break;
-                }
-
-                colChunk = _step.value;
-                colMetadata = colChunk.meta_data;
-                colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
-
-                if (!(columnList.length > 0 && (0, _readUtils.fieldIndexOf)(columnList, colKey) < 0)) {
-                  _context2.next = 10;
-                  break;
-                }
-
-                return _context2.abrupt("continue", 13);
-
-              case 10:
-                _context2.next = 12;
-                return this.readColumnChunk(schema, colChunk);
-
-              case 12:
-                buffer.columnData[colKey.join()] = _context2.sent;
-
-              case 13:
-                _context2.next = 4;
-                break;
-
-              case 15:
-                _context2.next = 20;
-                break;
-
-              case 17:
-                _context2.prev = 17;
-                _context2.t0 = _context2["catch"](2);
-
-                _iterator.e(_context2.t0);
-
-              case 20:
-                _context2.prev = 20;
-
-                _iterator.f();
-
-                return _context2.finish(20);
-
-              case 23:
-                return _context2.abrupt("return", buffer);
-
-              case 24:
-              case "end":
-                return _context2.stop();
-            }
-          }
-        }, _callee2, this, [[2, 17, 20, 23]]);
-      }));
-
-      function readRowGroup(_x, _x2, _x3) {
-        return _readRowGroup.apply(this, arguments);
-      }
+    switch (magic) {
+      case _constants.PARQUET_MAGIC:
+        break;
 
-      return readRowGroup;
-    }()
-  }, {
-    key: "readColumnChunk",
-    value: function () {
-      var _readColumnChunk = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(schema, colChunk) {
-        var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
-
-        var field, type, compression, pagesOffset, pagesSize, _colChunk$meta_data6, options, dictionary, dictionaryPageOffset, dictionaryOffset, pagesBuf;
-
-        return _regenerator.default.wrap(function _callee3$(_context3) {
-          while (1) {
-            switch (_context3.prev = _context3.next) {
-              case 0:
-                if (!(colChunk.file_path !== undefined && colChunk.file_path !== null)) {
-                  _context3.next = 2;
-                  break;
-                }
-
-                throw new Error('external references are not supported');
-
-              case 2:
-                field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
-                type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
-
-                if (!(type !== field.primitiveType)) {
-                  _context3.next = 6;
-                  break;
-                }
-
-                throw new Error("chunk type not matching schema: ".concat(type));
-
-              case 6:
-                compression = (0, _readUtils.getThriftEnum)(_parquetThrift.CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
-                pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
-                pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
-
-                if (!colChunk.file_path) {
-                  pagesSize = Math.min(this.fileSize - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
-                }
-
-                options = {
-                  type: type,
-                  rLevelMax: field.rLevelMax,
-                  dLevelMax: field.dLevelMax,
-                  compression: compression,
-                  column: field,
-                  numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
-                  dictionary: []
-                };
-                dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
-
-                if (!dictionaryPageOffset) {
-                  _context3.next = 17;
-                  break;
-                }
-
-                dictionaryOffset = Number(dictionaryPageOffset);
-                _context3.next = 16;
-                return this.getDictionary(dictionaryOffset, options, pagesOffset);
-
-              case 16:
-                dictionary = _context3.sent;
-
-              case 17:
-                dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
-                _context3.next = 20;
-                return this.read(pagesOffset, pagesSize);
-
-              case 20:
-                pagesBuf = _context3.sent;
-                _context3.next = 23;
-                return (0, _decoders.decodeDataPages)(pagesBuf, _objectSpread(_objectSpread({}, options), {}, {
-                  dictionary: dictionary
-                }));
-
-              case 23:
-                return _context3.abrupt("return", _context3.sent);
-
-              case 24:
-              case "end":
-                return _context3.stop();
-            }
-          }
-        }, _callee3, this);
-      }));
-
-      function readColumnChunk(_x4, _x5) {
-        return _readColumnChunk.apply(this, arguments);
-      }
+      case _constants.PARQUET_MAGIC_ENCRYPTED:
+        throw new Error('Encrypted parquet file not supported');
 
-      return readColumnChunk;
-    }()
-  }, {
-    key: "getDictionary",
-    value: function () {
-      var _getDictionary = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4(dictionaryPageOffset, options, pagesOffset) {
-        var dictionarySize, pagesBuf, cursor, decodedPage;
-        return _regenerator.default.wrap(function _callee4$(_context4) {
-          while (1) {
-            switch (_context4.prev = _context4.next) {
-              case 0:
-                if (!(dictionaryPageOffset === 0)) {
-                  _context4.next = 2;
-                  break;
-                }
-
-                return _context4.abrupt("return", []);
-
-              case 2:
-                dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
-                _context4.next = 5;
-                return this.read(dictionaryPageOffset, dictionarySize);
-
-              case 5:
-                pagesBuf = _context4.sent;
-                cursor = {
-                  buffer: pagesBuf,
-                  offset: 0,
-                  size: pagesBuf.length
-                };
-                _context4.next = 9;
-                return (0, _decoders.decodePage)(cursor, options);
-
-              case 9:
-                decodedPage = _context4.sent;
-                return _context4.abrupt("return", decodedPage.dictionary);
-
-              case 11:
-              case "end":
-                return _context4.stop();
-            }
-          }
-        }, _callee4, this);
-      }));
-
-      function getDictionary(_x6, _x7, _x8) {
-        return _getDictionary.apply(this, arguments);
-      }
+      default:
+        throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
+    }
+  }
 
-      return getDictionary;
-    }()
-  }, {
-    key: "readFooter",
-    value: function () {
-      var _readFooter = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5() {
-        var trailerLen, trailerBuf, magic, metadataSize, metadataOffset, metadataBuf, _decodeFileMetadata, metadata;
-
-        return _regenerator.default.wrap(function _callee5$(_context5) {
-          while (1) {
-            switch (_context5.prev = _context5.next) {
-              case 0:
-                trailerLen = _constants.PARQUET_MAGIC.length + 4;
-                _context5.next = 3;
-                return this.read(this.fileSize - trailerLen, trailerLen);
-
-              case 3:
-                trailerBuf = _context5.sent;
-                magic = trailerBuf.slice(4).toString();
-
-                if (!(magic !== _constants.PARQUET_MAGIC)) {
-                  _context5.next = 7;
-                  break;
-                }
-
-                throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
-
-              case 7:
-                metadataSize = trailerBuf.readUInt32LE(0);
-                metadataOffset = this.fileSize - metadataSize - trailerLen;
-
-                if (!(metadataOffset < _constants.PARQUET_MAGIC.length)) {
-                  _context5.next = 11;
-                  break;
-                }
-
-                throw new Error("Invalid metadata size ".concat(metadataOffset));
-
-              case 11:
-                _context5.next = 13;
-                return this.read(metadataOffset, metadataSize);
-
-              case 13:
-                metadataBuf = _context5.sent;
-                _decodeFileMetadata = (0, _readUtils.decodeFileMetadata)(metadataBuf), metadata = _decodeFileMetadata.metadata;
-                return _context5.abrupt("return", metadata);
-
-              case 16:
-              case "end":
-                return _context5.stop();
-            }
-          }
-        }, _callee5, this);
-      }));
-
-      function readFooter() {
-        return _readFooter.apply(this, arguments);
-      }
+  async readRowGroup(schema, rowGroup, columnList) {
+    const buffer = {
+      rowCount: Number(rowGroup.num_rows),
+      columnData: {}
+    };
 
-      return readFooter;
-    }()
-  }], [{
-    key: "openFile",
-    value: function () {
-      var _openFile = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee6(filePath) {
-        var fileStat, fileDescriptor, readFn, closeFn;
-        return _regenerator.default.wrap(function _callee6$(_context6) {
-          while (1) {
-            switch (_context6.prev = _context6.next) {
-              case 0:
-                _context6.next = 2;
-                return (0, _fileUtils.fstat)(filePath);
-
-              case 2:
-                fileStat = _context6.sent;
-                _context6.next = 5;
-                return (0, _fileUtils.fopen)(filePath);
-
-              case 5:
-                fileDescriptor = _context6.sent;
-                readFn = _fileUtils.fread.bind(undefined, fileDescriptor);
-                closeFn = _fileUtils.fclose.bind(undefined, fileDescriptor);
-                return _context6.abrupt("return", new ParquetEnvelopeReader(readFn, closeFn, fileStat.size));
-
-              case 9:
-              case "end":
-                return _context6.stop();
-            }
-          }
-        }, _callee6);
-      }));
-
-      function openFile(_x9) {
-        return _openFile.apply(this, arguments);
-      }
+    for (const colChunk of rowGroup.columns) {
+      const colMetadata = colChunk.meta_data;
+      const colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
 
-      return openFile;
-    }()
-  }, {
-    key: "openBuffer",
-    value: function () {
-      var _openBuffer = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee7(buffer) {
-        var readFn, closeFn;
-        return _regenerator.default.wrap(function _callee7$(_context7) {
-          while (1) {
-            switch (_context7.prev = _context7.next) {
-              case 0:
-                readFn = function readFn(position, length) {
-                  return Promise.resolve(buffer.slice(position, position + length));
-                };
-
-                closeFn = function closeFn() {
-                  return Promise.resolve();
-                };
-
-                return _context7.abrupt("return", new ParquetEnvelopeReader(readFn, closeFn, buffer.length));
-
-              case 3:
-              case "end":
-                return _context7.stop();
-            }
-          }
-        }, _callee7);
-      }));
-
-      function openBuffer(_x10) {
-        return _openBuffer.apply(this, arguments);
+      if (columnList.length > 0 && (0, _readUtils.fieldIndexOf)(columnList, colKey) < 0) {
+        continue;
       }
 
-      return openBuffer;
-    }()
-  }]);
-  return ParquetEnvelopeReader;
-}();
+      buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
+    }
+
+    return buffer;
+  }
+
+  async readColumnChunk(schema, colChunk) {
+    var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
+
+    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
+      throw new Error('external references are not supported');
+    }
+
+    const field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
+    const type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
+
+    if (type !== field.primitiveType) {
+      throw new Error("chunk type not matching schema: ".concat(type));
+    }
+
+    const compression = (0, _readUtils.getThriftEnum)(_parquetThrift.CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
+    const pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
+    let pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
+
+    if (!colChunk.file_path) {
+      var _colChunk$meta_data6;
+
+      pagesSize = Math.min(this.fileSize - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
+    }
+
+    const options = {
+      type,
+      rLevelMax: field.rLevelMax,
+      dLevelMax: field.dLevelMax,
+      compression,
+      column: field,
+      numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
+      dictionary: []
+    };
+    let dictionary;
+    const dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
+
+    if (dictionaryPageOffset) {
+      const dictionaryOffset = Number(dictionaryPageOffset);
+      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+    }
+
+    dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
+    const pagesBuf = await this.read(pagesOffset, pagesSize);
+    return await (0, _decoders.decodeDataPages)(pagesBuf, { ...options,
+      dictionary
+    });
+  }
+
+  async getDictionary(dictionaryPageOffset, options, pagesOffset) {
+    if (dictionaryPageOffset === 0) {
+      return [];
+    }
+
+    const dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
+    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);
+    const cursor = {
+      buffer: pagesBuf,
+      offset: 0,
+      size: pagesBuf.length
+    };
+    const decodedPage = await (0, _decoders.decodePage)(cursor, options);
+    return decodedPage.dictionary;
+  }
+
+  async readFooter() {
+    const trailerLen = _constants.PARQUET_MAGIC.length + 4;
+    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
+    const magic = trailerBuf.slice(4).toString();
+
+    if (magic !== _constants.PARQUET_MAGIC) {
+      throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
+    }
+
+    const metadataSize = trailerBuf.readUInt32LE(0);
+    const metadataOffset = this.fileSize - metadataSize - trailerLen;
+
+    if (metadataOffset < _constants.PARQUET_MAGIC.length) {
+      throw new Error("Invalid metadata size ".concat(metadataOffset));
+    }
+
+    const metadataBuf = await this.read(metadataOffset, metadataSize);
+    const {
+      metadata
+    } = (0, _readUtils.decodeFileMetadata)(metadataBuf);
+    return metadata;
+  }
+
+}
 
 exports.ParquetEnvelopeReader = ParquetEnvelopeReader;
 //# sourceMappingURL=parquet-envelope-reader.js.map

package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":["DEFAULT_DICTIONARY_SIZE","ParquetEnvelopeReader","read","close","fileSize","options","defaultDictionarySize","PARQUET_MAGIC","length","buffer","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","schema","rowGroup","columnList","rowCount","Number","num_rows","columnData","columns","colChunk","colMetadata","meta_data","colKey","path_in_schema","readColumnChunk","join","file_path","undefined","field","findField","type","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","size","decodedPage","trailerLen","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","metadata","filePath","fileStat","fileDescriptor","readFn","fread","bind","closeFn","fclose","position","Promise","resolve"],"mappings":";;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AAQA;;AACA;;AACA;;;;;;;;;;;;AAEA,IAAMA,uBAAuB,GAAG,GAAhC;;IAQaC,qB;AA2BX,iCACEC,IADF,EAEEC,KAFF,EAGEC,QAHF,EAIEC,OAJF,EAKE;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,IAAL,GAAYA,IAAZ;AACA,SAAKC,KAAL,GAAaA,KAAb;AACA,SAAKC,QAAL,GAAgBA,QAAhB;AACA,SAAKE,qBAAL,GAA6B,CAAAD,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEC,qBAAT,KAAkCN,uBAA/D;AACD;;;;;kFAED;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACuB,KAAKE,IAAL,CAAU,CAAV,EAAaK,yBAAcC,MAA3B,CADvB;;AAAA;AACQC,gBAAAA,MADR;AAGQC,gBAAAA,KAHR,GAGgBD,MAAM,CAACE,QAAP,EAHhB;AAAA,8BAIUD,KAJV;AAAA,gDAKSH,wBALT,uBAOSK,kCAPT;AAAA;;AAAA;AAAA;;AAAA;AAAA,sBAQY,IAAIC,KAAJ,CAAU,sCAAV,CARZ;;AAAA;AAAA,sBAUY,IAAIA,KAAJ,uCAAyCH,KAAzC,OAVZ;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;oFAcA,kBACEI,MADF,EAEEC,QAFF,EAGEC,UAHF;AAAA;;AAAA;AAAA;AAAA;AAAA;AAKQP,gBAAAA,MALR,GAKgC;AAC5BQ,kBAAAA,QAAQ,EAAEC,MAAM,CAACH,QAAQ,CAACI,QAAV,CADY;AAE5BC,kBAAAA,UAAU,EAAE;AAFgB,iBALhC;AAAA,uDASyBL,QAAQ,CAACM,OATlC;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AASaC,gBAAAA,QATb;AAUUC,gBAAAA,WAVV,GAUwBD,QAAQ,CAACE,SAVjC;AAWUC,gBAAAA,MAXV,GAWmBF,WAXnB,aAWmBA,WAXnB,uBAWmBA,WAAW,CAAEG,cAXhC;;AAAA,sBAYQV,UAAU,CAACR,MAAX,GAAoB,CAApB,IAAyB,6BAAaQ,UAAb,EAAyBS,MAAzB,IAAoC,CAZrE;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;AAAA,uBAe8C,KAAKE,eAAL,CAAqBb,MAArB,EAA6BQ,QAA7B,CAf9C;;AAAA;AAeIb,gBAAAA,MAAM,CAACW,UAAP,CAAkBK,MAAM,CAAEG,IAAR,EAAlB,CAfJ;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAAA,kDAiBSnB,MAjBT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;uFAyBA,kBAAsBK,MAAtB,EAA6CQ,QAA7C;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA,sBACMA,QAAQ,CAACO,SAAT,KAAuBC,SAAvB,IAAoCR,QAAQ,CAACO,SAAT,KAAuB,IADjE;AAAA;AAAA;AAAA;;AAAA,sBAEU,IAAIhB,KAAJ,CAAU,uCAAV,CAFV;;AAAA;AAKQkB,gBAAAA,KALR,GAKgBjB,MAAM,CAACkB,SAAP,wBAAiBV,QAAQ,CAACE,SAA1B,wDAAiB,oBAAoBE,cAArC,CALhB;AAMQO,gBAAAA,IANR,GAM8B,8BAAcC,mBAAd,0BAAoBZ,QAAQ,CAACE,SAA7B,yDAAoB,qBAAoBS,IAAxC,CAN9B;;AAAA,sBAQMA,IAAI,KAAKF,KAAK,CAACI,aARrB;AAAA;AAAA;AAAA;;AAAA,sBASU,IAAItB,KAAJ,2CAA6CoB,IAA7C,EATV;;AAAA;AAYQG,gBAAAA,WAZR,GAY0C,8BACtCC,+BADsC,0BAEtCf,QAAQ,CAACE,SAF6B,yDAEtC,qBAAoBc,KAFkB,CAZ1C;AAiBQC,gBAAAA,WAjBR,GAiBsBrB,MAAM,yBAACI,QAAQ,CAACE,SAAV,yDAAC,qBAAoBgB,gBAArB,CAjB5B;AAkBMC,gBAAAA,SAlBN,GAkBkBvB,MAAM,yBAACI,QAAQ,CAACE,SAAV,yDAAC,qBAAoBkB,qBAArB,CAlBxB;;AAoBE,oBAAI,CAACpB,QAAQ,CAACO,SAAd,EAAyB;AACvBY,kBAAAA,SAAS,GAAGE,IAAI,CAACC,GAAL,CACV,KAAKxC,QAAL,GAAgBmC,WADN,EAEVrB,MAAM,yBAACI,QAAQ,CAACE,SAAV,yDAAC,qBAAoBkB,qBAArB,CAFI,CAAZ;AAID;;AAEKrC,gBAAAA,OA3BR,GA2BkC;AAC9B4B,kBAAAA,IAAI,EAAJA,IAD8B;AAE9BY,kBAAAA,SAAS,EAAEd,KAAK,CAACc,SAFa;AAG9BC,kBAAAA,SAAS,EAAEf,KAAK,CAACe,SAHa;AAI9BV,kBAAAA,WAAW,EAAXA,WAJ8B;AAK9BW,kBAAAA,MAAM,EAAEhB,KALsB;AAM9BiB,kBAAAA,SAAS,0BAAE1B,QAAQ,CAACE,SAAX,yDAAE,qBAAoByB,UAND;AAO9BC,kBAAAA,UAAU,EAAE;AAPkB,iBA3BlC;AAuCQC,gBAAAA,oBAvCR,GAuC+B7B,QAvC/B,aAuC+BA,QAvC/B,+CAuC+BA,QAAQ,CAAEE,SAvCzC,yDAuC+B,qBAAqB4B,sBAvCpD;;AAAA,qBAyCMD,oBAzCN;AAAA;AAAA;AAAA;;AA0CUE,gBAAAA,gBA1CV,GA0C6BnC,MAAM,CAACiC,oBAAD,CA1CnC;AAAA;AAAA,uBA4CuB,KAAKG,aAAL,CAAmBD,gBAAnB,EAAqChD,OAArC,EAA8CkC,WAA9C,CA5CvB;;AAAA;AA4CIW,gBAAAA,UA5CJ;;AAAA;AA+CEA,gBAAAA,UAAU,GAAG,uBAAA7C,OAAO,CAAC6C,UAAR,oEAAoB1C,MAApB,GAA6BH,OAAO,CAAC6C,UAArC,GAAkDA,UAA/D;AA/CF;AAAA,uBAgDyB,KAAKhD,IAAL,CAAUqC,WAAV,EAAuBE,SAAvB,CAhDzB;;AAAA;AAgDQc,gBAAAA,QAhDR;AAAA;AAAA,uBAiDe,+BAAgBA,QAAhB,kCAA8BlD,OAA9B;AAAuC6C,kBAAAA,UAAU,EAAVA;AAAvC,mBAjDf;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;qFA2DA,kBACEC,oBADF,EAEE9C,OAFF,EAGEkC,WAHF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBAKMY,oBAAoB,KAAK,CAL/B;AAAA;AAAA;AAAA;;AAAA,kDAaW,EAbX;;AAAA;AAgBQK,gBAAAA,cAhBR,GAgByBb,IAAI,CAACC,GAAL,CACrB,KAAKxC,QAAL,GAAgB+C,oBADK,EAErB,KAAK7C,qBAFgB,CAhBzB;AAAA;AAAA,uBAoByB,KAAKJ,IAAL,CAAUiD,oBAAV,EAAgCK,cAAhC,CApBzB;;AAAA;AAoBQD,gBAAAA,QApBR;AAsBQE,gBAAAA,MAtBR,GAsBiB;AAAChD,kBAAAA,MAAM,EAAE8C,QAAT;AAAmBG,kBAAAA,MAAM,EAAE,CAA3B;AAA8BC,kBAAAA,IAAI,EAAEJ,QAAQ,CAAC/C;AAA7C,iBAtBjB;AAAA;AAAA,uBAuB4B,0BAAWiD,MAAX,EAAmBpD,OAAnB,CAvB5B;;AAAA;AAuBQuD,gBAAAA,WAvBR;AAAA,kDAyBSA,WAAW,CAACV,UAzBrB;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFA4BA;AAAA;;AAAA;AAAA;AAAA;AAAA;AACQW,gBAAAA,UADR,GACqBtD,yBAAcC,MAAd,GAAuB,CAD5C;AAAA;AAAA,uBAE2B,KAAKN,IAAL,CAAU,KAAKE,QAAL,GAAgByD,UAA1B,EAAsCA,UAAtC,CAF3B;;AAAA;AAEQC,gBAAAA,UAFR;AAIQpD,gBAAAA,KAJR,GAIgBoD,UAAU,CAACC,KAAX,CAAiB,CAAjB,EAAoBpD,QAApB,EAJhB;;AAAA,sBAKMD,KAAK,KAAKH,wBALhB;AAAA;AAAA;AAAA;;AAAA,sBAMU,IAAIM,KAAJ,6CAA8CH,KAA9C,OANV;;AAAA;AASQsD,gBAAAA,YATR,GASuBF,UAAU,CAACG,YAAX,CAAwB,CAAxB,CATvB;AAUQC,gBAAAA,cAVR,GAUyB,KAAK9D,QAAL,GAAgB4D,YAAhB,GAA+BH,UAVxD;;AAAA,sBAWMK,cAAc,GAAG3D,yBAAcC,MAXrC;AAAA;AAAA;AAAA;;AAAA,sBAYU,IAAIK,KAAJ,iCAAmCqD,cAAnC,EAZV;;AAAA;AAAA;AAAA,uBAe4B,KAAKhE,IAAL,CAAUgE,cAAV,EAA0BF,YAA1B,CAf5B;;AAAA;AAeQG,gBAAAA,WAfR;AAAA,sCAkBqB,mCAAmBA,WAAnB,CAlBrB,EAkBSC,QAlBT,uBAkBSA,QAlBT;AAAA,kDAmBSA,QAnBT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;gFA3JA,kBAAsBC,QAAtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACyB,sBAAMA,QAAN,CADzB;;AAAA;AACQC,gBAAAA,QADR;AAAA;AAAA,uBAE+B,sBAAMD,QAAN,CAF/B;;AAAA;AAEQE,gBAAAA,cAFR;AAIQC,gBAAAA,MAJR,GAIiBC,iBAAMC,IAAN,CAAW5C,SAAX,EAAsByC,cAAtB,CAJjB;AAKQI,gBAAAA,OALR,GAKkBC,kBAAOF,IAAP,CAAY5C,SAAZ,EAAuByC,cAAvB,CALlB;AAAA,kDAOS,IAAItE,qBAAJ,CAA0BuE,MAA1B,EAAkCG,OAAlC,EAA2CL,QAAQ,CAACX,IAApD,CAPT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFAUA,kBAAwBlD,MAAxB;AAAA;AAAA;AAAA;AAAA;AAAA;AACQ+D,gBAAAA,MADR,GACiB,SAATA,MAAS,CAACK,QAAD,EAAmBrE,MAAnB;AAAA,yBACbsE,OAAO,CAACC,OAAR,CAAgBtE,MAAM,CAACsD,KAAP,CAAac,QAAb,EAAuBA,QAAQ,GAAGrE,MAAlC,CAAhB,CADa;AAAA,iBADjB;;AAGQmE,gBAAAA,OAHR,GAGkB,SAAVA,OAAU;AAAA,yBAAMG,OAAO,CAACC,OAAR,EAAN;AAAA,iBAHlB;;AAAA,kDAIS,IAAI9E,qBAAJ,CAA0BuE,MAA1B,EAAkCG,OAAlC,EAA2ClE,MAAM,CAACD,MAAlD,CAJT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetSchema} from '../schema/schema';\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {fstat, fopen, fread, fclose} from '../utils/file-utils';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nconst DEFAULT_DICTIONARY_SIZE = 1e6;\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n  public defaultDictionarySize: number;\n\n  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {\n    const fileStat = await fstat(filePath);\n    const fileDescriptor = await fopen(filePath);\n\n    const readFn = fread.bind(undefined, fileDescriptor);\n    const closeFn = fclose.bind(undefined, fileDescriptor);\n\n    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);\n  }\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number,\n    options?: any\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;\n  }\n\n  async readHeader(): Promise<void> {\n    const buffer = await this.read(0, PARQUET_MAGIC.length);\n\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Do reading of parquet file's column chunk\n   * @param schema\n   * @param colChunk\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.fileSize - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.fileSize - dictionaryPageOffset,\n      this.defaultDictionarySize\n    );\n    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n"],"file":"parquet-envelope-reader.js"}
+{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":["DEFAULT_DICTIONARY_SIZE","ParquetEnvelopeReader","openBuffer","buffer","readFn","position","length","Promise","resolve","slice","closeFn","constructor","read","close","fileSize","options","defaultDictionarySize","readHeader","PARQUET_MAGIC","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","readRowGroup","schema","rowGroup","columnList","rowCount","Number","num_rows","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","undefined","field","findField","type","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","size","decodedPage","readFooter","trailerLen","trailerBuf","metadataSize","readUInt32LE","metadataOffset","metadataBuf","metadata"],"mappings":";;;;;;;;;;;AAEA;;AACA;;AAQA;;AACA;;AAEA,MAAMA,uBAAuB,GAAG,GAAhC;;AAQO,MAAMC,qBAAN,CAA4B;AAUV,eAAVC,UAAU,CAACC,MAAD,EAAiD;AACtE,UAAMC,MAAM,GAAG,CAACC,QAAD,EAAmBC,MAAnB,KACbC,OAAO,CAACC,OAAR,CAAgBL,MAAM,CAACM,KAAP,CAAaJ,QAAb,EAAuBA,QAAQ,GAAGC,MAAlC,CAAhB,CADF;;AAEA,UAAMI,OAAO,GAAG,MAAMH,OAAO,CAACC,OAAR,EAAtB;;AACA,WAAO,IAAIP,qBAAJ,CAA0BG,MAA1B,EAAkCM,OAAlC,EAA2CP,MAAM,CAACG,MAAlD,CAAP;AACD;;AAEDK,EAAAA,WAAW,CACTC,IADS,EAETC,KAFS,EAGTC,QAHS,EAITC,OAJS,EAKT;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,IAAL,GAAYA,IAAZ;AACA,SAAKC,KAAL,GAAaA,KAAb;AACA,SAAKC,QAAL,GAAgBA,QAAhB;AACA,SAAKE,qBAAL,GAA6B,CAAAD,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEC,qBAAT,KAAkChB,uBAA/D;AACD;;AAEe,QAAViB,UAAU,GAAkB;AAChC,UAAMd,MAAM,GAAG,MAAM,KAAKS,IAAL,CAAU,CAAV,EAAaM,yBAAcZ,MAA3B,CAArB;AAEA,UAAMa,KAAK,GAAGhB,MAAM,CAACiB,QAAP,EAAd;;AACA,YAAQD,KAAR;AACE,WAAKD,wBAAL;AACE;;AACF,WAAKG,kCAAL;AACE,cAAM,IAAIC,KAAJ,CAAU,sCAAV,CAAN;;AACF;AACE,cAAM,IAAIA,KAAJ,uCAAyCH,KAAzC,OAAN;AANJ;AAQD;;AAEiB,QAAZI,YAAY,CAChBC,MADgB,EAEhBC,QAFgB,EAGhBC,UAHgB,EAIQ;AACxB,UAAMvB,MAAqB,GAAG;AAC5BwB,MAAAA,QAAQ,EAAEC,MAAM,CAACH,QAAQ,CAACI,QAAV,CADY;AAE5BC,MAAAA,UAAU,EAAE;AAFgB,KAA9B;;AAIA,SAAK,MAAMC,QAAX,IAAuBN,QAAQ,CAACO,OAAhC,EAAyC;AACvC,YAAMC,WAAW,GAAGF,QAAQ,CAACG,SAA7B;AACA,YAAMC,MAAM,GAAGF,WAAH,aAAGA,WAAH,uBAAGA,WAAW,CAAEG,cAA5B;;AACA,UAAIV,UAAU,CAACpB,MAAX,GAAoB,CAApB,IAAyB,6BAAaoB,UAAb,EAAyBS,MAAzB,IAAoC,CAAjE,EAAoE;AAClE;AACD;;AACDhC,MAAAA,MAAM,CAAC2B,UAAP,CAAkBK,MAAM,CAAEE,IAAR,EAAlB,IAAoC,MAAM,KAAKC,eAAL,CAAqBd,MAArB,EAA6BO,QAA7B,CAA1C;AACD;;AACD,WAAO5B,MAAP;AACD;;AAOoB,QAAfmC,eAAe,CAACd,MAAD,EAAwBO,QAAxB,EAAqE;AAAA;;AACxF,QAAIA,QAAQ,CAACQ,SAAT,KAAuBC,SAAvB,IAAoCT,QAAQ,CAACQ,SAAT,KAAuB,IAA/D,EAAqE;AACnE,YAAM,IAAIjB,KAAJ,CAAU,uCAAV,CAAN;AACD;;AAED,UAAMmB,KAAK,GAAGjB,MAAM,CAACkB,SAAP,wBAAiBX,QAAQ,CAACG,SAA1B,wDAAiB,oBAAoBE,cAArC,CAAd;AACA,UAAMO,IAAmB,GAAG,8BAAcC,mBAAd,0BAAoBb,QAAQ,CAACG,SAA7B,yDAAoB,qBAAoBS,IAAxC,CAA5B;;AAEA,QAAIA,IAAI,KAAKF,KAAK,CAACI,aAAnB,EAAkC;AAChC,YAAM,IAAIvB,KAAJ,2CAA6CqB,IAA7C,EAAN;AACD;;AAED,UAAMG,WAA+B,GAAG,8BACtCC,+BADsC,0BAEtChB,QAAQ,CAACG,SAF6B,yDAEtC,qBAAoBc,KAFkB,CAAxC;AAKA,UAAMC,WAAW,GAAGrB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBgB,gBAArB,CAA1B;AACA,QAAIC,SAAS,GAAGvB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBkB,qBAArB,CAAtB;;AAEA,QAAI,CAACrB,QAAQ,CAACQ,SAAd,EAAyB;AAAA;;AACvBY,MAAAA,SAAS,GAAGE,IAAI,CAACC,GAAL,CACV,KAAKxC,QAAL,GAAgBmC,WADN,EAEVrB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBkB,qBAArB,CAFI,CAAZ;AAID;;AAED,UAAMrC,OAAuB,GAAG;AAC9B4B,MAAAA,IAD8B;AAE9BY,MAAAA,SAAS,EAAEd,KAAK,CAACc,SAFa;AAG9BC,MAAAA,SAAS,EAAEf,KAAK,CAACe,SAHa;AAI9BV,MAAAA,WAJ8B;AAK9BW,MAAAA,MAAM,EAAEhB,KALsB;AAM9BiB,MAAAA,SAAS,0BAAE3B,QAAQ,CAACG,SAAX,yDAAE,qBAAoByB,UAND;AAO9BC,MAAAA,UAAU,EAAE;AAPkB,KAAhC;AAUA,QAAIA,UAAJ;AAEA,UAAMC,oBAAoB,GAAG9B,QAAH,aAAGA,QAAH,+CAAGA,QAAQ,CAAEG,SAAb,yDAAG,qBAAqB4B,sBAAlD;;AAEA,QAAID,oBAAJ,EAA0B;AACxB,YAAME,gBAAgB,GAAGnC,MAAM,CAACiC,oBAAD,CAA/B;AAEAD,MAAAA,UAAU,GAAG,MAAM,KAAKI,aAAL,CAAmBD,gBAAnB,EAAqChD,OAArC,EAA8CkC,WAA9C,CAAnB;AACD;;AAEDW,IAAAA,UAAU,GAAG,uBAAA7C,OAAO,CAAC6C,UAAR,oEAAoBtD,MAApB,GAA6BS,OAAO,CAAC6C,UAArC,GAAkDA,UAA/D;AACA,UAAMK,QAAQ,GAAG,MAAM,KAAKrD,IAAL,CAAUqC,WAAV,EAAuBE,SAAvB,CAAvB;AACA,WAAO,MAAM,+BAAgBc,QAAhB,EAA0B,EAAC,GAAGlD,OAAJ;AAAa6C,MAAAA;AAAb,KAA1B,CAAb;AACD;;AASkB,QAAbI,aAAa,CACjBH,oBADiB,EAEjB9C,OAFiB,EAGjBkC,WAHiB,EAIE;AACnB,QAAIY,oBAAoB,KAAK,CAA7B,EAAgC;AAQ9B,aAAO,EAAP;AACD;;AAED,UAAMK,cAAc,GAAGb,IAAI,CAACC,GAAL,CACrB,KAAKxC,QAAL,GAAgB+C,oBADK,EAErB,KAAK7C,qBAFgB,CAAvB;AAIA,UAAMiD,QAAQ,GAAG,MAAM,KAAKrD,IAAL,CAAUiD,oBAAV,EAAgCK,cAAhC,CAAvB;AAEA,UAAMC,MAAM,GAAG;AAAChE,MAAAA,MAAM,EAAE8D,QAAT;AAAmBG,MAAAA,MAAM,EAAE,CAA3B;AAA8BC,MAAAA,IAAI,EAAEJ,QAAQ,CAAC3D;AAA7C,KAAf;AACA,UAAMgE,WAAW,GAAG,MAAM,0BAAWH,MAAX,EAAmBpD,OAAnB,CAA1B;AAEA,WAAOuD,WAAW,CAACV,UAAnB;AACD;;AAEe,QAAVW,UAAU,GAA0B;AACxC,UAAMC,UAAU,GAAGtD,yBAAcZ,MAAd,GAAuB,CAA1C;AACA,UAAMmE,UAAU,GAAG,MAAM,KAAK7D,IAAL,CAAU,KAAKE,QAAL,GAAgB0D,UAA1B,EAAsCA,UAAtC,CAAzB;AAEA,UAAMrD,KAAK,GAAGsD,UAAU,CAAChE,KAAX,CAAiB,CAAjB,EAAoBW,QAApB,EAAd;;AACA,QAAID,KAAK,KAAKD,wBAAd,EAA6B;AAC3B,YAAM,IAAII,KAAJ,6CAA8CH,KAA9C,OAAN;AACD;;AAED,UAAMuD,YAAY,GAAGD,UAAU,CAACE,YAAX,CAAwB,CAAxB,CAArB;AACA,UAAMC,cAAc,GAAG,KAAK9D,QAAL,GAAgB4D,YAAhB,GAA+BF,UAAtD;;AACA,QAAII,cAAc,GAAG1D,yBAAcZ,MAAnC,EAA2C;AACzC,YAAM,IAAIgB,KAAJ,iCAAmCsD,cAAnC,EAAN;AACD;;AAED,UAAMC,WAAW,GAAG,MAAM,KAAKjE,IAAL,CAAUgE,cAAV,EAA0BF,YAA1B,CAA1B;AAGA,UAAM;AAACI,MAAAA;AAAD,QAAa,mCAAmBD,WAAnB,CAAnB;AACA,WAAOC,QAAP;AACD;;AA/KgC","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetSchema} from '../schema/schema';\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nconst DEFAULT_DICTIONARY_SIZE = 1e6;\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n  public defaultDictionarySize: number;\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number,\n    options?: any\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;\n  }\n\n  async readHeader(): Promise<void> {\n    const buffer = await this.read(0, PARQUET_MAGIC.length);\n\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Do reading of parquet file's column chunk\n   * @param schema\n   * @param colChunk\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.fileSize - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.fileSize - dictionaryPageOffset,\n      this.defaultDictionarySize\n    );\n    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n"],"file":"parquet-envelope-reader.js"}