@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

package/dist/es5/parquetjs/parser/parquet-cursor.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-cursor.ts"],"names":["Symbol","asyncIterator","ParquetCursor","metadata","envelopeReader","schema","columnList","rowGroupIndex","rowGroup","length","row_groups","readRowGroup","rowBuffer","shift","done","next","value","return","throw"],"mappings":";;;;;;;;;;;;;;;;;;;AAKA;;;;wBAiEGA,MAAM,CAACC,a;;IA5DGC,a;AAcX,yBACEC,QADF,EAEEC,cAFF,EAGEC,MAHF,EAIEC,UAJF,EAKE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oDAdiC,EAcjC;AAAA;AACA,SAAKH,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,SAAKC,MAAL,GAAcA,MAAd;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACA,SAAKC,aAAL,GAAqB,CAArB;AACD;;;;;4EAMD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBACM,KAAKC,QAAL,CAAcC,MAAd,KAAyB,CAD/B;AAAA;AAAA;AAAA;;AAAA,sBAEQ,KAAKF,aAAL,IAAsB,KAAKJ,QAAL,CAAcO,UAAd,CAAyBD,MAFvD;AAAA;AAAA;AAAA;;AAAA,iDAIa,IAJb;;AAAA;AAAA;AAAA,uBAM4B,KAAKL,cAAL,CAAoBO,YAApB,CACtB,KAAKN,MADiB,EAEtB,KAAKF,QAAL,CAAcO,UAAd,CAAyB,KAAKH,aAA9B,CAFsB,EAGtB,KAAKD,UAHiB,CAN5B;;AAAA;AAMUM,gBAAAA,SANV;AAWI,qBAAKJ,QAAL,GAAgB,+BAAmB,KAAKH,MAAxB,EAAgCO,SAAhC,CAAhB;AACA,qBAAKL,aAAL;;AAZJ;AAAA,iDAcS,KAAKC,QAAL,CAAcK,KAAd,EAdT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;WAoBA,kBAAe;AACb,WAAKL,QAAL,GAAgB,EAAhB;AACA,WAAKD,aAAL,GAAqB,CAArB;AACD;;;WAMD,iBAA2C;AAAA;;AACzC,UAAIO,IAAI,GAAG,KAAX;AACA,aAAO;AACLC,QAAAA,IAAI;AAAA,iFAAE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBACAD,IADA;AAAA;AAAA;AAAA;;AAAA,sDAEK;AAACA,sBAAAA,IAAI,EAAJA,IAAD;AAAOE,sBAAAA,KAAK,EAAE;AAAd,qBAFL;;AAAA;AAAA;AAAA,2BAIgB,KAAI,CAACD,IAAL,EAJhB;;AAAA;AAIEC,oBAAAA,KAJF;;AAAA,0BAKAA,KAAK,KAAK,IALV;AAAA;AAAA;AAAA;;AAAA,sDAMK;AAACF,sBAAAA,IAAI,EAAE,IAAP;AAAaE,sBAAAA,KAAK,EAALA;AAAb,qBANL;;AAAA;AAAA,sDAQG;AAACF,sBAAAA,IAAI,EAAE,KAAP;AAAcE,sBAAAA,KAAK,EAALA;AAAd,qBARH;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,WADC;AAWLC,QAAAA,MAAM;AAAA,mFAAE;AAAA;AAAA;AAAA;AAAA;AACNH,oBAAAA,IAAI,GAAG,IAAP;AADM,sDAEC;AAACA,sBAAAA,IAAI,EAAJA,IAAD;AAAOE,sBAAAA,KAAK,EAAE;AAAd,qBAFD;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,WAXD;AAeLE,QAAAA,KAAK;AAAA,kFAAE;AAAA;AAAA;AAAA;AAAA;AACLJ,oBAAAA,IAAI,GAAG,IAAP;AADK,sDAEE;AAACA,sBAAAA,IAAI,EAAE,IAAP;AAAaE,sBAAAA,KAAK,EAAE;AAApB,qBAFF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAfA,OAAP;AAoBD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetRecord} from '../schema/declare';\nimport {materializeRecords} from '../schema/shred';\n\n/**\n * A parquet cursor is used to retrieve rows from a parquet file in order\n */\nexport class ParquetCursor<T> implements AsyncIterable<T> {\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n  public columnList: string[][];\n  public rowGroup: ParquetRecord[] = [];\n  public rowGroupIndex: number;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is usually not recommended to call this constructor directly except for\n   * advanced and internal use cases. Consider using getCursor() on the\n   * ParquetReader instead\n   */\n  constructor(\n    metadata: FileMetaData,\n    envelopeReader: ParquetEnvelopeReader,\n    schema: ParquetSchema,\n    columnList: string[][]\n  ) {\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    this.schema = schema;\n    this.columnList = columnList;\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Retrieve the next row from the cursor. Returns a row or NULL if the end\n   * of the file was reached\n   */\n  async next<T = any>(): Promise<T> {\n    if (this.rowGroup.length === 0) {\n      if (this.rowGroupIndex >= this.metadata.row_groups.length) {\n        // @ts-ignore\n        return null;\n      }\n      const rowBuffer = await this.envelopeReader.readRowGroup(\n        this.schema,\n        this.metadata.row_groups[this.rowGroupIndex],\n        this.columnList\n      );\n      this.rowGroup = materializeRecords(this.schema, rowBuffer);\n      this.rowGroupIndex++;\n    }\n    return this.rowGroup.shift() as any;\n  }\n\n  /**\n   * Rewind the cursor the the beginning of the file\n   */\n  rewind(): void {\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    let done = false;\n    return {\n      next: async () => {\n        if (done) {\n          return {done, value: null};\n        }\n        const value = await this.next();\n        if (value === null) {\n          return {done: true, value};\n        }\n        return {done: false, value};\n      },\n      return: async () => {\n        done = true;\n        return {done, value: null};\n      },\n      throw: async () => {\n        done = true;\n        return {done: true, value: null};\n      }\n    };\n  }\n}\n"],"file":"parquet-cursor.js"}

package/dist/es5/parquetjs/parser/parquet-envelope-reader.js

@@ -1,413 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.ParquetEnvelopeReader = void 0;
-
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _constants = require("../../constants");
-
-var _parquetThrift = require("../parquet-thrift");
-
-var _readUtils = require("../utils/read-utils");
-
-var _decoders = require("./decoders");
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
-var DEFAULT_DICTIONARY_SIZE = 1e6;
-
-var ParquetEnvelopeReader = function () {
-  function ParquetEnvelopeReader(read, close, fileSize, options) {
-    (0, _classCallCheck2.default)(this, ParquetEnvelopeReader);
-    (0, _defineProperty2.default)(this, "read", void 0);
-    (0, _defineProperty2.default)(this, "close", void 0);
-    (0, _defineProperty2.default)(this, "fileSize", void 0);
-    (0, _defineProperty2.default)(this, "defaultDictionarySize", void 0);
-    this.read = read;
-    this.close = close;
-    this.fileSize = fileSize;
-    this.defaultDictionarySize = (options === null || options === void 0 ? void 0 : options.defaultDictionarySize) || DEFAULT_DICTIONARY_SIZE;
-  }
-
-  (0, _createClass2.default)(ParquetEnvelopeReader, [{
-    key: "readHeader",
-    value: function () {
-      var _readHeader = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee() {
-        var buffer, magic;
-        return _regenerator.default.wrap(function _callee$(_context) {
-          while (1) {
-            switch (_context.prev = _context.next) {
-              case 0:
-                _context.next = 2;
-                return this.read(0, _constants.PARQUET_MAGIC.length);
-
-              case 2:
-                buffer = _context.sent;
-                magic = buffer.toString();
-                _context.t0 = magic;
-                _context.next = _context.t0 === _constants.PARQUET_MAGIC ? 7 : _context.t0 === _constants.PARQUET_MAGIC_ENCRYPTED ? 8 : 9;
-                break;
-
-              case 7:
-                return _context.abrupt("break", 10);
-
-              case 8:
-                throw new Error('Encrypted parquet file not supported');
-
-              case 9:
-                throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
-
-              case 10:
-              case "end":
-                return _context.stop();
-            }
-          }
-        }, _callee, this);
-      }));
-
-      function readHeader() {
-        return _readHeader.apply(this, arguments);
-      }
-
-      return readHeader;
-    }()
-  }, {
-    key: "readRowGroup",
-    value: function () {
-      var _readRowGroup = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(schema, rowGroup, columnList) {
-        var buffer, _iterator, _step, colChunk, colMetadata, colKey;
-
-        return _regenerator.default.wrap(function _callee2$(_context2) {
-          while (1) {
-            switch (_context2.prev = _context2.next) {
-              case 0:
-                buffer = {
-                  rowCount: Number(rowGroup.num_rows),
-                  columnData: {}
-                };
-                _iterator = _createForOfIteratorHelper(rowGroup.columns);
-                _context2.prev = 2;
-
-                _iterator.s();
-
-              case 4:
-                if ((_step = _iterator.n()).done) {
-                  _context2.next = 15;
-                  break;
-                }
-
-                colChunk = _step.value;
-                colMetadata = colChunk.meta_data;
-                colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
-
-                if (!(columnList.length > 0 && (0, _readUtils.fieldIndexOf)(columnList, colKey) < 0)) {
-                  _context2.next = 10;
-                  break;
-                }
-
-                return _context2.abrupt("continue", 13);
-
-              case 10:
-                _context2.next = 12;
-                return this.readColumnChunk(schema, colChunk);
-
-              case 12:
-                buffer.columnData[colKey.join()] = _context2.sent;
-
-              case 13:
-                _context2.next = 4;
-                break;
-
-              case 15:
-                _context2.next = 20;
-                break;
-
-              case 17:
-                _context2.prev = 17;
-                _context2.t0 = _context2["catch"](2);
-
-                _iterator.e(_context2.t0);
-
-              case 20:
-                _context2.prev = 20;
-
-                _iterator.f();
-
-                return _context2.finish(20);
-
-              case 23:
-                return _context2.abrupt("return", buffer);
-
-              case 24:
-              case "end":
-                return _context2.stop();
-            }
-          }
-        }, _callee2, this, [[2, 17, 20, 23]]);
-      }));
-
-      function readRowGroup(_x, _x2, _x3) {
-        return _readRowGroup.apply(this, arguments);
-      }
-
-      return readRowGroup;
-    }()
-  }, {
-    key: "readColumnChunk",
-    value: function () {
-      var _readColumnChunk = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(schema, colChunk) {
-        var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
-
-        var field, type, compression, pagesOffset, pagesSize, _colChunk$meta_data6, options, dictionary, dictionaryPageOffset, dictionaryOffset, pagesBuf;
-
-        return _regenerator.default.wrap(function _callee3$(_context3) {
-          while (1) {
-            switch (_context3.prev = _context3.next) {
-              case 0:
-                if (!(colChunk.file_path !== undefined && colChunk.file_path !== null)) {
-                  _context3.next = 2;
-                  break;
-                }
-
-                throw new Error('external references are not supported');
-
-              case 2:
-                field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
-                type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
-
-                if (!(type !== field.primitiveType)) {
-                  _context3.next = 6;
-                  break;
-                }
-
-                throw new Error("chunk type not matching schema: ".concat(type));
-
-              case 6:
-                compression = (0, _readUtils.getThriftEnum)(_parquetThrift.CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
-                pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
-                pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
-
-                if (!colChunk.file_path) {
-                  pagesSize = Math.min(this.fileSize - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
-                }
-
-                options = {
-                  type: type,
-                  rLevelMax: field.rLevelMax,
-                  dLevelMax: field.dLevelMax,
-                  compression: compression,
-                  column: field,
-                  numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
-                  dictionary: []
-                };
-                dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
-
-                if (!dictionaryPageOffset) {
-                  _context3.next = 17;
-                  break;
-                }
-
-                dictionaryOffset = Number(dictionaryPageOffset);
-                _context3.next = 16;
-                return this.getDictionary(dictionaryOffset, options, pagesOffset);
-
-              case 16:
-                dictionary = _context3.sent;
-
-              case 17:
-                dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
-                _context3.next = 20;
-                return this.read(pagesOffset, pagesSize);
-
-              case 20:
-                pagesBuf = _context3.sent;
-                _context3.next = 23;
-                return (0, _decoders.decodeDataPages)(pagesBuf, _objectSpread(_objectSpread({}, options), {}, {
-                  dictionary: dictionary
-                }));
-
-              case 23:
-                return _context3.abrupt("return", _context3.sent);
-
-              case 24:
-              case "end":
-                return _context3.stop();
-            }
-          }
-        }, _callee3, this);
-      }));
-
-      function readColumnChunk(_x4, _x5) {
-        return _readColumnChunk.apply(this, arguments);
-      }
-
-      return readColumnChunk;
-    }()
-  }, {
-    key: "getDictionary",
-    value: function () {
-      var _getDictionary = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4(dictionaryPageOffset, options, pagesOffset) {
-        var dictionarySize, pagesBuf, cursor, decodedPage;
-        return _regenerator.default.wrap(function _callee4$(_context4) {
-          while (1) {
-            switch (_context4.prev = _context4.next) {
-              case 0:
-                if (!(dictionaryPageOffset === 0)) {
-                  _context4.next = 2;
-                  break;
-                }
-
-                return _context4.abrupt("return", []);
-
-              case 2:
-                dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
-                _context4.next = 5;
-                return this.read(dictionaryPageOffset, dictionarySize);
-
-              case 5:
-                pagesBuf = _context4.sent;
-                cursor = {
-                  buffer: pagesBuf,
-                  offset: 0,
-                  size: pagesBuf.length
-                };
-                _context4.next = 9;
-                return (0, _decoders.decodePage)(cursor, options);
-
-              case 9:
-                decodedPage = _context4.sent;
-                return _context4.abrupt("return", decodedPage.dictionary);
-
-              case 11:
-              case "end":
-                return _context4.stop();
-            }
-          }
-        }, _callee4, this);
-      }));
-
-      function getDictionary(_x6, _x7, _x8) {
-        return _getDictionary.apply(this, arguments);
-      }
-
-      return getDictionary;
-    }()
-  }, {
-    key: "readFooter",
-    value: function () {
-      var _readFooter = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5() {
-        var trailerLen, trailerBuf, magic, metadataSize, metadataOffset, metadataBuf, _decodeFileMetadata, metadata;
-
-        return _regenerator.default.wrap(function _callee5$(_context5) {
-          while (1) {
-            switch (_context5.prev = _context5.next) {
-              case 0:
-                trailerLen = _constants.PARQUET_MAGIC.length + 4;
-                _context5.next = 3;
-                return this.read(this.fileSize - trailerLen, trailerLen);
-
-              case 3:
-                trailerBuf = _context5.sent;
-                magic = trailerBuf.slice(4).toString();
-
-                if (!(magic !== _constants.PARQUET_MAGIC)) {
-                  _context5.next = 7;
-                  break;
-                }
-
-                throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
-
-              case 7:
-                metadataSize = trailerBuf.readUInt32LE(0);
-                metadataOffset = this.fileSize - metadataSize - trailerLen;
-
-                if (!(metadataOffset < _constants.PARQUET_MAGIC.length)) {
-                  _context5.next = 11;
-                  break;
-                }
-
-                throw new Error("Invalid metadata size ".concat(metadataOffset));
-
-              case 11:
-                _context5.next = 13;
-                return this.read(metadataOffset, metadataSize);
-
-              case 13:
-                metadataBuf = _context5.sent;
-                _decodeFileMetadata = (0, _readUtils.decodeFileMetadata)(metadataBuf), metadata = _decodeFileMetadata.metadata;
-                return _context5.abrupt("return", metadata);
-
-              case 16:
-              case "end":
-                return _context5.stop();
-            }
-          }
-        }, _callee5, this);
-      }));
-
-      function readFooter() {
-        return _readFooter.apply(this, arguments);
-      }
-
-      return readFooter;
-    }()
-  }], [{
-    key: "openBuffer",
-    value: function () {
-      var _openBuffer = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee6(buffer) {
-        var readFn, closeFn;
-        return _regenerator.default.wrap(function _callee6$(_context6) {
-          while (1) {
-            switch (_context6.prev = _context6.next) {
-              case 0:
-                readFn = function readFn(position, length) {
-                  return Promise.resolve(buffer.slice(position, position + length));
-                };
-
-                closeFn = function closeFn() {
-                  return Promise.resolve();
-                };
-
-                return _context6.abrupt("return", new ParquetEnvelopeReader(readFn, closeFn, buffer.length));
-
-              case 3:
-              case "end":
-                return _context6.stop();
-            }
-          }
-        }, _callee6);
-      }));
-
-      function openBuffer(_x9) {
-        return _openBuffer.apply(this, arguments);
-      }
-
-      return openBuffer;
-    }()
-  }]);
-  return ParquetEnvelopeReader;
-}();
-
-exports.ParquetEnvelopeReader = ParquetEnvelopeReader;
-//# sourceMappingURL=parquet-envelope-reader.js.map

package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":["DEFAULT_DICTIONARY_SIZE","ParquetEnvelopeReader","read","close","fileSize","options","defaultDictionarySize","PARQUET_MAGIC","length","buffer","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","schema","rowGroup","columnList","rowCount","Number","num_rows","columnData","columns","colChunk","colMetadata","meta_data","colKey","path_in_schema","readColumnChunk","join","file_path","undefined","field","findField","type","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","size","decodedPage","trailerLen","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","metadata","readFn","position","Promise","resolve","closeFn"],"mappings":";;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AAQA;;AACA;;;;;;;;;;;;AAEA,IAAMA,uBAAuB,GAAG,GAAhC;;IAQaC,qB;AAiBX,iCACEC,IADF,EAEEC,KAFF,EAGEC,QAHF,EAIEC,OAJF,EAKE;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,IAAL,GAAYA,IAAZ;AACA,SAAKC,KAAL,GAAaA,KAAb;AACA,SAAKC,QAAL,GAAgBA,QAAhB;AACA,SAAKE,qBAAL,GAA6B,CAAAD,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEC,qBAAT,KAAkCN,uBAA/D;AACD;;;;;kFAED;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACuB,KAAKE,IAAL,CAAU,CAAV,EAAaK,yBAAcC,MAA3B,CADvB;;AAAA;AACQC,gBAAAA,MADR;AAGQC,gBAAAA,KAHR,GAGgBD,MAAM,CAACE,QAAP,EAHhB;AAAA,8BAIUD,KAJV;AAAA,gDAKSH,wBALT,uBAOSK,kCAPT;AAAA;;AAAA;AAAA;;AAAA;AAAA,sBAQY,IAAIC,KAAJ,CAAU,sCAAV,CARZ;;AAAA;AAAA,sBAUY,IAAIA,KAAJ,uCAAyCH,KAAzC,OAVZ;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;oFAcA,kBACEI,MADF,EAEEC,QAFF,EAGEC,UAHF;AAAA;;AAAA;AAAA;AAAA;AAAA;AAKQP,gBAAAA,MALR,GAKgC;AAC5BQ,kBAAAA,QAAQ,EAAEC,MAAM,CAACH,QAAQ,CAACI,QAAV,CADY;AAE5BC,kBAAAA,UAAU,EAAE;AAFgB,iBALhC;AAAA,uDASyBL,QAAQ,CAACM,OATlC;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AASaC,gBAAAA,QATb;AAUUC,gBAAAA,WAVV,GAUwBD,QAAQ,CAACE,SAVjC;AAWUC,gBAAAA,MAXV,GAWmBF,WAXnB,aAWmBA,WAXnB,uBAWmBA,WAAW,CAAEG,cAXhC;;AAAA,sBAYQV,UAAU,CAACR,MAAX,GAAoB,CAApB,IAAyB,6BAAaQ,UAAb,EAAyBS,MAAzB,IAAoC,CAZrE;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;AAAA,uBAe8C,KAAKE,eAAL,CAAqBb,MAArB,EAA6BQ,QAA7B,CAf9C;;AAAA;AAeIb,gBAAAA,MAAM,CAACW,UAAP,CAAkBK,MAAM,CAAEG,IAAR,EAAlB,CAfJ;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAAA,kDAiBSnB,MAjBT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;uFAyBA,kBAAsBK,MAAtB,EAA6CQ,QAA7C;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA,sBACMA,QAAQ,CAACO,SAAT,KAAuBC,SAAvB,IAAoCR,QAAQ,CAACO,SAAT,KAAuB,IADjE;AAAA;AAAA;AAAA;;AAAA,sBAEU,IAAIhB,KAAJ,CAAU,uCAAV,CAFV;;AAAA;AAKQkB,gBAAAA,KALR,GAKgBjB,MAAM,CAACkB,SAAP,wBAAiBV,QAAQ,CAACE,SAA1B,wDAAiB,oBAAoBE,cAArC,CALhB;AAMQO,gBAAAA,IANR,GAM8B,8BAAcC,mBAAd,0BAAoBZ,QAAQ,CAACE,SAA7B,yDAAoB,qBAAoBS,IAAxC,CAN9B;;AAAA,sBAQMA,IAAI,KAAKF,KAAK,CAACI,aARrB;AAAA;AAAA;AAAA;;AAAA,sBASU,IAAItB,KAAJ,2CAA6CoB,IAA7C,EATV;;AAAA;AAYQG,gBAAAA,WAZR,GAY0C,8BACtCC,+BADsC,0BAEtCf,QAAQ,CAACE,SAF6B,yDAEtC,qBAAoBc,KAFkB,CAZ1C;AAiBQC,gBAAAA,WAjBR,GAiBsBrB,MAAM,yBAACI,QAAQ,CAACE,SAAV,yDAAC,qBAAoBgB,gBAArB,CAjB5B;AAkBMC,gBAAAA,SAlBN,GAkBkBvB,MAAM,yBAACI,QAAQ,CAACE,SAAV,yDAAC,qBAAoBkB,qBAArB,CAlBxB;;AAoBE,oBAAI,CAACpB,QAAQ,CAACO,SAAd,EAAyB;AACvBY,kBAAAA,SAAS,GAAGE,IAAI,CAACC,GAAL,CACV,KAAKxC,QAAL,GAAgBmC,WADN,EAEVrB,MAAM,yBAACI,QAAQ,CAACE,SAAV,yDAAC,qBAAoBkB,qBAArB,CAFI,CAAZ;AAID;;AAEKrC,gBAAAA,OA3BR,GA2BkC;AAC9B4B,kBAAAA,IAAI,EAAJA,IAD8B;AAE9BY,kBAAAA,SAAS,EAAEd,KAAK,CAACc,SAFa;AAG9BC,kBAAAA,SAAS,EAAEf,KAAK,CAACe,SAHa;AAI9BV,kBAAAA,WAAW,EAAXA,WAJ8B;AAK9BW,kBAAAA,MAAM,EAAEhB,KALsB;AAM9BiB,kBAAAA,SAAS,0BAAE1B,QAAQ,CAACE,SAAX,yDAAE,qBAAoByB,UAND;AAO9BC,kBAAAA,UAAU,EAAE;AAPkB,iBA3BlC;AAuCQC,gBAAAA,oBAvCR,GAuC+B7B,QAvC/B,aAuC+BA,QAvC/B,+CAuC+BA,QAAQ,CAAEE,SAvCzC,yDAuC+B,qBAAqB4B,sBAvCpD;;AAAA,qBAyCMD,oBAzCN;AAAA;AAAA;AAAA;;AA0CUE,gBAAAA,gBA1CV,GA0C6BnC,MAAM,CAACiC,oBAAD,CA1CnC;AAAA;AAAA,uBA4CuB,KAAKG,aAAL,CAAmBD,gBAAnB,EAAqChD,OAArC,EAA8CkC,WAA9C,CA5CvB;;AAAA;AA4CIW,gBAAAA,UA5CJ;;AAAA;AA+CEA,gBAAAA,UAAU,GAAG,uBAAA7C,OAAO,CAAC6C,UAAR,oEAAoB1C,MAApB,GAA6BH,OAAO,CAAC6C,UAArC,GAAkDA,UAA/D;AA/CF;AAAA,uBAgDyB,KAAKhD,IAAL,CAAUqC,WAAV,EAAuBE,SAAvB,CAhDzB;;AAAA;AAgDQc,gBAAAA,QAhDR;AAAA;AAAA,uBAiDe,+BAAgBA,QAAhB,kCAA8BlD,OAA9B;AAAuC6C,kBAAAA,UAAU,EAAVA;AAAvC,mBAjDf;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;qFA2DA,kBACEC,oBADF,EAEE9C,OAFF,EAGEkC,WAHF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBAKMY,oBAAoB,KAAK,CAL/B;AAAA;AAAA;AAAA;;AAAA,kDAaW,EAbX;;AAAA;AAgBQK,gBAAAA,cAhBR,GAgByBb,IAAI,CAACC,GAAL,CACrB,KAAKxC,QAAL,GAAgB+C,oBADK,EAErB,KAAK7C,qBAFgB,CAhBzB;AAAA;AAAA,uBAoByB,KAAKJ,IAAL,CAAUiD,oBAAV,EAAgCK,cAAhC,CApBzB;;AAAA;AAoBQD,gBAAAA,QApBR;AAsBQE,gBAAAA,MAtBR,GAsBiB;AAAChD,kBAAAA,MAAM,EAAE8C,QAAT;AAAmBG,kBAAAA,MAAM,EAAE,CAA3B;AAA8BC,kBAAAA,IAAI,EAAEJ,QAAQ,CAAC/C;AAA7C,iBAtBjB;AAAA;AAAA,uBAuB4B,0BAAWiD,MAAX,EAAmBpD,OAAnB,CAvB5B;;AAAA;AAuBQuD,gBAAAA,WAvBR;AAAA,kDAyBSA,WAAW,CAACV,UAzBrB;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFA4BA;AAAA;;AAAA;AAAA;AAAA;AAAA;AACQW,gBAAAA,UADR,GACqBtD,yBAAcC,MAAd,GAAuB,CAD5C;AAAA;AAAA,uBAE2B,KAAKN,IAAL,CAAU,KAAKE,QAAL,GAAgByD,UAA1B,EAAsCA,UAAtC,CAF3B;;AAAA;AAEQC,gBAAAA,UAFR;AAIQpD,gBAAAA,KAJR,GAIgBoD,UAAU,CAACC,KAAX,CAAiB,CAAjB,EAAoBpD,QAApB,EAJhB;;AAAA,sBAKMD,KAAK,KAAKH,wBALhB;AAAA;AAAA;AAAA;;AAAA,sBAMU,IAAIM,KAAJ,6CAA8CH,KAA9C,OANV;;AAAA;AASQsD,gBAAAA,YATR,GASuBF,UAAU,CAACG,YAAX,CAAwB,CAAxB,CATvB;AAUQC,gBAAAA,cAVR,GAUyB,KAAK9D,QAAL,GAAgB4D,YAAhB,GAA+BH,UAVxD;;AAAA,sBAWMK,cAAc,GAAG3D,yBAAcC,MAXrC;AAAA;AAAA;AAAA;;AAAA,sBAYU,IAAIK,KAAJ,iCAAmCqD,cAAnC,EAZV;;AAAA;AAAA;AAAA,uBAe4B,KAAKhE,IAAL,CAAUgE,cAAV,EAA0BF,YAA1B,CAf5B;;AAAA;AAeQG,gBAAAA,WAfR;AAAA,sCAkBqB,mCAAmBA,WAAnB,CAlBrB,EAkBSC,QAlBT,uBAkBSA,QAlBT;AAAA,kDAmBSA,QAnBT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFAjJA,kBAAwB3D,MAAxB;AAAA;AAAA;AAAA;AAAA;AAAA;AACQ4D,gBAAAA,MADR,GACiB,SAATA,MAAS,CAACC,QAAD,EAAmB9D,MAAnB;AAAA,yBACb+D,OAAO,CAACC,OAAR,CAAgB/D,MAAM,CAACsD,KAAP,CAAaO,QAAb,EAAuBA,QAAQ,GAAG9D,MAAlC,CAAhB,CADa;AAAA,iBADjB;;AAGQiE,gBAAAA,OAHR,GAGkB,SAAVA,OAAU;AAAA,yBAAMF,OAAO,CAACC,OAAR,EAAN;AAAA,iBAHlB;;AAAA,kDAIS,IAAIvE,qBAAJ,CAA0BoE,MAA1B,EAAkCI,OAAlC,EAA2ChE,MAAM,CAACD,MAAlD,CAJT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetSchema} from '../schema/schema';\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nconst DEFAULT_DICTIONARY_SIZE = 1e6;\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n  public defaultDictionarySize: number;\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number,\n    options?: any\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;\n  }\n\n  async readHeader(): Promise<void> {\n    const buffer = await this.read(0, PARQUET_MAGIC.length);\n\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Do reading of parquet file's column chunk\n   * @param schema\n   * @param colChunk\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.fileSize - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.fileSize - dictionaryPageOffset,\n      this.defaultDictionarySize\n    );\n    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n"],"file":"parquet-envelope-reader.js"}