@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/es5/parquetjs/codecs/rle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rle.js","names":["_varint","_interopRequireDefault","require","encodeValues","type","values","opts","Error","map","x","parseInt","concat","buf","Buffer","alloc","run","repeats","i","length","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","_values","apply","_toConsumableArray2","default","decodeRunBitpacked","_values2","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"sources":["../../../../src/parquetjs/codecs/rle.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"mappings":";;;;;;;;;AAIA,IAAAA,OAAA,GAAAC,sBAAA,CAAAC,OAAA;AAGO,SAASC,YAAYA,CAC1BC,IAAmB,EACnBC,MAAa,EACbC,IAAyB,EACjB;EACR,IAAI,EAAE,UAAU,IAAIA,IAAI,CAAC,EAAE;IACzB,MAAM,IAAIC,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,QAAQH,IAAI;IACV,KAAK,SAAS;IACd,KAAK,OAAO;IACZ,KAAK,OAAO;MAEVC,MAAM,GAAGA,MAAM,CAACG,GAAG,CAAC,UAACC,CAAC;QAAA,OAAKC,QAAQ,CAACD,CAAC,EAAE,EAAE,CAAC;MAAA,EAAC;MAC3C;IAEF;MACE,MAAM,IAAIF,KAAK,sBAAAI,MAAA,CAAsBP,IAAI,CAAE,CAAC;EAChD;EAEA,IAAIQ,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,CAAC;EACzB,IAAIC,GAAU,GAAG,EAAE;EACnB,IAAIC,OAAO,GAAG,CAAC;EAEf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGZ,MAAM,CAACa,MAAM,EAAED,CAAC,EAAE,EAAE;IAGtC,IAAID,OAAO,KAAK,CAAC,IAAID,GAAG,CAACG,MAAM,GAAG,CAAC,KAAK,CAAC,IAAIb,MAAM,CAACY,CAAC,CAAC,KAAKZ,MAAM,CAACY,CAAC,GAAG,CAAC,CAAC,EAAE;MAExE,IAAIF,GAAG,CAACG,MAAM,EAAE;QACdN,GAAG,GAAGC,MAAM,CAACF,MAAM,CAAC,CAACC,GAAG,EAAEO,kBAAkB,CAACJ,GAAG,EAAET,IAAI,CAAC,CAAC,CAAC;QACzDS,GAAG,GAAG,EAAE;MACV;MACAC,OAAO,GAAG,CAAC;IACb,CAAC,MAAM,IAAIA,OAAO,GAAG,CAAC,IAAIX,MAAM,CAACY,CAAC,CAAC,KAAKZ,MAAM,CAACY,CAAC,GAAG,CAAC,CAAC,EAAE;MACrDD,OAAO,IAAI,CAAC;IACd,CAAC,MAAM;MAEL,IAAIA,OAAO,EAAE;QACXJ,GAAG,GAAGC,MAAM,CAACF,MAAM,CAAC,CAACC,GAAG,EAAEQ,iBAAiB,CAACf,MAAM,CAACY,CAAC,GAAG,CAAC,CAAC,EAAED,OAAO,EAAEV,IAAI,CAAC,CAAC,CAAC;QAC3EU,OAAO,GAAG,CAAC;MACb;MACAD,GAAG,CAACM,IAAI,CAAChB,MAAM,CAACY,CAAC,CAAC,CAAC;IACrB;EACF;EAEA,IAAID,OAAO,EAAE;IACXJ,GAAG,GAAGC,MAAM,CAACF,MAAM,CAAC,CAACC,GAAG,EAAEQ,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACa,MAAM,GAAG,CAAC,CAAC,EAAEF,OAAO,EAAEV,IAAI,CAAC,CAAC,CAAC;EACzF,CAAC,MAAM,IAAIS,GAAG,CAACG,MAAM,EAAE;IACrBN,GAAG,GAAGC,MAAM,CAACF,MAAM,CAAC,CAACC,GAAG,EAAEO,kBAAkB,CAACJ,GAAG,EAAET,IAAI,CAAC,CAAC,CAAC;EAC3D;EAEA,IAAIA,IAAI,CAACgB,eAAe,EAAE;IACxB,OAAOV,GAAG;EACZ;EAEA,IAAMW,QAAQ,GAAGV,MAAM,CAACC,KAAK,CAACF,GAAG,CAACM,MAAM,GAAG,CAAC,CAAC;EAC7CK,QAAQ,CAACC,aAAa,CAACZ,GAAG,CAACM,MAAM,EAAEO,SAAS,CAAC;EAC7Cb,GAAG,CAACc,IAAI,CAACH,QAAQ,EAAE,CAAC,CAAC;EAErB,OAAOA,QAAQ;AACjB;AAEO,SAASI,YAAYA,CAC1BvB,IAAmB,EACnBwB,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EACV,IAAI,EAAE,UAAU,IAAIA,IAAI,CAAC,EAAE;IACzB,MAAM,IAAIC,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,IAAI,CAACD,IAAI,CAACgB,eAAe,EAAE;IACzBM,MAAM,CAACE,MAAM,IAAI,CAAC;EACpB;EAEA,IAAIzB,MAAgB,GAAG,EAAE;EACzB,OAAOA,MAAM,CAACa,MAAM,GAAGW,KAAK,EAAE;IAC5B,IAAME,MAAM,GAAGC,eAAM,CAACC,MAAM,CAACL,MAAM,CAACM,MAAM,EAAEN,MAAM,CAACE,MAAM,CAAC;IAC1DF,MAAM,CAACE,MAAM,IAAIE,eAAM,CAACG,cAAc,CAACJ,MAAM,CAAC;IAC9C,IAAIA,MAAM,GAAG,CAAC,EAAE;MAAA,IAAAK,OAAA;MACd,IAAMP,MAAK,GAAG,CAACE,MAAM,IAAI,CAAC,IAAI,CAAC;MAC/B,CAAAK,OAAA,GAAA/B,MAAM,EAACgB,IAAI,CAAAgB,KAAA,CAAAD,OAAA,MAAAE,mBAAA,CAAAC,OAAA,EAAIC,kBAAkB,CAACZ,MAAM,EAAEC,MAAK,EAAEvB,IAAI,CAAC,EAAC;IACzD,CAAC,MAAM;MAAA,IAAAmC,QAAA;MACL,IAAMZ,OAAK,GAAGE,MAAM,IAAI,CAAC;MACzB,CAAAU,QAAA,GAAApC,MAAM,EAACgB,IAAI,CAAAgB,KAAA,CAAAI,QAAA,MAAAH,mBAAA,CAAAC,OAAA,EAAIG,iBAAiB,CAACd,MAAM,EAAEC,OAAK,EAAEvB,IAAI,CAAC,EAAC;IACxD;EACF;EACAD,MAAM,GAAGA,MAAM,CAACsC,KAAK,CAAC,CAAC,EAAEd,KAAK,CAAC;EAE/B,IAAIxB,MAAM,CAACa,MAAM,KAAKW,KAAK,EAAE;IAC3B,MAAM,IAAItB,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,OAAOF,MAAM;AACf;AAEA,SAASmC,kBAAkBA,CACzBZ,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EAEV,IAAMsC,QAAgB,GAAGtC,IAAI,CAACsC,QAAQ;EAEtC,IAAIf,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE;IACnB,MAAM,IAAItB,KAAK,CAAC,yBAAyB,CAAC;EAC5C;EAGA,IAAMF,MAAM,GAAG,IAAIwC,KAAK,CAAChB,KAAK,CAAC,CAACiB,IAAI,CAAC,CAAC,CAAC;EACvC,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,QAAQ,GAAGf,KAAK,EAAEkB,CAAC,EAAE,EAAE;IACzC,IAAInB,MAAM,CAACM,MAAM,CAACN,MAAM,CAACE,MAAM,GAAGkB,IAAI,CAACC,KAAK,CAACF,CAAC,GAAG,CAAC,CAAC,CAAC,GAAI,CAAC,IAAIA,CAAC,GAAG,CAAE,EAAE;MACnE1C,MAAM,CAAC2C,IAAI,CAACC,KAAK,CAACF,CAAC,GAAGH,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAIG,CAAC,GAAGH,QAAQ;IACvD;EACF;EAEAhB,MAAM,CAACE,MAAM,IAAIc,QAAQ,IAAIf,KAAK,GAAG,CAAC,CAAC;EACvC,OAAOxB,MAAM;AACf;AAEA,SAASqC,iBAAiBA,CACxBd,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EAEV,IAAMsC,QAAgB,GAAGtC,IAAI,CAACsC,QAAQ;EAEtC,IAAIM,KAAK,GAAG,CAAC;EACb,KAAK,IAAIjC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG+B,IAAI,CAACG,IAAI,CAACP,QAAQ,GAAG,CAAC,CAAC,EAAE3B,CAAC,EAAE,EAAE;IAEhDiC,KAAK,IAAI,CAAC;IACVA,KAAK,IAAItB,MAAM,CAACM,MAAM,CAACN,MAAM,CAACE,MAAM,CAAC;IACrCF,MAAM,CAACE,MAAM,IAAI,CAAC;EACpB;EAGA,OAAO,IAAIe,KAAK,CAAChB,KAAK,CAAC,CAACiB,IAAI,CAACI,KAAK,CAAC;AACrC;AAEA,SAAS/B,kBAAkBA,CAACd,MAAgB,EAAEC,IAAyB,EAAU;EAE/E,IAAMsC,QAAgB,GAAGtC,IAAI,CAACsC,QAAQ;EAEtC,KAAK,IAAI3B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGZ,MAAM,CAACa,MAAM,GAAG,CAAC,EAAED,CAAC,EAAE,EAAE;IAC1CZ,MAAM,CAACgB,IAAI,CAAC,CAAC,CAAC;EAChB;EAEA,IAAMT,GAAG,GAAGC,MAAM,CAACC,KAAK,CAACkC,IAAI,CAACG,IAAI,CAACP,QAAQ,IAAIvC,MAAM,CAACa,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;EACnE,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,QAAQ,GAAGvC,MAAM,CAACa,MAAM,EAAE6B,CAAC,EAAE,EAAE;IACjD,IAAI,CAAC1C,MAAM,CAAC2C,IAAI,CAACC,KAAK,CAACF,CAAC,GAAGH,QAAQ,CAAC,CAAC,GAAI,CAAC,IAAIG,CAAC,GAAGH,QAAS,IAAI,CAAC,EAAE;MAChEhC,GAAG,CAACoC,IAAI,CAACC,KAAK,CAACF,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAIA,CAAC,GAAG,CAAC;IACtC;EACF;EAEA,OAAOlC,MAAM,CAACF,MAAM,CAAC,CAACE,MAAM,CAACuC,IAAI,CAACpB,eAAM,CAACqB,MAAM,CAAGhD,MAAM,CAACa,MAAM,GAAG,CAAC,IAAK,CAAC,GAAI,CAAC,CAAC,CAAC,EAAEN,GAAG,CAAC,CAAC;AACzF;AAEA,SAASQ,iBAAiBA,CAAC8B,KAAa,EAAErB,KAAa,EAAEvB,IAAyB,EAAU;EAE1F,IAAMsC,QAAgB,GAAGtC,IAAI,CAACsC,QAAQ;EAEtC,IAAMhC,GAAG,GAAGC,MAAM,CAACC,KAAK,CAACkC,IAAI,CAACG,IAAI,CAACP,QAAQ,GAAG,CAAC,CAAC,CAAC;EAEjD,KAAK,IAAI3B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGL,GAAG,CAACM,MAAM,EAAED,CAAC,EAAE,EAAE;IACnCL,GAAG,CAAC0C,UAAU,CAACJ,KAAK,GAAG,IAAI,EAAEjC,CAAC,CAAC;IAE/BiC,KAAK,IAAI,CAAC;EACZ;EAEA,OAAOrC,MAAM,CAACF,MAAM,CAAC,CAACE,MAAM,CAACuC,IAAI,CAACpB,eAAM,CAACqB,MAAM,CAACxB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAEjB,GAAG,CAAC,CAAC;AACrE"}

package/dist/es5/parquetjs/compression.js

@@ -0,0 +1,137 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.PARQUET_COMPRESSION_METHODS = void 0;
+exports.decompress = decompress;
+exports.deflate = deflate;
+exports.inflate = inflate;
+exports.preloadCompressions = preloadCompressions;
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+var _compression = require("@loaders.gl/compression");
+var _lz4js = _interopRequireDefault(require("lz4js"));
+function toBuffer(arrayBuffer) {
+  return Buffer.from(arrayBuffer);
+}
+function toArrayBuffer(buffer) {
+  if (Buffer.isBuffer(buffer)) {
+    var typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+    return typedArray.slice().buffer;
+  }
+  return buffer;
+}
+var modules = {
+  lz4js: _lz4js.default
+};
+var PARQUET_COMPRESSION_METHODS = {
+  UNCOMPRESSED: new _compression.NoCompression(),
+  GZIP: new _compression.GZipCompression(),
+  SNAPPY: new _compression.SnappyCompression(),
+  BROTLI: new _compression.BrotliCompression({
+    modules: modules
+  }),
+  LZ4: new _compression.LZ4Compression({
+    modules: modules
+  }),
+  LZ4_RAW: new _compression.LZ4Compression({
+    modules: modules
+  }),
+  ZSTD: new _compression.ZstdCompression({
+    modules: modules
+  })
+};
+exports.PARQUET_COMPRESSION_METHODS = PARQUET_COMPRESSION_METHODS;
+function preloadCompressions(_x) {
+  return _preloadCompressions.apply(this, arguments);
+}
+function _preloadCompressions() {
+  _preloadCompressions = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(options) {
+    var compressions;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          compressions = Object.values(PARQUET_COMPRESSION_METHODS);
+          _context.next = 3;
+          return Promise.all(compressions.map(function (compression) {
+            return compression.preload();
+          }));
+        case 3:
+          return _context.abrupt("return", _context.sent);
+        case 4:
+        case "end":
+          return _context.stop();
+      }
+    }, _callee);
+  }));
+  return _preloadCompressions.apply(this, arguments);
+}
+function deflate(_x2, _x3) {
+  return _deflate.apply(this, arguments);
+}
+function _deflate() {
+  _deflate = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(method, value) {
+    var compression, inputArrayBuffer, compressedArrayBuffer;
+    return _regenerator.default.wrap(function _callee2$(_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
+        case 0:
+          compression = PARQUET_COMPRESSION_METHODS[method];
+          if (compression) {
+            _context2.next = 3;
+            break;
+          }
+          throw new Error("parquet: invalid compression method: ".concat(method));
+        case 3:
+          inputArrayBuffer = toArrayBuffer(value);
+          _context2.next = 6;
+          return compression.compress(inputArrayBuffer);
+        case 6:
+          compressedArrayBuffer = _context2.sent;
+          return _context2.abrupt("return", toBuffer(compressedArrayBuffer));
+        case 8:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2);
+  }));
+  return _deflate.apply(this, arguments);
+}
+function decompress(_x4, _x5, _x6) {
+  return _decompress.apply(this, arguments);
+}
+function _decompress() {
+  _decompress = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(method, value, size) {
+    var compression, inputArrayBuffer, compressedArrayBuffer;
+    return _regenerator.default.wrap(function _callee3$(_context3) {
+      while (1) switch (_context3.prev = _context3.next) {
+        case 0:
+          compression = PARQUET_COMPRESSION_METHODS[method];
+          if (compression) {
+            _context3.next = 3;
+            break;
+          }
+          throw new Error("parquet: invalid compression method: ".concat(method));
+        case 3:
+          inputArrayBuffer = toArrayBuffer(value);
+          _context3.next = 6;
+          return compression.decompress(inputArrayBuffer, size);
+        case 6:
+          compressedArrayBuffer = _context3.sent;
+          return _context3.abrupt("return", toBuffer(compressedArrayBuffer));
+        case 8:
+        case "end":
+          return _context3.stop();
+      }
+    }, _callee3);
+  }));
+  return _decompress.apply(this, arguments);
+}
+function inflate(method, value, size) {
+  if (!(method in PARQUET_COMPRESSION_METHODS)) {
+    throw new Error("invalid compression method: ".concat(method));
+  }
+  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
+}
+//# sourceMappingURL=compression.js.map

package/dist/es5/parquetjs/compression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compression.js","names":["_compression","require","_lz4js","_interopRequireDefault","toBuffer","arrayBuffer","Buffer","from","toArrayBuffer","buffer","isBuffer","typedArray","Uint8Array","byteOffset","length","slice","modules","lz4js","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","NoCompression","GZIP","GZipCompression","SNAPPY","SnappyCompression","BROTLI","BrotliCompression","LZ4","LZ4Compression","LZ4_RAW","ZSTD","ZstdCompression","exports","preloadCompressions","_x","_preloadCompressions","apply","arguments","_asyncToGenerator2","default","_regenerator","mark","_callee","options","compressions","wrap","_callee$","_context","prev","next","Object","values","Promise","all","map","compression","preload","abrupt","sent","stop","deflate","_x2","_x3","_deflate","_callee2","method","value","inputArrayBuffer","compressedArrayBuffer","_callee2$","_context2","Error","concat","compress","decompress","_x4","_x5","_x6","_decompress","_callee3","size","_callee3$","_context3","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  // LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\n\n/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */\nfunction toBuffer(arrayBuffer: ArrayBuffer): Buffer {\n  return Buffer.from(arrayBuffer);\n}\n\nfunction toArrayBuffer(buffer: Buffer): ArrayBuffer {\n  // TODO - per docs we should just be able to call buffer.buffer, but there are issues\n  if (Buffer.isBuffer(buffer)) {\n    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);\n    return typedArray.slice().buffer;\n  }\n  return buffer;\n}\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\n// import lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js\n  // lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n/**\n * See https://github.com/apache/parquet-format/blob/master/Compression.md\n */\n// @ts-expect-error\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  //\n  // LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":";;;;;;;;;;;;;AAIA,IAAAA,YAAA,GAAAC,OAAA;AA+BA,IAAAC,MAAA,GAAAC,sBAAA,CAAAF,OAAA;AAjBA,SAASG,QAAQA,CAACC,WAAwB,EAAU;EAClD,OAAOC,MAAM,CAACC,IAAI,CAACF,WAAW,CAAC;AACjC;AAEA,SAASG,aAAaA,CAACC,MAAc,EAAe;EAElD,IAAIH,MAAM,CAACI,QAAQ,CAACD,MAAM,CAAC,EAAE;IAC3B,IAAME,UAAU,GAAG,IAAIC,UAAU,CAACH,MAAM,CAACA,MAAM,EAAEA,MAAM,CAACI,UAAU,EAAEJ,MAAM,CAACK,MAAM,CAAC;IAClF,OAAOH,UAAU,CAACI,KAAK,CAAC,CAAC,CAACN,MAAM;EAClC;EACA,OAAOA,MAAM;AACf;AAWA,IAAMO,OAAO,GAAG;EAQdC,KAAK,EAALA;AAGF,CAAC;AAMM,IAAMC,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIC,0BAAa,CAAC,CAAC;EACjCC,IAAI,EAAE,IAAIC,4BAAe,CAAC,CAAC;EAC3BC,MAAM,EAAE,IAAIC,8BAAiB,CAAC,CAAC;EAC/BC,MAAM,EAAE,IAAIC,8BAAiB,CAAC;IAACV,OAAO,EAAPA;EAAO,CAAC,CAAC;EAExCW,GAAG,EAAE,IAAIC,2BAAc,CAAC;IAACZ,OAAO,EAAPA;EAAO,CAAC,CAAC;EAClCa,OAAO,EAAE,IAAID,2BAAc,CAAC;IAACZ,OAAO,EAAPA;EAAO,CAAC,CAAC;EAGtCc,IAAI,EAAE,IAAIC,4BAAe,CAAC;IAACf,OAAO,EAAPA;EAAO,CAAC;AACrC,CAAC;AAACgB,OAAA,CAAAd,2BAAA,GAAAA,2BAAA;AAAA,SAMoBe,mBAAmBA,CAAAC,EAAA;EAAA,OAAAC,oBAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAF,qBAAA;EAAAA,oBAAA,OAAAG,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAlC,SAAAC,QAAmCC,OAAyC;IAAA,IAAAC,YAAA;IAAA,OAAAJ,YAAA,CAAAD,OAAA,CAAAM,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;QAAA;UAC3EL,YAAY,GAAGM,MAAM,CAACC,MAAM,CAACjC,2BAA2B,CAAC;UAAA6B,QAAA,CAAAE,IAAA;UAAA,OAClDG,OAAO,CAACC,GAAG,CAACT,YAAY,CAACU,GAAG,CAAC,UAACC,WAAW;YAAA,OAAKA,WAAW,CAACC,OAAO,CAAC,CAAC;UAAA,EAAC,CAAC;QAAA;UAAA,OAAAT,QAAA,CAAAU,MAAA,WAAAV,QAAA,CAAAW,IAAA;QAAA;QAAA;UAAA,OAAAX,QAAA,CAAAY,IAAA;MAAA;IAAA,GAAAjB,OAAA;EAAA,CACnF;EAAA,OAAAP,oBAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAKqBuB,OAAOA,CAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,QAAA,CAAA3B,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAA0B,SAAA;EAAAA,QAAA,OAAAzB,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAtB,SAAAuB,SAAuBC,MAA0B,EAAEC,KAAa;IAAA,IAAAX,WAAA,EAAAY,gBAAA,EAAAC,qBAAA;IAAA,OAAA5B,YAAA,CAAAD,OAAA,CAAAM,IAAA,UAAAwB,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAtB,IAAA,GAAAsB,SAAA,CAAArB,IAAA;QAAA;UAC/DM,WAAW,GAAGrC,2BAA2B,CAAC+C,MAAM,CAAC;UAAA,IAClDV,WAAW;YAAAe,SAAA,CAAArB,IAAA;YAAA;UAAA;UAAA,MACR,IAAIsB,KAAK,yCAAAC,MAAA,CAAyCP,MAAM,CAAE,CAAC;QAAA;UAE7DE,gBAAgB,GAAG3D,aAAa,CAAC0D,KAAK,CAAC;UAAAI,SAAA,CAAArB,IAAA;UAAA,OACTM,WAAW,CAACkB,QAAQ,CAACN,gBAAgB,CAAC;QAAA;UAApEC,qBAAqB,GAAAE,SAAA,CAAAZ,IAAA;UAAA,OAAAY,SAAA,CAAAb,MAAA,WACpBrD,QAAQ,CAACgE,qBAAqB,CAAC;QAAA;QAAA;UAAA,OAAAE,SAAA,CAAAX,IAAA;MAAA;IAAA,GAAAK,QAAA;EAAA,CACvC;EAAA,OAAAD,QAAA,CAAA3B,KAAA,OAAAC,SAAA;AAAA;AAAA,SAKqBqC,UAAUA,CAAAC,GAAA,EAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,WAAA,CAAA1C,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAyC,YAAA;EAAAA,WAAA,OAAAxC,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAzB,SAAAsC,SACLd,MAA0B,EAC1BC,KAAa,EACbc,IAAY;IAAA,IAAAzB,WAAA,EAAAY,gBAAA,EAAAC,qBAAA;IAAA,OAAA5B,YAAA,CAAAD,OAAA,CAAAM,IAAA,UAAAoC,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAlC,IAAA,GAAAkC,SAAA,CAAAjC,IAAA;QAAA;UAENM,WAAW,GAAGrC,2BAA2B,CAAC+C,MAAM,CAAC;UAAA,IAClDV,WAAW;YAAA2B,SAAA,CAAAjC,IAAA;YAAA;UAAA;UAAA,MACR,IAAIsB,KAAK,yCAAAC,MAAA,CAAyCP,MAAM,CAAE,CAAC;QAAA;UAE7DE,gBAAgB,GAAG3D,aAAa,CAAC0D,KAAK,CAAC;UAAAgB,SAAA,CAAAjC,IAAA;UAAA,OACTM,WAAW,CAACmB,UAAU,CAACP,gBAAgB,EAAEa,IAAI,CAAC;QAAA;UAA5EZ,qBAAqB,GAAAc,SAAA,CAAAxB,IAAA;UAAA,OAAAwB,SAAA,CAAAzB,MAAA,WACpBrD,QAAQ,CAACgE,qBAAqB,CAAC;QAAA;QAAA;UAAA,OAAAc,SAAA,CAAAvB,IAAA;MAAA;IAAA,GAAAoB,QAAA;EAAA,CACvC;EAAA,OAAAD,WAAA,CAAA1C,KAAA,OAAAC,SAAA;AAAA;AAKM,SAAS8C,OAAOA,CAAClB,MAA0B,EAAEC,KAAa,EAAEc,IAAY,EAAU;EACvF,IAAI,EAAEf,MAAM,IAAI/C,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAIqD,KAAK,gCAAAC,MAAA,CAAgCP,MAAM,CAAE,CAAC;EAC1D;EAEA,OAAO/C,2BAA2B,CAAC+C,MAAM,CAAC,CAACkB,OAAO,CAACjB,KAAK,EAAEc,IAAI,CAAC;AACjE"}