@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

package/dist/es5/parquetjs/codecs/plain.js

@@ -1,287 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.encodeValues = encodeValues;
-exports.decodeValues = decodeValues;
-
-var _int = _interopRequireDefault(require("int53"));
-
-function encodeValues(type, values, opts) {
-  switch (type) {
-    case 'BOOLEAN':
-      return encodeValues_BOOLEAN(values);
-
-    case 'INT32':
-      return encodeValues_INT32(values);
-
-    case 'INT64':
-      return encodeValues_INT64(values);
-
-    case 'INT96':
-      return encodeValues_INT96(values);
-
-    case 'FLOAT':
-      return encodeValues_FLOAT(values);
-
-    case 'DOUBLE':
-      return encodeValues_DOUBLE(values);
-
-    case 'BYTE_ARRAY':
-      return encodeValues_BYTE_ARRAY(values);
-
-    case 'FIXED_LEN_BYTE_ARRAY':
-      return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
-
-    default:
-      throw new Error("unsupported type: ".concat(type));
-  }
-}
-
-function decodeValues(type, cursor, count, opts) {
-  switch (type) {
-    case 'BOOLEAN':
-      return decodeValues_BOOLEAN(cursor, count);
-
-    case 'INT32':
-      return decodeValues_INT32(cursor, count);
-
-    case 'INT64':
-      return decodeValues_INT64(cursor, count);
-
-    case 'INT96':
-      return decodeValues_INT96(cursor, count);
-
-    case 'FLOAT':
-      return decodeValues_FLOAT(cursor, count);
-
-    case 'DOUBLE':
-      return decodeValues_DOUBLE(cursor, count);
-
-    case 'BYTE_ARRAY':
-      return decodeValues_BYTE_ARRAY(cursor, count);
-
-    case 'FIXED_LEN_BYTE_ARRAY':
-      return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
-
-    default:
-      throw new Error("unsupported type: ".concat(type));
-  }
-}
-
-function encodeValues_BOOLEAN(values) {
-  var buf = Buffer.alloc(Math.ceil(values.length / 8));
-  buf.fill(0);
-
-  for (var i = 0; i < values.length; i++) {
-    if (values[i]) {
-      buf[Math.floor(i / 8)] |= 1 << i % 8;
-    }
-  }
-
-  return buf;
-}
-
-function decodeValues_BOOLEAN(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    var b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
-    values.push((b & 1 << i % 8) > 0);
-  }
-
-  cursor.offset += Math.ceil(count / 8);
-  return values;
-}
-
-function encodeValues_INT32(values) {
-  var buf = Buffer.alloc(4 * values.length);
-
-  for (var i = 0; i < values.length; i++) {
-    buf.writeInt32LE(values[i], i * 4);
-  }
-
-  return buf;
-}
-
-function decodeValues_INT32(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    values.push(cursor.buffer.readInt32LE(cursor.offset));
-    cursor.offset += 4;
-  }
-
-  return values;
-}
-
-function encodeValues_INT64(values) {
-  var buf = Buffer.alloc(8 * values.length);
-
-  for (var i = 0; i < values.length; i++) {
-    _int.default.writeInt64LE(values[i], buf, i * 8);
-  }
-
-  return buf;
-}
-
-function decodeValues_INT64(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    values.push(_int.default.readInt64LE(cursor.buffer, cursor.offset));
-    cursor.offset += 8;
-  }
-
-  return values;
-}
-
-function encodeValues_INT96(values) {
-  var buf = Buffer.alloc(12 * values.length);
-
-  for (var i = 0; i < values.length; i++) {
-    if (values[i] >= 0) {
-      _int.default.writeInt64LE(values[i], buf, i * 12);
-
-      buf.writeUInt32LE(0, i * 12 + 8);
-    } else {
-      _int.default.writeInt64LE(~-values[i] + 1, buf, i * 12);
-
-      buf.writeUInt32LE(0xffffffff, i * 12 + 8);
-    }
-  }
-
-  return buf;
-}
-
-function decodeValues_INT96(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    var low = _int.default.readInt64LE(cursor.buffer, cursor.offset);
-
-    var high = cursor.buffer.readUInt32LE(cursor.offset + 8);
-
-    if (high === 0xffffffff) {
-      values.push(~-low + 1);
-    } else {
-      values.push(low);
-    }
-
-    cursor.offset += 12;
-  }
-
-  return values;
-}
-
-function encodeValues_FLOAT(values) {
-  var buf = Buffer.alloc(4 * values.length);
-
-  for (var i = 0; i < values.length; i++) {
-    buf.writeFloatLE(values[i], i * 4);
-  }
-
-  return buf;
-}
-
-function decodeValues_FLOAT(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    values.push(cursor.buffer.readFloatLE(cursor.offset));
-    cursor.offset += 4;
-  }
-
-  return values;
-}
-
-function encodeValues_DOUBLE(values) {
-  var buf = Buffer.alloc(8 * values.length);
-
-  for (var i = 0; i < values.length; i++) {
-    buf.writeDoubleLE(values[i], i * 8);
-  }
-
-  return buf;
-}
-
-function decodeValues_DOUBLE(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    values.push(cursor.buffer.readDoubleLE(cursor.offset));
-    cursor.offset += 8;
-  }
-
-  return values;
-}
-
-function encodeValues_BYTE_ARRAY(values) {
-  var buf_len = 0;
-
-  for (var i = 0; i < values.length; i++) {
-    values[i] = Buffer.from(values[i]);
-    buf_len += 4 + values[i].length;
-  }
-
-  var buf = Buffer.alloc(buf_len);
-  var buf_pos = 0;
-
-  for (var _i = 0; _i < values.length; _i++) {
-    buf.writeUInt32LE(values[_i].length, buf_pos);
-
-    values[_i].copy(buf, buf_pos + 4);
-
-    buf_pos += 4 + values[_i].length;
-  }
-
-  return buf;
-}
-
-function decodeValues_BYTE_ARRAY(cursor, count) {
-  var values = [];
-
-  for (var i = 0; i < count; i++) {
-    var len = cursor.buffer.readUInt32LE(cursor.offset);
-    cursor.offset += 4;
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
-    cursor.offset += len;
-  }
-
-  return values;
-}
-
-function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
-  if (!opts.typeLength) {
-    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
-  }
-
-  for (var i = 0; i < values.length; i++) {
-    values[i] = Buffer.from(values[i]);
-
-    if (values[i].length !== opts.typeLength) {
-      throw new Error("invalid value for FIXED_LEN_BYTE_ARRAY: ".concat(values[i]));
-    }
-  }
-
-  return Buffer.concat(values);
-}
-
-function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
-  var values = [];
-
-  if (!opts.typeLength) {
-    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
-  }
-
-  for (var i = 0; i < count; i++) {
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
-    cursor.offset += opts.typeLength;
-  }
-
-  return values;
-}
-//# sourceMappingURL=plain.js.map

package/dist/es5/parquetjs/codecs/plain.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/plain.ts"],"names":["encodeValues","type","values","opts","encodeValues_BOOLEAN","encodeValues_INT32","encodeValues_INT64","encodeValues_INT96","encodeValues_FLOAT","encodeValues_DOUBLE","encodeValues_BYTE_ARRAY","encodeValues_FIXED_LEN_BYTE_ARRAY","Error","decodeValues","cursor","count","decodeValues_BOOLEAN","decodeValues_INT32","decodeValues_INT64","decodeValues_INT96","decodeValues_FLOAT","decodeValues_DOUBLE","decodeValues_BYTE_ARRAY","decodeValues_FIXED_LEN_BYTE_ARRAY","buf","Buffer","alloc","Math","ceil","length","fill","i","floor","b","buffer","offset","push","writeInt32LE","readInt32LE","INT53","writeInt64LE","readInt64LE","writeUInt32LE","low","high","readUInt32LE","writeFloatLE","readFloatLE","writeDoubleLE","readDoubleLE","buf_len","from","buf_pos","copy","len","slice","typeLength","concat"],"mappings":";;;;;;;;;;AAIA;;AAEO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOG,oBAAoB,CAACF,MAAD,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACH,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACJ,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACL,MAAD,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOM,kBAAkB,CAACN,MAAD,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOO,mBAAmB,CAACP,MAAD,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOQ,uBAAuB,CAACR,MAAD,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOS,iCAAiC,CAACT,MAAD,EAASC,IAAT,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAEM,SAASY,YAAT,CACLZ,IADK,EAELa,MAFK,EAGLC,KAHK,EAILZ,IAJK,EAKE;AACP,UAAQF,IAAR;AACE,SAAK,SAAL;AACE,aAAOe,oBAAoB,CAACF,MAAD,EAASC,KAAT,CAA3B;;AACF,SAAK,OAAL;AACE,aAAOE,kBAAkB,CAACH,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOG,kBAAkB,CAACJ,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOI,kBAAkB,CAACL,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,OAAL;AACE,aAAOK,kBAAkB,CAACN,MAAD,EAASC,KAAT,CAAzB;;AACF,SAAK,QAAL;AACE,aAAOM,mBAAmB,CAACP,MAAD,EAASC,KAAT,CAA1B;;AACF,SAAK,YAAL;AACE,aAAOO,uBAAuB,CAACR,MAAD,EAASC,KAAT,CAA9B;;AACF,SAAK,sBAAL;AACE,aAAOQ,iCAAiC,CAACT,MAAD,EAASC,KAAT,EAAgBZ,IAAhB,CAAxC;;AACF;AACE,YAAM,IAAIS,KAAJ,6BAA+BX,IAA/B,EAAN;AAlBJ;AAoBD;;AAED,SAASG,oBAAT,CAA8BF,MAA9B,EAAyD;AACvD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAaC,IAAI,CAACC,IAAL,CAAU1B,MAAM,CAAC2B,MAAP,GAAgB,CAA1B,CAAb,CAAZ;AACAL,EAAAA,GAAG,CAACM,IAAJ,CAAS,CAAT;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAV,EAAe;AACbP,MAAAA,GAAG,CAACG,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASR,oBAAT,CAA8BF,MAA9B,EAAoDC,KAApD,EAA8E;AAC5E,MAAMb,MAAiB,GAAG,EAA1B;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAME,CAAC,GAAGnB,MAAM,CAACoB,MAAP,CAAcpB,MAAM,CAACqB,MAAP,GAAgBR,IAAI,CAACK,KAAL,CAAWD,CAAC,GAAG,CAAf,CAA9B,CAAV;AACA7B,IAAAA,MAAM,CAACkC,IAAP,CAAY,CAACH,CAAC,GAAI,KAAKF,CAAC,GAAG,CAAf,IAAqB,CAAjC;AACD;;AACDjB,EAAAA,MAAM,CAACqB,MAAP,IAAiBR,IAAI,CAACC,IAAL,CAAUb,KAAK,GAAG,CAAlB,CAAjB;AACA,SAAOb,MAAP;AACD;;AAED,SAASG,kBAAT,CAA4BH,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACa,YAAJ,CAAiBnC,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASP,kBAAT,CAA4BH,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcI,WAAd,CAA0BxB,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASI,kBAAT,CAA4BJ,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCQ,iBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,CAAvC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASN,kBAAT,CAA4BJ,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYG,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASK,kBAAT,CAA4BL,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,KAAKxB,MAAM,CAAC2B,MAAzB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,IAAa,CAAjB,EAAoB;AAClBQ,mBAAMC,YAAN,CAAmBtC,MAAM,CAAC6B,CAAD,CAAzB,EAA8BP,GAA9B,EAAmCO,CAAC,GAAG,EAAvC;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,CAAlB,EAAqBX,CAAC,GAAG,EAAJ,GAAS,CAA9B;AACD,KAHD,MAGO;AACLQ,mBAAMC,YAAN,CAAmB,CAAC,CAACtC,MAAM,CAAC6B,CAAD,CAAR,GAAc,CAAjC,EAAoCP,GAApC,EAAyCO,CAAC,GAAG,EAA7C;;AACAP,MAAAA,GAAG,CAACkB,aAAJ,CAAkB,UAAlB,EAA8BX,CAAC,GAAG,EAAJ,GAAS,CAAvC;AACD;AACF;;AACD,SAAOP,GAAP;AACD;;AAED,SAASL,kBAAT,CAA4BL,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAMY,GAAG,GAAGJ,aAAME,WAAN,CAAkB3B,MAAM,CAACoB,MAAzB,EAAiCpB,MAAM,CAACqB,MAAxC,CAAZ;;AACA,QAAMS,IAAI,GAAG9B,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAP,GAAgB,CAA3C,CAAb;;AACA,QAAIS,IAAI,KAAK,UAAb,EAAyB;AACvB1C,MAAAA,MAAM,CAACkC,IAAP,CAAY,CAAC,CAACO,GAAF,GAAQ,CAApB;AACD,KAFD,MAEO;AACLzC,MAAAA,MAAM,CAACkC,IAAP,CAAYO,GAAZ;AACD;;AACD7B,IAAAA,MAAM,CAACqB,MAAP,IAAiB,EAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASM,kBAAT,CAA4BN,MAA5B,EAAsD;AACpD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACsB,YAAJ,CAAiB5C,MAAM,CAAC6B,CAAD,CAAvB,EAA4BA,CAAC,GAAG,CAAhC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASJ,kBAAT,CAA4BN,MAA5B,EAAkDC,KAAlD,EAA2E;AACzE,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAca,WAAd,CAA0BjC,MAAM,CAACqB,MAAjC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASO,mBAAT,CAA6BP,MAA7B,EAAuD;AACrD,MAAMsB,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,IAAIxB,MAAM,CAAC2B,MAAxB,CAAZ;;AACA,OAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACwB,aAAJ,CAAkB9C,MAAM,CAAC6B,CAAD,CAAxB,EAA6BA,CAAC,GAAG,CAAjC;AACD;;AACD,SAAOP,GAAP;AACD;;AAED,SAASH,mBAAT,CAA6BP,MAA7B,EAAmDC,KAAnD,EAA4E;AAC1E,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAce,YAAd,CAA2BnC,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACD;;AACD,SAAOjC,MAAP;AACD;;AAED,SAASQ,uBAAT,CAAiCR,MAAjC,EAA2D;AAEzD,MAAIgD,OAAO,GAAG,CAAd;;AACA,OAAK,IAAInB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;AACAmB,IAAAA,OAAO,IAAI,IAAIhD,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,MAAML,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAawB,OAAb,CAAZ;AAEA,MAAIE,OAAO,GAAG,CAAd;;AACA,OAAK,IAAIrB,EAAC,GAAG,CAAb,EAAgBA,EAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,EAAC,EAApC,EAAwC;AACtCP,IAAAA,GAAG,CAACkB,aAAJ,CAAkBxC,MAAM,CAAC6B,EAAD,CAAN,CAAUF,MAA5B,EAAoCuB,OAApC;;AACAlD,IAAAA,MAAM,CAAC6B,EAAD,CAAN,CAAUsB,IAAV,CAAe7B,GAAf,EAAoB4B,OAAO,GAAG,CAA9B;;AACAA,IAAAA,OAAO,IAAI,IAAIlD,MAAM,CAAC6B,EAAD,CAAN,CAAUF,MAAzB;AACD;;AACD,SAAOL,GAAP;AACD;;AAED,SAASF,uBAAT,CAAiCR,MAAjC,EAAuDC,KAAvD,EAAgF;AAC9E,MAAMb,MAAgB,GAAG,EAAzB;;AACA,OAAK,IAAI6B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B,QAAMuB,GAAG,GAAGxC,MAAM,CAACoB,MAAP,CAAcW,YAAd,CAA2B/B,MAAM,CAACqB,MAAlC,CAAZ;AACArB,IAAAA,MAAM,CAACqB,MAAP,IAAiB,CAAjB;AACAjC,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBmB,GAAnD,CAAZ;AACAxC,IAAAA,MAAM,CAACqB,MAAP,IAAiBmB,GAAjB;AACD;;AACD,SAAOpD,MAAP;AACD;;AAED,SAASS,iCAAT,CAA2CT,MAA3C,EAA6DC,IAA7D,EAAgG;AAC9F,MAAI,CAACA,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG7B,MAAM,CAAC2B,MAA3B,EAAmCE,CAAC,EAApC,EAAwC;AACtC7B,IAAAA,MAAM,CAAC6B,CAAD,CAAN,GAAYN,MAAM,CAAC0B,IAAP,CAAYjD,MAAM,CAAC6B,CAAD,CAAlB,CAAZ;;AACA,QAAI7B,MAAM,CAAC6B,CAAD,CAAN,CAAUF,MAAV,KAAqB1B,IAAI,CAACqD,UAA9B,EAA0C;AACxC,YAAM,IAAI5C,KAAJ,mDAAqDV,MAAM,CAAC6B,CAAD,CAA3D,EAAN;AACD;AACF;;AACD,SAAON,MAAM,CAACgC,MAAP,CAAcvD,MAAd,CAAP;AACD;;AAED,SAASqB,iCAAT,CACET,MADF,EAEEC,KAFF,EAGEZ,IAHF,EAIY;AACV,MAAMD,MAAgB,GAAG,EAAzB;;AACA,MAAI,CAACC,IAAI,CAACqD,UAAV,EAAsB;AACpB,UAAM,IAAI5C,KAAJ,CAAU,gEAAV,CAAN;AACD;;AACD,OAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGhB,KAApB,EAA2BgB,CAAC,EAA5B,EAAgC;AAC9B7B,IAAAA,MAAM,CAACkC,IAAP,CAAYtB,MAAM,CAACoB,MAAP,CAAcqB,KAAd,CAAoBzC,MAAM,CAACqB,MAA3B,EAAmCrB,MAAM,CAACqB,MAAP,GAAgBhC,IAAI,CAACqD,UAAxD,CAAZ;AACA1C,IAAAA,MAAM,CAACqB,MAAP,IAAiBhC,IAAI,CAACqD,UAAtB;AACD;;AACD,SAAOtD,MAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport INT53 from 'int53';\n\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  switch (type) {\n    case 'BOOLEAN':\n      return encodeValues_BOOLEAN(values);\n    case 'INT32':\n      return encodeValues_INT32(values);\n    case 'INT64':\n      return encodeValues_INT64(values);\n    case 'INT96':\n      return encodeValues_INT96(values);\n    case 'FLOAT':\n      return encodeValues_FLOAT(values);\n    case 'DOUBLE':\n      return encodeValues_DOUBLE(values);\n    case 'BYTE_ARRAY':\n      return encodeValues_BYTE_ARRAY(values);\n    case 'FIXED_LEN_BYTE_ARRAY':\n      return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  switch (type) {\n    case 'BOOLEAN':\n      return decodeValues_BOOLEAN(cursor, count);\n    case 'INT32':\n      return decodeValues_INT32(cursor, count);\n    case 'INT64':\n      return decodeValues_INT64(cursor, count);\n    case 'INT96':\n      return decodeValues_INT96(cursor, count);\n    case 'FLOAT':\n      return decodeValues_FLOAT(cursor, count);\n    case 'DOUBLE':\n      return decodeValues_DOUBLE(cursor, count);\n    case 'BYTE_ARRAY':\n      return decodeValues_BYTE_ARRAY(cursor, count);\n    case 'FIXED_LEN_BYTE_ARRAY':\n      return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n}\n\nfunction encodeValues_BOOLEAN(values: boolean[]): Buffer {\n  const buf = Buffer.alloc(Math.ceil(values.length / 8));\n  buf.fill(0);\n  for (let i = 0; i < values.length; i++) {\n    if (values[i]) {\n      buf[Math.floor(i / 8)] |= 1 << i % 8;\n    }\n  }\n  return buf;\n}\n\nfunction decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {\n  const values: boolean[] = [];\n  for (let i = 0; i < count; i++) {\n    const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];\n    values.push((b & (1 << i % 8)) > 0);\n  }\n  cursor.offset += Math.ceil(count / 8);\n  return values;\n}\n\nfunction encodeValues_INT32(values: number[]): Buffer {\n  const buf = Buffer.alloc(4 * values.length);\n  for (let i = 0; i < values.length; i++) {\n    buf.writeInt32LE(values[i], i * 4);\n  }\n  return buf;\n}\n\nfunction decodeValues_INT32(cursor: CursorBuffer, count: number): number[] {\n  const values: number[] = [];\n  for (let i = 0; i < count; i++) {\n    values.push(cursor.buffer.readInt32LE(cursor.offset));\n    cursor.offset += 4;\n  }\n  return values;\n}\n\nfunction encodeValues_INT64(values: number[]): Buffer {\n  const buf = Buffer.alloc(8 * values.length);\n  for (let i = 0; i < values.length; i++) {\n    INT53.writeInt64LE(values[i], buf, i * 8);\n  }\n  return buf;\n}\n\nfunction decodeValues_INT64(cursor: CursorBuffer, count: number): number[] {\n  const values: number[] = [];\n  for (let i = 0; i < count; i++) {\n    values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));\n    cursor.offset += 8;\n  }\n  return values;\n}\n\nfunction encodeValues_INT96(values: number[]): Buffer {\n  const buf = Buffer.alloc(12 * values.length);\n  for (let i = 0; i < values.length; i++) {\n    if (values[i] >= 0) {\n      INT53.writeInt64LE(values[i], buf, i * 12);\n      buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision\n    } else {\n      INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);\n      buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision\n    }\n  }\n  return buf;\n}\n\nfunction decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {\n  const values: number[] = [];\n  for (let i = 0; i < count; i++) {\n    const low = INT53.readInt64LE(cursor.buffer, cursor.offset);\n    const high = cursor.buffer.readUInt32LE(cursor.offset + 8);\n    if (high === 0xffffffff) {\n      values.push(~-low + 1); // truncate to 64 actual precision\n    } else {\n      values.push(low); // truncate to 64 actual precision\n    }\n    cursor.offset += 12;\n  }\n  return values;\n}\n\nfunction encodeValues_FLOAT(values: number[]): Buffer {\n  const buf = Buffer.alloc(4 * values.length);\n  for (let i = 0; i < values.length; i++) {\n    buf.writeFloatLE(values[i], i * 4);\n  }\n  return buf;\n}\n\nfunction decodeValues_FLOAT(cursor: CursorBuffer, count: number): number[] {\n  const values: number[] = [];\n  for (let i = 0; i < count; i++) {\n    values.push(cursor.buffer.readFloatLE(cursor.offset));\n    cursor.offset += 4;\n  }\n  return values;\n}\n\nfunction encodeValues_DOUBLE(values: number[]): Buffer {\n  const buf = Buffer.alloc(8 * values.length);\n  for (let i = 0; i < values.length; i++) {\n    buf.writeDoubleLE(values[i], i * 8);\n  }\n  return buf;\n}\n\nfunction decodeValues_DOUBLE(cursor: CursorBuffer, count: number): number[] {\n  const values: number[] = [];\n  for (let i = 0; i < count; i++) {\n    values.push(cursor.buffer.readDoubleLE(cursor.offset));\n    cursor.offset += 8;\n  }\n  return values;\n}\n\nfunction encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {\n  // tslint:disable-next-line:variable-name\n  let buf_len = 0;\n  for (let i = 0; i < values.length; i++) {\n    values[i] = Buffer.from(values[i]);\n    buf_len += 4 + values[i].length;\n  }\n  const buf = Buffer.alloc(buf_len);\n  // tslint:disable-next-line:variable-name\n  let buf_pos = 0;\n  for (let i = 0; i < values.length; i++) {\n    buf.writeUInt32LE(values[i].length, buf_pos);\n    values[i].copy(buf, buf_pos + 4);\n    buf_pos += 4 + values[i].length;\n  }\n  return buf;\n}\n\nfunction decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[] {\n  const values: Buffer[] = [];\n  for (let i = 0; i < count; i++) {\n    const len = cursor.buffer.readUInt32LE(cursor.offset);\n    cursor.offset += 4;\n    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));\n    cursor.offset += len;\n  }\n  return values;\n}\n\nfunction encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecOptions): Buffer {\n  if (!opts.typeLength) {\n    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n  }\n  for (let i = 0; i < values.length; i++) {\n    values[i] = Buffer.from(values[i]);\n    if (values[i].length !== opts.typeLength) {\n      throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);\n    }\n  }\n  return Buffer.concat(values);\n}\n\nfunction decodeValues_FIXED_LEN_BYTE_ARRAY(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): Buffer[] {\n  const values: Buffer[] = [];\n  if (!opts.typeLength) {\n    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');\n  }\n  for (let i = 0; i < count; i++) {\n    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));\n    cursor.offset += opts.typeLength;\n  }\n  return values;\n}\n"],"file":"plain.js"}

package/dist/es5/parquetjs/codecs/rle.js

@@ -1,174 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.encodeValues = encodeValues;
-exports.decodeValues = decodeValues;
-
-var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
-
-var _varint = _interopRequireDefault(require("varint"));
-
-function encodeValues(type, values, opts) {
-  if (!('bitWidth' in opts)) {
-    throw new Error('bitWidth is required');
-  }
-
-  switch (type) {
-    case 'BOOLEAN':
-    case 'INT32':
-    case 'INT64':
-      values = values.map(function (x) {
-        return parseInt(x, 10);
-      });
-      break;
-
-    default:
-      throw new Error("unsupported type: ".concat(type));
-  }
-
-  var buf = Buffer.alloc(0);
-  var run = [];
-  var repeats = 0;
-
-  for (var i = 0; i < values.length; i++) {
-    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
-      if (run.length) {
-        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
-        run = [];
-      }
-
-      repeats = 1;
-    } else if (repeats > 0 && values[i] === values[i - 1]) {
-      repeats += 1;
-    } else {
-      if (repeats) {
-        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
-        repeats = 0;
-      }
-
-      run.push(values[i]);
-    }
-  }
-
-  if (repeats) {
-    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
-  } else if (run.length) {
-    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
-  }
-
-  if (opts.disableEnvelope) {
-    return buf;
-  }
-
-  var envelope = Buffer.alloc(buf.length + 4);
-  envelope.writeUInt32LE(buf.length, undefined);
-  buf.copy(envelope, 4);
-  return envelope;
-}
-
-function decodeValues(type, cursor, count, opts) {
-  if (!('bitWidth' in opts)) {
-    throw new Error('bitWidth is required');
-  }
-
-  if (!opts.disableEnvelope) {
-    cursor.offset += 4;
-  }
-
-  var values = [];
-
-  while (values.length < count) {
-    var header = _varint.default.decode(cursor.buffer, cursor.offset);
-
-    cursor.offset += _varint.default.encodingLength(header);
-
-    if (header & 1) {
-      var _values;
-
-      var _count = (header >> 1) * 8;
-
-      (_values = values).push.apply(_values, (0, _toConsumableArray2.default)(decodeRunBitpacked(cursor, _count, opts)));
-    } else {
-      var _values2;
-
-      var _count2 = header >> 1;
-
-      (_values2 = values).push.apply(_values2, (0, _toConsumableArray2.default)(decodeRunRepeated(cursor, _count2, opts)));
-    }
-  }
-
-  values = values.slice(0, count);
-
-  if (values.length !== count) {
-    throw new Error('invalid RLE encoding');
-  }
-
-  return values;
-}
-
-function decodeRunBitpacked(cursor, count, opts) {
-  var bitWidth = opts.bitWidth;
-
-  if (count % 8 !== 0) {
-    throw new Error('must be a multiple of 8');
-  }
-
-  var values = new Array(count).fill(0);
-
-  for (var b = 0; b < bitWidth * count; b++) {
-    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
-      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
-    }
-  }
-
-  cursor.offset += bitWidth * (count / 8);
-  return values;
-}
-
-function decodeRunRepeated(cursor, count, opts) {
-  var bitWidth = opts.bitWidth;
-  var value = 0;
-
-  for (var i = 0; i < Math.ceil(bitWidth / 8); i++) {
-    value << 8;
-    value += cursor.buffer[cursor.offset];
-    cursor.offset += 1;
-  }
-
-  return new Array(count).fill(value);
-}
-
-function encodeRunBitpacked(values, opts) {
-  var bitWidth = opts.bitWidth;
-
-  for (var i = 0; i < values.length % 8; i++) {
-    values.push(0);
-  }
-
-  var buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
-
-  for (var b = 0; b < bitWidth * values.length; b++) {
-    if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
-      buf[Math.floor(b / 8)] |= 1 << b % 8;
-    }
-  }
-
-  return Buffer.concat([Buffer.from(_varint.default.encode(values.length / 8 << 1 | 1)), buf]);
-}
-
-function encodeRunRepeated(value, count, opts) {
-  var bitWidth = opts.bitWidth;
-  var buf = Buffer.alloc(Math.ceil(bitWidth / 8));
-
-  for (var i = 0; i < buf.length; i++) {
-    buf.writeUInt8(value & 0xff, i);
-    value >> 8;
-  }
-
-  return Buffer.concat([Buffer.from(_varint.default.encode(count << 1)), buf]);
-}
-//# sourceMappingURL=rle.js.map

package/dist/es5/parquetjs/codecs/rle.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAEhD6B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AAEA6B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}

package/dist/es5/parquetjs/compression.js

@@ -1,167 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.preloadCompressions = preloadCompressions;
-exports.deflate = deflate;
-exports.decompress = decompress;
-exports.inflate = inflate;
-exports.PARQUET_COMPRESSION_METHODS = void 0;
-
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-
-var _compression = require("@loaders.gl/compression");
-
-var _bufferUtils = require("./utils/buffer-utils");
-
-var _lz4js = _interopRequireDefault(require("lz4js"));
-
-var _lzo = _interopRequireDefault(require("lzo"));
-
-var modules = {
-  lz4js: _lz4js.default,
-  lzo: _lzo.default
-};
-var PARQUET_COMPRESSION_METHODS = {
-  UNCOMPRESSED: new _compression.NoCompression(),
-  GZIP: new _compression.GZipCompression(),
-  SNAPPY: new _compression.SnappyCompression(),
-  BROTLI: new _compression.BrotliCompression({
-    modules: modules
-  }),
-  LZ4: new _compression.LZ4Compression({
-    modules: modules
-  }),
-  LZ4_RAW: new _compression.LZ4Compression({
-    modules: modules
-  }),
-  LZO: new _compression.LZOCompression({
-    modules: modules
-  }),
-  ZSTD: new _compression.ZstdCompression({
-    modules: modules
-  })
-};
-exports.PARQUET_COMPRESSION_METHODS = PARQUET_COMPRESSION_METHODS;
-
-function preloadCompressions(_x) {
-  return _preloadCompressions.apply(this, arguments);
-}
-
-function _preloadCompressions() {
-  _preloadCompressions = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(options) {
-    var compressions;
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) {
-        switch (_context.prev = _context.next) {
-          case 0:
-            compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-            _context.next = 3;
-            return Promise.all(compressions.map(function (compression) {
-              return compression.preload();
-            }));
-
-          case 3:
-            return _context.abrupt("return", _context.sent);
-
-          case 4:
-          case "end":
-            return _context.stop();
-        }
-      }
-    }, _callee);
-  }));
-  return _preloadCompressions.apply(this, arguments);
-}
-
-function deflate(_x2, _x3) {
-  return _deflate.apply(this, arguments);
-}
-
-function _deflate() {
-  _deflate = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(method, value) {
-    var compression, inputArrayBuffer, compressedArrayBuffer;
-    return _regenerator.default.wrap(function _callee2$(_context2) {
-      while (1) {
-        switch (_context2.prev = _context2.next) {
-          case 0:
-            compression = PARQUET_COMPRESSION_METHODS[method];
-
-            if (compression) {
-              _context2.next = 3;
-              break;
-            }
-
-            throw new Error("parquet: invalid compression method: ".concat(method));
-
-          case 3:
-            inputArrayBuffer = (0, _bufferUtils.toArrayBuffer)(value);
-            _context2.next = 6;
-            return compression.compress(inputArrayBuffer);
-
-          case 6:
-            compressedArrayBuffer = _context2.sent;
-            return _context2.abrupt("return", (0, _bufferUtils.toBuffer)(compressedArrayBuffer));
-
-          case 8:
-          case "end":
-            return _context2.stop();
-        }
-      }
-    }, _callee2);
-  }));
-  return _deflate.apply(this, arguments);
-}
-
-function decompress(_x4, _x5, _x6) {
-  return _decompress.apply(this, arguments);
-}
-
-function _decompress() {
-  _decompress = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(method, value, size) {
-    var compression, inputArrayBuffer, compressedArrayBuffer;
-    return _regenerator.default.wrap(function _callee3$(_context3) {
-      while (1) {
-        switch (_context3.prev = _context3.next) {
-          case 0:
-            compression = PARQUET_COMPRESSION_METHODS[method];
-
-            if (compression) {
-              _context3.next = 3;
-              break;
-            }
-
-            throw new Error("parquet: invalid compression method: ".concat(method));
-
-          case 3:
-            inputArrayBuffer = (0, _bufferUtils.toArrayBuffer)(value);
-            _context3.next = 6;
-            return compression.decompress(inputArrayBuffer, size);
-
-          case 6:
-            compressedArrayBuffer = _context3.sent;
-            return _context3.abrupt("return", (0, _bufferUtils.toBuffer)(compressedArrayBuffer));
-
-          case 8:
-          case "end":
-            return _context3.stop();
-        }
-      }
-    }, _callee3);
-  }));
-  return _decompress.apply(this, arguments);
-}
-
-function inflate(method, value, size) {
-  if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw new Error("invalid compression method: ".concat(method));
-  }
-
-  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
-}
-//# sourceMappingURL=compression.js.map