@loaders.gl/parquet 3.3.0-alpha.5 → 3.3.0-alpha.7

package/dist/esm/parquetjs/codecs/rle.js

@@ -1,31 +1,29 @@
+
+
 import varint from 'varint';
+
 export function encodeValues(type, values, opts) {
   if (!('bitWidth' in opts)) {
     throw new Error('bitWidth is required');
   }
-
   switch (type) {
     case 'BOOLEAN':
     case 'INT32':
     case 'INT64':
       values = values.map(x => parseInt(x, 10));
       break;
-
     default:
       throw new Error("unsupported type: ".concat(type));
   }
-
   let buf = Buffer.alloc(0);
   let run = [];
   let repeats = 0;
-
   for (let i = 0; i < values.length; i++) {
     if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
       if (run.length) {
         buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
         run = [];
       }
-
       repeats = 1;
     } else if (repeats > 0 && values[i] === values[i - 1]) {
       repeats += 1;
@@ -34,21 +32,17 @@ export function encodeValues(type, values, opts) {
         buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
         repeats = 0;
       }
-
       run.push(values[i]);
     }
   }
-
   if (repeats) {
     buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
   } else if (run.length) {
     buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
   }
-
   if (opts.disableEnvelope) {
     return buf;
   }
-
   const envelope = Buffer.alloc(buf.length + 4);
   envelope.writeUInt32LE(buf.length, undefined);
   buf.copy(envelope, 4);
@@ -58,17 +52,13 @@ export function decodeValues(type, cursor, count, opts) {
   if (!('bitWidth' in opts)) {
     throw new Error('bitWidth is required');
   }
-
   if (!opts.disableEnvelope) {
     cursor.offset += 4;
   }
-
   let values = [];
-
   while (values.length < count) {
     const header = varint.decode(cursor.buffer, cursor.offset);
     cursor.offset += varint.encodingLength(header);
-
     if (header & 1) {
       const count = (header >> 1) * 8;
       values.push(...decodeRunBitpacked(cursor, count, opts));
@@ -77,39 +67,30 @@ export function decodeValues(type, cursor, count, opts) {
       values.push(...decodeRunRepeated(cursor, count, opts));
     }
   }
-
   values = values.slice(0, count);
-
   if (values.length !== count) {
     throw new Error('invalid RLE encoding');
   }
-
   return values;
 }
-
 function decodeRunBitpacked(cursor, count, opts) {
   const bitWidth = opts.bitWidth;
-
   if (count % 8 !== 0) {
     throw new Error('must be a multiple of 8');
   }
 
   const values = new Array(count).fill(0);
-
   for (let b = 0; b < bitWidth * count; b++) {
     if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
       values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
     }
   }
-
   cursor.offset += bitWidth * (count / 8);
   return values;
 }
-
 function decodeRunRepeated(cursor, count, opts) {
   const bitWidth = opts.bitWidth;
   let value = 0;
-
   for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
     value << 8;
     value += cursor.buffer[cursor.offset];
@@ -118,29 +99,22 @@ function decodeRunRepeated(cursor, count, opts) {
 
   return new Array(count).fill(value);
 }
-
 function encodeRunBitpacked(values, opts) {
   const bitWidth = opts.bitWidth;
-
   for (let i = 0; i < values.length % 8; i++) {
     values.push(0);
   }
-
   const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
-
   for (let b = 0; b < bitWidth * values.length; b++) {
     if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
       buf[Math.floor(b / 8)] |= 1 << b % 8;
     }
   }
-
   return Buffer.concat([Buffer.from(varint.encode(values.length / 8 << 1 | 1)), buf]);
 }
-
 function encodeRunRepeated(value, count, opts) {
   const bitWidth = opts.bitWidth;
   const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
-
   for (let i = 0; i < buf.length; i++) {
     buf.writeUInt8(value & 0xff, i);
     value >> 8;

package/dist/esm/parquetjs/codecs/rle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["varint","encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":"AAIA,OAAOA,MAAP,MAAmB,QAAnB;AAGA,OAAO,SAASC,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;AAED,OAAO,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAG7B,MAAM,CAAC8B,MAAP,CAAcJ,MAAM,CAACK,MAArB,EAA6BL,MAAM,CAACE,MAApC,CAAf;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB5B,MAAM,CAACgC,cAAP,CAAsBH,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGc,kBAAkB,CAACP,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,iBAAiB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACgC,KAAP,CAAa,CAAb,EAAgBR,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS8B,kBAAT,CACEP,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,MAAIT,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAIkC,KAAJ,CAAUV,KAAV,EAAiBW,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGT,KAA/B,EAAsCY,CAAC,EAAvC,EAA2C;AACzC,QAAIb,MAAM,CAACK,MAAP,CAAcL,MAAM,CAACE,MAAP,GAAgBY,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnEpC,MAAAA,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDV,EAAAA,MAAM,CAACE,MAAP,IAAiBQ,QAAQ,IAAIT,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAAS+B,iBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI5B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG0B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CtB,CAAC,EAA9C,EAAkD;AAEhD4B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIhB,MAAM,CAACK,MAAP,CAAcL,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIS,KAAJ,CAAUV,KAAV,EAAiBW,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAASzB,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIjC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIwB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGjC,MAAM,CAACY,MAAtC,EAA8CwB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACpC,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE3B,MAAAA,GAAG,CAAC+B,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO7B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAY5C,MAAM,CAAC6C,MAAP,CAAgB1C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2BwB,KAA3B,EAA0Cf,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,QAAM3B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACqC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B5B,CAA7B;AAEA4B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOhC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAY5C,MAAM,CAAC6C,MAAP,CAAclB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
+{"version":3,"file":"rle.js","names":["varint","encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"sources":["../../../../src/parquetjs/codecs/rle.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"mappings":";;AAIA,OAAOA,MAAM,MAAM,QAAQ;;AAG3B,OAAO,SAASC,YAAY,CAC1BC,IAAmB,EACnBC,MAAa,EACbC,IAAyB,EACjB;EACR,IAAI,EAAE,UAAU,IAAIA,IAAI,CAAC,EAAE;IACzB,MAAM,IAAIC,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,QAAQH,IAAI;IACV,KAAK,SAAS;IACd,KAAK,OAAO;IACZ,KAAK,OAAO;MAEVC,MAAM,GAAGA,MAAM,CAACG,GAAG,CAAEC,CAAC,IAAKC,QAAQ,CAACD,CAAC,EAAE,EAAE,CAAC,CAAC;MAC3C;IAEF;MACE,MAAM,IAAIF,KAAK,6BAAsBH,IAAI,EAAG;EAAC;EAGjD,IAAIO,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,CAAC;EACzB,IAAIC,GAAU,GAAG,EAAE;EACnB,IAAIC,OAAO,GAAG,CAAC;EAEf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGX,MAAM,CAACY,MAAM,EAAED,CAAC,EAAE,EAAE;IAGtC,IAAID,OAAO,KAAK,CAAC,IAAID,GAAG,CAACG,MAAM,GAAG,CAAC,KAAK,CAAC,IAAIZ,MAAM,CAACW,CAAC,CAAC,KAAKX,MAAM,CAACW,CAAC,GAAG,CAAC,CAAC,EAAE;MAExE,IAAIF,GAAG,CAACG,MAAM,EAAE;QACdN,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAEQ,kBAAkB,CAACL,GAAG,EAAER,IAAI,CAAC,CAAC,CAAC;QACzDQ,GAAG,GAAG,EAAE;MACV;MACAC,OAAO,GAAG,CAAC;IACb,CAAC,MAAM,IAAIA,OAAO,GAAG,CAAC,IAAIV,MAAM,CAACW,CAAC,CAAC,KAAKX,MAAM,CAACW,CAAC,GAAG,CAAC,CAAC,EAAE;MACrDD,OAAO,IAAI,CAAC;IACd,CAAC,MAAM;MAEL,IAAIA,OAAO,EAAE;QACXJ,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAES,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAC,CAAC,EAAED,OAAO,EAAET,IAAI,CAAC,CAAC,CAAC;QAC3ES,OAAO,GAAG,CAAC;MACb;MACAD,GAAG,CAACO,IAAI,CAAChB,MAAM,CAACW,CAAC,CAAC,CAAC;IACrB;EACF;EAEA,IAAID,OAAO,EAAE;IACXJ,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAES,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAM,GAAG,CAAC,CAAC,EAAEF,OAAO,EAAET,IAAI,CAAC,CAAC,CAAC;EACzF,CAAC,MAAM,IAAIQ,GAAG,CAACG,MAAM,EAAE;IACrBN,GAAG,GAAGC,MAAM,CAACM,MAAM,CAAC,CAACP,GAAG,EAAEQ,kBAAkB,CAACL,GAAG,EAAER,IAAI,CAAC,CAAC,CAAC;EAC3D;EAEA,IAAIA,IAAI,CAACgB,eAAe,EAAE;IACxB,OAAOX,GAAG;EACZ;EAEA,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAK,CAACF,GAAG,CAACM,MAAM,GAAG,CAAC,CAAC;EAC7CM,QAAQ,CAACC,aAAa,CAACb,GAAG,CAACM,MAAM,EAAEQ,SAAS,CAAC;EAC7Cd,GAAG,CAACe,IAAI,CAACH,QAAQ,EAAE,CAAC,CAAC;EAErB,OAAOA,QAAQ;AACjB;AAEA,OAAO,SAASI,YAAY,CAC1BvB,IAAmB,EACnBwB,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EACV,IAAI,EAAE,UAAU,IAAIA,IAAI,CAAC,EAAE;IACzB,MAAM,IAAIC,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,IAAI,CAACD,IAAI,CAACgB,eAAe,EAAE;IACzBM,MAAM,CAACE,MAAM,IAAI,CAAC;EACpB;EAEA,IAAIzB,MAAgB,GAAG,EAAE;EACzB,OAAOA,MAAM,CAACY,MAAM,GAAGY,KAAK,EAAE;IAC5B,MAAME,MAAM,GAAG7B,MAAM,CAAC8B,MAAM,CAACJ,MAAM,CAACK,MAAM,EAAEL,MAAM,CAACE,MAAM,CAAC;IAC1DF,MAAM,CAACE,MAAM,IAAI5B,MAAM,CAACgC,cAAc,CAACH,MAAM,CAAC;IAC9C,IAAIA,MAAM,GAAG,CAAC,EAAE;MACd,MAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAC,IAAI,CAAC;MAC/B1B,MAAM,CAACgB,IAAI,CAAC,GAAGc,kBAAkB,CAACP,MAAM,EAAEC,KAAK,EAAEvB,IAAI,CAAC,CAAC;IACzD,CAAC,MAAM;MACL,MAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAC;MACzB1B,MAAM,CAACgB,IAAI,CAAC,GAAGe,iBAAiB,CAACR,MAAM,EAAEC,KAAK,EAAEvB,IAAI,CAAC,CAAC;IACxD;EACF;EACAD,MAAM,GAAGA,MAAM,CAACgC,KAAK,CAAC,CAAC,EAAER,KAAK,CAAC;EAE/B,IAAIxB,MAAM,CAACY,MAAM,KAAKY,KAAK,EAAE;IAC3B,MAAM,IAAItB,KAAK,CAAC,sBAAsB,CAAC;EACzC;EAEA,OAAOF,MAAM;AACf;AAEA,SAAS8B,kBAAkB,CACzBP,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EAEV,MAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAAQ;EAEtC,IAAIT,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE;IACnB,MAAM,IAAItB,KAAK,CAAC,yBAAyB,CAAC;EAC5C;;EAGA,MAAMF,MAAM,GAAG,IAAIkC,KAAK,CAACV,KAAK,CAAC,CAACW,IAAI,CAAC,CAAC,CAAC;EACvC,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,QAAQ,GAAGT,KAAK,EAAEY,CAAC,EAAE,EAAE;IACzC,IAAIb,MAAM,CAACK,MAAM,CAACL,MAAM,CAACE,MAAM,GAAGY,IAAI,CAACC,KAAK,CAACF,CAAC,GAAG,CAAC,CAAC,CAAC,GAAI,CAAC,IAAIA,CAAC,GAAG,CAAE,EAAE;MACnEpC,MAAM,CAACqC,IAAI,CAACC,KAAK,CAACF,CAAC,GAAGH,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAIG,CAAC,GAAGH,QAAQ;IACvD;EACF;EAEAV,MAAM,CAACE,MAAM,IAAIQ,QAAQ,IAAIT,KAAK,GAAG,CAAC,CAAC;EACvC,OAAOxB,MAAM;AACf;AAEA,SAAS+B,iBAAiB,CACxBR,MAAoB,EACpBC,KAAa,EACbvB,IAAyB,EACf;EAEV,MAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAAQ;EAEtC,IAAIM,KAAK,GAAG,CAAC;EACb,KAAK,IAAI5B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG0B,IAAI,CAACG,IAAI,CAACP,QAAQ,GAAG,CAAC,CAAC,EAAEtB,CAAC,EAAE,EAAE;IAEhD4B,KAAK,IAAI,CAAC;IACVA,KAAK,IAAIhB,MAAM,CAACK,MAAM,CAACL,MAAM,CAACE,MAAM,CAAC;IACrCF,MAAM,CAACE,MAAM,IAAI,CAAC;EACpB;;EAGA,OAAO,IAAIS,KAAK,CAACV,KAAK,CAAC,CAACW,IAAI,CAACI,KAAK,CAAC;AACrC;AAEA,SAASzB,kBAAkB,CAACd,MAAgB,EAAEC,IAAyB,EAAU;EAE/E,MAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAAQ;EAEtC,KAAK,IAAItB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGX,MAAM,CAACY,MAAM,GAAG,CAAC,EAAED,CAAC,EAAE,EAAE;IAC1CX,MAAM,CAACgB,IAAI,CAAC,CAAC,CAAC;EAChB;EAEA,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC6B,IAAI,CAACG,IAAI,CAACP,QAAQ,IAAIjC,MAAM,CAACY,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;EACnE,KAAK,IAAIwB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,QAAQ,GAAGjC,MAAM,CAACY,MAAM,EAAEwB,CAAC,EAAE,EAAE;IACjD,IAAI,CAACpC,MAAM,CAACqC,IAAI,CAACC,KAAK,CAACF,CAAC,GAAGH,QAAQ,CAAC,CAAC,GAAI,CAAC,IAAIG,CAAC,GAAGH,QAAS,IAAI,CAAC,EAAE;MAChE3B,GAAG,CAAC+B,IAAI,CAACC,KAAK,CAACF,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAIA,CAAC,GAAG,CAAC;IACtC;EACF;EAEA,OAAO7B,MAAM,CAACM,MAAM,CAAC,CAACN,MAAM,CAACkC,IAAI,CAAC5C,MAAM,CAAC6C,MAAM,CAAG1C,MAAM,CAACY,MAAM,GAAG,CAAC,IAAK,CAAC,GAAI,CAAC,CAAC,CAAC,EAAEN,GAAG,CAAC,CAAC;AACzF;AAEA,SAASS,iBAAiB,CAACwB,KAAa,EAAEf,KAAa,EAAEvB,IAAyB,EAAU;EAE1F,MAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAAQ;EAEtC,MAAM3B,GAAG,GAAGC,MAAM,CAACC,KAAK,CAAC6B,IAAI,CAACG,IAAI,CAACP,QAAQ,GAAG,CAAC,CAAC,CAAC;EAEjD,KAAK,IAAItB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGL,GAAG,CAACM,MAAM,EAAED,CAAC,EAAE,EAAE;IACnCL,GAAG,CAACqC,UAAU,CAACJ,KAAK,GAAG,IAAI,EAAE5B,CAAC,CAAC;IAE/B4B,KAAK,IAAI,CAAC;EACZ;;EAEA,OAAOhC,MAAM,CAACM,MAAM,CAAC,CAACN,MAAM,CAACkC,IAAI,CAAC5C,MAAM,CAAC6C,MAAM,CAAClB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAElB,GAAG,CAAC,CAAC;AACrE"}

package/dist/esm/parquetjs/compression.js

@@ -1,11 +1,16 @@
+
+
 import { NoCompression, GZipCompression, SnappyCompression, BrotliCompression, LZOCompression, LZ4Compression, ZstdCompression } from '@loaders.gl/compression';
 import { toArrayBuffer, toBuffer } from './utils/buffer-utils';
+
 import lz4js from 'lz4js';
 import lzo from 'lzo';
+
 const modules = {
   lz4js,
   lzo
 };
+
 export const PARQUET_COMPRESSION_METHODS = {
   UNCOMPRESSED: new NoCompression(),
   GZIP: new GZipCompression(),
@@ -26,37 +31,36 @@ export const PARQUET_COMPRESSION_METHODS = {
     modules
   })
 };
+
 export async function preloadCompressions(options) {
   const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
   return await Promise.all(compressions.map(compression => compression.preload()));
 }
+
 export async function deflate(method, value) {
   const compression = PARQUET_COMPRESSION_METHODS[method];
-
   if (!compression) {
     throw new Error("parquet: invalid compression method: ".concat(method));
   }
-
   const inputArrayBuffer = toArrayBuffer(value);
   const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
   return toBuffer(compressedArrayBuffer);
 }
+
 export async function decompress(method, value, size) {
   const compression = PARQUET_COMPRESSION_METHODS[method];
-
   if (!compression) {
     throw new Error("parquet: invalid compression method: ".concat(method));
   }
-
   const inputArrayBuffer = toArrayBuffer(value);
   const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
   return toBuffer(compressedArrayBuffer);
 }
+
 export function inflate(method, value, size) {
   if (!(method in PARQUET_COMPRESSION_METHODS)) {
     throw new Error("invalid compression method: ".concat(method));
   }
-
   return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
 }
 //# sourceMappingURL=compression.js.map

package/dist/esm/parquetjs/compression.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["NoCompression","GZipCompression","SnappyCompression","BrotliCompression","LZOCompression","LZ4Compression","ZstdCompression","toArrayBuffer","toBuffer","lz4js","lzo","modules","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","GZIP","SNAPPY","BROTLI","LZ4","LZ4_RAW","LZO","ZSTD","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compressedArrayBuffer","compress","decompress","size","inflate"],"mappings":"AAIA,SAEEA,aAFF,EAGEC,eAHF,EAIEC,iBAJF,EAKEC,iBALF,EAMEC,cANF,EAOEC,cAPF,EAQEC,eARF,QASO,yBATP;AAYA,SAAQC,aAAR,EAAuBC,QAAvB,QAAsC,sBAAtC;AAMA,OAAOC,KAAP,MAAkB,OAAlB;AACA,OAAOC,GAAP,MAAgB,KAAhB;AAIA,MAAMC,OAAO,GAAG;AAQdF,EAAAA,KARc;AASdC,EAAAA;AATc,CAAhB;AAcA,OAAO,MAAME,2BAAoE,GAAG;AAClFC,EAAAA,YAAY,EAAE,IAAIb,aAAJ,EADoE;AAElFc,EAAAA,IAAI,EAAE,IAAIb,eAAJ,EAF4E;AAGlFc,EAAAA,MAAM,EAAE,IAAIb,iBAAJ,EAH0E;AAIlFc,EAAAA,MAAM,EAAE,IAAIb,iBAAJ,CAAsB;AAACQ,IAAAA;AAAD,GAAtB,CAJ0E;AAMlFM,EAAAA,GAAG,EAAE,IAAIZ,cAAJ,CAAmB;AAACM,IAAAA;AAAD,GAAnB,CAN6E;AAOlFO,EAAAA,OAAO,EAAE,IAAIb,cAAJ,CAAmB;AAACM,IAAAA;AAAD,GAAnB,CAPyE;AAQlFQ,EAAAA,GAAG,EAAE,IAAIf,cAAJ,CAAmB;AAACO,IAAAA;AAAD,GAAnB,CAR6E;AASlFS,EAAAA,IAAI,EAAE,IAAId,eAAJ,CAAoB;AAACK,IAAAA;AAAD,GAApB;AAT4E,CAA7E;AAgBP,OAAO,eAAeU,mBAAf,CAAmCC,OAAnC,EAA8E;AACnF,QAAMC,YAAY,GAAGC,MAAM,CAACC,MAAP,CAAcb,2BAAd,CAArB;AACA,SAAO,MAAMc,OAAO,CAACC,GAAR,CAAYJ,YAAY,CAACK,GAAb,CAAkBC,WAAD,IAAiBA,WAAW,CAACC,OAAZ,EAAlC,CAAZ,CAAb;AACD;AAKD,OAAO,eAAeC,OAAf,CAAuBC,MAAvB,EAAmDC,KAAnD,EAAmF;AACxF,QAAMJ,WAAW,GAAGjB,2BAA2B,CAACoB,MAAD,CAA/C;;AACA,MAAI,CAACH,WAAL,EAAkB;AAChB,UAAM,IAAIK,KAAJ,gDAAkDF,MAAlD,EAAN;AACD;;AACD,QAAMG,gBAAgB,GAAG5B,aAAa,CAAC0B,KAAD,CAAtC;AACA,QAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACQ,QAAZ,CAAqBF,gBAArB,CAApC;AACA,SAAO3B,QAAQ,CAAC4B,qBAAD,CAAf;AACD;AAKD,OAAO,eAAeE,UAAf,CACLN,MADK,EAELC,KAFK,EAGLM,IAHK,EAIY;AACjB,QAAMV,WAAW,GAAGjB,2BAA2B,CAACoB,MAAD,CAA/C;;AACA,MAAI,CAACH,WAAL,EAAkB;AAChB,UAAM,IAAIK,KAAJ,gDAAkDF,MAAlD,EAAN;AACD;;AACD,QAAMG,gBAAgB,GAAG5B,aAAa,CAAC0B,KAAD,CAAtC;AACA,QAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACS,UAAZ,CAAuBH,gBAAvB,EAAyCI,IAAzC,CAApC;AACA,SAAO/B,QAAQ,CAAC4B,qBAAD,CAAf;AACD;AAKD,OAAO,SAASI,OAAT,CAAiBR,MAAjB,EAA6CC,KAA7C,EAA4DM,IAA5D,EAAkF;AACvF,MAAI,EAAEP,MAAM,IAAIpB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIsB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOpB,2BAA2B,CAACoB,MAAD,CAA3B,CAAoCQ,OAApC,CAA4CP,KAA5C,EAAmDM,IAAnD,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"file":"compression.js"}
+{"version":3,"file":"compression.js","names":["NoCompression","GZipCompression","SnappyCompression","BrotliCompression","LZOCompression","LZ4Compression","ZstdCompression","toArrayBuffer","toBuffer","lz4js","lzo","modules","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","GZIP","SNAPPY","BROTLI","LZ4","LZ4_RAW","LZO","ZSTD","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compressedArrayBuffer","compress","decompress","size","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":";;AAIA,SAEEA,aAAa,EACbC,eAAe,EACfC,iBAAiB,EACjBC,iBAAiB,EACjBC,cAAc,EACdC,cAAc,EACdC,eAAe,QACV,yBAAyB;AAGhC,SAAQC,aAAa,EAAEC,QAAQ,QAAO,sBAAsB;;AAM5D,OAAOC,KAAK,MAAM,OAAO;AACzB,OAAOC,GAAG,MAAM,KAAK;;AAIrB,MAAMC,OAAO,GAAG;EAQdF,KAAK;EACLC;AAEF,CAAC;;AAGD,OAAO,MAAME,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIb,aAAa,EAAE;EACjCc,IAAI,EAAE,IAAIb,eAAe,EAAE;EAC3Bc,MAAM,EAAE,IAAIb,iBAAiB,EAAE;EAC/Bc,MAAM,EAAE,IAAIb,iBAAiB,CAAC;IAACQ;EAAO,CAAC,CAAC;EAExCM,GAAG,EAAE,IAAIZ,cAAc,CAAC;IAACM;EAAO,CAAC,CAAC;EAClCO,OAAO,EAAE,IAAIb,cAAc,CAAC;IAACM;EAAO,CAAC,CAAC;EACtCQ,GAAG,EAAE,IAAIf,cAAc,CAAC;IAACO;EAAO,CAAC,CAAC;EAClCS,IAAI,EAAE,IAAId,eAAe,CAAC;IAACK;EAAO,CAAC;AACrC,CAAC;;AAMD,OAAO,eAAeU,mBAAmB,CAACC,OAAyC,EAAE;EACnF,MAAMC,YAAY,GAAGC,MAAM,CAACC,MAAM,CAACb,2BAA2B,CAAC;EAC/D,OAAO,MAAMc,OAAO,CAACC,GAAG,CAACJ,YAAY,CAACK,GAAG,CAAEC,WAAW,IAAKA,WAAW,CAACC,OAAO,EAAE,CAAC,CAAC;AACpF;;AAKA,OAAO,eAAeC,OAAO,CAACC,MAA0B,EAAEC,KAAa,EAAmB;EACxF,MAAMJ,WAAW,GAAGjB,2BAA2B,CAACoB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;EACnE;EACA,MAAMG,gBAAgB,GAAG5B,aAAa,CAAC0B,KAAK,CAAC;EAC7C,MAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACQ,QAAQ,CAACF,gBAAgB,CAAC;EAC1E,OAAO3B,QAAQ,CAAC4B,qBAAqB,CAAC;AACxC;;AAKA,OAAO,eAAeE,UAAU,CAC9BN,MAA0B,EAC1BC,KAAa,EACbM,IAAY,EACK;EACjB,MAAMV,WAAW,GAAGjB,2BAA2B,CAACoB,MAAM,CAAC;EACvD,IAAI,CAACH,WAAW,EAAE;IAChB,MAAM,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;EACnE;EACA,MAAMG,gBAAgB,GAAG5B,aAAa,CAAC0B,KAAK,CAAC;EAC7C,MAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACS,UAAU,CAACH,gBAAgB,EAAEI,IAAI,CAAC;EAClF,OAAO/B,QAAQ,CAAC4B,qBAAqB,CAAC;AACxC;;AAKA,OAAO,SAASI,OAAO,CAACR,MAA0B,EAAEC,KAAa,EAAEM,IAAY,EAAU;EACvF,IAAI,EAAEP,MAAM,IAAIpB,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAIsB,KAAK,uCAAgCF,MAAM,EAAG;EAC1D;EAEA,OAAOpB,2BAA2B,CAACoB,MAAM,CAAC,CAACQ,OAAO,CAACP,KAAK,EAAEM,IAAI,CAAC;AACjE"}

package/dist/esm/parquetjs/encoder/writer.js

@@ -7,10 +7,14 @@ import { ColumnChunk, ColumnMetaData, CompressionCodec, ConvertedType, DataPageH
 import { osopen, oswrite, osclose } from '../utils/file-utils';
 import { getBitWidth, serializeThrift } from '../utils/read-utils';
 import Int64 from 'node-int64';
+
 const PARQUET_MAGIC = 'PAR1';
+
 const PARQUET_VERSION = 1;
+
 const PARQUET_DEFAULT_PAGE_SIZE = 8192;
 const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
+
 const PARQUET_RDLVL_TYPE = 'INT32';
 const PARQUET_RDLVL_ENCODING = 'RLE';
 export class ParquetWriter {
@@ -23,33 +27,25 @@ export class ParquetWriter {
     if (!opts) {
       opts = {};
     }
-
     const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
     return new ParquetWriter(schema, envelopeWriter, opts);
   }
-
   constructor(schema, envelopeWriter, opts) {
     _defineProperty(this, "schema", void 0);
-
     _defineProperty(this, "envelopeWriter", void 0);
-
     _defineProperty(this, "rowBuffer", void 0);
-
     _defineProperty(this, "rowGroupSize", void 0);
-
     _defineProperty(this, "closed", void 0);
-
     _defineProperty(this, "userMetadata", void 0);
-
     this.schema = schema;
     this.envelopeWriter = envelopeWriter;
     this.rowBuffer = {};
     this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
     this.closed = false;
     this.userMetadata = {};
+
     this.writeHeader();
   }
-
   async writeHeader() {
     try {
       await this.envelopeWriter.writeHeader();
@@ -63,9 +59,7 @@ export class ParquetWriter {
     if (this.closed) {
       throw new Error('writer was closed');
     }
-
     Shred.shredRecord(this.schema, row, this.rowBuffer);
-
     if (this.rowBuffer.rowCount >= this.rowGroupSize) {
       this.rowBuffer = {};
     }
@@ -75,13 +69,10 @@ export class ParquetWriter {
     if (this.closed) {
       throw new Error('writer was closed');
     }
-
     this.closed = true;
-
     if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
       this.rowBuffer = {};
     }
-
     await this.envelopeWriter.writeFooter(this.userMetadata);
     await this.envelopeWriter.close();
 
@@ -101,32 +92,23 @@ export class ParquetWriter {
   setPageSize(cnt) {
     this.envelopeWriter.setPageSize(cnt);
   }
-
 }
+
 export class ParquetEnvelopeWriter {
   static async openStream(schema, outputStream, opts) {
     const writeFn = oswrite.bind(undefined, outputStream);
     const closeFn = osclose.bind(undefined, outputStream);
     return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);
   }
-
   constructor(schema, writeFn, closeFn, fileOffset, opts) {
     _defineProperty(this, "schema", void 0);
-
     _defineProperty(this, "write", void 0);
-
     _defineProperty(this, "close", void 0);
-
     _defineProperty(this, "offset", void 0);
-
     _defineProperty(this, "rowCount", void 0);
-
     _defineProperty(this, "rowGroups", void 0);
-
     _defineProperty(this, "pageSize", void 0);
-
     _defineProperty(this, "useDataPageV2", void 0);
-
     this.schema = schema;
     this.write = writeFn;
     this.close = closeFn;
@@ -136,7 +118,6 @@ export class ParquetEnvelopeWriter {
     this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
     this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
   }
-
   writeSection(buf) {
     this.offset += buf.length;
     return this.write(buf);
@@ -161,29 +142,26 @@ export class ParquetEnvelopeWriter {
     if (!userMetadata) {
       userMetadata = {};
     }
-
     return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
   }
 
   setPageSize(cnt) {
     this.pageSize = cnt;
   }
-
 }
+
 export class ParquetTransformer extends Transform {
-  constructor(schema, opts = {}) {
+  constructor(schema) {
+    let opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
     super({
       objectMode: true
     });
-
     _defineProperty(this, "writer", void 0);
-
     const writeProxy = function (t) {
       return async function (b) {
         t.push(b);
       };
     }(this);
-
     this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts), opts);
   }
 
@@ -191,7 +169,6 @@ export class ParquetTransformer extends Transform {
     if (row) {
       return this.writer.appendRow(row).then(callback);
     }
-
     callback();
     return Promise.resolve();
   }
@@ -199,20 +176,17 @@ export class ParquetTransformer extends Transform {
   async _flush(callback) {
     await this.writer.close(callback);
   }
-
 }
 
 function encodeValues(type, encoding, values, opts) {
   if (!(encoding in PARQUET_CODECS)) {
     throw new Error("invalid encoding: ".concat(encoding));
   }
-
   return PARQUET_CODECS[encoding].encodeValues(type, values, opts);
 }
 
 async function encodeDataPage(column, data) {
   let rLevelsBuf = Buffer.alloc(0);
-
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax)
@@ -220,7 +194,6 @@ async function encodeDataPage(column, data) {
   }
 
   let dLevelsBuf = Buffer.alloc(0);
-
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax)
@@ -232,7 +205,9 @@ async function encodeDataPage(column, data) {
     bitWidth: column.typeLength
   });
   const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
+
   const compressedBuf = await Compression.deflate(column.compression, dataBuf);
+
   const header = new PageHeader({
     type: PageType.DATA_PAGE,
     data_page_header: new DataPageHeader({
@@ -241,9 +216,11 @@ async function encodeDataPage(column, data) {
       definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING],
       repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING]
     }),
+
     uncompressed_page_size: dataBuf.length,
     compressed_page_size: compressedBuf.length
   });
+
   const headerBuf = serializeThrift(header);
   const page = Buffer.concat([headerBuf, compressedBuf]);
   return {
@@ -258,18 +235,17 @@ async function encodeDataPageV2(column, data, rowCount) {
     typeLength: column.typeLength,
     bitWidth: column.typeLength
   });
+
   const compressedBuf = await Compression.deflate(column.compression, valuesBuf);
-  let rLevelsBuf = Buffer.alloc(0);
 
+  let rLevelsBuf = Buffer.alloc(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax),
       disableEnvelope: true
     });
   }
-
   let dLevelsBuf = Buffer.alloc(0);
-
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax),
@@ -291,6 +267,7 @@ async function encodeDataPageV2(column, data, rowCount) {
     uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,
     compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length
   });
+
   const headerBuf = serializeThrift(header);
   const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
   return {
@@ -312,6 +289,7 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
     total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
     total_compressed_size += result.header.compressed_page_size + result.headerSize;
   }
+
   const metadata = new ColumnMetaData({
     path_in_schema: column.path,
     num_values: data.count,
@@ -322,8 +300,10 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
     type: Type[column.primitiveType],
     codec: CompressionCodec[column.compression]
   });
+
   metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);
   metadata.encodings.push(Encoding[column.encoding]);
+
   const metadataOffset = baseOffset + pageBuf.length;
   const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);
   return {
@@ -340,7 +320,6 @@ async function encodeRowGroup(schema, data, opts) {
     total_byte_size: 0
   });
   let body = Buffer.alloc(0);
-
   for (const field of schema.fieldList) {
     if (field.isNested) {
       continue;
@@ -355,7 +334,6 @@ async function encodeRowGroup(schema, data, opts) {
     metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
     body = Buffer.concat([body, cchunkData.body]);
   }
-
   return {
     body,
     metadata
@@ -371,17 +349,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
     schema: [],
     key_value_metadata: []
   });
-
   for (const key in userMetadata) {
-    var _metadata$key_value_m, _metadata$key_value_m2;
-
+    var _metadata$key_value_m, _metadata$key_value_m2, _metadata$key_value_m3;
     const kv = new KeyValue({
       key,
       value: userMetadata[key]
     });
-    (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = _metadata$key_value_m.push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m, kv);
+    (_metadata$key_value_m = metadata.key_value_metadata) === null || _metadata$key_value_m === void 0 ? void 0 : (_metadata$key_value_m2 = (_metadata$key_value_m3 = _metadata$key_value_m).push) === null || _metadata$key_value_m2 === void 0 ? void 0 : _metadata$key_value_m2.call(_metadata$key_value_m3, kv);
   }
-
   {
     const schemaRoot = new SchemaElement({
       name: 'root',
@@ -389,28 +364,23 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
     });
     metadata.schema.push(schemaRoot);
   }
-
   for (const field of schema.fieldList) {
     const relt = FieldRepetitionType[field.repetitionType];
     const schemaElem = new SchemaElement({
       name: field.name,
       repetition_type: relt
     });
-
     if (field.isNested) {
       schemaElem.num_children = field.fieldCount;
     } else {
       schemaElem.type = Type[field.primitiveType];
     }
-
     if (field.originalType) {
       schemaElem.converted_type = ConvertedType[field.originalType];
     }
-
     schemaElem.type_length = field.typeLength;
     metadata.schema.push(schemaElem);
   }
-
   const metadataEncoded = serializeThrift(metadata);
   const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
   metadataEncoded.copy(footerEncoded);