@loaders.gl/parquet 3.1.0-alpha.5 → 3.1.0-beta.5

package/dist/es5/index.js

@@ -21,10 +21,10 @@ Object.defineProperty(exports, "preloadCompressions", {
     return _compression.preloadCompressions;
   }
 });
-Object.defineProperty(exports, "ParquetEnvelopeReader", {
+Object.defineProperty(exports, "ParquetSchema", {
   enumerable: true,
   get: function () {
-    return _parquetEnvelopeReader.ParquetEnvelopeReader;
+    return _schema.ParquetSchema;
   }
 });
 Object.defineProperty(exports, "ParquetReader", {
@@ -33,28 +33,10 @@ Object.defineProperty(exports, "ParquetReader", {
     return _parquetReader.ParquetReader;
   }
 });
-Object.defineProperty(exports, "ParquetWriter", {
-  enumerable: true,
-  get: function () {
-    return _writer.ParquetWriter;
-  }
-});
-Object.defineProperty(exports, "ParquetEnvelopeWriter", {
-  enumerable: true,
-  get: function () {
-    return _writer.ParquetEnvelopeWriter;
-  }
-});
-Object.defineProperty(exports, "ParquetTransformer", {
-  enumerable: true,
-  get: function () {
-    return _writer.ParquetTransformer;
-  }
-});
-Object.defineProperty(exports, "ParquetSchema", {
+Object.defineProperty(exports, "ParquetEnvelopeReader", {
   enumerable: true,
   get: function () {
-    return _schema.ParquetSchema;
+    return _parquetEnvelopeReader.ParquetEnvelopeReader;
   }
 });
 Object.defineProperty(exports, "convertParquetToArrowSchema", {
@@ -73,13 +55,11 @@ var _parquetWriter = require("./parquet-writer");
 
 var _compression = require("./parquetjs/compression");
 
-var _parquetEnvelopeReader = require("./parquetjs/parser/parquet-envelope-reader");
+var _schema = require("./parquetjs/schema/schema");
 
 var _parquetReader = require("./parquetjs/parser/parquet-reader");
 
-var _writer = require("./parquetjs/encoder/writer");
-
-var _schema = require("./parquetjs/schema/schema");
+var _parquetEnvelopeReader = require("./parquetjs/parser/parquet-envelope-reader");
 
 var _convertSchema = require("./lib/convert-schema");
 

package/dist/es5/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parse","parseParquet","parseFileInBatches","parseParquetFileInBatches","_typecheckParquetLoader"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA;;AACA;;AAaA;;AAIA;;AAEA;;AACA;;AACA;;AACA;;AACA;;AAlBO,MAAMA,aAAa,GAAG,EAC3B,GAAGC,4BADwB;AAE3BC,EAAAA,KAAK,EAAEC,0BAFoB;AAG3BC,EAAAA,kBAAkB,EAAEC;AAHO,CAAtB;;AAqBA,MAAMC,uBAAyC,GAAGN,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}
+{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parse","parseParquet","parseFileInBatches","parseParquetFileInBatches","_typecheckParquetLoader"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA;;AACA;;AAaA;;AAIA;;AAEA;;AACA;;AACA;;AAEA;;AAlBO,MAAMA,aAAa,GAAG,EAC3B,GAAGC,4BADwB;AAE3BC,EAAAA,KAAK,EAAEC,0BAFoB;AAG3BC,EAAAA,kBAAkB,EAAEC;AAHO,CAAtB;;AAqBA,MAAMC,uBAAyC,GAAGN,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}

package/dist/es5/parquet-loader.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
-const VERSION = typeof "3.1.0-alpha.5" !== 'undefined' ? "3.1.0-alpha.5" : 'latest';
+const VERSION = typeof "3.1.0-beta.5" !== 'undefined' ? "3.1.0-beta.5" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',

package/dist/es5/parquet-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":";;;;;;AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQO,MAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;;AAcA,MAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}
+{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":";;;;;;AAIA,MAAMA,OAAO,GAAG,0BAAuB,WAAvB,oBAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQO,MAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;;AAcA,MAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}

package/dist/es5/parquet-writer.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetWriter = void 0;
-const VERSION = typeof "3.1.0-alpha.5" !== 'undefined' ? "3.1.0-alpha.5" : 'latest';
+const VERSION = typeof "3.1.0-beta.5" !== 'undefined' ? "3.1.0-beta.5" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {};
 const ParquetWriter = {
   name: 'Apache Parquet',

package/dist/es5/parquet-writer.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":";;;;;;AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
+{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":";;;;;;AAIA,MAAMA,OAAO,GAAG,0BAAuB,WAAvB,oBAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}

package/dist/es5/parquetjs/codecs/rle.js

@@ -61,7 +61,7 @@ function encodeValues(type, values, opts) {
   }
 
   const envelope = Buffer.alloc(buf.length + 4);
-  envelope.writeUInt32LE(buf.length, 0);
+  envelope.writeUInt32LE(buf.length, undefined);
   buf.copy(envelope, 4);
   return envelope;
 }

package/dist/es5/parquetjs/codecs/rle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmC,CAAnC;AACAN,EAAAA,GAAG,CAACc,IAAJ,CAASF,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASG,YAAT,CACLtB,IADK,EAELuB,MAFK,EAGLC,KAHK,EAILtB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBK,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIxB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBW,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACAzB,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGc,kBAAkB,CAACR,MAAD,EAASC,KAAT,EAAgBtB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMsB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACAzB,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,iBAAiB,CAACT,MAAD,EAASC,KAAT,EAAgBtB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACgC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIvB,MAAM,CAACY,MAAP,KAAkBW,KAAtB,EAA6B;AAC3B,UAAM,IAAIrB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS8B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEtB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAIrB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAIkC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnEpC,MAAAA,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOvB,MAAP;AACD;;AAED,SAAS+B,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEtB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI5B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG0B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CtB,CAAC,EAA9C,EAAkD;AAEhD4B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAASzB,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIjC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIwB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGjC,MAAM,CAACY,MAAtC,EAA8CwB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACpC,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE3B,MAAAA,GAAG,CAAC+B,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO7B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB1C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2BwB,KAA3B,EAA0ChB,KAA1C,EAAyDtB,IAAzD,EAA4F;AAE1F,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,QAAM3B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACqC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B5B,CAA7B;AAEA4B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOhC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyCjB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, 0);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
+{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,kBAAkB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGgB,iBAAiB,CAACT,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAEhD6B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,QAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AAEA6B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}

package/dist/es5/parquetjs/compression.js

@@ -15,24 +15,13 @@ var _compression = require("@loaders.gl/compression");
 
 var _bufferUtils = require("./utils/buffer-utils");
 
-var _decompress = _interopRequireDefault(require("brotli/decompress"));
-
 var _lz4js = _interopRequireDefault(require("lz4js"));
 
 var _lzo = _interopRequireDefault(require("lzo"));
 
-var _zstdCodec = require("zstd-codec");
-
 const modules = {
-  brotli: {
-    decompress: _decompress.default,
-    compress: () => {
-      throw new Error('brotli compress');
-    }
-  },
   lz4js: _lz4js.default,
-  lzo: _lzo.default,
-  'zstd-codec': _zstdCodec.ZstdCodec
+  lzo: _lzo.default
 };
 const PARQUET_COMPRESSION_METHODS = {
   UNCOMPRESSED: new _compression.NoCompression(),

package/dist/es5/parquetjs/compression.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["modules","brotli","decompress","brotliDecompress","compress","Error","lz4js","lzo","ZstdCodec","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","NoCompression","GZIP","GZipCompression","SNAPPY","SnappyCompression","BROTLI","BrotliCompression","LZ4","LZ4Compression","LZ4_RAW","LZO","LZOCompression","ZSTD","ZstdCompression","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","inputArrayBuffer","compressedArrayBuffer","size","inflate"],"mappings":";;;;;;;;;;;;;AAIA;;AAYA;;AAKA;;AACA;;AACA;;AACA;;AAGA,MAAMA,OAAO,GAAG;AAEdC,EAAAA,MAAM,EAAE;AACNC,IAAAA,UAAU,EAAEC,mBADN;AAENC,IAAAA,QAAQ,EAAE,MAAM;AACd,YAAM,IAAIC,KAAJ,CAAU,iBAAV,CAAN;AACD;AAJK,GAFM;AAQdC,EAAAA,KAAK,EAALA,cARc;AASdC,EAAAA,GAAG,EAAHA,YATc;AAUd,gBAAcC;AAVA,CAAhB;AAcO,MAAMC,2BAAoE,GAAG;AAClFC,EAAAA,YAAY,EAAE,IAAIC,0BAAJ,EADoE;AAElFC,EAAAA,IAAI,EAAE,IAAIC,4BAAJ,EAF4E;AAGlFC,EAAAA,MAAM,EAAE,IAAIC,8BAAJ,EAH0E;AAIlFC,EAAAA,MAAM,EAAE,IAAIC,8BAAJ,CAAsB;AAACjB,IAAAA;AAAD,GAAtB,CAJ0E;AAMlFkB,EAAAA,GAAG,EAAE,IAAIC,2BAAJ,CAAmB;AAACnB,IAAAA;AAAD,GAAnB,CAN6E;AAOlFoB,EAAAA,OAAO,EAAE,IAAID,2BAAJ,CAAmB;AAACnB,IAAAA;AAAD,GAAnB,CAPyE;AAQlFqB,EAAAA,GAAG,EAAE,IAAIC,2BAAJ,CAAmB;AAACtB,IAAAA;AAAD,GAAnB,CAR6E;AASlFuB,EAAAA,IAAI,EAAE,IAAIC,4BAAJ,CAAoB;AAACxB,IAAAA;AAAD,GAApB;AAT4E,CAA7E;;;AAgBA,eAAeyB,mBAAf,CAAmCC,OAAnC,EAA8E;AACnF,QAAMC,YAAY,GAAGC,MAAM,CAACC,MAAP,CAAcpB,2BAAd,CAArB;AACA,SAAO,MAAMqB,OAAO,CAACC,GAAR,CAAYJ,YAAY,CAACK,GAAb,CAAkBC,WAAD,IAAiBA,WAAW,CAACC,OAAZ,EAAlC,CAAZ,CAAb;AACD;;AAKM,eAAeC,OAAf,CAAuBC,MAAvB,EAAmDC,KAAnD,EAAmF;AACxF,QAAMJ,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAD,CAA/C;;AACA,MAAI,CAACH,WAAL,EAAkB;AAChB,UAAM,IAAI5B,KAAJ,gDAAkD+B,MAAlD,EAAN;AACD;;AACD,QAAME,gBAAgB,GAAG,gCAAcD,KAAd,CAAzB;AACA,QAAME,qBAAqB,GAAG,MAAMN,WAAW,CAAC7B,QAAZ,CAAqBkC,gBAArB,CAApC;AACA,SAAO,2BAASC,qBAAT,CAAP;AACD;;AAKM,eAAerC,UAAf,CACLkC,MADK,EAELC,KAFK,EAGLG,IAHK,EAIY;AACjB,QAAMP,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAD,CAA/C;;AACA,MAAI,CAACH,WAAL,EAAkB;AAChB,UAAM,IAAI5B,KAAJ,gDAAkD+B,MAAlD,EAAN;AACD;;AACD,QAAME,gBAAgB,GAAG,gCAAcD,KAAd,CAAzB;AACA,QAAME,qBAAqB,GAAG,MAAMN,WAAW,CAAC/B,UAAZ,CAAuBoC,gBAAvB,EAAyCE,IAAzC,CAApC;AACA,SAAO,2BAASD,qBAAT,CAAP;AACD;;AAKM,SAASE,OAAT,CAAiBL,MAAjB,EAA6CC,KAA7C,EAA4DG,IAA5D,EAAkF;AACvF,MAAI,EAAEJ,MAAM,IAAI3B,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIJ,KAAJ,uCAAyC+B,MAAzC,EAAN;AACD;;AAED,SAAO3B,2BAA2B,CAAC2B,MAAD,CAA3B,CAAoCK,OAApC,CAA4CJ,KAA5C,EAAmDG,IAAnD,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\nimport brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\nimport {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  brotli: {\n    decompress: brotliDecompress,\n    compress: () => {\n      throw new Error('brotli compress');\n    }\n  },\n  lz4js,\n  lzo,\n  'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"file":"compression.js"}
+{"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["modules","lz4js","lzo","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","NoCompression","GZIP","GZipCompression","SNAPPY","SnappyCompression","BROTLI","BrotliCompression","LZ4","LZ4Compression","LZ4_RAW","LZO","LZOCompression","ZSTD","ZstdCompression","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compressedArrayBuffer","compress","decompress","size","inflate"],"mappings":";;;;;;;;;;;;;AAIA;;AAYA;;AAMA;;AACA;;AAIA,MAAMA,OAAO,GAAG;AAQdC,EAAAA,KAAK,EAALA,cARc;AASdC,EAAAA,GAAG,EAAHA;AATc,CAAhB;AAcO,MAAMC,2BAAoE,GAAG;AAClFC,EAAAA,YAAY,EAAE,IAAIC,0BAAJ,EADoE;AAElFC,EAAAA,IAAI,EAAE,IAAIC,4BAAJ,EAF4E;AAGlFC,EAAAA,MAAM,EAAE,IAAIC,8BAAJ,EAH0E;AAIlFC,EAAAA,MAAM,EAAE,IAAIC,8BAAJ,CAAsB;AAACX,IAAAA;AAAD,GAAtB,CAJ0E;AAMlFY,EAAAA,GAAG,EAAE,IAAIC,2BAAJ,CAAmB;AAACb,IAAAA;AAAD,GAAnB,CAN6E;AAOlFc,EAAAA,OAAO,EAAE,IAAID,2BAAJ,CAAmB;AAACb,IAAAA;AAAD,GAAnB,CAPyE;AAQlFe,EAAAA,GAAG,EAAE,IAAIC,2BAAJ,CAAmB;AAAChB,IAAAA;AAAD,GAAnB,CAR6E;AASlFiB,EAAAA,IAAI,EAAE,IAAIC,4BAAJ,CAAoB;AAAClB,IAAAA;AAAD,GAApB;AAT4E,CAA7E;;;AAgBA,eAAemB,mBAAf,CAAmCC,OAAnC,EAA8E;AACnF,QAAMC,YAAY,GAAGC,MAAM,CAACC,MAAP,CAAcpB,2BAAd,CAArB;AACA,SAAO,MAAMqB,OAAO,CAACC,GAAR,CAAYJ,YAAY,CAACK,GAAb,CAAkBC,WAAD,IAAiBA,WAAW,CAACC,OAAZ,EAAlC,CAAZ,CAAb;AACD;;AAKM,eAAeC,OAAf,CAAuBC,MAAvB,EAAmDC,KAAnD,EAAmF;AACxF,QAAMJ,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAD,CAA/C;;AACA,MAAI,CAACH,WAAL,EAAkB;AAChB,UAAM,IAAIK,KAAJ,gDAAkDF,MAAlD,EAAN;AACD;;AACD,QAAMG,gBAAgB,GAAG,gCAAcF,KAAd,CAAzB;AACA,QAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACQ,QAAZ,CAAqBF,gBAArB,CAApC;AACA,SAAO,2BAASC,qBAAT,CAAP;AACD;;AAKM,eAAeE,UAAf,CACLN,MADK,EAELC,KAFK,EAGLM,IAHK,EAIY;AACjB,QAAMV,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAD,CAA/C;;AACA,MAAI,CAACH,WAAL,EAAkB;AAChB,UAAM,IAAIK,KAAJ,gDAAkDF,MAAlD,EAAN;AACD;;AACD,QAAMG,gBAAgB,GAAG,gCAAcF,KAAd,CAAzB;AACA,QAAMG,qBAAqB,GAAG,MAAMP,WAAW,CAACS,UAAZ,CAAuBH,gBAAvB,EAAyCI,IAAzC,CAApC;AACA,SAAO,2BAASH,qBAAT,CAAP;AACD;;AAKM,SAASI,OAAT,CAAiBR,MAAjB,EAA6CC,KAA7C,EAA4DM,IAA5D,EAAkF;AACvF,MAAI,EAAEP,MAAM,IAAI3B,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAI6B,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAO3B,2BAA2B,CAAC2B,MAAD,CAA3B,CAAoCQ,OAApC,CAA4CP,KAA5C,EAAmDM,IAAnD,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"file":"compression.js"}

package/dist/es5/parquetjs/parser/decoders.js

@@ -66,7 +66,7 @@ async function decodePage(cursor, options) {
   const {
     pageHeader,
     length
-  } = await (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset);
+  } = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset);
   cursor.offset += length;
   const pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
 

package/dist/es5/parquetjs/parser/decoders.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/decoders.ts"],"names":["decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","PageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","num_children","res","name","fields","Type","logicalType","converted_type","ConvertedType","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","PARQUET_CODECS","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","PARQUET_RDLVL_TYPE","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","data_page_header_v2","num_nulls","PARQUET_RDLVL_ENCODING","valuesBufCursor","is_compressed","dictCursor","dictionary_page_header","d","toString"],"mappings":";;;;;;;;;AAUA;;AACA;;AASA;;AACA;;AACA;;AASO,eAAeA,eAAf,CACLC,MADK,EAELC,OAFK,EAGiB;AACtB,QAAMC,MAAoB,GAAG;AAC3BF,IAAAA,MAD2B;AAE3BG,IAAAA,MAAM,EAAE,CAFmB;AAG3BC,IAAAA,IAAI,EAAEJ,MAAM,CAACK;AAHc,GAA7B;AAMA,QAAMC,IAAiB,GAAG;AACxBC,IAAAA,OAAO,EAAE,EADe;AAExBC,IAAAA,OAAO,EAAE,EAFe;AAGxBC,IAAAA,MAAM,EAAE,EAHgB;AAIxBC,IAAAA,WAAW,EAAE,EAJW;AAKxBC,IAAAA,KAAK,EAAE;AALiB,GAA1B;AAQA,MAAIC,UAAU,GAAGX,OAAO,CAACW,UAAR,IAAsB,EAAvC;;AAEA,SAEEV,MAAM,CAACC,MAAP,GAAgBD,MAAM,CAACE,IAAvB,KACC,CAACH,OAAO,CAACY,SAAT,IAAsBP,IAAI,CAACE,OAAL,CAAaH,MAAb,GAAsBS,MAAM,CAACb,OAAO,CAACY,SAAT,CADnD,CAFF,EAIE;AAEA,UAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAD,EAASD,OAAT,CAA7B;;AAEA,QAAIc,IAAI,CAACH,UAAT,EAAqB;AACnBA,MAAAA,UAAU,GAAGG,IAAI,CAACH,UAAlB;AAEA;AACD;;AAED,QAAIA,UAAU,CAACP,MAAf,EAAuB;AAErBU,MAAAA,IAAI,CAACN,MAAL,GAAcM,IAAI,CAACN,MAAL,CAAYQ,GAAZ,CAAiBC,KAAD,IAAWN,UAAU,CAACM,KAAD,CAArC,CAAd;AACD;;AAED,SAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGJ,IAAI,CAACR,OAAL,CAAaF,MAAzC,EAAiDc,KAAK,EAAtD,EAA0D;AACxDb,MAAAA,IAAI,CAACC,OAAL,CAAaa,IAAb,CAAkBL,IAAI,CAACR,OAAL,CAAaY,KAAb,CAAlB;AACAb,MAAAA,IAAI,CAACE,OAAL,CAAaY,IAAb,CAAkBL,IAAI,CAACP,OAAL,CAAaW,KAAb,CAAlB;AACA,YAAMD,KAAK,GAAGH,IAAI,CAACN,MAAL,CAAYU,KAAZ,CAAd;;AAEA,UAAID,KAAK,KAAKG,SAAd,EAAyB;AACvBf,QAAAA,IAAI,CAACG,MAAL,CAAYW,IAAZ,CAAiBF,KAAjB;AACD;AACF;;AAEDZ,IAAAA,IAAI,CAACK,KAAL,IAAcI,IAAI,CAACJ,KAAnB;AACAL,IAAAA,IAAI,CAACI,WAAL,CAAiBU,IAAjB,CAAsBL,IAAI,CAACO,UAA3B;AACD;;AAED,SAAOhB,IAAP;AACD;;AAOM,eAAeU,UAAf,CACLd,MADK,EAELD,OAFK,EAGqB;AAC1B,MAAIc,IAAJ;AACA,QAAM;AAACO,IAAAA,UAAD;AAAajB,IAAAA;AAAb,MAAuB,MAAM,iCAAiBH,MAAM,CAACF,MAAxB,EAAgCE,MAAM,CAACC,MAAvC,CAAnC;AACAD,EAAAA,MAAM,CAACC,MAAP,IAAiBE,MAAjB;AAEA,QAAMkB,QAAQ,GAAG,8BAAcC,uBAAd,EAAwBF,UAAU,CAACG,IAAnC,CAAjB;;AAEA,UAAQF,QAAR;AACE,SAAK,WAAL;AACER,MAAAA,IAAI,GAAG,MAAMW,cAAc,CAACxB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAA3B;AACA;;AACF,SAAK,cAAL;AACEc,MAAAA,IAAI,GAAG,MAAMY,gBAAgB,CAACzB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAA7B;AACA;;AACF,SAAK,iBAAL;AACEc,MAAAA,IAAI,GAAG;AACLH,QAAAA,UAAU,EAAE,MAAMgB,oBAAoB,CAAC1B,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CADjC;AAELqB,QAAAA;AAFK,OAAP;AAIA;;AACF;AACE,YAAM,IAAIO,KAAJ,8BAAgCN,QAAhC,EAAN;AAdJ;;AAiBA,SAAOR,IAAP;AACD;;AAYM,SAASe,YAAT,CACLC,cADK,EAEL5B,MAFK,EAGL6B,GAHK,EAQL;AACA,QAAMC,MAAwB,GAAG,EAAjC;AACA,MAAIC,IAAI,GAAG/B,MAAX;;AACA,OAAK,IAAIgC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,GAApB,EAAyBG,CAAC,EAA1B,EAA8B;AAC5B,UAAMC,aAAa,GAAGL,cAAc,CAACG,IAAD,CAApC;AAEA,UAAMG,cAAc,GAClBH,IAAI,GAAG,CAAP,GAAW,8BAAcI,kCAAd,EAAmCF,aAAa,CAACG,eAAjD,CAAX,GAAgF,MADlF;AAGA,QAAIC,QAAQ,GAAG,KAAf;AACA,QAAIC,QAAQ,GAAG,KAAf;;AACA,YAAQJ,cAAR;AACE,WAAK,UAAL;AACE;;AACF,WAAK,UAAL;AACEG,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF,WAAK,UAAL;AACEC,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF;AACE,cAAM,IAAIZ,KAAJ,CAAU,kCAAV,CAAN;AAVJ;;AAaA,QAAIO,aAAa,CAACM,YAAd,GAA8B,CAAlC,EAAqC;AACnC,YAAMC,GAAG,GAAGb,YAAY,CAACC,cAAD,EAAiBG,IAAI,GAAG,CAAxB,EAA2BE,aAAa,CAACM,YAAzC,CAAxB;AACAR,MAAAA,IAAI,GAAGS,GAAG,CAACT,IAAX;AACAD,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAE3BJ,QAAAA,QAF2B;AAG3BC,QAAAA,QAH2B;AAI3BI,QAAAA,MAAM,EAAEF,GAAG,CAACV;AAJe,OAA7B;AAMD,KATD,MASO;AACL,YAAMR,IAAI,GAAG,8BAAcqB,mBAAd,EAAoBV,aAAa,CAACX,IAAlC,CAAb;AACA,UAAIsB,WAAW,GAAGtB,IAAlB;;AAEA,UAAIW,aAAa,CAACY,cAAlB,EAAkC;AAChCD,QAAAA,WAAW,GAAG,8BAAcE,4BAAd,EAA6Bb,aAAa,CAACY,cAA3C,CAAd;AACD;;AAED,cAAQD,WAAR;AACE,aAAK,SAAL;AACEA,UAAAA,WAAW,aAAMA,WAAN,cAAqBtB,IAArB,CAAX;AACA;;AACF;AAJF;;AAOAQ,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAC3BnB,QAAAA,IAAI,EAAEsB,WADqB;AAE3BG,QAAAA,UAAU,EAAEd,aAAa,CAACe,WAFC;AAG3BC,QAAAA,SAAS,EAAEhB,aAAa,CAACiB,SAHE;AAI3BC,QAAAA,KAAK,EAAElB,aAAa,CAACkB,KAJM;AAK3Bd,QAAAA,QAL2B;AAM3BC,QAAAA;AAN2B,OAA7B;AAQAP,MAAAA,IAAI;AACL;AACF;;AACD,SAAO;AAACD,IAAAA,MAAD;AAAS9B,IAAAA,MAAT;AAAiB+B,IAAAA;AAAjB,GAAP;AACD;;AAKD,SAASqB,YAAT,CACE9B,IADF,EAEE+B,QAFF,EAGEtD,MAHF,EAIES,KAJF,EAKE8C,IALF,EAMS;AACP,MAAI,EAAED,QAAQ,IAAIE,sBAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI7B,KAAJ,6BAA+B2B,QAA/B,EAAN;AACD;;AACD,SAAOE,uBAAeF,QAAf,EAAyBD,YAAzB,CAAsC9B,IAAtC,EAA4CvB,MAA5C,EAAoDS,KAApD,EAA2D8C,IAA3D,CAAP;AACD;;AAQD,eAAe/B,cAAf,CACExB,MADF,EAEEyD,MAFF,EAGE1D,OAHF,EAI4B;AAAA;;AAC1B,QAAM2D,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAAzC;AACA,QAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAV,0DAAG,sBAAyBC,UAA5C;AAGA,MAAIC,UAAU,GAAG/D,MAAjB;;AAEA,MAAID,OAAO,CAACiE,WAAR,KAAwB,cAA5B,EAA4C;AAC1C,UAAMC,SAAS,GAAG,MAAM,6BACtBlE,OAAO,CAACiE,WADc,EAEtBhE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACU,sBAHe,CAAxB;AAKAJ,IAAAA,UAAU,GAAG;AACXjE,MAAAA,MAAM,EAAEmE,SADG;AAEXhE,MAAAA,MAAM,EAAE,CAFG;AAGXC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHL,KAAb;AAKAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAGD,QAAMU,cAAc,GAAG,8BACrBC,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBS,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIC,KAAJ,CAAUZ,UAAV,CAAd;;AAEA,MAAI7D,OAAO,CAAC0E,MAAR,CAAeC,SAAf,GAA2B,CAA/B,EAAkC;AAChCH,IAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBP,cAArB,EAAqCL,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,MAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeC,SAA3B,CADwE;AAElFG,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLN,IAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAGD,QAAMC,cAAc,GAAG,8BACrBV,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBmB,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIT,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAI7D,OAAO,CAAC0E,MAAR,CAAeS,SAAf,GAA2B,CAA/B,EAAkC;AAChCD,IAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBI,cAArB,EAAqChB,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,MAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeS,SAA3B,CADwE;AAElFL,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AACD,MAAIK,iBAAiB,GAAG,CAAxB;;AACA,OAAK,MAAMC,IAAX,IAAmBH,OAAnB,EAA4B;AAC1B,QAAIG,IAAI,KAAKrF,OAAO,CAAC0E,MAAR,CAAeS,SAA5B,EAAuC;AACrCC,MAAAA,iBAAiB;AAClB;AACF;;AAGD,QAAME,aAAa,GAAG,8BAAchB,uBAAd,4BAAwBZ,MAAM,CAACI,gBAA/B,2DAAwB,uBAAyBP,QAAjD,CAAtB;AACA,QAAMgC,aAAa,GAAG;AACpBtC,IAAAA,UAAU,EAAEjD,OAAO,CAAC0E,MAAR,CAAezB,UADP;AAEpB4B,IAAAA,QAAQ,EAAE7E,OAAO,CAAC0E,MAAR,CAAezB;AAFL,GAAtB;AAKA,QAAMzC,MAAM,GAAG8C,YAAY,CACzBtD,OAAO,CAAC0E,MAAR,CAAec,aADU,EAEzBF,aAFyB,EAGzBtB,UAHyB,EAIzBoB,iBAJyB,EAKzBG,aALyB,CAA3B;AAQA,SAAO;AACLhF,IAAAA,OAAO,EAAE2E,OADJ;AAEL5E,IAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,IAAAA,MAHK;AAILE,IAAAA,KAAK,EAAEmD,UAJF;AAKLxC,IAAAA,UAAU,EAAEqC;AALP,GAAP;AAOD;;AASD,eAAehC,gBAAf,CACEzB,MADF,EAEEyD,MAFF,EAGEF,IAHF,EAI4B;AAAA;;AAC1B,QAAMG,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAAzC;AAEA,QAAMC,UAAU,6BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4B1B,UAA/C;AAEA,QAAMqB,iBAAiB,GAAGvB,UAAU,8BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4BC,SAA/B,CAApC;AACA,QAAMJ,aAAa,GAAG,8BACpBhB,uBADoB,4BAEpBZ,MAAM,CAAC+B,mBAFa,2DAEpB,uBAA4BlC,QAFR,CAAtB;AAOA,MAAIiB,OAAO,GAAG,IAAIC,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAIL,IAAI,CAACkB,MAAL,CAAYC,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BH,IAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,MAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYC,SAAxB,CAD4E;AAEtFG,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLN,IAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAID,MAAIG,OAAO,GAAG,IAAIT,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAIL,IAAI,CAACkB,MAAL,CAAYS,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BD,IAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,MAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYS,SAAxB,CAD4E;AAEtFL,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AAGD,MAAIa,eAAe,GAAG3F,MAAtB;;AAEA,gCAAIyD,MAAM,CAAC+B,mBAAX,mDAAI,uBAA4BI,aAAhC,EAA+C;AAC7C,UAAM3B,SAAS,GAAG,MAAM,6BACtBV,IAAI,CAACS,WADiB,EAEtBhE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACU,sBAHe,CAAxB;AAMAwB,IAAAA,eAAe,GAAG;AAChB7F,MAAAA,MAAM,EAAEmE,SADQ;AAEhBhE,MAAAA,MAAM,EAAE,CAFQ;AAGhBC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHA,KAAlB;AAMAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAED,QAAM4B,aAAa,GAAG;AACpBtC,IAAAA,UAAU,EAAEO,IAAI,CAACkB,MAAL,CAAYzB,UADJ;AAEpB4B,IAAAA,QAAQ,EAAErB,IAAI,CAACkB,MAAL,CAAYzB;AAFF,GAAtB;AAKA,QAAMzC,MAAM,GAAG8C,YAAY,CACzBE,IAAI,CAACkB,MAAL,CAAYc,aADa,EAEzBF,aAFyB,EAGzBM,eAHyB,EAIzBR,iBAJyB,EAKzBG,aALyB,CAA3B;AAQA,SAAO;AACLhF,IAAAA,OAAO,EAAE2E,OADJ;AAEL5E,IAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,IAAAA,MAHK;AAILE,IAAAA,KAAK,EAAEmD,UAJF;AAKLxC,IAAAA,UAAU,EAAEqC;AALP,GAAP;AAOD;;AAQD,eAAe/B,oBAAf,CACE1B,MADF,EAEEoB,UAFF,EAGErB,OAHF,EAIqB;AAAA;;AACnB,QAAM2D,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBmB,UAAU,CAACuC,oBAA7C;AAEA,MAAIkC,UAAU,GAAG;AACf5F,IAAAA,MAAM,EAAE,CADO;AAEfH,IAAAA,MAAM,EAAEE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFO;AAGfxD,IAAAA,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;AAHV,GAAjB;AAMAD,EAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AAEA,MAAI3D,OAAO,CAACiE,WAAR,KAAwB,cAA5B,EAA4C;AAC1C,UAAMC,SAAS,GAAG,MAAM,6BACtBlE,OAAO,CAACiE,WADc,EAEtB6B,UAAU,CAAC/F,MAAX,CAAkBoE,KAAlB,CAAwB2B,UAAU,CAAC5F,MAAnC,EAA2CyD,SAA3C,CAFsB,EAGtBtC,UAAU,CAAC+C,sBAHW,CAAxB;AAMA0B,IAAAA,UAAU,GAAG;AACX/F,MAAAA,MAAM,EAAEmE,SADG;AAEXhE,MAAAA,MAAM,EAAE,CAFG;AAGXC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHL,KAAb;AAMAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAED,QAAM/C,SAAS,GAAG,CAAAS,UAAU,SAAV,IAAAA,UAAU,WAAV,qCAAAA,UAAU,CAAE0E,sBAAZ,gFAAoChC,UAApC,KAAkD,CAApE;AAEA,SAAOT,YAAY,CACjBtD,OAAO,CAAC0E,MAAR,CAAec,aADE,EAEjBxF,OAAO,CAAC0E,MAAR,CAAenB,QAFE,EAGjBuC,UAHiB,EAIjBlF,SAJiB,EAKjBZ,OALiB,CAAZ,CAMLgB,GANK,CAMAgF,CAAD,IAAOA,CAAC,CAACC,QAAF,EANN,CAAP;AAOD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetData,\n  ParquetOptions,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  options: ParquetOptions\n): Promise<ParquetData> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetData = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = options.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!options.numValues || data.dlevels.length < Number(options.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, options);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param options\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  let page;\n  const {pageHeader, length} = await decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, options);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, options);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (options.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (options.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === options.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions = {\n    typeLength: options.column.typeLength,\n    bitWidth: options.column.typeLength\n  };\n\n  const values = decodeValues(\n    options.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  opts: any\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (opts.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (opts.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      opts.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: opts.column.typeLength,\n    bitWidth: opts.column.typeLength\n  };\n\n  const values = decodeValues(\n    opts.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  options: ParquetOptions\n): Promise<string[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  return decodeValues(\n    options.column.primitiveType!,\n    options.column.encoding!,\n    dictCursor,\n    numValues,\n    options as ParquetCodecOptions\n  ).map((d) => d.toString());\n}\n"],"file":"decoders.js"}
+{"version":3,"sources":["../../../../src/parquetjs/parser/decoders.ts"],"names":["decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","PageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","num_children","res","name","fields","Type","logicalType","converted_type","ConvertedType","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","PARQUET_CODECS","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","PARQUET_RDLVL_TYPE","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","data_page_header_v2","num_nulls","PARQUET_RDLVL_ENCODING","valuesBufCursor","is_compressed","dictCursor","dictionary_page_header","d","toString"],"mappings":";;;;;;;;;AAUA;;AACA;;AASA;;AACA;;AACA;;AASO,eAAeA,eAAf,CACLC,MADK,EAELC,OAFK,EAGiB;AACtB,QAAMC,MAAoB,GAAG;AAC3BF,IAAAA,MAD2B;AAE3BG,IAAAA,MAAM,EAAE,CAFmB;AAG3BC,IAAAA,IAAI,EAAEJ,MAAM,CAACK;AAHc,GAA7B;AAMA,QAAMC,IAAiB,GAAG;AACxBC,IAAAA,OAAO,EAAE,EADe;AAExBC,IAAAA,OAAO,EAAE,EAFe;AAGxBC,IAAAA,MAAM,EAAE,EAHgB;AAIxBC,IAAAA,WAAW,EAAE,EAJW;AAKxBC,IAAAA,KAAK,EAAE;AALiB,GAA1B;AAQA,MAAIC,UAAU,GAAGX,OAAO,CAACW,UAAR,IAAsB,EAAvC;;AAEA,SAEEV,MAAM,CAACC,MAAP,GAAgBD,MAAM,CAACE,IAAvB,KACC,CAACH,OAAO,CAACY,SAAT,IAAsBP,IAAI,CAACE,OAAL,CAAaH,MAAb,GAAsBS,MAAM,CAACb,OAAO,CAACY,SAAT,CADnD,CAFF,EAIE;AAEA,UAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAD,EAASD,OAAT,CAA7B;;AAEA,QAAIc,IAAI,CAACH,UAAT,EAAqB;AACnBA,MAAAA,UAAU,GAAGG,IAAI,CAACH,UAAlB;AAEA;AACD;;AAED,QAAIA,UAAU,CAACP,MAAf,EAAuB;AAErBU,MAAAA,IAAI,CAACN,MAAL,GAAcM,IAAI,CAACN,MAAL,CAAYQ,GAAZ,CAAiBC,KAAD,IAAWN,UAAU,CAACM,KAAD,CAArC,CAAd;AACD;;AAED,SAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGJ,IAAI,CAACR,OAAL,CAAaF,MAAzC,EAAiDc,KAAK,EAAtD,EAA0D;AACxDb,MAAAA,IAAI,CAACC,OAAL,CAAaa,IAAb,CAAkBL,IAAI,CAACR,OAAL,CAAaY,KAAb,CAAlB;AACAb,MAAAA,IAAI,CAACE,OAAL,CAAaY,IAAb,CAAkBL,IAAI,CAACP,OAAL,CAAaW,KAAb,CAAlB;AACA,YAAMD,KAAK,GAAGH,IAAI,CAACN,MAAL,CAAYU,KAAZ,CAAd;;AAEA,UAAID,KAAK,KAAKG,SAAd,EAAyB;AACvBf,QAAAA,IAAI,CAACG,MAAL,CAAYW,IAAZ,CAAiBF,KAAjB;AACD;AACF;;AAEDZ,IAAAA,IAAI,CAACK,KAAL,IAAcI,IAAI,CAACJ,KAAnB;AACAL,IAAAA,IAAI,CAACI,WAAL,CAAiBU,IAAjB,CAAsBL,IAAI,CAACO,UAA3B;AACD;;AAED,SAAOhB,IAAP;AACD;;AAOM,eAAeU,UAAf,CACLd,MADK,EAELD,OAFK,EAGqB;AAC1B,MAAIc,IAAJ;AACA,QAAM;AAACO,IAAAA,UAAD;AAAajB,IAAAA;AAAb,MAAuB,iCAAiBH,MAAM,CAACF,MAAxB,EAAgCE,MAAM,CAACC,MAAvC,CAA7B;AACAD,EAAAA,MAAM,CAACC,MAAP,IAAiBE,MAAjB;AAEA,QAAMkB,QAAQ,GAAG,8BAAcC,uBAAd,EAAwBF,UAAU,CAACG,IAAnC,CAAjB;;AAEA,UAAQF,QAAR;AACE,SAAK,WAAL;AACER,MAAAA,IAAI,GAAG,MAAMW,cAAc,CAACxB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAA3B;AACA;;AACF,SAAK,cAAL;AACEc,MAAAA,IAAI,GAAG,MAAMY,gBAAgB,CAACzB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAA7B;AACA;;AACF,SAAK,iBAAL;AACEc,MAAAA,IAAI,GAAG;AACLH,QAAAA,UAAU,EAAE,MAAMgB,oBAAoB,CAAC1B,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CADjC;AAELqB,QAAAA;AAFK,OAAP;AAIA;;AACF;AACE,YAAM,IAAIO,KAAJ,8BAAgCN,QAAhC,EAAN;AAdJ;;AAiBA,SAAOR,IAAP;AACD;;AAYM,SAASe,YAAT,CACLC,cADK,EAEL5B,MAFK,EAGL6B,GAHK,EAQL;AACA,QAAMC,MAAwB,GAAG,EAAjC;AACA,MAAIC,IAAI,GAAG/B,MAAX;;AACA,OAAK,IAAIgC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,GAApB,EAAyBG,CAAC,EAA1B,EAA8B;AAC5B,UAAMC,aAAa,GAAGL,cAAc,CAACG,IAAD,CAApC;AAEA,UAAMG,cAAc,GAClBH,IAAI,GAAG,CAAP,GAAW,8BAAcI,kCAAd,EAAmCF,aAAa,CAACG,eAAjD,CAAX,GAAgF,MADlF;AAGA,QAAIC,QAAQ,GAAG,KAAf;AACA,QAAIC,QAAQ,GAAG,KAAf;;AACA,YAAQJ,cAAR;AACE,WAAK,UAAL;AACE;;AACF,WAAK,UAAL;AACEG,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF,WAAK,UAAL;AACEC,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF;AACE,cAAM,IAAIZ,KAAJ,CAAU,kCAAV,CAAN;AAVJ;;AAaA,QAAIO,aAAa,CAACM,YAAd,GAA8B,CAAlC,EAAqC;AACnC,YAAMC,GAAG,GAAGb,YAAY,CAACC,cAAD,EAAiBG,IAAI,GAAG,CAAxB,EAA2BE,aAAa,CAACM,YAAzC,CAAxB;AACAR,MAAAA,IAAI,GAAGS,GAAG,CAACT,IAAX;AACAD,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAE3BJ,QAAAA,QAF2B;AAG3BC,QAAAA,QAH2B;AAI3BI,QAAAA,MAAM,EAAEF,GAAG,CAACV;AAJe,OAA7B;AAMD,KATD,MASO;AACL,YAAMR,IAAI,GAAG,8BAAcqB,mBAAd,EAAoBV,aAAa,CAACX,IAAlC,CAAb;AACA,UAAIsB,WAAW,GAAGtB,IAAlB;;AAEA,UAAIW,aAAa,CAACY,cAAlB,EAAkC;AAChCD,QAAAA,WAAW,GAAG,8BAAcE,4BAAd,EAA6Bb,aAAa,CAACY,cAA3C,CAAd;AACD;;AAED,cAAQD,WAAR;AACE,aAAK,SAAL;AACEA,UAAAA,WAAW,aAAMA,WAAN,cAAqBtB,IAArB,CAAX;AACA;;AACF;AAJF;;AAOAQ,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAC3BnB,QAAAA,IAAI,EAAEsB,WADqB;AAE3BG,QAAAA,UAAU,EAAEd,aAAa,CAACe,WAFC;AAG3BC,QAAAA,SAAS,EAAEhB,aAAa,CAACiB,SAHE;AAI3BC,QAAAA,KAAK,EAAElB,aAAa,CAACkB,KAJM;AAK3Bd,QAAAA,QAL2B;AAM3BC,QAAAA;AAN2B,OAA7B;AAQAP,MAAAA,IAAI;AACL;AACF;;AACD,SAAO;AAACD,IAAAA,MAAD;AAAS9B,IAAAA,MAAT;AAAiB+B,IAAAA;AAAjB,GAAP;AACD;;AAKD,SAASqB,YAAT,CACE9B,IADF,EAEE+B,QAFF,EAGEtD,MAHF,EAIES,KAJF,EAKE8C,IALF,EAMS;AACP,MAAI,EAAED,QAAQ,IAAIE,sBAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI7B,KAAJ,6BAA+B2B,QAA/B,EAAN;AACD;;AACD,SAAOE,uBAAeF,QAAf,EAAyBD,YAAzB,CAAsC9B,IAAtC,EAA4CvB,MAA5C,EAAoDS,KAApD,EAA2D8C,IAA3D,CAAP;AACD;;AAQD,eAAe/B,cAAf,CACExB,MADF,EAEEyD,MAFF,EAGE1D,OAHF,EAI4B;AAAA;;AAC1B,QAAM2D,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAAzC;AACA,QAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAV,0DAAG,sBAAyBC,UAA5C;AAGA,MAAIC,UAAU,GAAG/D,MAAjB;;AAEA,MAAID,OAAO,CAACiE,WAAR,KAAwB,cAA5B,EAA4C;AAC1C,UAAMC,SAAS,GAAG,MAAM,6BACtBlE,OAAO,CAACiE,WADc,EAEtBhE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACU,sBAHe,CAAxB;AAKAJ,IAAAA,UAAU,GAAG;AACXjE,MAAAA,MAAM,EAAEmE,SADG;AAEXhE,MAAAA,MAAM,EAAE,CAFG;AAGXC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHL,KAAb;AAKAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAGD,QAAMU,cAAc,GAAG,8BACrBC,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBS,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIC,KAAJ,CAAUZ,UAAV,CAAd;;AAEA,MAAI7D,OAAO,CAAC0E,MAAR,CAAeC,SAAf,GAA2B,CAA/B,EAAkC;AAChCH,IAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBP,cAArB,EAAqCL,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,MAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeC,SAA3B,CADwE;AAElFG,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLN,IAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAGD,QAAMC,cAAc,GAAG,8BACrBV,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBmB,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIT,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAI7D,OAAO,CAAC0E,MAAR,CAAeS,SAAf,GAA2B,CAA/B,EAAkC;AAChCD,IAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBI,cAArB,EAAqChB,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,MAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeS,SAA3B,CADwE;AAElFL,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AACD,MAAIK,iBAAiB,GAAG,CAAxB;;AACA,OAAK,MAAMC,IAAX,IAAmBH,OAAnB,EAA4B;AAC1B,QAAIG,IAAI,KAAKrF,OAAO,CAAC0E,MAAR,CAAeS,SAA5B,EAAuC;AACrCC,MAAAA,iBAAiB;AAClB;AACF;;AAGD,QAAME,aAAa,GAAG,8BAAchB,uBAAd,4BAAwBZ,MAAM,CAACI,gBAA/B,2DAAwB,uBAAyBP,QAAjD,CAAtB;AACA,QAAMgC,aAAa,GAAG;AACpBtC,IAAAA,UAAU,EAAEjD,OAAO,CAAC0E,MAAR,CAAezB,UADP;AAEpB4B,IAAAA,QAAQ,EAAE7E,OAAO,CAAC0E,MAAR,CAAezB;AAFL,GAAtB;AAKA,QAAMzC,MAAM,GAAG8C,YAAY,CACzBtD,OAAO,CAAC0E,MAAR,CAAec,aADU,EAEzBF,aAFyB,EAGzBtB,UAHyB,EAIzBoB,iBAJyB,EAKzBG,aALyB,CAA3B;AAQA,SAAO;AACLhF,IAAAA,OAAO,EAAE2E,OADJ;AAEL5E,IAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,IAAAA,MAHK;AAILE,IAAAA,KAAK,EAAEmD,UAJF;AAKLxC,IAAAA,UAAU,EAAEqC;AALP,GAAP;AAOD;;AASD,eAAehC,gBAAf,CACEzB,MADF,EAEEyD,MAFF,EAGEF,IAHF,EAI4B;AAAA;;AAC1B,QAAMG,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAAzC;AAEA,QAAMC,UAAU,6BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4B1B,UAA/C;AAEA,QAAMqB,iBAAiB,GAAGvB,UAAU,8BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4BC,SAA/B,CAApC;AACA,QAAMJ,aAAa,GAAG,8BACpBhB,uBADoB,4BAEpBZ,MAAM,CAAC+B,mBAFa,2DAEpB,uBAA4BlC,QAFR,CAAtB;AAOA,MAAIiB,OAAO,GAAG,IAAIC,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAIL,IAAI,CAACkB,MAAL,CAAYC,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BH,IAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,MAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYC,SAAxB,CAD4E;AAEtFG,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLN,IAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAID,MAAIG,OAAO,GAAG,IAAIT,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAIL,IAAI,CAACkB,MAAL,CAAYS,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BD,IAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,MAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYS,SAAxB,CAD4E;AAEtFL,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AAGD,MAAIa,eAAe,GAAG3F,MAAtB;;AAEA,gCAAIyD,MAAM,CAAC+B,mBAAX,mDAAI,uBAA4BI,aAAhC,EAA+C;AAC7C,UAAM3B,SAAS,GAAG,MAAM,6BACtBV,IAAI,CAACS,WADiB,EAEtBhE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACU,sBAHe,CAAxB;AAMAwB,IAAAA,eAAe,GAAG;AAChB7F,MAAAA,MAAM,EAAEmE,SADQ;AAEhBhE,MAAAA,MAAM,EAAE,CAFQ;AAGhBC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHA,KAAlB;AAMAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAED,QAAM4B,aAAa,GAAG;AACpBtC,IAAAA,UAAU,EAAEO,IAAI,CAACkB,MAAL,CAAYzB,UADJ;AAEpB4B,IAAAA,QAAQ,EAAErB,IAAI,CAACkB,MAAL,CAAYzB;AAFF,GAAtB;AAKA,QAAMzC,MAAM,GAAG8C,YAAY,CACzBE,IAAI,CAACkB,MAAL,CAAYc,aADa,EAEzBF,aAFyB,EAGzBM,eAHyB,EAIzBR,iBAJyB,EAKzBG,aALyB,CAA3B;AAQA,SAAO;AACLhF,IAAAA,OAAO,EAAE2E,OADJ;AAEL5E,IAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,IAAAA,MAHK;AAILE,IAAAA,KAAK,EAAEmD,UAJF;AAKLxC,IAAAA,UAAU,EAAEqC;AALP,GAAP;AAOD;;AAQD,eAAe/B,oBAAf,CACE1B,MADF,EAEEoB,UAFF,EAGErB,OAHF,EAIqB;AAAA;;AACnB,QAAM2D,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBmB,UAAU,CAACuC,oBAA7C;AAEA,MAAIkC,UAAU,GAAG;AACf5F,IAAAA,MAAM,EAAE,CADO;AAEfH,IAAAA,MAAM,EAAEE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFO;AAGfxD,IAAAA,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;AAHV,GAAjB;AAMAD,EAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AAEA,MAAI3D,OAAO,CAACiE,WAAR,KAAwB,cAA5B,EAA4C;AAC1C,UAAMC,SAAS,GAAG,MAAM,6BACtBlE,OAAO,CAACiE,WADc,EAEtB6B,UAAU,CAAC/F,MAAX,CAAkBoE,KAAlB,CAAwB2B,UAAU,CAAC5F,MAAnC,EAA2CyD,SAA3C,CAFsB,EAGtBtC,UAAU,CAAC+C,sBAHW,CAAxB;AAMA0B,IAAAA,UAAU,GAAG;AACX/F,MAAAA,MAAM,EAAEmE,SADG;AAEXhE,MAAAA,MAAM,EAAE,CAFG;AAGXC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHL,KAAb;AAMAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAED,QAAM/C,SAAS,GAAG,CAAAS,UAAU,SAAV,IAAAA,UAAU,WAAV,qCAAAA,UAAU,CAAE0E,sBAAZ,gFAAoChC,UAApC,KAAkD,CAApE;AAEA,SAAOT,YAAY,CACjBtD,OAAO,CAAC0E,MAAR,CAAec,aADE,EAEjBxF,OAAO,CAAC0E,MAAR,CAAenB,QAFE,EAGjBuC,UAHiB,EAIjBlF,SAJiB,EAKjBZ,OALiB,CAAZ,CAMLgB,GANK,CAMAgF,CAAD,IAAOA,CAAC,CAACC,QAAF,EANN,CAAP;AAOD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetData,\n  ParquetOptions,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  options: ParquetOptions\n): Promise<ParquetData> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetData = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = options.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!options.numValues || data.dlevels.length < Number(options.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, options);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param options\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  let page;\n  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, options);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, options);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (options.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (options.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === options.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions = {\n    typeLength: options.column.typeLength,\n    bitWidth: options.column.typeLength\n  };\n\n  const values = decodeValues(\n    options.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  opts: any\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (opts.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (opts.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      opts.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: opts.column.typeLength,\n    bitWidth: opts.column.typeLength\n  };\n\n  const values = decodeValues(\n    opts.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  options: ParquetOptions\n): Promise<string[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  return decodeValues(\n    options.column.primitiveType!,\n    options.column.encoding!,\n    dictCursor,\n    numValues,\n    options as ParquetCodecOptions\n  ).map((d) => d.toString());\n}\n"],"file":"decoders.js"}

package/dist/es5/parquetjs/parser/parquet-envelope-reader.js

@@ -13,8 +13,6 @@ var _constants = require("../../constants");
 
 var _parquetThrift = require("../parquet-thrift");
 
-var _fileUtils = require("../utils/file-utils");
-
 var _readUtils = require("../utils/read-utils");
 
 var _decoders = require("./decoders");
@@ -22,17 +20,6 @@ var _decoders = require("./decoders");
 const DEFAULT_DICTIONARY_SIZE = 1e6;
 
 class ParquetEnvelopeReader {
-  static async openFile(filePath) {
-    const fileStat = await (0, _fileUtils.fstat)(filePath);
-    const fileDescriptor = await (0, _fileUtils.fopen)(filePath);
-
-    const readFn = _fileUtils.fread.bind(undefined, fileDescriptor);
-
-    const closeFn = _fileUtils.fclose.bind(undefined, fileDescriptor);
-
-    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);
-  }
-
   static async openBuffer(buffer) {
     const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
 

package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":["DEFAULT_DICTIONARY_SIZE","ParquetEnvelopeReader","openFile","filePath","fileStat","fileDescriptor","readFn","fread","bind","undefined","closeFn","fclose","size","openBuffer","buffer","position","length","Promise","resolve","slice","constructor","read","close","fileSize","options","defaultDictionarySize","readHeader","PARQUET_MAGIC","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","readRowGroup","schema","rowGroup","columnList","rowCount","Number","num_rows","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","field","findField","type","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage","readFooter","trailerLen","trailerBuf","metadataSize","readUInt32LE","metadataOffset","metadataBuf","metadata"],"mappings":";;;;;;;;;;;AAEA;;AACA;;AAQA;;AACA;;AACA;;AAEA,MAAMA,uBAAuB,GAAG,GAAhC;;AAQO,MAAMC,qBAAN,CAA4B;AAUZ,eAARC,QAAQ,CAACC,QAAD,EAAmD;AACtE,UAAMC,QAAQ,GAAG,MAAM,sBAAMD,QAAN,CAAvB;AACA,UAAME,cAAc,GAAG,MAAM,sBAAMF,QAAN,CAA7B;;AAEA,UAAMG,MAAM,GAAGC,iBAAMC,IAAN,CAAWC,SAAX,EAAsBJ,cAAtB,CAAf;;AACA,UAAMK,OAAO,GAAGC,kBAAOH,IAAP,CAAYC,SAAZ,EAAuBJ,cAAvB,CAAhB;;AAEA,WAAO,IAAIJ,qBAAJ,CAA0BK,MAA1B,EAAkCI,OAAlC,EAA2CN,QAAQ,CAACQ,IAApD,CAAP;AACD;;AAEsB,eAAVC,UAAU,CAACC,MAAD,EAAiD;AACtE,UAAMR,MAAM,GAAG,CAACS,QAAD,EAAmBC,MAAnB,KACbC,OAAO,CAACC,OAAR,CAAgBJ,MAAM,CAACK,KAAP,CAAaJ,QAAb,EAAuBA,QAAQ,GAAGC,MAAlC,CAAhB,CADF;;AAEA,UAAMN,OAAO,GAAG,MAAMO,OAAO,CAACC,OAAR,EAAtB;;AACA,WAAO,IAAIjB,qBAAJ,CAA0BK,MAA1B,EAAkCI,OAAlC,EAA2CI,MAAM,CAACE,MAAlD,CAAP;AACD;;AAEDI,EAAAA,WAAW,CACTC,IADS,EAETC,KAFS,EAGTC,QAHS,EAITC,OAJS,EAKT;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,IAAL,GAAYA,IAAZ;AACA,SAAKC,KAAL,GAAaA,KAAb;AACA,SAAKC,QAAL,GAAgBA,QAAhB;AACA,SAAKE,qBAAL,GAA6B,CAAAD,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEC,qBAAT,KAAkCzB,uBAA/D;AACD;;AAEe,QAAV0B,UAAU,GAAkB;AAChC,UAAMZ,MAAM,GAAG,MAAM,KAAKO,IAAL,CAAU,CAAV,EAAaM,yBAAcX,MAA3B,CAArB;AAEA,UAAMY,KAAK,GAAGd,MAAM,CAACe,QAAP,EAAd;;AACA,YAAQD,KAAR;AACE,WAAKD,wBAAL;AACE;;AACF,WAAKG,kCAAL;AACE,cAAM,IAAIC,KAAJ,CAAU,sCAAV,CAAN;;AACF;AACE,cAAM,IAAIA,KAAJ,uCAAyCH,KAAzC,OAAN;AANJ;AAQD;;AAEiB,QAAZI,YAAY,CAChBC,MADgB,EAEhBC,QAFgB,EAGhBC,UAHgB,EAIQ;AACxB,UAAMrB,MAAqB,GAAG;AAC5BsB,MAAAA,QAAQ,EAAEC,MAAM,CAACH,QAAQ,CAACI,QAAV,CADY;AAE5BC,MAAAA,UAAU,EAAE;AAFgB,KAA9B;;AAIA,SAAK,MAAMC,QAAX,IAAuBN,QAAQ,CAACO,OAAhC,EAAyC;AACvC,YAAMC,WAAW,GAAGF,QAAQ,CAACG,SAA7B;AACA,YAAMC,MAAM,GAAGF,WAAH,aAAGA,WAAH,uBAAGA,WAAW,CAAEG,cAA5B;;AACA,UAAIV,UAAU,CAACnB,MAAX,GAAoB,CAApB,IAAyB,6BAAamB,UAAb,EAAyBS,MAAzB,IAAoC,CAAjE,EAAoE;AAClE;AACD;;AACD9B,MAAAA,MAAM,CAACyB,UAAP,CAAkBK,MAAM,CAAEE,IAAR,EAAlB,IAAoC,MAAM,KAAKC,eAAL,CAAqBd,MAArB,EAA6BO,QAA7B,CAA1C;AACD;;AACD,WAAO1B,MAAP;AACD;;AAOoB,QAAfiC,eAAe,CAACd,MAAD,EAAwBO,QAAxB,EAAqE;AAAA;;AACxF,QAAIA,QAAQ,CAACQ,SAAT,KAAuBvC,SAAvB,IAAoC+B,QAAQ,CAACQ,SAAT,KAAuB,IAA/D,EAAqE;AACnE,YAAM,IAAIjB,KAAJ,CAAU,uCAAV,CAAN;AACD;;AAED,UAAMkB,KAAK,GAAGhB,MAAM,CAACiB,SAAP,wBAAiBV,QAAQ,CAACG,SAA1B,wDAAiB,oBAAoBE,cAArC,CAAd;AACA,UAAMM,IAAmB,GAAG,8BAAcC,mBAAd,0BAAoBZ,QAAQ,CAACG,SAA7B,yDAAoB,qBAAoBQ,IAAxC,CAA5B;;AAEA,QAAIA,IAAI,KAAKF,KAAK,CAACI,aAAnB,EAAkC;AAChC,YAAM,IAAItB,KAAJ,2CAA6CoB,IAA7C,EAAN;AACD;;AAED,UAAMG,WAA+B,GAAG,8BACtCC,+BADsC,0BAEtCf,QAAQ,CAACG,SAF6B,yDAEtC,qBAAoBa,KAFkB,CAAxC;AAKA,UAAMC,WAAW,GAAGpB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBe,gBAArB,CAA1B;AACA,QAAIC,SAAS,GAAGtB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBiB,qBAArB,CAAtB;;AAEA,QAAI,CAACpB,QAAQ,CAACQ,SAAd,EAAyB;AAAA;;AACvBW,MAAAA,SAAS,GAAGE,IAAI,CAACC,GAAL,CACV,KAAKvC,QAAL,GAAgBkC,WADN,EAEVpB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBiB,qBAArB,CAFI,CAAZ;AAID;;AAED,UAAMpC,OAAuB,GAAG;AAC9B2B,MAAAA,IAD8B;AAE9BY,MAAAA,SAAS,EAAEd,KAAK,CAACc,SAFa;AAG9BC,MAAAA,SAAS,EAAEf,KAAK,CAACe,SAHa;AAI9BV,MAAAA,WAJ8B;AAK9BW,MAAAA,MAAM,EAAEhB,KALsB;AAM9BiB,MAAAA,SAAS,0BAAE1B,QAAQ,CAACG,SAAX,yDAAE,qBAAoBwB,UAND;AAO9BC,MAAAA,UAAU,EAAE;AAPkB,KAAhC;AAUA,QAAIA,UAAJ;AAEA,UAAMC,oBAAoB,GAAG7B,QAAH,aAAGA,QAAH,+CAAGA,QAAQ,CAAEG,SAAb,yDAAG,qBAAqB2B,sBAAlD;;AAEA,QAAID,oBAAJ,EAA0B;AACxB,YAAME,gBAAgB,GAAGlC,MAAM,CAACgC,oBAAD,CAA/B;AAEAD,MAAAA,UAAU,GAAG,MAAM,KAAKI,aAAL,CAAmBD,gBAAnB,EAAqC/C,OAArC,EAA8CiC,WAA9C,CAAnB;AACD;;AAEDW,IAAAA,UAAU,GAAG,uBAAA5C,OAAO,CAAC4C,UAAR,oEAAoBpD,MAApB,GAA6BQ,OAAO,CAAC4C,UAArC,GAAkDA,UAA/D;AACA,UAAMK,QAAQ,GAAG,MAAM,KAAKpD,IAAL,CAAUoC,WAAV,EAAuBE,SAAvB,CAAvB;AACA,WAAO,MAAM,+BAAgBc,QAAhB,EAA0B,EAAC,GAAGjD,OAAJ;AAAa4C,MAAAA;AAAb,KAA1B,CAAb;AACD;;AASkB,QAAbI,aAAa,CACjBH,oBADiB,EAEjB7C,OAFiB,EAGjBiC,WAHiB,EAIE;AACnB,QAAIY,oBAAoB,KAAK,CAA7B,EAAgC;AAQ9B,aAAO,EAAP;AACD;;AAED,UAAMK,cAAc,GAAGb,IAAI,CAACC,GAAL,CACrB,KAAKvC,QAAL,GAAgB8C,oBADK,EAErB,KAAK5C,qBAFgB,CAAvB;AAIA,UAAMgD,QAAQ,GAAG,MAAM,KAAKpD,IAAL,CAAUgD,oBAAV,EAAgCK,cAAhC,CAAvB;AAEA,UAAMC,MAAM,GAAG;AAAC7D,MAAAA,MAAM,EAAE2D,QAAT;AAAmBG,MAAAA,MAAM,EAAE,CAA3B;AAA8BhE,MAAAA,IAAI,EAAE6D,QAAQ,CAACzD;AAA7C,KAAf;AACA,UAAM6D,WAAW,GAAG,MAAM,0BAAWF,MAAX,EAAmBnD,OAAnB,CAA1B;AAEA,WAAOqD,WAAW,CAACT,UAAnB;AACD;;AAEe,QAAVU,UAAU,GAA0B;AACxC,UAAMC,UAAU,GAAGpD,yBAAcX,MAAd,GAAuB,CAA1C;AACA,UAAMgE,UAAU,GAAG,MAAM,KAAK3D,IAAL,CAAU,KAAKE,QAAL,GAAgBwD,UAA1B,EAAsCA,UAAtC,CAAzB;AAEA,UAAMnD,KAAK,GAAGoD,UAAU,CAAC7D,KAAX,CAAiB,CAAjB,EAAoBU,QAApB,EAAd;;AACA,QAAID,KAAK,KAAKD,wBAAd,EAA6B;AAC3B,YAAM,IAAII,KAAJ,6CAA8CH,KAA9C,OAAN;AACD;;AAED,UAAMqD,YAAY,GAAGD,UAAU,CAACE,YAAX,CAAwB,CAAxB,CAArB;AACA,UAAMC,cAAc,GAAG,KAAK5D,QAAL,GAAgB0D,YAAhB,GAA+BF,UAAtD;;AACA,QAAII,cAAc,GAAGxD,yBAAcX,MAAnC,EAA2C;AACzC,YAAM,IAAIe,KAAJ,iCAAmCoD,cAAnC,EAAN;AACD;;AAED,UAAMC,WAAW,GAAG,MAAM,KAAK/D,IAAL,CAAU8D,cAAV,EAA0BF,YAA1B,CAA1B;AAGA,UAAM;AAACI,MAAAA;AAAD,QAAa,mCAAmBD,WAAnB,CAAnB;AACA,WAAOC,QAAP;AACD;;AAzLgC","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetSchema} from '../schema/schema';\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {fstat, fopen, fread, fclose} from '../utils/file-utils';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nconst DEFAULT_DICTIONARY_SIZE = 1e6;\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n  public defaultDictionarySize: number;\n\n  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {\n    const fileStat = await fstat(filePath);\n    const fileDescriptor = await fopen(filePath);\n\n    const readFn = fread.bind(undefined, fileDescriptor);\n    const closeFn = fclose.bind(undefined, fileDescriptor);\n\n    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);\n  }\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number,\n    options?: any\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;\n  }\n\n  async readHeader(): Promise<void> {\n    const buffer = await this.read(0, PARQUET_MAGIC.length);\n\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Do reading of parquet file's column chunk\n   * @param schema\n   * @param colChunk\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.fileSize - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.fileSize - dictionaryPageOffset,\n      this.defaultDictionarySize\n    );\n    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n"],"file":"parquet-envelope-reader.js"}
+{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":["DEFAULT_DICTIONARY_SIZE","ParquetEnvelopeReader","openBuffer","buffer","readFn","position","length","Promise","resolve","slice","closeFn","constructor","read","close","fileSize","options","defaultDictionarySize","readHeader","PARQUET_MAGIC","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","readRowGroup","schema","rowGroup","columnList","rowCount","Number","num_rows","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","undefined","field","findField","type","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","size","decodedPage","readFooter","trailerLen","trailerBuf","metadataSize","readUInt32LE","metadataOffset","metadataBuf","metadata"],"mappings":";;;;;;;;;;;AAEA;;AACA;;AAQA;;AACA;;AAEA,MAAMA,uBAAuB,GAAG,GAAhC;;AAQO,MAAMC,qBAAN,CAA4B;AAUV,eAAVC,UAAU,CAACC,MAAD,EAAiD;AACtE,UAAMC,MAAM,GAAG,CAACC,QAAD,EAAmBC,MAAnB,KACbC,OAAO,CAACC,OAAR,CAAgBL,MAAM,CAACM,KAAP,CAAaJ,QAAb,EAAuBA,QAAQ,GAAGC,MAAlC,CAAhB,CADF;;AAEA,UAAMI,OAAO,GAAG,MAAMH,OAAO,CAACC,OAAR,EAAtB;;AACA,WAAO,IAAIP,qBAAJ,CAA0BG,MAA1B,EAAkCM,OAAlC,EAA2CP,MAAM,CAACG,MAAlD,CAAP;AACD;;AAEDK,EAAAA,WAAW,CACTC,IADS,EAETC,KAFS,EAGTC,QAHS,EAITC,OAJS,EAKT;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,IAAL,GAAYA,IAAZ;AACA,SAAKC,KAAL,GAAaA,KAAb;AACA,SAAKC,QAAL,GAAgBA,QAAhB;AACA,SAAKE,qBAAL,GAA6B,CAAAD,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEC,qBAAT,KAAkChB,uBAA/D;AACD;;AAEe,QAAViB,UAAU,GAAkB;AAChC,UAAMd,MAAM,GAAG,MAAM,KAAKS,IAAL,CAAU,CAAV,EAAaM,yBAAcZ,MAA3B,CAArB;AAEA,UAAMa,KAAK,GAAGhB,MAAM,CAACiB,QAAP,EAAd;;AACA,YAAQD,KAAR;AACE,WAAKD,wBAAL;AACE;;AACF,WAAKG,kCAAL;AACE,cAAM,IAAIC,KAAJ,CAAU,sCAAV,CAAN;;AACF;AACE,cAAM,IAAIA,KAAJ,uCAAyCH,KAAzC,OAAN;AANJ;AAQD;;AAEiB,QAAZI,YAAY,CAChBC,MADgB,EAEhBC,QAFgB,EAGhBC,UAHgB,EAIQ;AACxB,UAAMvB,MAAqB,GAAG;AAC5BwB,MAAAA,QAAQ,EAAEC,MAAM,CAACH,QAAQ,CAACI,QAAV,CADY;AAE5BC,MAAAA,UAAU,EAAE;AAFgB,KAA9B;;AAIA,SAAK,MAAMC,QAAX,IAAuBN,QAAQ,CAACO,OAAhC,EAAyC;AACvC,YAAMC,WAAW,GAAGF,QAAQ,CAACG,SAA7B;AACA,YAAMC,MAAM,GAAGF,WAAH,aAAGA,WAAH,uBAAGA,WAAW,CAAEG,cAA5B;;AACA,UAAIV,UAAU,CAACpB,MAAX,GAAoB,CAApB,IAAyB,6BAAaoB,UAAb,EAAyBS,MAAzB,IAAoC,CAAjE,EAAoE;AAClE;AACD;;AACDhC,MAAAA,MAAM,CAAC2B,UAAP,CAAkBK,MAAM,CAAEE,IAAR,EAAlB,IAAoC,MAAM,KAAKC,eAAL,CAAqBd,MAArB,EAA6BO,QAA7B,CAA1C;AACD;;AACD,WAAO5B,MAAP;AACD;;AAOoB,QAAfmC,eAAe,CAACd,MAAD,EAAwBO,QAAxB,EAAqE;AAAA;;AACxF,QAAIA,QAAQ,CAACQ,SAAT,KAAuBC,SAAvB,IAAoCT,QAAQ,CAACQ,SAAT,KAAuB,IAA/D,EAAqE;AACnE,YAAM,IAAIjB,KAAJ,CAAU,uCAAV,CAAN;AACD;;AAED,UAAMmB,KAAK,GAAGjB,MAAM,CAACkB,SAAP,wBAAiBX,QAAQ,CAACG,SAA1B,wDAAiB,oBAAoBE,cAArC,CAAd;AACA,UAAMO,IAAmB,GAAG,8BAAcC,mBAAd,0BAAoBb,QAAQ,CAACG,SAA7B,yDAAoB,qBAAoBS,IAAxC,CAA5B;;AAEA,QAAIA,IAAI,KAAKF,KAAK,CAACI,aAAnB,EAAkC;AAChC,YAAM,IAAIvB,KAAJ,2CAA6CqB,IAA7C,EAAN;AACD;;AAED,UAAMG,WAA+B,GAAG,8BACtCC,+BADsC,0BAEtChB,QAAQ,CAACG,SAF6B,yDAEtC,qBAAoBc,KAFkB,CAAxC;AAKA,UAAMC,WAAW,GAAGrB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBgB,gBAArB,CAA1B;AACA,QAAIC,SAAS,GAAGvB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBkB,qBAArB,CAAtB;;AAEA,QAAI,CAACrB,QAAQ,CAACQ,SAAd,EAAyB;AAAA;;AACvBY,MAAAA,SAAS,GAAGE,IAAI,CAACC,GAAL,CACV,KAAKxC,QAAL,GAAgBmC,WADN,EAEVrB,MAAM,yBAACG,QAAQ,CAACG,SAAV,yDAAC,qBAAoBkB,qBAArB,CAFI,CAAZ;AAID;;AAED,UAAMrC,OAAuB,GAAG;AAC9B4B,MAAAA,IAD8B;AAE9BY,MAAAA,SAAS,EAAEd,KAAK,CAACc,SAFa;AAG9BC,MAAAA,SAAS,EAAEf,KAAK,CAACe,SAHa;AAI9BV,MAAAA,WAJ8B;AAK9BW,MAAAA,MAAM,EAAEhB,KALsB;AAM9BiB,MAAAA,SAAS,0BAAE3B,QAAQ,CAACG,SAAX,yDAAE,qBAAoByB,UAND;AAO9BC,MAAAA,UAAU,EAAE;AAPkB,KAAhC;AAUA,QAAIA,UAAJ;AAEA,UAAMC,oBAAoB,GAAG9B,QAAH,aAAGA,QAAH,+CAAGA,QAAQ,CAAEG,SAAb,yDAAG,qBAAqB4B,sBAAlD;;AAEA,QAAID,oBAAJ,EAA0B;AACxB,YAAME,gBAAgB,GAAGnC,MAAM,CAACiC,oBAAD,CAA/B;AAEAD,MAAAA,UAAU,GAAG,MAAM,KAAKI,aAAL,CAAmBD,gBAAnB,EAAqChD,OAArC,EAA8CkC,WAA9C,CAAnB;AACD;;AAEDW,IAAAA,UAAU,GAAG,uBAAA7C,OAAO,CAAC6C,UAAR,oEAAoBtD,MAApB,GAA6BS,OAAO,CAAC6C,UAArC,GAAkDA,UAA/D;AACA,UAAMK,QAAQ,GAAG,MAAM,KAAKrD,IAAL,CAAUqC,WAAV,EAAuBE,SAAvB,CAAvB;AACA,WAAO,MAAM,+BAAgBc,QAAhB,EAA0B,EAAC,GAAGlD,OAAJ;AAAa6C,MAAAA;AAAb,KAA1B,CAAb;AACD;;AASkB,QAAbI,aAAa,CACjBH,oBADiB,EAEjB9C,OAFiB,EAGjBkC,WAHiB,EAIE;AACnB,QAAIY,oBAAoB,KAAK,CAA7B,EAAgC;AAQ9B,aAAO,EAAP;AACD;;AAED,UAAMK,cAAc,GAAGb,IAAI,CAACC,GAAL,CACrB,KAAKxC,QAAL,GAAgB+C,oBADK,EAErB,KAAK7C,qBAFgB,CAAvB;AAIA,UAAMiD,QAAQ,GAAG,MAAM,KAAKrD,IAAL,CAAUiD,oBAAV,EAAgCK,cAAhC,CAAvB;AAEA,UAAMC,MAAM,GAAG;AAAChE,MAAAA,MAAM,EAAE8D,QAAT;AAAmBG,MAAAA,MAAM,EAAE,CAA3B;AAA8BC,MAAAA,IAAI,EAAEJ,QAAQ,CAAC3D;AAA7C,KAAf;AACA,UAAMgE,WAAW,GAAG,MAAM,0BAAWH,MAAX,EAAmBpD,OAAnB,CAA1B;AAEA,WAAOuD,WAAW,CAACV,UAAnB;AACD;;AAEe,QAAVW,UAAU,GAA0B;AACxC,UAAMC,UAAU,GAAGtD,yBAAcZ,MAAd,GAAuB,CAA1C;AACA,UAAMmE,UAAU,GAAG,MAAM,KAAK7D,IAAL,CAAU,KAAKE,QAAL,GAAgB0D,UAA1B,EAAsCA,UAAtC,CAAzB;AAEA,UAAMrD,KAAK,GAAGsD,UAAU,CAAChE,KAAX,CAAiB,CAAjB,EAAoBW,QAApB,EAAd;;AACA,QAAID,KAAK,KAAKD,wBAAd,EAA6B;AAC3B,YAAM,IAAII,KAAJ,6CAA8CH,KAA9C,OAAN;AACD;;AAED,UAAMuD,YAAY,GAAGD,UAAU,CAACE,YAAX,CAAwB,CAAxB,CAArB;AACA,UAAMC,cAAc,GAAG,KAAK9D,QAAL,GAAgB4D,YAAhB,GAA+BF,UAAtD;;AACA,QAAII,cAAc,GAAG1D,yBAAcZ,MAAnC,EAA2C;AACzC,YAAM,IAAIgB,KAAJ,iCAAmCsD,cAAnC,EAAN;AACD;;AAED,UAAMC,WAAW,GAAG,MAAM,KAAKjE,IAAL,CAAUgE,cAAV,EAA0BF,YAA1B,CAA1B;AAGA,UAAM;AAACI,MAAAA;AAAD,QAAa,mCAAmBD,WAAnB,CAAnB;AACA,WAAOC,QAAP;AACD;;AA/KgC","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetSchema} from '../schema/schema';\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nconst DEFAULT_DICTIONARY_SIZE = 1e6;\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n  public defaultDictionarySize: number;\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number,\n    options?: any\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;\n  }\n\n  async readHeader(): Promise<void> {\n    const buffer = await this.read(0, PARQUET_MAGIC.length);\n\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Do reading of parquet file's column chunk\n   * @param schema\n   * @param colChunk\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.fileSize - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.fileSize - dictionaryPageOffset,\n      this.defaultDictionarySize\n    );\n    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n"],"file":"parquet-envelope-reader.js"}

package/dist/es5/parquetjs/parser/parquet-reader.js

@@ -63,19 +63,6 @@ class ParquetReader {
     }
   }
 
-  static async openFile(filePath) {
-    const envelopeReader = await _parquetEnvelopeReader.ParquetEnvelopeReader.openFile(filePath);
-
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
-  }
-
   static async openBuffer(buffer) {
     const envelopeReader = await _parquetEnvelopeReader.ParquetEnvelopeReader.openBuffer(buffer);
 

package/dist/es5/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"names":["Symbol","asyncIterator","ParquetReader","openBlob","blob","readFn","start","length","arrayBuffer","slice","Buffer","from","closeFn","size","envelopeReader","ParquetEnvelopeReader","readHeader","metadata","readFooter","err","close","openArrayBuffer","byteLength","openFile","filePath","openBuffer","buffer","constructor","version","PARQUET_VERSION","Error","root","schema","num_children","ParquetSchema","getCursor","columnList","map","x","Array","isArray","ParquetCursor","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","value"],"mappings":";;;;;;;;;;;AACA;;AAEA;;AACA;;AACA;;AACA;;;;wBAyKGA,MAAM,CAACC,a;;AAhKH,MAAMC,aAAN,CAAmD;AAInC,eAARC,QAAQ,CAAIC,IAAJ,EAA2C;AAC9D,UAAMC,MAAM,GAAG,OAAOC,KAAP,EAAsBC,MAAtB,KAAyC;AACtD,YAAMC,WAAW,GAAG,MAAMJ,IAAI,CAACK,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,EAAkCC,WAAlC,EAA1B;AACA,aAAOE,MAAM,CAACC,IAAP,CAAYH,WAAZ,CAAP;AACD,KAHD;;AAIA,UAAMI,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGT,IAAI,CAACS,IAAlB;AACA,UAAMC,cAAc,GAAG,IAAIC,4CAAJ,CAA0BV,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMC,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAkBe,QAAlB,EAA4BH,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAK2B,eAAfE,eAAe,CAAIb,WAAJ,EAAyD;AACnF,UAAMH,MAAM,GAAG,OAAOC,KAAP,EAAsBC,MAAtB,KAAyCG,MAAM,CAACC,IAAP,CAAYH,WAAZ,EAAyBF,KAAzB,EAAgCC,MAAhC,CAAxD;;AACA,UAAMK,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGL,WAAW,CAACc,UAAzB;AACA,UAAMR,cAAc,GAAG,IAAIC,4CAAJ,CAA0BV,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMC,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAkBe,QAAlB,EAA4BH,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAMoB,eAARI,QAAQ,CAAIC,QAAJ,EAAiD;AACpE,UAAMV,cAAc,GAAG,MAAMC,6CAAsBQ,QAAtB,CAA+BC,QAA/B,CAA7B;;AACA,QAAI;AACF,YAAMV,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAqBe,QAArB,EAA+BH,cAA/B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAEsB,eAAVM,UAAU,CAAIC,MAAJ,EAA+C;AACpE,UAAMZ,cAAc,GAAG,MAAMC,6CAAsBU,UAAtB,CAAiCC,MAAjC,CAA7B;;AACA,QAAI;AACF,YAAMZ,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAqBe,QAArB,EAA+BH,cAA/B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAYDQ,EAAAA,WAAW,CAACV,QAAD,EAAyBH,cAAzB,EAAgE;AAAA;AAAA;AAAA;;AACzE,QAAIG,QAAQ,CAACW,OAAT,KAAqBC,0BAAzB,EAA0C;AACxC,YAAM,IAAIC,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAED,SAAKb,QAAL,GAAgBA,QAAhB;AACA,SAAKH,cAAL,GAAsBA,cAAtB;AACA,UAAMiB,IAAI,GAAG,KAAKd,QAAL,CAAce,MAAd,CAAqB,CAArB,CAAb;AACA,UAAM;AAACA,MAAAA;AAAD,QAAW,4BAAa,KAAKf,QAAL,CAAce,MAA3B,EAAmC,CAAnC,EAAsCD,IAAI,CAACE,YAA3C,CAAjB;AACA,SAAKD,MAAL,GAAc,IAAIE,qBAAJ,CAAkBF,MAAlB,CAAd;AACD;;AAMU,QAALZ,KAAK,GAAkB;AAC3B,UAAM,KAAKN,cAAL,CAAoBM,KAApB,EAAN;AAGD;;AAeDe,EAAAA,SAAS,CAACC,UAAD,EAAgE;AACvE,QAAI,CAACA,UAAL,EAAiB;AAEfA,MAAAA,UAAU,GAAG,EAAb;AACD;;AAGDA,IAAAA,UAAU,GAAGA,UAAU,CAACC,GAAX,CAAgBC,CAAD,IAAQC,KAAK,CAACC,OAAN,CAAcF,CAAd,IAAmBA,CAAnB,GAAuB,CAACA,CAAD,CAA9C,CAAb;AAEA,WAAO,IAAIG,4BAAJ,CACL,KAAKxB,QADA,EAEL,KAAKH,cAFA,EAGL,KAAKkB,MAHA,EAILI,UAJK,CAAP;AAMD;;AAMDM,EAAAA,WAAW,GAAW;AACpB,WAAOC,MAAM,CAAC,KAAK1B,QAAL,CAAc2B,QAAf,CAAb;AACD;;AAKDC,EAAAA,SAAS,GAAkB;AACzB,WAAO,KAAKb,MAAZ;AACD;;AAKDc,EAAAA,WAAW,GAA2B;AACpC,UAAMC,EAA0B,GAAG,EAAnC;;AACA,SAAK,MAAMC,EAAX,IAAiB,KAAK/B,QAAL,CAAcgC,kBAA/B,EAAoD;AAClDF,MAAAA,EAAE,CAACC,EAAE,CAACE,GAAJ,CAAF,GAAaF,EAAE,CAACG,KAAhB;AACD;;AACD,WAAOJ,EAAP;AACD;;AAMD,4BAA2C;AACzC,WAAO,KAAKZ,SAAL,GAAiBnC,MAAM,CAACC,aAAxB,GAAP;AACD;;AAlKuD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * Open the parquet file pointed to by the specified path and return a new\n   * parquet reader\n   */\n  static async openFile<T>(filePath: string): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"file":"parquet-reader.js"}
+{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"names":["Symbol","asyncIterator","ParquetReader","openBlob","blob","readFn","start","length","arrayBuffer","slice","Buffer","from","closeFn","size","envelopeReader","ParquetEnvelopeReader","readHeader","metadata","readFooter","err","close","openArrayBuffer","byteLength","openBuffer","buffer","constructor","version","PARQUET_VERSION","Error","root","schema","num_children","ParquetSchema","getCursor","columnList","map","x","Array","isArray","ParquetCursor","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","value"],"mappings":";;;;;;;;;;;AACA;;AAEA;;AACA;;AACA;;AACA;;;;wBAyJGA,MAAM,CAACC,a;;AAhJH,MAAMC,aAAN,CAAmD;AAInC,eAARC,QAAQ,CAAIC,IAAJ,EAA2C;AAC9D,UAAMC,MAAM,GAAG,OAAOC,KAAP,EAAsBC,MAAtB,KAAyC;AACtD,YAAMC,WAAW,GAAG,MAAMJ,IAAI,CAACK,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,EAAkCC,WAAlC,EAA1B;AACA,aAAOE,MAAM,CAACC,IAAP,CAAYH,WAAZ,CAAP;AACD,KAHD;;AAIA,UAAMI,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGT,IAAI,CAACS,IAAlB;AACA,UAAMC,cAAc,GAAG,IAAIC,4CAAJ,CAA0BV,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMC,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAkBe,QAAlB,EAA4BH,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAK2B,eAAfE,eAAe,CAAIb,WAAJ,EAAyD;AACnF,UAAMH,MAAM,GAAG,OAAOC,KAAP,EAAsBC,MAAtB,KAAyCG,MAAM,CAACC,IAAP,CAAYH,WAAZ,EAAyBF,KAAzB,EAAgCC,MAAhC,CAAxD;;AACA,UAAMK,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGL,WAAW,CAACc,UAAzB;AACA,UAAMR,cAAc,GAAG,IAAIC,4CAAJ,CAA0BV,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMC,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAkBe,QAAlB,EAA4BH,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAEsB,eAAVI,UAAU,CAAIC,MAAJ,EAA+C;AACpE,UAAMV,cAAc,GAAG,MAAMC,6CAAsBQ,UAAtB,CAAiCC,MAAjC,CAA7B;;AACA,QAAI;AACF,YAAMV,cAAc,CAACE,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMH,cAAc,CAACI,UAAf,EAAvB;AACA,aAAO,IAAIhB,aAAJ,CAAqBe,QAArB,EAA+BH,cAA/B,CAAP;AACD,KAJD,CAIE,OAAOK,GAAP,EAAY;AACZ,YAAML,cAAc,CAACM,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAYDM,EAAAA,WAAW,CAACR,QAAD,EAAyBH,cAAzB,EAAgE;AAAA;AAAA;AAAA;;AACzE,QAAIG,QAAQ,CAACS,OAAT,KAAqBC,0BAAzB,EAA0C;AACxC,YAAM,IAAIC,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAED,SAAKX,QAAL,GAAgBA,QAAhB;AACA,SAAKH,cAAL,GAAsBA,cAAtB;AACA,UAAMe,IAAI,GAAG,KAAKZ,QAAL,CAAca,MAAd,CAAqB,CAArB,CAAb;AACA,UAAM;AAACA,MAAAA;AAAD,QAAW,4BAAa,KAAKb,QAAL,CAAca,MAA3B,EAAmC,CAAnC,EAAsCD,IAAI,CAACE,YAA3C,CAAjB;AACA,SAAKD,MAAL,GAAc,IAAIE,qBAAJ,CAAkBF,MAAlB,CAAd;AACD;;AAMU,QAALV,KAAK,GAAkB;AAC3B,UAAM,KAAKN,cAAL,CAAoBM,KAApB,EAAN;AAGD;;AAeDa,EAAAA,SAAS,CAACC,UAAD,EAAgE;AACvE,QAAI,CAACA,UAAL,EAAiB;AAEfA,MAAAA,UAAU,GAAG,EAAb;AACD;;AAGDA,IAAAA,UAAU,GAAGA,UAAU,CAACC,GAAX,CAAgBC,CAAD,IAAQC,KAAK,CAACC,OAAN,CAAcF,CAAd,IAAmBA,CAAnB,GAAuB,CAACA,CAAD,CAA9C,CAAb;AAEA,WAAO,IAAIG,4BAAJ,CACL,KAAKtB,QADA,EAEL,KAAKH,cAFA,EAGL,KAAKgB,MAHA,EAILI,UAJK,CAAP;AAMD;;AAMDM,EAAAA,WAAW,GAAW;AACpB,WAAOC,MAAM,CAAC,KAAKxB,QAAL,CAAcyB,QAAf,CAAb;AACD;;AAKDC,EAAAA,SAAS,GAAkB;AACzB,WAAO,KAAKb,MAAZ;AACD;;AAKDc,EAAAA,WAAW,GAA2B;AACpC,UAAMC,EAA0B,GAAG,EAAnC;;AACA,SAAK,MAAMC,EAAX,IAAiB,KAAK7B,QAAL,CAAc8B,kBAA/B,EAAoD;AAClDF,MAAAA,EAAE,CAACC,EAAE,CAACE,GAAJ,CAAF,GAAaF,EAAE,CAACG,KAAhB;AACD;;AACD,WAAOJ,EAAP;AACD;;AAMD,4BAA2C;AACzC,WAAO,KAAKZ,SAAL,GAAiBjC,MAAM,CAACC,aAAxB,GAAP;AACD;;AAlJuD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"file":"parquet-reader.js"}