@loaders.gl/parquet 3.1.0-beta.5 → 3.1.2

package/dist/es5/parquetjs/parser/decoders.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/decoders.ts"],"names":["decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","PageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","num_children","res","name","fields","Type","logicalType","converted_type","ConvertedType","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","PARQUET_CODECS","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","PARQUET_RDLVL_TYPE","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","data_page_header_v2","num_nulls","PARQUET_RDLVL_ENCODING","valuesBufCursor","is_compressed","dictCursor","dictionary_page_header","d","toString"],"mappings":";;;;;;;;;AAUA;;AACA;;AASA;;AACA;;AACA;;AASO,eAAeA,eAAf,CACLC,MADK,EAELC,OAFK,EAGiB;AACtB,QAAMC,MAAoB,GAAG;AAC3BF,IAAAA,MAD2B;AAE3BG,IAAAA,MAAM,EAAE,CAFmB;AAG3BC,IAAAA,IAAI,EAAEJ,MAAM,CAACK;AAHc,GAA7B;AAMA,QAAMC,IAAiB,GAAG;AACxBC,IAAAA,OAAO,EAAE,EADe;AAExBC,IAAAA,OAAO,EAAE,EAFe;AAGxBC,IAAAA,MAAM,EAAE,EAHgB;AAIxBC,IAAAA,WAAW,EAAE,EAJW;AAKxBC,IAAAA,KAAK,EAAE;AALiB,GAA1B;AAQA,MAAIC,UAAU,GAAGX,OAAO,CAACW,UAAR,IAAsB,EAAvC;;AAEA,SAEEV,MAAM,CAACC,MAAP,GAAgBD,MAAM,CAACE,IAAvB,KACC,CAACH,OAAO,CAACY,SAAT,IAAsBP,IAAI,CAACE,OAAL,CAAaH,MAAb,GAAsBS,MAAM,CAACb,OAAO,CAACY,SAAT,CADnD,CAFF,EAIE;AAEA,UAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAD,EAASD,OAAT,CAA7B;;AAEA,QAAIc,IAAI,CAACH,UAAT,EAAqB;AACnBA,MAAAA,UAAU,GAAGG,IAAI,CAACH,UAAlB;AAEA;AACD;;AAED,QAAIA,UAAU,CAACP,MAAf,EAAuB;AAErBU,MAAAA,IAAI,CAACN,MAAL,GAAcM,IAAI,CAACN,MAAL,CAAYQ,GAAZ,CAAiBC,KAAD,IAAWN,UAAU,CAACM,KAAD,CAArC,CAAd;AACD;;AAED,SAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGJ,IAAI,CAACR,OAAL,CAAaF,MAAzC,EAAiDc,KAAK,EAAtD,EAA0D;AACxDb,MAAAA,IAAI,CAACC,OAAL,CAAaa,IAAb,CAAkBL,IAAI,CAACR,OAAL,CAAaY,KAAb,CAAlB;AACAb,MAAAA,IAAI,CAACE,OAAL,CAAaY,IAAb,CAAkBL,IAAI,CAACP,OAAL,CAAaW,KAAb,CAAlB;AACA,YAAMD,KAAK,GAAGH,IAAI,CAACN,MAAL,CAAYU,KAAZ,CAAd;;AAEA,UAAID,KAAK,KAAKG,SAAd,EAAyB;AACvBf,QAAAA,IAAI,CAACG,MAAL,CAAYW,IAAZ,CAAiBF,KAAjB;AACD;AACF;;AAEDZ,IAAAA,IAAI,CAACK,KAAL,IAAcI,IAAI,CAACJ,KAAnB;AACAL,IAAAA,IAAI,CAACI,WAAL,CAAiBU,IAAjB,CAAsBL,IAAI,CAACO,UAA3B;AACD;;AAED,SAAOhB,IAAP;AACD;;AAOM,eAAeU,UAAf,CACLd,MADK,EAELD,OAFK,EAGqB;AAC1B,MAAIc,IAAJ;AACA,QAAM;AAACO,IAAAA,UAAD;AAAajB,IAAAA;AAAb,MAAuB,iCAAiBH,MAAM,CAACF,MAAxB,EAAgCE,MAAM,CAACC,MAAvC,CAA7B;AACAD,EAAAA,MAAM,CAACC,MAAP,IAAiBE,MAAjB;AAEA,QAAMkB,QAAQ,GAAG,8BAAcC,uBAAd,EAAwBF,UAAU,CAACG,IAAnC,CAAjB;;AAEA,UAAQF,QAAR;AACE,SAAK,WAAL;AACER,MAAAA,IAAI,GAAG,MAAMW,cAAc,CAACxB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAA3B;AACA;;AACF,SAAK,cAAL;AACEc,MAAAA,IAAI,GAAG,MAAMY,gBAAgB,CAACzB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAA7B;AACA;;AACF,SAAK,iBAAL;AACEc,MAAAA,IAAI,GAAG;AACLH,QAAAA,UAAU,EAAE,MAAMgB,oBAAoB,CAAC1B,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CADjC;AAELqB,QAAAA;AAFK,OAAP;AAIA;;AACF;AACE,YAAM,IAAIO,KAAJ,8BAAgCN,QAAhC,EAAN;AAdJ;;AAiBA,SAAOR,IAAP;AACD;;AAYM,SAASe,YAAT,CACLC,cADK,EAEL5B,MAFK,EAGL6B,GAHK,EAQL;AACA,QAAMC,MAAwB,GAAG,EAAjC;AACA,MAAIC,IAAI,GAAG/B,MAAX;;AACA,OAAK,IAAIgC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,GAApB,EAAyBG,CAAC,EAA1B,EAA8B;AAC5B,UAAMC,aAAa,GAAGL,cAAc,CAACG,IAAD,CAApC;AAEA,UAAMG,cAAc,GAClBH,IAAI,GAAG,CAAP,GAAW,8BAAcI,kCAAd,EAAmCF,aAAa,CAACG,eAAjD,CAAX,GAAgF,MADlF;AAGA,QAAIC,QAAQ,GAAG,KAAf;AACA,QAAIC,QAAQ,GAAG,KAAf;;AACA,YAAQJ,cAAR;AACE,WAAK,UAAL;AACE;;AACF,WAAK,UAAL;AACEG,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF,WAAK,UAAL;AACEC,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF;AACE,cAAM,IAAIZ,KAAJ,CAAU,kCAAV,CAAN;AAVJ;;AAaA,QAAIO,aAAa,CAACM,YAAd,GAA8B,CAAlC,EAAqC;AACnC,YAAMC,GAAG,GAAGb,YAAY,CAACC,cAAD,EAAiBG,IAAI,GAAG,CAAxB,EAA2BE,aAAa,CAACM,YAAzC,CAAxB;AACAR,MAAAA,IAAI,GAAGS,GAAG,CAACT,IAAX;AACAD,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAE3BJ,QAAAA,QAF2B;AAG3BC,QAAAA,QAH2B;AAI3BI,QAAAA,MAAM,EAAEF,GAAG,CAACV;AAJe,OAA7B;AAMD,KATD,MASO;AACL,YAAMR,IAAI,GAAG,8BAAcqB,mBAAd,EAAoBV,aAAa,CAACX,IAAlC,CAAb;AACA,UAAIsB,WAAW,GAAGtB,IAAlB;;AAEA,UAAIW,aAAa,CAACY,cAAlB,EAAkC;AAChCD,QAAAA,WAAW,GAAG,8BAAcE,4BAAd,EAA6Bb,aAAa,CAACY,cAA3C,CAAd;AACD;;AAED,cAAQD,WAAR;AACE,aAAK,SAAL;AACEA,UAAAA,WAAW,aAAMA,WAAN,cAAqBtB,IAArB,CAAX;AACA;;AACF;AAJF;;AAOAQ,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAC3BnB,QAAAA,IAAI,EAAEsB,WADqB;AAE3BG,QAAAA,UAAU,EAAEd,aAAa,CAACe,WAFC;AAG3BC,QAAAA,SAAS,EAAEhB,aAAa,CAACiB,SAHE;AAI3BC,QAAAA,KAAK,EAAElB,aAAa,CAACkB,KAJM;AAK3Bd,QAAAA,QAL2B;AAM3BC,QAAAA;AAN2B,OAA7B;AAQAP,MAAAA,IAAI;AACL;AACF;;AACD,SAAO;AAACD,IAAAA,MAAD;AAAS9B,IAAAA,MAAT;AAAiB+B,IAAAA;AAAjB,GAAP;AACD;;AAKD,SAASqB,YAAT,CACE9B,IADF,EAEE+B,QAFF,EAGEtD,MAHF,EAIES,KAJF,EAKE8C,IALF,EAMS;AACP,MAAI,EAAED,QAAQ,IAAIE,sBAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI7B,KAAJ,6BAA+B2B,QAA/B,EAAN;AACD;;AACD,SAAOE,uBAAeF,QAAf,EAAyBD,YAAzB,CAAsC9B,IAAtC,EAA4CvB,MAA5C,EAAoDS,KAApD,EAA2D8C,IAA3D,CAAP;AACD;;AAQD,eAAe/B,cAAf,CACExB,MADF,EAEEyD,MAFF,EAGE1D,OAHF,EAI4B;AAAA;;AAC1B,QAAM2D,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAAzC;AACA,QAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAV,0DAAG,sBAAyBC,UAA5C;AAGA,MAAIC,UAAU,GAAG/D,MAAjB;;AAEA,MAAID,OAAO,CAACiE,WAAR,KAAwB,cAA5B,EAA4C;AAC1C,UAAMC,SAAS,GAAG,MAAM,6BACtBlE,OAAO,CAACiE,WADc,EAEtBhE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACU,sBAHe,CAAxB;AAKAJ,IAAAA,UAAU,GAAG;AACXjE,MAAAA,MAAM,EAAEmE,SADG;AAEXhE,MAAAA,MAAM,EAAE,CAFG;AAGXC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHL,KAAb;AAKAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAGD,QAAMU,cAAc,GAAG,8BACrBC,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBS,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIC,KAAJ,CAAUZ,UAAV,CAAd;;AAEA,MAAI7D,OAAO,CAAC0E,MAAR,CAAeC,SAAf,GAA2B,CAA/B,EAAkC;AAChCH,IAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBP,cAArB,EAAqCL,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,MAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeC,SAA3B,CADwE;AAElFG,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLN,IAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAGD,QAAMC,cAAc,GAAG,8BACrBV,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBmB,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIT,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAI7D,OAAO,CAAC0E,MAAR,CAAeS,SAAf,GAA2B,CAA/B,EAAkC;AAChCD,IAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBI,cAArB,EAAqChB,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,MAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeS,SAA3B,CADwE;AAElFL,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AACD,MAAIK,iBAAiB,GAAG,CAAxB;;AACA,OAAK,MAAMC,IAAX,IAAmBH,OAAnB,EAA4B;AAC1B,QAAIG,IAAI,KAAKrF,OAAO,CAAC0E,MAAR,CAAeS,SAA5B,EAAuC;AACrCC,MAAAA,iBAAiB;AAClB;AACF;;AAGD,QAAME,aAAa,GAAG,8BAAchB,uBAAd,4BAAwBZ,MAAM,CAACI,gBAA/B,2DAAwB,uBAAyBP,QAAjD,CAAtB;AACA,QAAMgC,aAAa,GAAG;AACpBtC,IAAAA,UAAU,EAAEjD,OAAO,CAAC0E,MAAR,CAAezB,UADP;AAEpB4B,IAAAA,QAAQ,EAAE7E,OAAO,CAAC0E,MAAR,CAAezB;AAFL,GAAtB;AAKA,QAAMzC,MAAM,GAAG8C,YAAY,CACzBtD,OAAO,CAAC0E,MAAR,CAAec,aADU,EAEzBF,aAFyB,EAGzBtB,UAHyB,EAIzBoB,iBAJyB,EAKzBG,aALyB,CAA3B;AAQA,SAAO;AACLhF,IAAAA,OAAO,EAAE2E,OADJ;AAEL5E,IAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,IAAAA,MAHK;AAILE,IAAAA,KAAK,EAAEmD,UAJF;AAKLxC,IAAAA,UAAU,EAAEqC;AALP,GAAP;AAOD;;AASD,eAAehC,gBAAf,CACEzB,MADF,EAEEyD,MAFF,EAGEF,IAHF,EAI4B;AAAA;;AAC1B,QAAMG,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAAzC;AAEA,QAAMC,UAAU,6BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4B1B,UAA/C;AAEA,QAAMqB,iBAAiB,GAAGvB,UAAU,8BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4BC,SAA/B,CAApC;AACA,QAAMJ,aAAa,GAAG,8BACpBhB,uBADoB,4BAEpBZ,MAAM,CAAC+B,mBAFa,2DAEpB,uBAA4BlC,QAFR,CAAtB;AAOA,MAAIiB,OAAO,GAAG,IAAIC,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAIL,IAAI,CAACkB,MAAL,CAAYC,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BH,IAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,MAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYC,SAAxB,CAD4E;AAEtFG,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLN,IAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAID,MAAIG,OAAO,GAAG,IAAIT,KAAJ,CAAUZ,UAAV,CAAd;;AACA,MAAIL,IAAI,CAACkB,MAAL,CAAYS,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BD,IAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,MAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYS,SAAxB,CAD4E;AAEtFL,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AAGD,MAAIa,eAAe,GAAG3F,MAAtB;;AAEA,gCAAIyD,MAAM,CAAC+B,mBAAX,mDAAI,uBAA4BI,aAAhC,EAA+C;AAC7C,UAAM3B,SAAS,GAAG,MAAM,6BACtBV,IAAI,CAACS,WADiB,EAEtBhE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACU,sBAHe,CAAxB;AAMAwB,IAAAA,eAAe,GAAG;AAChB7F,MAAAA,MAAM,EAAEmE,SADQ;AAEhBhE,MAAAA,MAAM,EAAE,CAFQ;AAGhBC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHA,KAAlB;AAMAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAED,QAAM4B,aAAa,GAAG;AACpBtC,IAAAA,UAAU,EAAEO,IAAI,CAACkB,MAAL,CAAYzB,UADJ;AAEpB4B,IAAAA,QAAQ,EAAErB,IAAI,CAACkB,MAAL,CAAYzB;AAFF,GAAtB;AAKA,QAAMzC,MAAM,GAAG8C,YAAY,CACzBE,IAAI,CAACkB,MAAL,CAAYc,aADa,EAEzBF,aAFyB,EAGzBM,eAHyB,EAIzBR,iBAJyB,EAKzBG,aALyB,CAA3B;AAQA,SAAO;AACLhF,IAAAA,OAAO,EAAE2E,OADJ;AAEL5E,IAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,IAAAA,MAHK;AAILE,IAAAA,KAAK,EAAEmD,UAJF;AAKLxC,IAAAA,UAAU,EAAEqC;AALP,GAAP;AAOD;;AAQD,eAAe/B,oBAAf,CACE1B,MADF,EAEEoB,UAFF,EAGErB,OAHF,EAIqB;AAAA;;AACnB,QAAM2D,SAAS,GAAG1D,MAAM,CAACC,MAAP,GAAgBmB,UAAU,CAACuC,oBAA7C;AAEA,MAAIkC,UAAU,GAAG;AACf5F,IAAAA,MAAM,EAAE,CADO;AAEfH,IAAAA,MAAM,EAAEE,MAAM,CAACF,MAAP,CAAcoE,KAAd,CAAoBlE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFO;AAGfxD,IAAAA,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;AAHV,GAAjB;AAMAD,EAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AAEA,MAAI3D,OAAO,CAACiE,WAAR,KAAwB,cAA5B,EAA4C;AAC1C,UAAMC,SAAS,GAAG,MAAM,6BACtBlE,OAAO,CAACiE,WADc,EAEtB6B,UAAU,CAAC/F,MAAX,CAAkBoE,KAAlB,CAAwB2B,UAAU,CAAC5F,MAAnC,EAA2CyD,SAA3C,CAFsB,EAGtBtC,UAAU,CAAC+C,sBAHW,CAAxB;AAMA0B,IAAAA,UAAU,GAAG;AACX/F,MAAAA,MAAM,EAAEmE,SADG;AAEXhE,MAAAA,MAAM,EAAE,CAFG;AAGXC,MAAAA,IAAI,EAAE+D,SAAS,CAAC9D;AAHL,KAAb;AAMAH,IAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;AACD;;AAED,QAAM/C,SAAS,GAAG,CAAAS,UAAU,SAAV,IAAAA,UAAU,WAAV,qCAAAA,UAAU,CAAE0E,sBAAZ,gFAAoChC,UAApC,KAAkD,CAApE;AAEA,SAAOT,YAAY,CACjBtD,OAAO,CAAC0E,MAAR,CAAec,aADE,EAEjBxF,OAAO,CAAC0E,MAAR,CAAenB,QAFE,EAGjBuC,UAHiB,EAIjBlF,SAJiB,EAKjBZ,OALiB,CAAZ,CAMLgB,GANK,CAMAgF,CAAD,IAAOA,CAAC,CAACC,QAAF,EANN,CAAP;AAOD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetData,\n  ParquetOptions,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  options: ParquetOptions\n): Promise<ParquetData> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetData = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = options.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!options.numValues || data.dlevels.length < Number(options.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, options);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param options\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  let page;\n  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, options);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, options);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (options.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (options.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === options.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions = {\n    typeLength: options.column.typeLength,\n    bitWidth: options.column.typeLength\n  };\n\n  const values = decodeValues(\n    options.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  opts: any\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (opts.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (opts.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      opts.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: opts.column.typeLength,\n    bitWidth: opts.column.typeLength\n  };\n\n  const values = decodeValues(\n    opts.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  options: ParquetOptions\n): Promise<string[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  return decodeValues(\n    options.column.primitiveType!,\n    options.column.encoding!,\n    dictCursor,\n    numValues,\n    options as ParquetCodecOptions\n  ).map((d) => d.toString());\n}\n"],"file":"decoders.js"}
+{"version":3,"sources":["../../../../src/parquetjs/parser/decoders.ts"],"names":["decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","decodePage","page","map","value","index","push","undefined","pageHeader","pageType","PageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","num_children","res","name","fields","Type","logicalType","converted_type","ConvertedType","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","PARQUET_CODECS","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","slice","uncompressed_page_size","valuesBuf","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","PARQUET_RDLVL_TYPE","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","data_page_header_v2","num_nulls","PARQUET_RDLVL_ENCODING","valuesBufCursor","is_compressed","dictCursor","dictionary_page_header","d","toString"],"mappings":";;;;;;;;;;;;;;;AAUA;;AACA;;AASA;;AACA;;AACA;;;;;;;;SASsBA,e;;;;;+EAAf,iBACLC,MADK,EAELC,OAFK;AAAA;AAAA;AAAA;AAAA;AAAA;AAICC,YAAAA,MAJD,GAIwB;AAC3BF,cAAAA,MAAM,EAANA,MAD2B;AAE3BG,cAAAA,MAAM,EAAE,CAFmB;AAG3BC,cAAAA,IAAI,EAAEJ,MAAM,CAACK;AAHc,aAJxB;AAUCC,YAAAA,IAVD,GAUqB;AACxBC,cAAAA,OAAO,EAAE,EADe;AAExBC,cAAAA,OAAO,EAAE,EAFe;AAGxBC,cAAAA,MAAM,EAAE,EAHgB;AAIxBC,cAAAA,WAAW,EAAE,EAJW;AAKxBC,cAAAA,KAAK,EAAE;AALiB,aAVrB;AAkBDC,YAAAA,UAlBC,GAkBYX,OAAO,CAACW,UAAR,IAAsB,EAlBlC;;AAAA;AAAA,kBAsBHV,MAAM,CAACC,MAAP,GAAgBD,MAAM,CAACE,IAAvB,KACC,CAACH,OAAO,CAACY,SAAT,IAAsBP,IAAI,CAACE,OAAL,CAAaH,MAAb,GAAsBS,MAAM,CAACb,OAAO,CAACY,SAAT,CADnD,CAtBG;AAAA;AAAA;AAAA;;AAAA;AAAA,mBA0BgBE,UAAU,CAACb,MAAD,EAASD,OAAT,CA1B1B;;AAAA;AA0BGe,YAAAA,IA1BH;;AAAA,iBA4BCA,IAAI,CAACJ,UA5BN;AAAA;AAAA;AAAA;;AA6BDA,YAAAA,UAAU,GAAGI,IAAI,CAACJ,UAAlB;AA7BC;;AAAA;AAkCH,gBAAIA,UAAU,CAACP,MAAf,EAAuB;AAErBW,cAAAA,IAAI,CAACP,MAAL,GAAcO,IAAI,CAACP,MAAL,CAAYQ,GAAZ,CAAgB,UAACC,KAAD;AAAA,uBAAWN,UAAU,CAACM,KAAD,CAArB;AAAA,eAAhB,CAAd;AACD;;AAED,iBAASC,KAAT,GAAiB,CAAjB,EAAoBA,KAAK,GAAGH,IAAI,CAACT,OAAL,CAAaF,MAAzC,EAAiDc,KAAK,EAAtD,EAA0D;AACxDb,cAAAA,IAAI,CAACC,OAAL,CAAaa,IAAb,CAAkBJ,IAAI,CAACT,OAAL,CAAaY,KAAb,CAAlB;AACAb,cAAAA,IAAI,CAACE,OAAL,CAAaY,IAAb,CAAkBJ,IAAI,CAACR,OAAL,CAAaW,KAAb,CAAlB;AACMD,cAAAA,KAHkD,GAG1CF,IAAI,CAACP,MAAL,CAAYU,KAAZ,CAH0C;;AAKxD,kBAAID,KAAK,KAAKG,SAAd,EAAyB;AACvBf,gBAAAA,IAAI,CAACG,MAAL,CAAYW,IAAZ,CAAiBF,KAAjB;AACD;AACF;;AAEDZ,YAAAA,IAAI,CAACK,KAAL,IAAcK,IAAI,CAACL,KAAnB;AACAL,YAAAA,IAAI,CAACI,WAAL,CAAiBU,IAAjB,CAAsBJ,IAAI,CAACM,UAA3B;AAlDG;AAAA;;AAAA;AAAA,6CAqDEhB,IArDF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SA6DeS,U;;;;;0EAAf,kBACLb,MADK,EAELD,OAFK;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA,gCAKwB,iCAAiBC,MAAM,CAACF,MAAxB,EAAgCE,MAAM,CAACC,MAAvC,CALxB,EAKEmB,UALF,qBAKEA,UALF,EAKcjB,MALd,qBAKcA,MALd;AAMLH,YAAAA,MAAM,CAACC,MAAP,IAAiBE,MAAjB;AAEMkB,YAAAA,QARD,GAQY,8BAAcC,uBAAd,EAAwBF,UAAU,CAACG,IAAnC,CARZ;AAAA,2BAUGF,QAVH;AAAA,8CAWE,WAXF,wBAcE,cAdF,yBAiBE,iBAjBF;AAAA;;AAAA;AAAA;AAAA,mBAYYG,cAAc,CAACxB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAZ1B;;AAAA;AAYDe,YAAAA,IAZC;AAAA;;AAAA;AAAA;AAAA,mBAeYW,gBAAgB,CAACzB,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAf5B;;AAAA;AAeDe,YAAAA,IAfC;AAAA;;AAAA;AAAA;AAAA,mBAmBmBY,oBAAoB,CAAC1B,MAAD,EAASoB,UAAT,EAAqBrB,OAArB,CAnBvC;;AAAA;AAAA;AAAA,2BAoBCqB,UApBD;AAkBDN,YAAAA,IAlBC;AAmBCJ,cAAAA,UAnBD;AAoBCU,cAAAA,UApBD;AAAA;AAAA;;AAAA;AAAA,kBAwBK,IAAIO,KAAJ,8BAAgCN,QAAhC,EAxBL;;AAAA;AAAA,8CA2BEP,IA3BF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAwCA,SAASc,YAAT,CACLC,cADK,EAEL5B,MAFK,EAGL6B,GAHK,EAQL;AACA,MAAMC,MAAwB,GAAG,EAAjC;AACA,MAAIC,IAAI,GAAG/B,MAAX;;AACA,OAAK,IAAIgC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,GAApB,EAAyBG,CAAC,EAA1B,EAA8B;AAC5B,QAAMC,aAAa,GAAGL,cAAc,CAACG,IAAD,CAApC;AAEA,QAAMG,cAAc,GAClBH,IAAI,GAAG,CAAP,GAAW,8BAAcI,kCAAd,EAAmCF,aAAa,CAACG,eAAjD,CAAX,GAAgF,MADlF;AAGA,QAAIC,QAAQ,GAAG,KAAf;AACA,QAAIC,QAAQ,GAAG,KAAf;;AACA,YAAQJ,cAAR;AACE,WAAK,UAAL;AACE;;AACF,WAAK,UAAL;AACEG,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF,WAAK,UAAL;AACEC,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF;AACE,cAAM,IAAIZ,KAAJ,CAAU,kCAAV,CAAN;AAVJ;;AAaA,QAAIO,aAAa,CAACM,YAAd,GAA8B,CAAlC,EAAqC;AACnC,UAAMC,GAAG,GAAGb,YAAY,CAACC,cAAD,EAAiBG,IAAI,GAAG,CAAxB,EAA2BE,aAAa,CAACM,YAAzC,CAAxB;AACAR,MAAAA,IAAI,GAAGS,GAAG,CAACT,IAAX;AACAD,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAE3BJ,QAAAA,QAAQ,EAARA,QAF2B;AAG3BC,QAAAA,QAAQ,EAARA,QAH2B;AAI3BI,QAAAA,MAAM,EAAEF,GAAG,CAACV;AAJe,OAA7B;AAMD,KATD,MASO;AACL,UAAMR,IAAI,GAAG,8BAAcqB,mBAAd,EAAoBV,aAAa,CAACX,IAAlC,CAAb;AACA,UAAIsB,WAAW,GAAGtB,IAAlB;;AAEA,UAAIW,aAAa,CAACY,cAAlB,EAAkC;AAChCD,QAAAA,WAAW,GAAG,8BAAcE,4BAAd,EAA6Bb,aAAa,CAACY,cAA3C,CAAd;AACD;;AAED,cAAQD,WAAR;AACE,aAAK,SAAL;AACEA,UAAAA,WAAW,aAAMA,WAAN,cAAqBtB,IAArB,CAAX;AACA;;AACF;AAJF;;AAOAQ,MAAAA,MAAM,CAACG,aAAa,CAACQ,IAAf,CAAN,GAA6B;AAC3BnB,QAAAA,IAAI,EAAEsB,WADqB;AAE3BG,QAAAA,UAAU,EAAEd,aAAa,CAACe,WAFC;AAG3BC,QAAAA,SAAS,EAAEhB,aAAa,CAACiB,SAHE;AAI3BC,QAAAA,KAAK,EAAElB,aAAa,CAACkB,KAJM;AAK3Bd,QAAAA,QAAQ,EAARA,QAL2B;AAM3BC,QAAAA,QAAQ,EAARA;AAN2B,OAA7B;AAQAP,MAAAA,IAAI;AACL;AACF;;AACD,SAAO;AAACD,IAAAA,MAAM,EAANA,MAAD;AAAS9B,IAAAA,MAAM,EAANA,MAAT;AAAiB+B,IAAAA,IAAI,EAAJA;AAAjB,GAAP;AACD;;AAKD,SAASqB,YAAT,CACE9B,IADF,EAEE+B,QAFF,EAGEtD,MAHF,EAIES,KAJF,EAKE8C,IALF,EAMS;AACP,MAAI,EAAED,QAAQ,IAAIE,sBAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI7B,KAAJ,6BAA+B2B,QAA/B,EAAN;AACD;;AACD,SAAOE,uBAAeF,QAAf,EAAyBD,YAAzB,CAAsC9B,IAAtC,EAA4CvB,MAA5C,EAAoDS,KAApD,EAA2D8C,IAA3D,CAAP;AACD;;SAQc/B,c;;;;;8EAAf,kBACExB,MADF,EAEEyD,MAFF,EAGE1D,OAHF;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAKQ2D,YAAAA,SALR,GAKoB1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAL3C;AAMQC,YAAAA,UANR,4BAMqBH,MAAM,CAACI,gBAN5B,0DAMqB,sBAAyBC,UAN9C;AASMC,YAAAA,UATN,GASmB/D,MATnB;;AAAA,kBAWMD,OAAO,CAACiE,WAAR,KAAwB,cAX9B;AAAA;AAAA;AAAA;;AAAA;AAAA,mBAY4B,6BACtBjE,OAAO,CAACiE,WADc,EAEtBhE,MAAM,CAACF,MAAP,CAAcmE,KAAd,CAAoBjE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACS,sBAHe,CAZ5B;;AAAA;AAYUC,YAAAA,SAZV;AAiBIJ,YAAAA,UAAU,GAAG;AACXjE,cAAAA,MAAM,EAAEqE,SADG;AAEXlE,cAAAA,MAAM,EAAE,CAFG;AAGXC,cAAAA,IAAI,EAAEiE,SAAS,CAAChE;AAHL,aAAb;AAKAH,YAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AAtBJ;AA0BQU,YAAAA,cA1BR,GA0ByB,8BACrBC,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBS,yBAFJ,CA1BzB;AA+BMC,YAAAA,OA/BN,GA+BgB,IAAIC,KAAJ,CAAUZ,UAAV,CA/BhB;;AAiCE,gBAAI7D,OAAO,CAAC0E,MAAR,CAAeC,SAAf,GAA2B,CAA/B,EAAkC;AAChCH,cAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBP,cAArB,EAAqCL,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,gBAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeC,SAA3B,CADwE;AAElFG,gBAAAA,eAAe,EAAE;AAFiE,eAA9D,CAAtB;AAKD,aAND,MAMO;AACLN,cAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAGKC,YAAAA,cA5CR,GA4CyB,8BACrBV,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBmB,yBAFJ,CA5CzB;AAiDMC,YAAAA,OAjDN,GAiDgB,IAAIT,KAAJ,CAAUZ,UAAV,CAjDhB;;AAkDE,gBAAI7D,OAAO,CAAC0E,MAAR,CAAeS,SAAf,GAA2B,CAA/B,EAAkC;AAChCD,cAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBI,cAArB,EAAqChB,UAArC,EAAiDH,UAAjD,EAA8D;AAClFgB,gBAAAA,QAAQ,EAAE,4BAAY7E,OAAO,CAAC0E,MAAR,CAAeS,SAA3B,CADwE;AAElFL,gBAAAA,eAAe,EAAE;AAFiE,eAA9D,CAAtB;AAKD,aAND,MAMO;AACLI,cAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AACGK,YAAAA,iBA3DN,GA2D0B,CA3D1B;AAAA,mDA4DqBF,OA5DrB;;AAAA;AA4DE,kEAA4B;AAAjBG,gBAAAA,IAAiB;;AAC1B,oBAAIA,IAAI,KAAKrF,OAAO,CAAC0E,MAAR,CAAeS,SAA5B,EAAuC;AACrCC,kBAAAA,iBAAiB;AAClB;AACF;AAhEH;AAAA;AAAA;AAAA;AAAA;;AAmEQE,YAAAA,aAnER,GAmEwB,8BAAchB,uBAAd,4BAAwBZ,MAAM,CAACI,gBAA/B,2DAAwB,uBAAyBP,QAAjD,CAnExB;AAoEQgC,YAAAA,aApER,GAoEwB;AACpBtC,cAAAA,UAAU,EAAEjD,OAAO,CAAC0E,MAAR,CAAezB,UADP;AAEpB4B,cAAAA,QAAQ,EAAE7E,OAAO,CAAC0E,MAAR,CAAezB;AAFL,aApExB;AAyEQzC,YAAAA,MAzER,GAyEiB8C,YAAY,CACzBtD,OAAO,CAAC0E,MAAR,CAAec,aADU,EAEzBF,aAFyB,EAGzBtB,UAHyB,EAIzBoB,iBAJyB,EAKzBG,aALyB,CAzE7B;AAAA,8CAiFS;AACLhF,cAAAA,OAAO,EAAE2E,OADJ;AAEL5E,cAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,cAAAA,MAAM,EAANA,MAHK;AAILE,cAAAA,KAAK,EAAEmD,UAJF;AAKLxC,cAAAA,UAAU,EAAEqC;AALP,aAjFT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SAiGehC,gB;;;;;+EAAf,kBACEzB,MADF,EAEEyD,MAFF,EAGEF,IAHF;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAKQG,YAAAA,SALR,GAKoB1D,MAAM,CAACC,MAAP,GAAgBwD,MAAM,CAACE,oBAL3C;AAOQC,YAAAA,UAPR,6BAOqBH,MAAM,CAAC+B,mBAP5B,2DAOqB,uBAA4B1B,UAPjD;AASQqB,YAAAA,iBATR,GAS4BvB,UAAU,8BAAGH,MAAM,CAAC+B,mBAAV,2DAAG,uBAA4BC,SAA/B,CATtC;AAUQJ,YAAAA,aAVR,GAUwB,8BACpBhB,uBADoB,4BAEpBZ,MAAM,CAAC+B,mBAFa,2DAEpB,uBAA4BlC,QAFR,CAVxB;AAiBMiB,YAAAA,OAjBN,GAiBgB,IAAIC,KAAJ,CAAUZ,UAAV,CAjBhB;;AAkBE,gBAAIL,IAAI,CAACkB,MAAL,CAAYC,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BH,cAAAA,OAAO,GAAGlB,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,gBAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYC,SAAxB,CAD4E;AAEtFG,gBAAAA,eAAe,EAAE;AAFqE,eAAlE,CAAtB;AAID,aALD,MAKO;AACLN,cAAAA,OAAO,CAACO,IAAR,CAAa,CAAb;AACD;;AAIGG,YAAAA,OA7BN,GA6BgB,IAAIT,KAAJ,CAAUZ,UAAV,CA7BhB;;AA8BE,gBAAIL,IAAI,CAACkB,MAAL,CAAYS,SAAZ,GAAwB,CAA5B,EAA+B;AAC7BD,cAAAA,OAAO,GAAG5B,YAAY,CAACsB,6BAAD,EAAqBe,iCAArB,EAA6C1F,MAA7C,EAAqD4D,UAArD,EAAkE;AACtFgB,gBAAAA,QAAQ,EAAE,4BAAYrB,IAAI,CAACkB,MAAL,CAAYS,SAAxB,CAD4E;AAEtFL,gBAAAA,eAAe,EAAE;AAFqE,eAAlE,CAAtB;AAID,aALD,MAKO;AACLI,cAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AAGGa,YAAAA,eAxCN,GAwCwB3F,MAxCxB;;AAAA,4CA0CMyD,MAAM,CAAC+B,mBA1Cb,mDA0CM,uBAA4BI,aA1ClC;AAAA;AAAA;AAAA;;AAAA;AAAA,mBA2C4B,6BACtBrC,IAAI,CAACS,WADiB,EAEtBhE,MAAM,CAACF,MAAP,CAAcmE,KAAd,CAAoBjE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFsB,EAGtBD,MAAM,CAACS,sBAHe,CA3C5B;;AAAA;AA2CUC,YAAAA,SA3CV;AAiDIwB,YAAAA,eAAe,GAAG;AAChB7F,cAAAA,MAAM,EAAEqE,SADQ;AAEhBlE,cAAAA,MAAM,EAAE,CAFQ;AAGhBC,cAAAA,IAAI,EAAEiE,SAAS,CAAChE;AAHA,aAAlB;AAMAH,YAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AAvDJ;AA0DQ4B,YAAAA,aA1DR,GA0DwB;AACpBtC,cAAAA,UAAU,EAAEO,IAAI,CAACkB,MAAL,CAAYzB,UADJ;AAEpB4B,cAAAA,QAAQ,EAAErB,IAAI,CAACkB,MAAL,CAAYzB;AAFF,aA1DxB;AA+DQzC,YAAAA,MA/DR,GA+DiB8C,YAAY,CACzBE,IAAI,CAACkB,MAAL,CAAYc,aADa,EAEzBF,aAFyB,EAGzBM,eAHyB,EAIzBR,iBAJyB,EAKzBG,aALyB,CA/D7B;AAAA,8CAuES;AACLhF,cAAAA,OAAO,EAAE2E,OADJ;AAEL5E,cAAAA,OAAO,EAAEkE,OAFJ;AAGLhE,cAAAA,MAAM,EAANA,MAHK;AAILE,cAAAA,KAAK,EAAEmD,UAJF;AAKLxC,cAAAA,UAAU,EAAEqC;AALP,aAvET;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;SAsFe/B,oB;;;;;oFAAf,kBACE1B,MADF,EAEEoB,UAFF,EAGErB,OAHF;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAKQ2D,YAAAA,SALR,GAKoB1D,MAAM,CAACC,MAAP,GAAgBmB,UAAU,CAACuC,oBAL/C;AAOMkC,YAAAA,UAPN,GAOmB;AACf5F,cAAAA,MAAM,EAAE,CADO;AAEfH,cAAAA,MAAM,EAAEE,MAAM,CAACF,MAAP,CAAcmE,KAAd,CAAoBjE,MAAM,CAACC,MAA3B,EAAmCyD,SAAnC,CAFO;AAGfxD,cAAAA,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;AAHV,aAPnB;AAaED,YAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AAbF,kBAeM3D,OAAO,CAACiE,WAAR,KAAwB,cAf9B;AAAA;AAAA;AAAA;;AAAA;AAAA,mBAgB4B,6BACtBjE,OAAO,CAACiE,WADc,EAEtB6B,UAAU,CAAC/F,MAAX,CAAkBmE,KAAlB,CAAwB4B,UAAU,CAAC5F,MAAnC,EAA2CyD,SAA3C,CAFsB,EAGtBtC,UAAU,CAAC8C,sBAHW,CAhB5B;;AAAA;AAgBUC,YAAAA,SAhBV;AAsBI0B,YAAAA,UAAU,GAAG;AACX/F,cAAAA,MAAM,EAAEqE,SADG;AAEXlE,cAAAA,MAAM,EAAE,CAFG;AAGXC,cAAAA,IAAI,EAAEiE,SAAS,CAAChE;AAHL,aAAb;AAMAH,YAAAA,MAAM,CAACC,MAAP,GAAgByD,SAAhB;;AA5BJ;AA+BQ/C,YAAAA,SA/BR,GA+BoB,CAAAS,UAAU,SAAV,IAAAA,UAAU,WAAV,qCAAAA,UAAU,CAAE0E,sBAAZ,gFAAoChC,UAApC,KAAkD,CA/BtE;AAAA,8CAiCST,YAAY,CACjBtD,OAAO,CAAC0E,MAAR,CAAec,aADE,EAEjBxF,OAAO,CAAC0E,MAAR,CAAenB,QAFE,EAGjBuC,UAHiB,EAIjBlF,SAJiB,EAKjBZ,OALiB,CAAZ,CAMLgB,GANK,CAMD,UAACgF,CAAD;AAAA,qBAAOA,CAAC,CAACC,QAAF,EAAP;AAAA,aANC,CAjCT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetData,\n  ParquetOptions,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  options: ParquetOptions\n): Promise<ParquetData> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetData = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = options.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!options.numValues || data.dlevels.length < Number(options.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, options);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param options\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  let page;\n  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, options);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, options);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  options: ParquetOptions\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (options.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (options.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(options.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === options.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions = {\n    typeLength: options.column.typeLength,\n    bitWidth: options.column.typeLength\n  };\n\n  const values = decodeValues(\n    options.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  opts: any\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (opts.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (opts.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(opts.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      opts.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: opts.column.typeLength,\n    bitWidth: opts.column.typeLength\n  };\n\n  const values = decodeValues(\n    opts.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  options: ParquetOptions\n): Promise<string[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (options.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      options.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  return decodeValues(\n    options.column.primitiveType!,\n    options.column.encoding!,\n    dictCursor,\n    numValues,\n    options as ParquetCodecOptions\n  ).map((d) => d.toString());\n}\n"],"file":"decoders.js"}

package/dist/es5/parquetjs/parser/parquet-cursor.js

@@ -7,16 +7,25 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ParquetCursor = void 0;
 
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
 var _shred = require("../schema/shred");
 
-let _Symbol$asyncIterator;
+var _Symbol$asyncIterator;
 
 _Symbol$asyncIterator = Symbol.asyncIterator;
 
-class ParquetCursor {
-  constructor(metadata, envelopeReader, schema, columnList) {
+var ParquetCursor = function () {
+  function ParquetCursor(metadata, envelopeReader, schema, columnList) {
+    (0, _classCallCheck2.default)(this, ParquetCursor);
     (0, _defineProperty2.default)(this, "metadata", void 0);
     (0, _defineProperty2.default)(this, "envelopeReader", void 0);
     (0, _defineProperty2.default)(this, "schema", void 0);
@@ -30,68 +39,177 @@ class ParquetCursor {
     this.rowGroupIndex = 0;
   }
 
-  async next() {
-    if (this.rowGroup.length === 0) {
-      if (this.rowGroupIndex >= this.metadata.row_groups.length) {
-        return null;
+  (0, _createClass2.default)(ParquetCursor, [{
+    key: "next",
+    value: function () {
+      var _next = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee() {
+        var rowBuffer;
+        return _regenerator.default.wrap(function _callee$(_context) {
+          while (1) {
+            switch (_context.prev = _context.next) {
+              case 0:
+                if (!(this.rowGroup.length === 0)) {
+                  _context.next = 8;
+                  break;
+                }
+
+                if (!(this.rowGroupIndex >= this.metadata.row_groups.length)) {
+                  _context.next = 3;
+                  break;
+                }
+
+                return _context.abrupt("return", null);
+
+              case 3:
+                _context.next = 5;
+                return this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
+
+              case 5:
+                rowBuffer = _context.sent;
+                this.rowGroup = (0, _shred.materializeRecords)(this.schema, rowBuffer);
+                this.rowGroupIndex++;
+
+              case 8:
+                return _context.abrupt("return", this.rowGroup.shift());
+
+              case 9:
+              case "end":
+                return _context.stop();
+            }
+          }
+        }, _callee, this);
+      }));
+
+      function next() {
+        return _next.apply(this, arguments);
       }
 
-      const rowBuffer = await this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
-      this.rowGroup = (0, _shred.materializeRecords)(this.schema, rowBuffer);
-      this.rowGroupIndex++;
+      return next;
+    }()
+  }, {
+    key: "rewind",
+    value: function rewind() {
+      this.rowGroup = [];
+      this.rowGroupIndex = 0;
     }
-
-    return this.rowGroup.shift();
-  }
-
-  rewind() {
-    this.rowGroup = [];
-    this.rowGroupIndex = 0;
-  }
-
-  [_Symbol$asyncIterator]() {
-    let done = false;
-    return {
-      next: async () => {
-        if (done) {
-          return {
-            done,
-            value: null
-          };
-        }
-
-        const value = await this.next();
-
-        if (value === null) {
-          return {
-            done: true,
-            value
-          };
-        }
-
-        return {
-          done: false,
-          value
-        };
-      },
-      return: async () => {
-        done = true;
-        return {
-          done,
-          value: null
-        };
-      },
-      throw: async () => {
-        done = true;
-        return {
-          done: true,
-          value: null
-        };
-      }
-    };
-  }
-
-}
+  }, {
+    key: _Symbol$asyncIterator,
+    value: function value() {
+      var _this = this;
+
+      var done = false;
+      return {
+        next: function () {
+          var _next2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2() {
+            var value;
+            return _regenerator.default.wrap(function _callee2$(_context2) {
+              while (1) {
+                switch (_context2.prev = _context2.next) {
+                  case 0:
+                    if (!done) {
+                      _context2.next = 2;
+                      break;
+                    }
+
+                    return _context2.abrupt("return", {
+                      done: done,
+                      value: null
+                    });
+
+                  case 2:
+                    _context2.next = 4;
+                    return _this.next();
+
+                  case 4:
+                    value = _context2.sent;
+
+                    if (!(value === null)) {
+                      _context2.next = 7;
+                      break;
+                    }
+
+                    return _context2.abrupt("return", {
+                      done: true,
+                      value: value
+                    });
+
+                  case 7:
+                    return _context2.abrupt("return", {
+                      done: false,
+                      value: value
+                    });
+
+                  case 8:
+                  case "end":
+                    return _context2.stop();
+                }
+              }
+            }, _callee2);
+          }));
+
+          function next() {
+            return _next2.apply(this, arguments);
+          }
+
+          return next;
+        }(),
+        return: function () {
+          var _return2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3() {
+            return _regenerator.default.wrap(function _callee3$(_context3) {
+              while (1) {
+                switch (_context3.prev = _context3.next) {
+                  case 0:
+                    done = true;
+                    return _context3.abrupt("return", {
+                      done: done,
+                      value: null
+                    });
+
+                  case 2:
+                  case "end":
+                    return _context3.stop();
+                }
+              }
+            }, _callee3);
+          }));
+
+          function _return() {
+            return _return2.apply(this, arguments);
+          }
+
+          return _return;
+        }(),
+        throw: function () {
+          var _throw2 = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4() {
+            return _regenerator.default.wrap(function _callee4$(_context4) {
+              while (1) {
+                switch (_context4.prev = _context4.next) {
+                  case 0:
+                    done = true;
+                    return _context4.abrupt("return", {
+                      done: true,
+                      value: null
+                    });
+
+                  case 2:
+                  case "end":
+                    return _context4.stop();
+                }
+              }
+            }, _callee4);
+          }));
+
+          function _throw() {
+            return _throw2.apply(this, arguments);
+          }
+
+          return _throw;
+        }()
+      };
+    }
+  }]);
+  return ParquetCursor;
+}();
 
 exports.ParquetCursor = ParquetCursor;
 //# sourceMappingURL=parquet-cursor.js.map

package/dist/es5/parquetjs/parser/parquet-cursor.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-cursor.ts"],"names":["Symbol","asyncIterator","ParquetCursor","constructor","metadata","envelopeReader","schema","columnList","rowGroupIndex","next","rowGroup","length","row_groups","rowBuffer","readRowGroup","shift","rewind","done","value","return","throw"],"mappings":";;;;;;;;;;;AAKA;;;;wBAiEGA,MAAM,CAACC,a;;AA5DH,MAAMC,aAAN,CAAmD;AAcxDC,EAAAA,WAAW,CACTC,QADS,EAETC,cAFS,EAGTC,MAHS,EAITC,UAJS,EAKT;AAAA;AAAA;AAAA;AAAA;AAAA,oDAdiC,EAcjC;AAAA;AACA,SAAKH,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,SAAKC,MAAL,GAAcA,MAAd;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACA,SAAKC,aAAL,GAAqB,CAArB;AACD;;AAMS,QAAJC,IAAI,GAAwB;AAChC,QAAI,KAAKC,QAAL,CAAcC,MAAd,KAAyB,CAA7B,EAAgC;AAC9B,UAAI,KAAKH,aAAL,IAAsB,KAAKJ,QAAL,CAAcQ,UAAd,CAAyBD,MAAnD,EAA2D;AAEzD,eAAO,IAAP;AACD;;AACD,YAAME,SAAS,GAAG,MAAM,KAAKR,cAAL,CAAoBS,YAApB,CACtB,KAAKR,MADiB,EAEtB,KAAKF,QAAL,CAAcQ,UAAd,CAAyB,KAAKJ,aAA9B,CAFsB,EAGtB,KAAKD,UAHiB,CAAxB;AAKA,WAAKG,QAAL,GAAgB,+BAAmB,KAAKJ,MAAxB,EAAgCO,SAAhC,CAAhB;AACA,WAAKL,aAAL;AACD;;AACD,WAAO,KAAKE,QAAL,CAAcK,KAAd,EAAP;AACD;;AAKDC,EAAAA,MAAM,GAAS;AACb,SAAKN,QAAL,GAAgB,EAAhB;AACA,SAAKF,aAAL,GAAqB,CAArB;AACD;;AAMD,4BAA2C;AACzC,QAAIS,IAAI,GAAG,KAAX;AACA,WAAO;AACLR,MAAAA,IAAI,EAAE,YAAY;AAChB,YAAIQ,IAAJ,EAAU;AACR,iBAAO;AAACA,YAAAA,IAAD;AAAOC,YAAAA,KAAK,EAAE;AAAd,WAAP;AACD;;AACD,cAAMA,KAAK,GAAG,MAAM,KAAKT,IAAL,EAApB;;AACA,YAAIS,KAAK,KAAK,IAAd,EAAoB;AAClB,iBAAO;AAACD,YAAAA,IAAI,EAAE,IAAP;AAAaC,YAAAA;AAAb,WAAP;AACD;;AACD,eAAO;AAACD,UAAAA,IAAI,EAAE,KAAP;AAAcC,UAAAA;AAAd,SAAP;AACD,OAVI;AAWLC,MAAAA,MAAM,EAAE,YAAY;AAClBF,QAAAA,IAAI,GAAG,IAAP;AACA,eAAO;AAACA,UAAAA,IAAD;AAAOC,UAAAA,KAAK,EAAE;AAAd,SAAP;AACD,OAdI;AAeLE,MAAAA,KAAK,EAAE,YAAY;AACjBH,QAAAA,IAAI,GAAG,IAAP;AACA,eAAO;AAACA,UAAAA,IAAI,EAAE,IAAP;AAAaC,UAAAA,KAAK,EAAE;AAApB,SAAP;AACD;AAlBI,KAAP;AAoBD;;AAlFuD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetRecord} from '../schema/declare';\nimport {materializeRecords} from '../schema/shred';\n\n/**\n * A parquet cursor is used to retrieve rows from a parquet file in order\n */\nexport class ParquetCursor<T> implements AsyncIterable<T> {\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n  public columnList: string[][];\n  public rowGroup: ParquetRecord[] = [];\n  public rowGroupIndex: number;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is usually not recommended to call this constructor directly except for\n   * advanced and internal use cases. Consider using getCursor() on the\n   * ParquetReader instead\n   */\n  constructor(\n    metadata: FileMetaData,\n    envelopeReader: ParquetEnvelopeReader,\n    schema: ParquetSchema,\n    columnList: string[][]\n  ) {\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    this.schema = schema;\n    this.columnList = columnList;\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Retrieve the next row from the cursor. Returns a row or NULL if the end\n   * of the file was reached\n   */\n  async next<T = any>(): Promise<T> {\n    if (this.rowGroup.length === 0) {\n      if (this.rowGroupIndex >= this.metadata.row_groups.length) {\n        // @ts-ignore\n        return null;\n      }\n      const rowBuffer = await this.envelopeReader.readRowGroup(\n        this.schema,\n        this.metadata.row_groups[this.rowGroupIndex],\n        this.columnList\n      );\n      this.rowGroup = materializeRecords(this.schema, rowBuffer);\n      this.rowGroupIndex++;\n    }\n    return this.rowGroup.shift() as any;\n  }\n\n  /**\n   * Rewind the cursor the the beginning of the file\n   */\n  rewind(): void {\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    let done = false;\n    return {\n      next: async () => {\n        if (done) {\n          return {done, value: null};\n        }\n        const value = await this.next();\n        if (value === null) {\n          return {done: true, value};\n        }\n        return {done: false, value};\n      },\n      return: async () => {\n        done = true;\n        return {done, value: null};\n      },\n      throw: async () => {\n        done = true;\n        return {done: true, value: null};\n      }\n    };\n  }\n}\n"],"file":"parquet-cursor.js"}
+{"version":3,"sources":["../../../../src/parquetjs/parser/parquet-cursor.ts"],"names":["Symbol","asyncIterator","ParquetCursor","metadata","envelopeReader","schema","columnList","rowGroupIndex","rowGroup","length","row_groups","readRowGroup","rowBuffer","shift","done","next","value","return","throw"],"mappings":";;;;;;;;;;;;;;;;;;;AAKA;;;;wBAiEGA,MAAM,CAACC,a;;IA5DGC,a;AAcX,yBACEC,QADF,EAEEC,cAFF,EAGEC,MAHF,EAIEC,UAJF,EAKE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oDAdiC,EAcjC;AAAA;AACA,SAAKH,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,SAAKC,MAAL,GAAcA,MAAd;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACA,SAAKC,aAAL,GAAqB,CAArB;AACD;;;;;4EAMD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBACM,KAAKC,QAAL,CAAcC,MAAd,KAAyB,CAD/B;AAAA;AAAA;AAAA;;AAAA,sBAEQ,KAAKF,aAAL,IAAsB,KAAKJ,QAAL,CAAcO,UAAd,CAAyBD,MAFvD;AAAA;AAAA;AAAA;;AAAA,iDAIa,IAJb;;AAAA;AAAA;AAAA,uBAM4B,KAAKL,cAAL,CAAoBO,YAApB,CACtB,KAAKN,MADiB,EAEtB,KAAKF,QAAL,CAAcO,UAAd,CAAyB,KAAKH,aAA9B,CAFsB,EAGtB,KAAKD,UAHiB,CAN5B;;AAAA;AAMUM,gBAAAA,SANV;AAWI,qBAAKJ,QAAL,GAAgB,+BAAmB,KAAKH,MAAxB,EAAgCO,SAAhC,CAAhB;AACA,qBAAKL,aAAL;;AAZJ;AAAA,iDAcS,KAAKC,QAAL,CAAcK,KAAd,EAdT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;WAoBA,kBAAe;AACb,WAAKL,QAAL,GAAgB,EAAhB;AACA,WAAKD,aAAL,GAAqB,CAArB;AACD;;;WAMD,iBAA2C;AAAA;;AACzC,UAAIO,IAAI,GAAG,KAAX;AACA,aAAO;AACLC,QAAAA,IAAI;AAAA,iFAAE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBACAD,IADA;AAAA;AAAA;AAAA;;AAAA,sDAEK;AAACA,sBAAAA,IAAI,EAAJA,IAAD;AAAOE,sBAAAA,KAAK,EAAE;AAAd,qBAFL;;AAAA;AAAA;AAAA,2BAIgB,KAAI,CAACD,IAAL,EAJhB;;AAAA;AAIEC,oBAAAA,KAJF;;AAAA,0BAKAA,KAAK,KAAK,IALV;AAAA;AAAA;AAAA;;AAAA,sDAMK;AAACF,sBAAAA,IAAI,EAAE,IAAP;AAAaE,sBAAAA,KAAK,EAALA;AAAb,qBANL;;AAAA;AAAA,sDAQG;AAACF,sBAAAA,IAAI,EAAE,KAAP;AAAcE,sBAAAA,KAAK,EAALA;AAAd,qBARH;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,WADC;AAWLC,QAAAA,MAAM;AAAA,mFAAE;AAAA;AAAA;AAAA;AAAA;AACNH,oBAAAA,IAAI,GAAG,IAAP;AADM,sDAEC;AAACA,sBAAAA,IAAI,EAAJA,IAAD;AAAOE,sBAAAA,KAAK,EAAE;AAAd,qBAFD;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,WAXD;AAeLE,QAAAA,KAAK;AAAA,kFAAE;AAAA;AAAA;AAAA;AAAA;AACLJ,oBAAAA,IAAI,GAAG,IAAP;AADK,sDAEE;AAACA,sBAAAA,IAAI,EAAE,IAAP;AAAaE,sBAAAA,KAAK,EAAE;AAApB,qBAFF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAfA,OAAP;AAoBD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetRecord} from '../schema/declare';\nimport {materializeRecords} from '../schema/shred';\n\n/**\n * A parquet cursor is used to retrieve rows from a parquet file in order\n */\nexport class ParquetCursor<T> implements AsyncIterable<T> {\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n  public columnList: string[][];\n  public rowGroup: ParquetRecord[] = [];\n  public rowGroupIndex: number;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is usually not recommended to call this constructor directly except for\n   * advanced and internal use cases. Consider using getCursor() on the\n   * ParquetReader instead\n   */\n  constructor(\n    metadata: FileMetaData,\n    envelopeReader: ParquetEnvelopeReader,\n    schema: ParquetSchema,\n    columnList: string[][]\n  ) {\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    this.schema = schema;\n    this.columnList = columnList;\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Retrieve the next row from the cursor. Returns a row or NULL if the end\n   * of the file was reached\n   */\n  async next<T = any>(): Promise<T> {\n    if (this.rowGroup.length === 0) {\n      if (this.rowGroupIndex >= this.metadata.row_groups.length) {\n        // @ts-ignore\n        return null;\n      }\n      const rowBuffer = await this.envelopeReader.readRowGroup(\n        this.schema,\n        this.metadata.row_groups[this.rowGroupIndex],\n        this.columnList\n      );\n      this.rowGroup = materializeRecords(this.schema, rowBuffer);\n      this.rowGroupIndex++;\n    }\n    return this.rowGroup.shift() as any;\n  }\n\n  /**\n   * Rewind the cursor the the beginning of the file\n   */\n  rewind(): void {\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    let done = false;\n    return {\n      next: async () => {\n        if (done) {\n          return {done, value: null};\n        }\n        const value = await this.next();\n        if (value === null) {\n          return {done: true, value};\n        }\n        return {done: false, value};\n      },\n      return: async () => {\n        done = true;\n        return {done, value: null};\n      },\n      throw: async () => {\n        done = true;\n        return {done: true, value: null};\n      }\n    };\n  }\n}\n"],"file":"parquet-cursor.js"}