@loaders.gl/parquet 3.0.13 → 3.0.14

package/dist/es5/parquetjs/reader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/parquetjs/reader.ts"],"names":["PARQUET_MAGIC","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","Symbol","asyncIterator","ParquetCursor","constructor","metadata","envelopeReader","schema","columnList","rowGroup","rowGroupIndex","next","length","row_groups","rowBuffer","readRowGroup","Shred","materializeRecords","shift","rewind","done","value","return","throw","ParquetReader","openFile","filePath","ParquetEnvelopeReader","readHeader","readFooter","err","close","openBuffer","buffer","openArrayBuffer","arrayBuffer","readFn","start","Buffer","from","closeFn","size","byteLength","version","Error","root","decodeSchema","num_children","ParquetSchema","getCursor","map","x","Array","isArray","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","fileStat","Util","fstat","fileDescriptor","fopen","fread","bind","undefined","fclose","position","Promise","resolve","slice","read","fileSize","buf","toString","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","fieldIndexOf","join","readColumnChunk","file_path","field","findField","type","getThriftEnum","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","pagesBuf","decodeDataPages","trailerLen","trailerBuf","metadataSize","readUInt32LE","metadataOffset","metadataBuf","decodeFileMetadata","decodeValues","encoding","cursor","count","opts","PARQUET_CODECS","column","offset","data","rlevels","dlevels","values","pageHeader","decodePageHeader","pageType","PageType","pageData","decodeDataPage","decodeDataPageV2","prototype","push","apply","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","valuesBuf","Compression","inflate","uncompressed_page_size","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","rLevelMax","bitWidth","getBitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","typeLength","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","schemaElements","len","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","res","name","fields","logicalType","converted_type","ConvertedType","type_length"],"mappings":";;;;;;;;;;;AACA;;AACA;;AAYA;;AACA;;AAEA;;AAaA;;;;;;;;AAMA,MAAMA,aAAa,GAAG,MAAtB;AAKA,MAAMC,eAAe,GAAG,CAAxB;AAKA,MAAMC,kBAAkB,GAAG,OAA3B;AACA,MAAMC,sBAAsB,GAAG,KAA/B;wBAkEGC,MAAM,CAACC,a;;AA7DH,MAAMC,aAAN,CAAmD;AAcxDC,EAAAA,WAAW,CACTC,QADS,EAETC,cAFS,EAGTC,MAHS,EAITC,UAJS,EAKT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,SAAKC,MAAL,GAAcA,MAAd;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACA,SAAKC,QAAL,GAAgB,EAAhB;AACA,SAAKC,aAAL,GAAqB,CAArB;AACD;;AAMS,QAAJC,IAAI,GAAwB;AAChC,QAAI,KAAKF,QAAL,CAAcG,MAAd,KAAyB,CAA7B,EAAgC;AAC9B,UAAI,KAAKF,aAAL,IAAsB,KAAKL,QAAL,CAAcQ,UAAd,CAAyBD,MAAnD,EAA2D;AAEzD,eAAO,IAAP;AACD;;AACD,YAAME,SAAS,GAAG,MAAM,KAAKR,cAAL,CAAoBS,YAApB,CACtB,KAAKR,MADiB,EAEtB,KAAKF,QAAL,CAAcQ,UAAd,CAAyB,KAAKH,aAA9B,CAFsB,EAGtB,KAAKF,UAHiB,CAAxB;AAKA,WAAKC,QAAL,GAAgBO,KAAK,CAACC,kBAAN,CAAyB,KAAKV,MAA9B,EAAsCO,SAAtC,CAAhB;AACA,WAAKJ,aAAL;AACD;;AACD,WAAO,KAAKD,QAAL,CAAcS,KAAd,EAAP;AACD;;AAKDC,EAAAA,MAAM,GAAS;AACb,SAAKV,QAAL,GAAgB,EAAhB;AACA,SAAKC,aAAL,GAAqB,CAArB;AACD;;AAMD,4BAA2C;AACzC,QAAIU,IAAI,GAAG,KAAX;AACA,WAAO;AACLT,MAAAA,IAAI,EAAE,YAAY;AAChB,YAAIS,IAAJ,EAAU;AACR,iBAAO;AAACA,YAAAA,IAAD;AAAOC,YAAAA,KAAK,EAAE;AAAd,WAAP;AACD;;AACD,cAAMA,KAAK,GAAG,MAAM,KAAKV,IAAL,EAApB;;AACA,YAAIU,KAAK,KAAK,IAAd,EAAoB;AAClB,iBAAO;AAACD,YAAAA,IAAI,EAAE,IAAP;AAAaC,YAAAA;AAAb,WAAP;AACD;;AACD,eAAO;AAACD,UAAAA,IAAI,EAAE,KAAP;AAAcC,UAAAA;AAAd,SAAP;AACD,OAVI;AAWLC,MAAAA,MAAM,EAAE,YAAY;AAClBF,QAAAA,IAAI,GAAG,IAAP;AACA,eAAO;AAACA,UAAAA,IAAD;AAAOC,UAAAA,KAAK,EAAE;AAAd,SAAP;AACD,OAdI;AAeLE,MAAAA,KAAK,EAAE,YAAY;AACjBH,QAAAA,IAAI,GAAG,IAAP;AACA,eAAO;AAACA,UAAAA,IAAI,EAAE,IAAP;AAAaC,UAAAA,KAAK,EAAE;AAApB,SAAP;AACD;AAlBI,KAAP;AAoBD;;AAnFuD;;;yBAwOvDpB,MAAM,CAACC,a;;AA3IH,MAAMsB,aAAN,CAAmD;AAKnC,eAARC,QAAQ,CAAIC,QAAJ,EAAiD;AACpE,UAAMpB,cAAc,GAAG,MAAMqB,qBAAqB,CAACF,QAAtB,CAA+BC,QAA/B,CAA7B;;AACA,QAAI;AACF,YAAMpB,cAAc,CAACsB,UAAf,EAAN;AACA,YAAMvB,QAAQ,GAAG,MAAMC,cAAc,CAACuB,UAAf,EAAvB;AACA,aAAO,IAAIL,aAAJ,CAAqBnB,QAArB,EAA+BC,cAA/B,CAAP;AACD,KAJD,CAIE,OAAOwB,GAAP,EAAY;AACZ,YAAMxB,cAAc,CAACyB,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAEsB,eAAVE,UAAU,CAAIC,MAAJ,EAA+C;AACpE,UAAM3B,cAAc,GAAG,MAAMqB,qBAAqB,CAACK,UAAtB,CAAiCC,MAAjC,CAA7B;;AACA,QAAI;AACF,YAAM3B,cAAc,CAACsB,UAAf,EAAN;AACA,YAAMvB,QAAQ,GAAG,MAAMC,cAAc,CAACuB,UAAf,EAAvB;AACA,aAAO,IAAIL,aAAJ,CAAqBnB,QAArB,EAA+BC,cAA/B,CAAP;AACD,KAJD,CAIE,OAAOwB,GAAP,EAAY;AACZ,YAAMxB,cAAc,CAACyB,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAK2B,eAAfI,eAAe,CAAIC,WAAJ,EAAyD;AACnF,UAAMC,MAAM,GAAG,OAAOC,KAAP,EAAsBzB,MAAtB,KAAyC0B,MAAM,CAACC,IAAP,CAAYJ,WAAZ,EAAyBE,KAAzB,EAAgCzB,MAAhC,CAAxD;;AACA,UAAM4B,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGN,WAAW,CAACO,UAAzB;AACA,UAAMpC,cAAc,GAAG,IAAIqB,qBAAJ,CAA0BS,MAA1B,EAAkCI,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMnC,cAAc,CAACsB,UAAf,EAAN;AACA,YAAMvB,QAAQ,GAAG,MAAMC,cAAc,CAACuB,UAAf,EAAvB;AACA,aAAO,IAAIL,aAAJ,CAAkBnB,QAAlB,EAA4BC,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOwB,GAAP,EAAY;AACZ,YAAMxB,cAAc,CAACyB,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAYD1B,EAAAA,WAAW,CAACC,QAAD,EAAyBC,cAAzB,EAAgE;AAAA;AAAA;AAAA;;AACzE,QAAID,QAAQ,CAACsC,OAAT,KAAqB7C,eAAzB,EAA0C;AACxC,YAAM,IAAI8C,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAED,SAAKvC,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,UAAMuC,IAAI,GAAG,KAAKxC,QAAL,CAAcE,MAAd,CAAqB,CAArB,CAAb;AACA,UAAM;AAACA,MAAAA;AAAD,QAAWuC,YAAY,CAAC,KAAKzC,QAAL,CAAcE,MAAf,EAAuB,CAAvB,EAA0BsC,IAAI,CAACE,YAA/B,CAA7B;AACA,SAAKxC,MAAL,GAAc,IAAIyC,qBAAJ,CAAkBzC,MAAlB,CAAd;AACD;;AAMU,QAALwB,KAAK,GAAkB;AAC3B,UAAM,KAAKzB,cAAL,CAAoByB,KAApB,EAAN;AAGD;;AAeDkB,EAAAA,SAAS,CAACzC,UAAD,EAAgE;AACvE,QAAI,CAACA,UAAL,EAAiB;AAEfA,MAAAA,UAAU,GAAG,EAAb;AACD;;AAGDA,IAAAA,UAAU,GAAGA,UAAU,CAAC0C,GAAX,CAAgBC,CAAD,IAAQC,KAAK,CAACC,OAAN,CAAcF,CAAd,IAAmBA,CAAnB,GAAuB,CAACA,CAAD,CAA9C,CAAb;AAEA,WAAO,IAAIhD,aAAJ,CACL,KAAKE,QADA,EAEL,KAAKC,cAFA,EAGL,KAAKC,MAHA,EAILC,UAJK,CAAP;AAMD;;AAMD8C,EAAAA,WAAW,GAAW;AACpB,WAAOC,MAAM,CAAC,KAAKlD,QAAL,CAAcmD,QAAf,CAAb;AACD;;AAKDC,EAAAA,SAAS,GAAkB;AACzB,WAAO,KAAKlD,MAAZ;AACD;;AAKDmD,EAAAA,WAAW,GAA2B;AACpC,UAAMC,EAA0B,GAAG,EAAnC;;AACA,SAAK,MAAMC,EAAX,IAAiB,KAAKvD,QAAL,CAAcwD,kBAA/B,EAAoD;AAClDF,MAAAA,EAAE,CAACC,EAAE,CAACE,GAAJ,CAAF,GAAaF,EAAE,CAACvC,KAAhB;AACD;;AACD,WAAOsC,EAAP;AACD;;AAMD,6BAA2C;AACzC,WAAO,KAAKV,SAAL,GAAiBhD,MAAM,CAACC,aAAxB,GAAP;AACD;;AA7IuD;;;;AAsJnD,MAAMyB,qBAAN,CAA4B;AASZ,eAARF,QAAQ,CAACC,QAAD,EAAmD;AACtE,UAAMqC,QAAQ,GAAG,MAAMC,IAAI,CAACC,KAAL,CAAWvC,QAAX,CAAvB;AACA,UAAMwC,cAAc,GAAG,MAAMF,IAAI,CAACG,KAAL,CAAWzC,QAAX,CAA7B;AAEA,UAAMU,MAAM,GAAG4B,IAAI,CAACI,KAAL,CAAWC,IAAX,CAAgBC,SAAhB,EAA2BJ,cAA3B,CAAf;AACA,UAAM1B,OAAO,GAAGwB,IAAI,CAACO,MAAL,CAAYF,IAAZ,CAAiBC,SAAjB,EAA4BJ,cAA5B,CAAhB;AAEA,WAAO,IAAIvC,qBAAJ,CAA0BS,MAA1B,EAAkCI,OAAlC,EAA2CuB,QAAQ,CAACtB,IAApD,CAAP;AACD;;AAEsB,eAAVT,UAAU,CAACC,MAAD,EAAiD;AACtE,UAAMG,MAAM,GAAG,CAACoC,QAAD,EAAmB5D,MAAnB,KACb6D,OAAO,CAACC,OAAR,CAAgBzC,MAAM,CAAC0C,KAAP,CAAaH,QAAb,EAAuBA,QAAQ,GAAG5D,MAAlC,CAAhB,CADF;;AAEA,UAAM4B,OAAO,GAAG,MAAMiC,OAAO,CAACC,OAAR,EAAtB;;AACA,WAAO,IAAI/C,qBAAJ,CAA0BS,MAA1B,EAAkCI,OAAlC,EAA2CP,MAAM,CAACrB,MAAlD,CAAP;AACD;;AAEDR,EAAAA,WAAW,CACTwE,IADS,EAET7C,KAFS,EAGT8C,QAHS,EAIT;AAAA;AAAA;AAAA;AACA,SAAKD,IAAL,GAAYA,IAAZ;AACA,SAAK7C,KAAL,GAAaA,KAAb;AACA,SAAK8C,QAAL,GAAgBA,QAAhB;AACD;;AAEe,QAAVjD,UAAU,GAAkB;AAChC,UAAMkD,GAAG,GAAG,MAAM,KAAKF,IAAL,CAAU,CAAV,EAAa/E,aAAa,CAACe,MAA3B,CAAlB;;AAEA,QAAIkE,GAAG,CAACC,QAAJ,OAAmBlF,aAAvB,EAAsC;AACpC,YAAM,IAAI+C,KAAJ,CAAU,wBAAV,CAAN;AACD;AACF;;AAEiB,QAAZ7B,YAAY,CAChBR,MADgB,EAEhBE,QAFgB,EAGhBD,UAHgB,EAIQ;AACxB,UAAMyB,MAAqB,GAAG;AAC5B+C,MAAAA,QAAQ,EAAEzB,MAAM,CAAC9C,QAAQ,CAAC+C,QAAV,CADY;AAE5ByB,MAAAA,UAAU,EAAE;AAFgB,KAA9B;;AAIA,SAAK,MAAMC,QAAX,IAAuBzE,QAAQ,CAAC0E,OAAhC,EAAyC;AACvC,YAAMC,WAAW,GAAGF,QAAQ,CAACG,SAA7B;AACA,YAAMC,MAAM,GAAGF,WAAH,aAAGA,WAAH,uBAAGA,WAAW,CAAEG,cAA5B;;AACA,UAAI/E,UAAU,CAACI,MAAX,GAAoB,CAApB,IAAyBoD,IAAI,CAACwB,YAAL,CAAkBhF,UAAlB,EAA8B8E,MAA9B,IAAyC,CAAtE,EAAyE;AACvE;AACD;;AACDrD,MAAAA,MAAM,CAACgD,UAAP,CAAkBK,MAAM,CAAEG,IAAR,EAAlB,IAAoC,MAAM,KAAKC,eAAL,CAAqBnF,MAArB,EAA6B2E,QAA7B,CAA1C;AACD;;AACD,WAAOjD,MAAP;AACD;;AAEoB,QAAfyD,eAAe,CAACnF,MAAD,EAAwB2E,QAAxB,EAAqE;AAAA;;AACxF,QAAIA,QAAQ,CAACS,SAAT,KAAuBrB,SAAvB,IAAoCY,QAAQ,CAACS,SAAT,KAAuB,IAA/D,EAAqE;AACnE,YAAM,IAAI/C,KAAJ,CAAU,uCAAV,CAAN;AACD;;AAED,UAAMgD,KAAK,GAAGrF,MAAM,CAACsF,SAAP,wBAAiBX,QAAQ,CAACG,SAA1B,wDAAiB,oBAAoBE,cAArC,CAAd;AACA,UAAMO,IAAmB,GAAG9B,IAAI,CAAC+B,aAAL,CAAmBC,mBAAnB,0BAAyBd,QAAQ,CAACG,SAAlC,yDAAyB,qBAAoBS,IAA7C,CAA5B;AACA,QAAIA,IAAI,KAAKF,KAAK,CAACK,aAAnB,EAAkC,MAAM,IAAIrD,KAAJ,2CAA6CkD,IAA7C,EAAN;AAElC,UAAMI,WAA+B,GAAGlC,IAAI,CAAC+B,aAAL,CACtCI,+BADsC,0BAEtCjB,QAAQ,CAACG,SAF6B,yDAEtC,qBAAoBe,KAFkB,CAAxC;AAKA,UAAMC,WAAW,GAAG9C,MAAM,yBAAC2B,QAAQ,CAACG,SAAV,yDAAC,qBAAoBiB,gBAArB,CAA1B;AACA,UAAMC,SAAS,GAAGhD,MAAM,yBAAC2B,QAAQ,CAACG,SAAV,yDAAC,qBAAoBmB,qBAArB,CAAxB;AACA,UAAMC,QAAQ,GAAG,MAAM,KAAK7B,IAAL,CAAUyB,WAAV,EAAuBE,SAAvB,CAAvB;AAEA,WAAOG,eAAe,CAACD,QAAD,EAAWb,KAAX,EAAkBM,WAAlB,CAAtB;AACD;;AAEe,QAAVrE,UAAU,GAA0B;AACxC,UAAM8E,UAAU,GAAG9G,aAAa,CAACe,MAAd,GAAuB,CAA1C;AACA,UAAMgG,UAAU,GAAG,MAAM,KAAKhC,IAAL,CAAU,KAAKC,QAAL,GAAgB8B,UAA1B,EAAsCA,UAAtC,CAAzB;;AAEA,QAAIC,UAAU,CAACjC,KAAX,CAAiB,CAAjB,EAAoBI,QAApB,OAAmClF,aAAvC,EAAsD;AACpD,YAAM,IAAI+C,KAAJ,CAAU,0BAAV,CAAN;AACD;;AAED,UAAMiE,YAAY,GAAGD,UAAU,CAACE,YAAX,CAAwB,CAAxB,CAArB;AACA,UAAMC,cAAc,GAAG,KAAKlC,QAAL,GAAgBgC,YAAhB,GAA+BF,UAAtD;;AACA,QAAII,cAAc,GAAGlH,aAAa,CAACe,MAAnC,EAA2C;AACzC,YAAM,IAAIgC,KAAJ,CAAU,uBAAV,CAAN;AACD;;AAED,UAAMoE,WAAW,GAAG,MAAM,KAAKpC,IAAL,CAAUmC,cAAV,EAA0BF,YAA1B,CAA1B;AAGA,UAAM;AAACxG,MAAAA;AAAD,QAAa2D,IAAI,CAACiD,kBAAL,CAAwBD,WAAxB,CAAnB;AACA,WAAO3G,QAAP;AACD;;AAxGgC;;;;AA8GnC,SAAS6G,YAAT,CACEpB,IADF,EAEEqB,QAFF,EAGEC,MAHF,EAIEC,KAJF,EAKEC,IALF,EAMS;AACP,MAAI,EAAEH,QAAQ,IAAII,sBAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI3E,KAAJ,6BAA+BuE,QAA/B,EAAN;AACD;;AACD,SAAOI,uBAAeJ,QAAf,EAAyBD,YAAzB,CAAsCpB,IAAtC,EAA4CsB,MAA5C,EAAoDC,KAApD,EAA2DC,IAA3D,CAAP;AACD;;AAED,SAASZ,eAAT,CACEzE,MADF,EAEEuF,MAFF,EAGEtB,WAHF,EAIe;AACb,QAAMkB,MAAoB,GAAG;AAC3BnF,IAAAA,MAD2B;AAE3BwF,IAAAA,MAAM,EAAE,CAFmB;AAG3BhF,IAAAA,IAAI,EAAER,MAAM,CAACrB;AAHc,GAA7B;AAMA,QAAM8G,IAAiB,GAAG;AACxBC,IAAAA,OAAO,EAAE,EADe;AAExBC,IAAAA,OAAO,EAAE,EAFe;AAGxBC,IAAAA,MAAM,EAAE,EAHgB;AAIxBR,IAAAA,KAAK,EAAE;AAJiB,GAA1B;;AAQA,SAAOD,MAAM,CAACK,MAAP,GAAgBL,MAAM,CAAC3E,IAA9B,EAAoC;AAIlC,UAAM;AAACqF,MAAAA,UAAD;AAAalH,MAAAA;AAAb,QAAuBoD,IAAI,CAAC+D,gBAAL,CAAsBX,MAAM,CAACnF,MAA7B,CAA7B;AACAmF,IAAAA,MAAM,CAACK,MAAP,IAAiB7G,MAAjB;AAEA,UAAMoH,QAAQ,GAAGhE,IAAI,CAAC+B,aAAL,CAAmBkC,uBAAnB,EAA6BH,UAAU,CAAChC,IAAxC,CAAjB;AAEA,QAAIoC,QAA4B,GAAG,IAAnC;;AACA,YAAQF,QAAR;AACE,WAAK,WAAL;AACEE,QAAAA,QAAQ,GAAGC,cAAc,CAACf,MAAD,EAASU,UAAT,EAAqBN,MAArB,EAA6BtB,WAA7B,CAAzB;AACA;;AACF,WAAK,cAAL;AACEgC,QAAAA,QAAQ,GAAGE,gBAAgB,CAAChB,MAAD,EAASU,UAAT,EAAqBN,MAArB,EAA6BtB,WAA7B,CAA3B;AACA;;AACF;AACE,cAAM,IAAItD,KAAJ,8BAAgCoF,QAAhC,EAAN;AARJ;;AAWA5E,IAAAA,KAAK,CAACiF,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2Bb,IAAI,CAACC,OAAhC,EAAyCO,QAAQ,CAACP,OAAlD;AACAvE,IAAAA,KAAK,CAACiF,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2Bb,IAAI,CAACE,OAAhC,EAAyCM,QAAQ,CAACN,OAAlD;AACAxE,IAAAA,KAAK,CAACiF,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2Bb,IAAI,CAACG,MAAhC,EAAwCK,QAAQ,CAACL,MAAjD;AACAH,IAAAA,IAAI,CAACL,KAAL,IAAca,QAAQ,CAACb,KAAvB;AACD;;AAED,SAAOK,IAAP;AACD;;AAED,SAASS,cAAT,CACEf,MADF,EAEEoB,MAFF,EAGEhB,MAHF,EAIEtB,WAJF,EAKe;AAAA;;AACb,QAAMuC,SAAS,GAAGrB,MAAM,CAACK,MAAP,GAAgBe,MAAM,CAACE,oBAAzC;AACA,QAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAV,0DAAG,sBAAyBC,UAA5C;AAmBA,MAAIC,UAAU,GAAG1B,MAAjB;;AACA,MAAIlB,WAAW,KAAK,cAApB,EAAoC;AAClC,UAAM6C,SAAS,GAAGC,WAAW,CAACC,OAAZ,CAChB/C,WADgB,EAEhBkB,MAAM,CAACnF,MAAP,CAAc0C,KAAd,CAAoByC,MAAM,CAACK,MAA3B,EAAmCgB,SAAnC,CAFgB,EAGhBD,MAAM,CAACU,sBAHS,CAAlB;AAKAJ,IAAAA,UAAU,GAAG;AACX7G,MAAAA,MAAM,EAAE8G,SADG;AAEXtB,MAAAA,MAAM,EAAE,CAFG;AAGXhF,MAAAA,IAAI,EAAEsG,SAAS,CAACnI;AAHL,KAAb;AAKAwG,IAAAA,MAAM,CAACK,MAAP,GAAgBgB,SAAhB;AACD;;AAGD,QAAMU,cAAc,GAAGnF,IAAI,CAAC+B,aAAL,CACrBqD,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBS,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIlG,KAAJ,CAAUuF,UAAV,CAAd;;AACA,MAAInB,MAAM,CAAC+B,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAGpC,YAAY,CAACnH,kBAAD,EAAqBoJ,cAArB,EAAqCL,UAArC,EAAiDH,UAAjD,EAA8D;AAClFa,MAAAA,QAAQ,EAAExF,IAAI,CAACyF,WAAL,CAAiBjC,MAAM,CAAC+B,SAAxB,CADwE;AAElFG,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLJ,IAAAA,OAAO,CAACK,IAAR,CAAa,CAAb;AACD;;AAGD,QAAMC,cAAc,GAAG5F,IAAI,CAAC+B,aAAL,CACrBqD,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBiB,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAI1G,KAAJ,CAAUuF,UAAV,CAAd;;AACA,MAAInB,MAAM,CAACuC,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAG5C,YAAY,CAACnH,kBAAD,EAAqB6J,cAArB,EAAqCd,UAArC,EAAiDH,UAAjD,EAA8D;AAClFa,MAAAA,QAAQ,EAAExF,IAAI,CAACyF,WAAL,CAAiBjC,MAAM,CAACuC,SAAxB,CADwE;AAElFL,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AACD,MAAIK,iBAAiB,GAAG,CAAxB;;AACA,OAAK,MAAMC,IAAX,IAAmBH,OAAnB,EAA4B;AAC1B,QAAIG,IAAI,KAAKzC,MAAM,CAACuC,SAApB,EAA+B;AAC7BC,MAAAA,iBAAiB;AAClB;AACF;;AAGD,QAAME,aAAa,GAAGlG,IAAI,CAAC+B,aAAL,CACpBqD,uBADoB,4BAEpBZ,MAAM,CAACI,gBAFa,2DAEpB,uBAAyBzB,QAFL,CAAtB;AAIA,QAAMU,MAAM,GAAGX,YAAY,CAACM,MAAM,CAACvB,aAAR,EAAwBiE,aAAxB,EAAuCpB,UAAvC,EAAmDkB,iBAAnD,EAAsE;AAC/FG,IAAAA,UAAU,EAAE3C,MAAM,CAAC2C,UAD4E;AAE/FX,IAAAA,QAAQ,EAAEhC,MAAM,CAAC2C;AAF8E,GAAtE,CAA3B;AASA,SAAO;AACLvC,IAAAA,OAAO,EAAEkC,OADJ;AAELnC,IAAAA,OAAO,EAAE2B,OAFJ;AAGLzB,IAAAA,MAHK;AAILR,IAAAA,KAAK,EAAEsB;AAJF,GAAP;AAMD;;AAED,SAASP,gBAAT,CACEhB,MADF,EAEEoB,MAFF,EAGEhB,MAHF,EAIEtB,WAJF,EAKe;AAAA;;AACb,QAAMuC,SAAS,GAAGrB,MAAM,CAACK,MAAP,GAAgBe,MAAM,CAACE,oBAAzC;AAEA,QAAMC,UAAU,6BAAGH,MAAM,CAAC4B,mBAAV,2DAAG,uBAA4BvB,UAA/C;AAEA,QAAMmB,iBAAiB,GAAGrB,UAAU,8BAAGH,MAAM,CAAC4B,mBAAV,2DAAG,uBAA4BC,SAA/B,CAApC;AACA,QAAMH,aAAa,GAAGlG,IAAI,CAAC+B,aAAL,CACpBqD,uBADoB,4BAEpBZ,MAAM,CAAC4B,mBAFa,2DAEpB,uBAA4BjD,QAFR,CAAtB;AAOA,MAAImC,OAAO,GAAG,IAAIlG,KAAJ,CAAUuF,UAAV,CAAd;;AACA,MAAInB,MAAM,CAAC+B,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAGpC,YAAY,CAACnH,kBAAD,EAAqBC,sBAArB,EAA6CoH,MAA7C,EAAqDuB,UAArD,EAAkE;AACtFa,MAAAA,QAAQ,EAAExF,IAAI,CAACyF,WAAL,CAAiBjC,MAAM,CAAC+B,SAAxB,CAD4E;AAEtFG,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLJ,IAAAA,OAAO,CAACK,IAAR,CAAa,CAAb;AACD;;AAID,MAAIG,OAAO,GAAG,IAAI1G,KAAJ,CAAUuF,UAAV,CAAd;;AACA,MAAInB,MAAM,CAACuC,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAG5C,YAAY,CAACnH,kBAAD,EAAqBC,sBAArB,EAA6CoH,MAA7C,EAAqDuB,UAArD,EAAkE;AACtFa,MAAAA,QAAQ,EAAExF,IAAI,CAACyF,WAAL,CAAiBjC,MAAM,CAACuC,SAAxB,CAD4E;AAEtFL,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AAGD,MAAIW,eAAe,GAAGlD,MAAtB;;AAEA,gCAAIoB,MAAM,CAAC4B,mBAAX,mDAAI,uBAA4BG,aAAhC,EAA+C;AAC7C,UAAMxB,SAAS,GAAGC,WAAW,CAACC,OAAZ,CAChB/C,WADgB,EAEhBkB,MAAM,CAACnF,MAAP,CAAc0C,KAAd,CAAoByC,MAAM,CAACK,MAA3B,EAAmCgB,SAAnC,CAFgB,EAGhBD,MAAM,CAACU,sBAHS,CAAlB;AAMAoB,IAAAA,eAAe,GAAG;AAChBrI,MAAAA,MAAM,EAAE8G,SADQ;AAEhBtB,MAAAA,MAAM,EAAE,CAFQ;AAGhBhF,MAAAA,IAAI,EAAEsG,SAAS,CAACnI;AAHA,KAAlB;AAMAwG,IAAAA,MAAM,CAACK,MAAP,GAAgBgB,SAAhB;AACD;;AAED,QAAMZ,MAAM,GAAGX,YAAY,CACzBM,MAAM,CAACvB,aADkB,EAEzBiE,aAFyB,EAGzBI,eAHyB,EAIzBN,iBAJyB,EAKzB;AACEG,IAAAA,UAAU,EAAE3C,MAAM,CAAC2C,UADrB;AAEEX,IAAAA,QAAQ,EAAEhC,MAAM,CAAC2C;AAFnB,GALyB,CAA3B;AAWA,SAAO;AACLvC,IAAAA,OAAO,EAAEkC,OADJ;AAELnC,IAAAA,OAAO,EAAE2B,OAFJ;AAGLzB,IAAAA,MAHK;AAILR,IAAAA,KAAK,EAAEsB;AAJF,GAAP;AAMD;;AAED,SAAS7F,YAAT,CACE0H,cADF,EAEE/C,MAFF,EAGEgD,GAHF,EAQE;AACA,QAAMlK,MAAwB,GAAG,EAAjC;AACA,MAAII,IAAI,GAAG8G,MAAX;;AACA,OAAK,IAAIiD,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGD,GAApB,EAAyBC,CAAC,EAA1B,EAA8B;AAC5B,UAAMC,aAAa,GAAGH,cAAc,CAAC7J,IAAD,CAApC;AAEA,UAAMiK,cAAc,GAClBjK,IAAI,GAAG,CAAP,GAAWqD,IAAI,CAAC+B,aAAL,CAAmB8E,kCAAnB,EAAwCF,aAAa,CAACG,eAAtD,CAAX,GAAqF,MADvF;AAGA,QAAIC,QAAQ,GAAG,KAAf;AACA,QAAIC,QAAQ,GAAG,KAAf;;AACA,YAAQJ,cAAR;AACE,WAAK,UAAL;AACE;;AACF,WAAK,UAAL;AACEG,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF,WAAK,UAAL;AACEC,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF;AACE,cAAM,IAAIpI,KAAJ,CAAU,kCAAV,CAAN;AAVJ;;AAaA,QAAI+H,aAAa,CAAC5H,YAAd,GAA8B,CAAlC,EAAqC;AACnC,YAAMkI,GAAG,GAAGnI,YAAY,CAAC0H,cAAD,EAAiB7J,IAAI,GAAG,CAAxB,EAA2BgK,aAAa,CAAC5H,YAAzC,CAAxB;AACApC,MAAAA,IAAI,GAAGsK,GAAG,CAACtK,IAAX;AACAJ,MAAAA,MAAM,CAACoK,aAAa,CAACO,IAAf,CAAN,GAA6B;AAE3BH,QAAAA,QAF2B;AAG3BC,QAAAA,QAH2B;AAI3BG,QAAAA,MAAM,EAAEF,GAAG,CAAC1K;AAJe,OAA7B;AAMD,KATD,MASO;AACL,UAAI6K,WAAW,GAAGpH,IAAI,CAAC+B,aAAL,CAAmBC,mBAAnB,EAAyB2E,aAAa,CAAC7E,IAAvC,CAAlB;;AAEA,UAAI6E,aAAa,CAACU,cAAlB,EAAkC;AAChCD,QAAAA,WAAW,GAAGpH,IAAI,CAAC+B,aAAL,CAAmBuF,4BAAnB,EAAkCX,aAAa,CAACU,cAAhD,CAAd;AACD;;AAED9K,MAAAA,MAAM,CAACoK,aAAa,CAACO,IAAf,CAAN,GAA6B;AAC3BpF,QAAAA,IAAI,EAAEsF,WADqB;AAE3BjB,QAAAA,UAAU,EAAEQ,aAAa,CAACY,WAFC;AAG3BR,QAAAA,QAH2B;AAI3BC,QAAAA;AAJ2B,OAA7B;AAMArK,MAAAA,IAAI;AACL;AACF;;AACD,SAAO;AAACJ,IAAAA,MAAD;AAASkH,IAAAA,MAAT;AAAiB9G,IAAAA;AAAjB,GAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from './codecs';\nimport * as Compression from './compression';\nimport {\n  ParquetBuffer,\n  ParquetCodec,\n  ParquetCompression,\n  ParquetData,\n  ParquetField,\n  ParquetRecord,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from './schema/declare';\nimport {ParquetSchema} from './schema/schema';\nimport * as Shred from './schema/shred';\n// tslint:disable-next-line:max-line-length\nimport {\n  ColumnChunk,\n  CompressionCodec,\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  FileMetaData,\n  PageHeader,\n  PageType,\n  RowGroup,\n  SchemaElement,\n  Type\n} from './parquet-thrift';\nimport * as Util from './util';\n// import Fs = require('fs');\n\n/**\n * Parquet File Magic String\n */\nconst PARQUET_MAGIC = 'PAR1';\n\n/**\n * Parquet File Format Version\n */\nconst PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\n/**\n * A parquet cursor is used to retrieve rows from a parquet file in order\n */\nexport class ParquetCursor<T> implements AsyncIterable<T> {\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n  public columnList: string[][];\n  public rowGroup: ParquetRecord[];\n  public rowGroupIndex: number;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is usually not recommended to call this constructor directly except for\n   * advanced and internal use cases. Consider using getCursor() on the\n   * ParquetReader instead\n   */\n  constructor(\n    metadata: FileMetaData,\n    envelopeReader: ParquetEnvelopeReader,\n    schema: ParquetSchema,\n    columnList: string[][]\n  ) {\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    this.schema = schema;\n    this.columnList = columnList;\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Retrieve the next row from the cursor. Returns a row or NULL if the end\n   * of the file was reached\n   */\n  async next<T = any>(): Promise<T> {\n    if (this.rowGroup.length === 0) {\n      if (this.rowGroupIndex >= this.metadata.row_groups.length) {\n        // @ts-ignore\n        return null;\n      }\n      const rowBuffer = await this.envelopeReader.readRowGroup(\n        this.schema,\n        this.metadata.row_groups[this.rowGroupIndex],\n        this.columnList\n      );\n      this.rowGroup = Shred.materializeRecords(this.schema, rowBuffer);\n      this.rowGroupIndex++;\n    }\n    return this.rowGroup.shift() as any;\n  }\n\n  /**\n   * Rewind the cursor the the beginning of the file\n   */\n  rewind(): void {\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    let done = false;\n    return {\n      next: async () => {\n        if (done) {\n          return {done, value: null};\n        }\n        const value = await this.next();\n        if (value === null) {\n          return {done: true, value};\n        }\n        return {done: false, value};\n      },\n      return: async () => {\n        done = true;\n        return {done, value: null};\n      },\n      throw: async () => {\n        done = true;\n        return {done: true, value: null};\n      }\n    };\n  }\n}\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * Open the parquet file pointed to by the specified path and return a new\n   * parquet reader\n   */\n  static async openFile<T>(filePath: string): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n\n  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {\n    const fileStat = await Util.fstat(filePath);\n    const fileDescriptor = await Util.fopen(filePath);\n\n    const readFn = Util.fread.bind(undefined, fileDescriptor);\n    const closeFn = Util.fclose.bind(undefined, fileDescriptor);\n\n    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);\n  }\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n  }\n\n  async readHeader(): Promise<void> {\n    const buf = await this.read(0, PARQUET_MAGIC.length);\n\n    if (buf.toString() !== PARQUET_MAGIC) {\n      throw new Error('not valid parquet file');\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && Util.fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = Util.getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n    if (type !== field.primitiveType) throw new Error(`chunk type not matching schema: ${type}`);\n\n    const compression: ParquetCompression = Util.getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    const pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n\n    return decodeDataPages(pagesBuf, field, compression);\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    if (trailerBuf.slice(4).toString() !== PARQUET_MAGIC) {\n      throw new Error('not a valid parquet file');\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error('invalid metadata size');\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = Util.decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\nfunction decodeDataPages(\n  buffer: Buffer,\n  column: ParquetField,\n  compression: ParquetCompression\n): ParquetData {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetData = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    count: 0\n  };\n\n  // @ts-ignore size can be undefined\n  while (cursor.offset < cursor.size) {\n    // const pageHeader = new parquet_thrift.PageHeader();\n    // cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer);\n\n    const {pageHeader, length} = Util.decodePageHeader(cursor.buffer);\n    cursor.offset += length;\n\n    const pageType = Util.getThriftEnum(PageType, pageHeader.type);\n\n    let pageData: ParquetData | null = null;\n    switch (pageType) {\n      case 'DATA_PAGE':\n        pageData = decodeDataPage(cursor, pageHeader, column, compression);\n        break;\n      case 'DATA_PAGE_V2':\n        pageData = decodeDataPageV2(cursor, pageHeader, column, compression);\n        break;\n      default:\n        throw new Error(`invalid page type: ${pageType}`);\n    }\n\n    Array.prototype.push.apply(data.rlevels, pageData.rlevels);\n    Array.prototype.push.apply(data.dlevels, pageData.dlevels);\n    Array.prototype.push.apply(data.values, pageData.values);\n    data.count += pageData.count;\n  }\n\n  return data;\n}\n\nfunction decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  column: ParquetField,\n  compression: ParquetCompression\n): ParquetData {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  // const info = {\n  //   path: opts.column.path.join('.'),\n  //   valueEncoding,\n  //   dLevelEncoding,\n  //   rLevelEncoding,\n  //   cursorOffset: cursor.offset,\n  //   cursorEnd,\n  //   cusrorSize: cursor.size,\n  //   header,\n  //   opts,\n  //   buffer: cursor.buffer.toJSON(),\n  //   values: null as any[],\n  //   valBuf: null as any\n  // };\n  // Fs.writeFileSync(`dump/${info.path}.ts.json`, JSON.stringify(info, null, 2));\n\n  /* uncompress page */\n  let dataCursor = cursor;\n  if (compression !== 'UNCOMPRESSED') {\n    const valuesBuf = Compression.inflate(\n      compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header?.encoding!\n  ) as ParquetCodec;\n  const values = decodeValues(column.primitiveType!, valueEncoding, dataCursor, valueCountNonNull, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  // info.valBuf = uncursor.buffer.toJSON();\n  // info.values = values;\n  // Fs.writeFileSync(`dump/${info.path}.ts.json`, JSON.stringify(info, null, 2));\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!\n  };\n}\n\nfunction decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  column: ParquetField,\n  compression: ParquetCompression\n): ParquetData {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = Compression.inflate(\n      compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const values = decodeValues(\n    column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    {\n      typeLength: column.typeLength,\n      bitWidth: column.typeLength\n    }\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!\n  };\n}\n\nfunction decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? Util.getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      let logicalType = Util.getThriftEnum(Type, schemaElement.type!);\n\n      if (schemaElement.converted_type) {\n        logicalType = Util.getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n"],"file":"reader.js"}
+{"version":3,"sources":["../../../src/parquetjs/reader.ts"],"names":["PARQUET_MAGIC","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","Symbol","asyncIterator","ParquetCursor","metadata","envelopeReader","schema","columnList","rowGroup","rowGroupIndex","length","row_groups","readRowGroup","rowBuffer","Shred","materializeRecords","shift","done","next","value","return","throw","ParquetReader","version","Error","root","decodeSchema","num_children","ParquetSchema","close","map","x","Array","isArray","Number","num_rows","md","key_value_metadata","kv","key","getCursor","filePath","ParquetEnvelopeReader","openFile","readHeader","readFooter","buffer","openBuffer","arrayBuffer","readFn","start","Buffer","from","closeFn","size","byteLength","read","fileSize","buf","toString","rowCount","columnData","columns","colChunk","colMetadata","meta_data","colKey","path_in_schema","Util","fieldIndexOf","readColumnChunk","join","file_path","undefined","field","findField","type","getThriftEnum","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","pagesBuf","decodeDataPages","trailerLen","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","decodeFileMetadata","fstat","fileStat","fopen","fileDescriptor","fread","bind","fclose","position","Promise","resolve","decodeValues","encoding","cursor","count","opts","PARQUET_CODECS","column","offset","data","rlevels","dlevels","values","decodePageHeader","pageHeader","pageType","PageType","pageData","decodeDataPage","decodeDataPageV2","prototype","push","apply","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","valuesBuf","Compression","inflate","uncompressed_page_size","rLevelEncoding","Encoding","repetition_level_encoding","rLevels","rLevelMax","bitWidth","getBitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","typeLength","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","schemaElements","len","i","schemaElement","repetitionType","FieldRepetitionType","repetition_type","optional","repeated","res","name","fields","logicalType","converted_type","ConvertedType","type_length"],"mappings":";;;;;;;;;;;;;;;;;;;;;AACA;;AACA;;AAYA;;AACA;;AAEA;;AAaA;;;;;;;;;;;;;;AAMA,IAAMA,aAAa,GAAG,MAAtB;AAKA,IAAMC,eAAe,GAAG,CAAxB;AAKA,IAAMC,kBAAkB,GAAG,OAA3B;AACA,IAAMC,sBAAsB,GAAG,KAA/B;wBAkEGC,MAAM,CAACC,a;;IA7DGC,a;AAcX,yBACEC,QADF,EAEEC,cAFF,EAGEC,MAHF,EAIEC,UAJF,EAKE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,SAAKH,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,SAAKC,MAAL,GAAcA,MAAd;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACA,SAAKC,QAAL,GAAgB,EAAhB;AACA,SAAKC,aAAL,GAAqB,CAArB;AACD;;;;;4EAMD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBACM,KAAKD,QAAL,CAAcE,MAAd,KAAyB,CAD/B;AAAA;AAAA;AAAA;;AAAA,sBAEQ,KAAKD,aAAL,IAAsB,KAAKL,QAAL,CAAcO,UAAd,CAAyBD,MAFvD;AAAA;AAAA;AAAA;;AAAA,iDAIa,IAJb;;AAAA;AAAA;AAAA,uBAM4B,KAAKL,cAAL,CAAoBO,YAApB,CACtB,KAAKN,MADiB,EAEtB,KAAKF,QAAL,CAAcO,UAAd,CAAyB,KAAKF,aAA9B,CAFsB,EAGtB,KAAKF,UAHiB,CAN5B;;AAAA;AAMUM,gBAAAA,SANV;AAWI,qBAAKL,QAAL,GAAgBM,KAAK,CAACC,kBAAN,CAAyB,KAAKT,MAA9B,EAAsCO,SAAtC,CAAhB;AACA,qBAAKJ,aAAL;;AAZJ;AAAA,iDAcS,KAAKD,QAAL,CAAcQ,KAAd,EAdT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;WAoBA,kBAAe;AACb,WAAKR,QAAL,GAAgB,EAAhB;AACA,WAAKC,aAAL,GAAqB,CAArB;AACD;;;WAMD,iBAA2C;AAAA;;AACzC,UAAIQ,IAAI,GAAG,KAAX;AACA,aAAO;AACLC,QAAAA,IAAI;AAAA,iFAAE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBACAD,IADA;AAAA;AAAA;AAAA;;AAAA,sDAEK;AAACA,sBAAAA,IAAI,EAAJA,IAAD;AAAOE,sBAAAA,KAAK,EAAE;AAAd,qBAFL;;AAAA;AAAA;AAAA,2BAIgB,KAAI,CAACD,IAAL,EAJhB;;AAAA;AAIEC,oBAAAA,KAJF;;AAAA,0BAKAA,KAAK,KAAK,IALV;AAAA;AAAA;AAAA;;AAAA,sDAMK;AAACF,sBAAAA,IAAI,EAAE,IAAP;AAAaE,sBAAAA,KAAK,EAALA;AAAb,qBANL;;AAAA;AAAA,sDAQG;AAACF,sBAAAA,IAAI,EAAE,KAAP;AAAcE,sBAAAA,KAAK,EAALA;AAAd,qBARH;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,WADC;AAWLC,QAAAA,MAAM;AAAA,mFAAE;AAAA;AAAA;AAAA;AAAA;AACNH,oBAAAA,IAAI,GAAG,IAAP;AADM,sDAEC;AAACA,sBAAAA,IAAI,EAAJA,IAAD;AAAOE,sBAAAA,KAAK,EAAE;AAAd,qBAFD;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA,WAXD;AAeLE,QAAAA,KAAK;AAAA,kFAAE;AAAA;AAAA;AAAA;AAAA;AACLJ,oBAAAA,IAAI,GAAG,IAAP;AADK,sDAEE;AAACA,sBAAAA,IAAI,EAAE,IAAP;AAAaE,sBAAAA,KAAK,EAAE;AAApB,qBAFF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAF;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAfA,OAAP;AAoBD;;;;;;yBAqJAlB,MAAM,CAACC,a;;IA3IGoB,a;AAyDX,yBAAYlB,QAAZ,EAAoCC,cAApC,EAA2E;AAAA;AAAA;AAAA;AAAA;;AACzE,QAAID,QAAQ,CAACmB,OAAT,KAAqBzB,eAAzB,EAA0C;AACxC,YAAM,IAAI0B,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAED,SAAKpB,QAAL,GAAgBA,QAAhB;AACA,SAAKC,cAAL,GAAsBA,cAAtB;AACA,QAAMoB,IAAI,GAAG,KAAKrB,QAAL,CAAcE,MAAd,CAAqB,CAArB,CAAb;;AACA,wBAAiBoB,YAAY,CAAC,KAAKtB,QAAL,CAAcE,MAAf,EAAuB,CAAvB,EAA0BmB,IAAI,CAACE,YAA/B,CAA7B;AAAA,QAAOrB,MAAP,iBAAOA,MAAP;;AACA,SAAKA,MAAL,GAAc,IAAIsB,qBAAJ,CAAkBtB,MAAlB,CAAd;AACD;;;;;6EAMD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACQ,KAAKD,cAAL,CAAoBwB,KAApB,EADR;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;WAmBA,mBAAUtB,UAAV,EAAyE;AACvE,UAAI,CAACA,UAAL,EAAiB;AAEfA,QAAAA,UAAU,GAAG,EAAb;AACD;;AAGDA,MAAAA,UAAU,GAAGA,UAAU,CAACuB,GAAX,CAAe,UAACC,CAAD;AAAA,eAAQC,KAAK,CAACC,OAAN,CAAcF,CAAd,IAAmBA,CAAnB,GAAuB,CAACA,CAAD,CAA/B;AAAA,OAAf,CAAb;AAEA,aAAO,IAAI5B,aAAJ,CACL,KAAKC,QADA,EAEL,KAAKC,cAFA,EAGL,KAAKC,MAHA,EAILC,UAJK,CAAP;AAMD;;;WAMD,uBAAsB;AACpB,aAAO2B,MAAM,CAAC,KAAK9B,QAAL,CAAc+B,QAAf,CAAb;AACD;;;WAKD,qBAA2B;AACzB,aAAO,KAAK7B,MAAZ;AACD;;;WAKD,uBAAsC;AACpC,UAAM8B,EAA0B,GAAG,EAAnC;;AADoC,iDAEnB,KAAKhC,QAAL,CAAciC,kBAFK;AAAA;;AAAA;AAEpC,4DAAoD;AAAA,cAAzCC,EAAyC;AAClDF,UAAAA,EAAE,CAACE,EAAE,CAACC,GAAJ,CAAF,GAAaD,EAAE,CAACnB,KAAhB;AACD;AAJmC;AAAA;AAAA;AAAA;AAAA;;AAKpC,aAAOiB,EAAP;AACD;;;WAMD,iBAA2C;AACzC,aAAO,KAAKI,SAAL,GAAiBvC,MAAM,CAACC,aAAxB,GAAP;AACD;;;;gFAxID,kBAAyBuC,QAAzB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAC+BC,qBAAqB,CAACC,QAAtB,CAA+BF,QAA/B,CAD/B;;AAAA;AACQpC,gBAAAA,cADR;AAAA;AAAA;AAAA,uBAGUA,cAAc,CAACuC,UAAf,EAHV;;AAAA;AAAA;AAAA,uBAI2BvC,cAAc,CAACwC,UAAf,EAJ3B;;AAAA;AAIUzC,gBAAAA,QAJV;AAAA,kDAKW,IAAIkB,aAAJ,CAAqBlB,QAArB,EAA+BC,cAA/B,CALX;;AAAA;AAAA;AAAA;AAAA;AAAA,uBAOUA,cAAc,CAACwB,KAAf,EAPV;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFAYA,kBAA2BiB,MAA3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAC+BJ,qBAAqB,CAACK,UAAtB,CAAiCD,MAAjC,CAD/B;;AAAA;AACQzC,gBAAAA,cADR;AAAA;AAAA;AAAA,uBAGUA,cAAc,CAACuC,UAAf,EAHV;;AAAA;AAAA;AAAA,uBAI2BvC,cAAc,CAACwC,UAAf,EAJ3B;;AAAA;AAIUzC,gBAAAA,QAJV;AAAA,kDAKW,IAAIkB,aAAJ,CAAqBlB,QAArB,EAA+BC,cAA/B,CALX;;AAAA;AAAA;AAAA;AAAA;AAAA,uBAOUA,cAAc,CAACwB,KAAf,EAPV;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;uFAeA,mBAAgCmB,WAAhC;AAAA;AAAA;AAAA;AAAA;AAAA;AACQC,gBAAAA,MADR;AAAA,uFACiB,kBAAOC,KAAP,EAAsBxC,MAAtB;AAAA;AAAA;AAAA;AAAA;AAAA,8DAAyCyC,MAAM,CAACC,IAAP,CAAYJ,WAAZ,EAAyBE,KAAzB,EAAgCxC,MAAhC,CAAzC;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBADjB;;AAAA,kCACQuC,MADR;AAAA;AAAA;AAAA;;AAEQI,gBAAAA,OAFR;AAAA,wFAEkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBAFlB;;AAAA,kCAEQA,OAFR;AAAA;AAAA;AAAA;;AAGQC,gBAAAA,IAHR,GAGeN,WAAW,CAACO,UAH3B;AAIQlD,gBAAAA,cAJR,GAIyB,IAAIqC,qBAAJ,CAA0BO,MAA1B,EAAkCI,OAAlC,EAA2CC,IAA3C,CAJzB;AAAA;AAAA;AAAA,uBAMUjD,cAAc,CAACuC,UAAf,EANV;;AAAA;AAAA;AAAA,uBAO2BvC,cAAc,CAACwC,UAAf,EAP3B;;AAAA;AAOUzC,gBAAAA,QAPV;AAAA,mDAQW,IAAIkB,aAAJ,CAAkBlB,QAAlB,EAA4BC,cAA5B,CARX;;AAAA;AAAA;AAAA;AAAA;AAAA,uBAUUA,cAAc,CAACwB,KAAf,EAVV;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;;;;IAsHWa,qB;AA0BX,iCACEc,IADF,EAEE3B,KAFF,EAGE4B,QAHF,EAIE;AAAA;AAAA;AAAA;AAAA;AACA,SAAKD,IAAL,GAAYA,IAAZ;AACA,SAAK3B,KAAL,GAAaA,KAAb;AACA,SAAK4B,QAAL,GAAgBA,QAAhB;AACD;;;;;kFAED;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACoB,KAAKD,IAAL,CAAU,CAAV,EAAa3D,aAAa,CAACa,MAA3B,CADpB;;AAAA;AACQgD,gBAAAA,GADR;;AAAA,sBAGMA,GAAG,CAACC,QAAJ,OAAmB9D,aAHzB;AAAA;AAAA;AAAA;;AAAA,sBAIU,IAAI2B,KAAJ,CAAU,wBAAV,CAJV;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;oFAQA,mBACElB,MADF,EAEEE,QAFF,EAGED,UAHF;AAAA;;AAAA;AAAA;AAAA;AAAA;AAKQuC,gBAAAA,MALR,GAKgC;AAC5Bc,kBAAAA,QAAQ,EAAE1B,MAAM,CAAC1B,QAAQ,CAAC2B,QAAV,CADY;AAE5B0B,kBAAAA,UAAU,EAAE;AAFgB,iBALhC;AAAA,wDASyBrD,QAAQ,CAACsD,OATlC;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AASaC,gBAAAA,QATb;AAUUC,gBAAAA,WAVV,GAUwBD,QAAQ,CAACE,SAVjC;AAWUC,gBAAAA,MAXV,GAWmBF,WAXnB,aAWmBA,WAXnB,uBAWmBA,WAAW,CAAEG,cAXhC;;AAAA,sBAYQ5D,UAAU,CAACG,MAAX,GAAoB,CAApB,IAAyB0D,IAAI,CAACC,YAAL,CAAkB9D,UAAlB,EAA8B2D,MAA9B,IAAyC,CAZ1E;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;AAAA,uBAe8C,KAAKI,eAAL,CAAqBhE,MAArB,EAA6ByD,QAA7B,CAf9C;;AAAA;AAeIjB,gBAAAA,MAAM,CAACe,UAAP,CAAkBK,MAAM,CAAEK,IAAR,EAAlB,CAfJ;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAAA,mDAiBSzB,MAjBT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;uFAoBA,mBAAsBxC,MAAtB,EAA6CyD,QAA7C;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBACMA,QAAQ,CAACS,SAAT,KAAuBC,SAAvB,IAAoCV,QAAQ,CAACS,SAAT,KAAuB,IADjE;AAAA;AAAA;AAAA;;AAAA,sBAEU,IAAIhD,KAAJ,CAAU,uCAAV,CAFV;;AAAA;AAKQkD,gBAAAA,KALR,GAKgBpE,MAAM,CAACqE,SAAP,wBAAiBZ,QAAQ,CAACE,SAA1B,wDAAiB,oBAAoBE,cAArC,CALhB;AAMQS,gBAAAA,IANR,GAM8BR,IAAI,CAACS,aAAL,CAAmBC,mBAAnB,0BAAyBf,QAAQ,CAACE,SAAlC,yDAAyB,qBAAoBW,IAA7C,CAN9B;;AAAA,sBAOMA,IAAI,KAAKF,KAAK,CAACK,aAPrB;AAAA;AAAA;AAAA;;AAAA,sBAO0C,IAAIvD,KAAJ,2CAA6CoD,IAA7C,EAP1C;;AAAA;AASQI,gBAAAA,WATR,GAS0CZ,IAAI,CAACS,aAAL,CACtCI,+BADsC,0BAEtClB,QAAQ,CAACE,SAF6B,yDAEtC,qBAAoBiB,KAFkB,CAT1C;AAcQC,gBAAAA,WAdR,GAcsBjD,MAAM,yBAAC6B,QAAQ,CAACE,SAAV,yDAAC,qBAAoBmB,gBAArB,CAd5B;AAeQC,gBAAAA,SAfR,GAeoBnD,MAAM,yBAAC6B,QAAQ,CAACE,SAAV,yDAAC,qBAAoBqB,qBAArB,CAf1B;AAAA;AAAA,uBAgByB,KAAK9B,IAAL,CAAU2B,WAAV,EAAuBE,SAAvB,CAhBzB;;AAAA;AAgBQE,gBAAAA,QAhBR;AAAA,mDAkBSC,eAAe,CAACD,QAAD,EAAWb,KAAX,EAAkBM,WAAlB,CAlBxB;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;kFAqBA;AAAA;;AAAA;AAAA;AAAA;AAAA;AACQS,gBAAAA,UADR,GACqB5F,aAAa,CAACa,MAAd,GAAuB,CAD5C;AAAA;AAAA,uBAE2B,KAAK8C,IAAL,CAAU,KAAKC,QAAL,GAAgBgC,UAA1B,EAAsCA,UAAtC,CAF3B;;AAAA;AAEQC,gBAAAA,UAFR;;AAAA,sBAIMA,UAAU,CAACC,KAAX,CAAiB,CAAjB,EAAoBhC,QAApB,OAAmC9D,aAJzC;AAAA;AAAA;AAAA;;AAAA,sBAKU,IAAI2B,KAAJ,CAAU,0BAAV,CALV;;AAAA;AAQQoE,gBAAAA,YARR,GAQuBF,UAAU,CAACG,YAAX,CAAwB,CAAxB,CARvB;AASQC,gBAAAA,cATR,GASyB,KAAKrC,QAAL,GAAgBmC,YAAhB,GAA+BH,UATxD;;AAAA,sBAUMK,cAAc,GAAGjG,aAAa,CAACa,MAVrC;AAAA;AAAA;AAAA;;AAAA,sBAWU,IAAIc,KAAJ,CAAU,uBAAV,CAXV;;AAAA;AAAA;AAAA,uBAc4B,KAAKgC,IAAL,CAAUsC,cAAV,EAA0BF,YAA1B,CAd5B;;AAAA;AAcQG,gBAAAA,WAdR;AAAA,wCAiBqB3B,IAAI,CAAC4B,kBAAL,CAAwBD,WAAxB,CAjBrB,EAiBS3F,QAjBT,yBAiBSA,QAjBT;AAAA,mDAkBSA,QAlBT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;iFA5EA,mBAAsBqC,QAAtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBACyB2B,IAAI,CAAC6B,KAAL,CAAWxD,QAAX,CADzB;;AAAA;AACQyD,gBAAAA,QADR;AAAA;AAAA,uBAE+B9B,IAAI,CAAC+B,KAAL,CAAW1D,QAAX,CAF/B;;AAAA;AAEQ2D,gBAAAA,cAFR;AAIQnD,gBAAAA,MAJR,GAIiBmB,IAAI,CAACiC,KAAL,CAAWC,IAAX,CAAgB7B,SAAhB,EAA2B2B,cAA3B,CAJjB;AAKQ/C,gBAAAA,OALR,GAKkBe,IAAI,CAACmC,MAAL,CAAYD,IAAZ,CAAiB7B,SAAjB,EAA4B2B,cAA5B,CALlB;AAAA,mDAOS,IAAI1D,qBAAJ,CAA0BO,MAA1B,EAAkCI,OAAlC,EAA2C6C,QAAQ,CAAC5C,IAApD,CAPT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;mFAUA,mBAAwBR,MAAxB;AAAA;AAAA;AAAA;AAAA;AAAA;AACQG,gBAAAA,MADR,GACiB,SAATA,MAAS,CAACuD,QAAD,EAAmB9F,MAAnB;AAAA,yBACb+F,OAAO,CAACC,OAAR,CAAgB5D,MAAM,CAAC6C,KAAP,CAAaa,QAAb,EAAuBA,QAAQ,GAAG9F,MAAlC,CAAhB,CADa;AAAA,iBADjB;;AAGQ2C,gBAAAA,OAHR,GAGkB,SAAVA,OAAU;AAAA,yBAAMoD,OAAO,CAACC,OAAR,EAAN;AAAA,iBAHlB;;AAAA,mDAIS,IAAIhE,qBAAJ,CAA0BO,MAA1B,EAAkCI,OAAlC,EAA2CP,MAAM,CAACpC,MAAlD,CAJT;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;;;;;AA2FF,SAASiG,YAAT,CACE/B,IADF,EAEEgC,QAFF,EAGEC,MAHF,EAIEC,KAJF,EAKEC,IALF,EAMS;AACP,MAAI,EAAEH,QAAQ,IAAII,sBAAd,CAAJ,EAAmC;AACjC,UAAM,IAAIxF,KAAJ,6BAA+BoF,QAA/B,EAAN;AACD;;AACD,SAAOI,uBAAeJ,QAAf,EAAyBD,YAAzB,CAAsC/B,IAAtC,EAA4CiC,MAA5C,EAAoDC,KAApD,EAA2DC,IAA3D,CAAP;AACD;;AAED,SAASvB,eAAT,CACE1C,MADF,EAEEmE,MAFF,EAGEjC,WAHF,EAIe;AACb,MAAM6B,MAAoB,GAAG;AAC3B/D,IAAAA,MAAM,EAANA,MAD2B;AAE3BoE,IAAAA,MAAM,EAAE,CAFmB;AAG3B5D,IAAAA,IAAI,EAAER,MAAM,CAACpC;AAHc,GAA7B;AAMA,MAAMyG,IAAiB,GAAG;AACxBC,IAAAA,OAAO,EAAE,EADe;AAExBC,IAAAA,OAAO,EAAE,EAFe;AAGxBC,IAAAA,MAAM,EAAE,EAHgB;AAIxBR,IAAAA,KAAK,EAAE;AAJiB,GAA1B;;AAQA,SAAOD,MAAM,CAACK,MAAP,GAAgBL,MAAM,CAACvD,IAA9B,EAAoC;AAIlC,gCAA6Bc,IAAI,CAACmD,gBAAL,CAAsBV,MAAM,CAAC/D,MAA7B,CAA7B;AAAA,QAAO0E,UAAP,yBAAOA,UAAP;AAAA,QAAmB9G,OAAnB,yBAAmBA,MAAnB;;AACAmG,IAAAA,MAAM,CAACK,MAAP,IAAiBxG,OAAjB;AAEA,QAAM+G,QAAQ,GAAGrD,IAAI,CAACS,aAAL,CAAmB6C,uBAAnB,EAA6BF,UAAU,CAAC5C,IAAxC,CAAjB;AAEA,QAAI+C,QAA4B,GAAG,IAAnC;;AACA,YAAQF,QAAR;AACE,WAAK,WAAL;AACEE,QAAAA,QAAQ,GAAGC,cAAc,CAACf,MAAD,EAASW,UAAT,EAAqBP,MAArB,EAA6BjC,WAA7B,CAAzB;AACA;;AACF,WAAK,cAAL;AACE2C,QAAAA,QAAQ,GAAGE,gBAAgB,CAAChB,MAAD,EAASW,UAAT,EAAqBP,MAArB,EAA6BjC,WAA7B,CAA3B;AACA;;AACF;AACE,cAAM,IAAIxD,KAAJ,8BAAgCiG,QAAhC,EAAN;AARJ;;AAWAzF,IAAAA,KAAK,CAAC8F,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2Bb,IAAI,CAACC,OAAhC,EAAyCO,QAAQ,CAACP,OAAlD;AACApF,IAAAA,KAAK,CAAC8F,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2Bb,IAAI,CAACE,OAAhC,EAAyCM,QAAQ,CAACN,OAAlD;AACArF,IAAAA,KAAK,CAAC8F,SAAN,CAAgBC,IAAhB,CAAqBC,KAArB,CAA2Bb,IAAI,CAACG,MAAhC,EAAwCK,QAAQ,CAACL,MAAjD;AACAH,IAAAA,IAAI,CAACL,KAAL,IAAca,QAAQ,CAACb,KAAvB;AACD;;AAED,SAAOK,IAAP;AACD;;AAED,SAASS,cAAT,CACEf,MADF,EAEEoB,MAFF,EAGEhB,MAHF,EAIEjC,WAJF,EAKe;AAAA;;AACb,MAAMkD,SAAS,GAAGrB,MAAM,CAACK,MAAP,GAAgBe,MAAM,CAACE,oBAAzC;AACA,MAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAV,0DAAG,sBAAyBC,UAA5C;AAmBA,MAAIC,UAAU,GAAG1B,MAAjB;;AACA,MAAI7B,WAAW,KAAK,cAApB,EAAoC;AAClC,QAAMwD,SAAS,GAAGC,WAAW,CAACC,OAAZ,CAChB1D,WADgB,EAEhB6B,MAAM,CAAC/D,MAAP,CAAc6C,KAAd,CAAoBkB,MAAM,CAACK,MAA3B,EAAmCgB,SAAnC,CAFgB,EAGhBD,MAAM,CAACU,sBAHS,CAAlB;AAKAJ,IAAAA,UAAU,GAAG;AACXzF,MAAAA,MAAM,EAAE0F,SADG;AAEXtB,MAAAA,MAAM,EAAE,CAFG;AAGX5D,MAAAA,IAAI,EAAEkF,SAAS,CAAC9H;AAHL,KAAb;AAKAmG,IAAAA,MAAM,CAACK,MAAP,GAAgBgB,SAAhB;AACD;;AAGD,MAAMU,cAAc,GAAGxE,IAAI,CAACS,aAAL,CACrBgE,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBS,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAI/G,KAAJ,CAAUoG,UAAV,CAAd;;AACA,MAAInB,MAAM,CAAC+B,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAGpC,YAAY,CAAC5G,kBAAD,EAAqB6I,cAArB,EAAqCL,UAArC,EAAiDH,UAAjD,EAA8D;AAClFa,MAAAA,QAAQ,EAAE7E,IAAI,CAAC8E,WAAL,CAAiBjC,MAAM,CAAC+B,SAAxB,CADwE;AAElFG,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLJ,IAAAA,OAAO,CAACK,IAAR,CAAa,CAAb;AACD;;AAGD,MAAMC,cAAc,GAAGjF,IAAI,CAACS,aAAL,CACrBgE,uBADqB,4BAErBZ,MAAM,CAACI,gBAFc,2DAErB,uBAAyBiB,yBAFJ,CAAvB;AAKA,MAAIC,OAAO,GAAG,IAAIvH,KAAJ,CAAUoG,UAAV,CAAd;;AACA,MAAInB,MAAM,CAACuC,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAG5C,YAAY,CAAC5G,kBAAD,EAAqBsJ,cAArB,EAAqCd,UAArC,EAAiDH,UAAjD,EAA8D;AAClFa,MAAAA,QAAQ,EAAE7E,IAAI,CAAC8E,WAAL,CAAiBjC,MAAM,CAACuC,SAAxB,CADwE;AAElFL,MAAAA,eAAe,EAAE;AAFiE,KAA9D,CAAtB;AAKD,GAND,MAMO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AACD,MAAIK,iBAAiB,GAAG,CAAxB;;AArEa,8CAsEMF,OAtEN;AAAA;;AAAA;AAsEb,2DAA4B;AAAA,UAAjBG,IAAiB;;AAC1B,UAAIA,IAAI,KAAKzC,MAAM,CAACuC,SAApB,EAA+B;AAC7BC,QAAAA,iBAAiB;AAClB;AACF;AA1EY;AAAA;AAAA;AAAA;AAAA;;AA6Eb,MAAME,aAAa,GAAGvF,IAAI,CAACS,aAAL,CACpBgE,uBADoB,4BAEpBZ,MAAM,CAACI,gBAFa,2DAEpB,uBAAyBzB,QAFL,CAAtB;AAIA,MAAMU,MAAM,GAAGX,YAAY,CAACM,MAAM,CAAClC,aAAR,EAAwB4E,aAAxB,EAAuCpB,UAAvC,EAAmDkB,iBAAnD,EAAsE;AAC/FG,IAAAA,UAAU,EAAE3C,MAAM,CAAC2C,UAD4E;AAE/FX,IAAAA,QAAQ,EAAEhC,MAAM,CAAC2C;AAF8E,GAAtE,CAA3B;AASA,SAAO;AACLvC,IAAAA,OAAO,EAAEkC,OADJ;AAELnC,IAAAA,OAAO,EAAE2B,OAFJ;AAGLzB,IAAAA,MAAM,EAANA,MAHK;AAILR,IAAAA,KAAK,EAAEsB;AAJF,GAAP;AAMD;;AAED,SAASP,gBAAT,CACEhB,MADF,EAEEoB,MAFF,EAGEhB,MAHF,EAIEjC,WAJF,EAKe;AAAA;;AACb,MAAMkD,SAAS,GAAGrB,MAAM,CAACK,MAAP,GAAgBe,MAAM,CAACE,oBAAzC;AAEA,MAAMC,UAAU,6BAAGH,MAAM,CAAC4B,mBAAV,2DAAG,uBAA4BvB,UAA/C;AAEA,MAAMmB,iBAAiB,GAAGrB,UAAU,8BAAGH,MAAM,CAAC4B,mBAAV,2DAAG,uBAA4BC,SAA/B,CAApC;AACA,MAAMH,aAAa,GAAGvF,IAAI,CAACS,aAAL,CACpBgE,uBADoB,4BAEpBZ,MAAM,CAAC4B,mBAFa,2DAEpB,uBAA4BjD,QAFR,CAAtB;AAOA,MAAImC,OAAO,GAAG,IAAI/G,KAAJ,CAAUoG,UAAV,CAAd;;AACA,MAAInB,MAAM,CAAC+B,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAGpC,YAAY,CAAC5G,kBAAD,EAAqBC,sBAArB,EAA6C6G,MAA7C,EAAqDuB,UAArD,EAAkE;AACtFa,MAAAA,QAAQ,EAAE7E,IAAI,CAAC8E,WAAL,CAAiBjC,MAAM,CAAC+B,SAAxB,CAD4E;AAEtFG,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLJ,IAAAA,OAAO,CAACK,IAAR,CAAa,CAAb;AACD;;AAID,MAAIG,OAAO,GAAG,IAAIvH,KAAJ,CAAUoG,UAAV,CAAd;;AACA,MAAInB,MAAM,CAACuC,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,OAAO,GAAG5C,YAAY,CAAC5G,kBAAD,EAAqBC,sBAArB,EAA6C6G,MAA7C,EAAqDuB,UAArD,EAAkE;AACtFa,MAAAA,QAAQ,EAAE7E,IAAI,CAAC8E,WAAL,CAAiBjC,MAAM,CAACuC,SAAxB,CAD4E;AAEtFL,MAAAA,eAAe,EAAE;AAFqE,KAAlE,CAAtB;AAID,GALD,MAKO;AACLI,IAAAA,OAAO,CAACH,IAAR,CAAa,CAAb;AACD;;AAGD,MAAIW,eAAe,GAAGlD,MAAtB;;AAEA,gCAAIoB,MAAM,CAAC4B,mBAAX,mDAAI,uBAA4BG,aAAhC,EAA+C;AAC7C,QAAMxB,SAAS,GAAGC,WAAW,CAACC,OAAZ,CAChB1D,WADgB,EAEhB6B,MAAM,CAAC/D,MAAP,CAAc6C,KAAd,CAAoBkB,MAAM,CAACK,MAA3B,EAAmCgB,SAAnC,CAFgB,EAGhBD,MAAM,CAACU,sBAHS,CAAlB;AAMAoB,IAAAA,eAAe,GAAG;AAChBjH,MAAAA,MAAM,EAAE0F,SADQ;AAEhBtB,MAAAA,MAAM,EAAE,CAFQ;AAGhB5D,MAAAA,IAAI,EAAEkF,SAAS,CAAC9H;AAHA,KAAlB;AAMAmG,IAAAA,MAAM,CAACK,MAAP,GAAgBgB,SAAhB;AACD;;AAED,MAAMZ,MAAM,GAAGX,YAAY,CACzBM,MAAM,CAAClC,aADkB,EAEzB4E,aAFyB,EAGzBI,eAHyB,EAIzBN,iBAJyB,EAKzB;AACEG,IAAAA,UAAU,EAAE3C,MAAM,CAAC2C,UADrB;AAEEX,IAAAA,QAAQ,EAAEhC,MAAM,CAAC2C;AAFnB,GALyB,CAA3B;AAWA,SAAO;AACLvC,IAAAA,OAAO,EAAEkC,OADJ;AAELnC,IAAAA,OAAO,EAAE2B,OAFJ;AAGLzB,IAAAA,MAAM,EAANA,MAHK;AAILR,IAAAA,KAAK,EAAEsB;AAJF,GAAP;AAMD;;AAED,SAAS1G,YAAT,CACEuI,cADF,EAEE/C,MAFF,EAGEgD,GAHF,EAQE;AACA,MAAM5J,MAAwB,GAAG,EAAjC;AACA,MAAIY,IAAI,GAAGgG,MAAX;;AACA,OAAK,IAAIiD,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGD,GAApB,EAAyBC,CAAC,EAA1B,EAA8B;AAC5B,QAAMC,aAAa,GAAGH,cAAc,CAAC/I,IAAD,CAApC;AAEA,QAAMmJ,cAAc,GAClBnJ,IAAI,GAAG,CAAP,GAAWkD,IAAI,CAACS,aAAL,CAAmByF,kCAAnB,EAAwCF,aAAa,CAACG,eAAtD,CAAX,GAAqF,MADvF;AAGA,QAAIC,QAAQ,GAAG,KAAf;AACA,QAAIC,QAAQ,GAAG,KAAf;;AACA,YAAQJ,cAAR;AACE,WAAK,UAAL;AACE;;AACF,WAAK,UAAL;AACEG,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF,WAAK,UAAL;AACEC,QAAAA,QAAQ,GAAG,IAAX;AACA;;AACF;AACE,cAAM,IAAIjJ,KAAJ,CAAU,kCAAV,CAAN;AAVJ;;AAaA,QAAI4I,aAAa,CAACzI,YAAd,GAA8B,CAAlC,EAAqC;AACnC,UAAM+I,GAAG,GAAGhJ,YAAY,CAACuI,cAAD,EAAiB/I,IAAI,GAAG,CAAxB,EAA2BkJ,aAAa,CAACzI,YAAzC,CAAxB;AACAT,MAAAA,IAAI,GAAGwJ,GAAG,CAACxJ,IAAX;AACAZ,MAAAA,MAAM,CAAC8J,aAAa,CAACO,IAAf,CAAN,GAA6B;AAE3BH,QAAAA,QAAQ,EAARA,QAF2B;AAG3BC,QAAAA,QAAQ,EAARA,QAH2B;AAI3BG,QAAAA,MAAM,EAAEF,GAAG,CAACpK;AAJe,OAA7B;AAMD,KATD,MASO;AACL,UAAIuK,WAAW,GAAGzG,IAAI,CAACS,aAAL,CAAmBC,mBAAnB,EAAyBsF,aAAa,CAACxF,IAAvC,CAAlB;;AAEA,UAAIwF,aAAa,CAACU,cAAlB,EAAkC;AAChCD,QAAAA,WAAW,GAAGzG,IAAI,CAACS,aAAL,CAAmBkG,4BAAnB,EAAkCX,aAAa,CAACU,cAAhD,CAAd;AACD;;AAEDxK,MAAAA,MAAM,CAAC8J,aAAa,CAACO,IAAf,CAAN,GAA6B;AAC3B/F,QAAAA,IAAI,EAAEiG,WADqB;AAE3BjB,QAAAA,UAAU,EAAEQ,aAAa,CAACY,WAFC;AAG3BR,QAAAA,QAAQ,EAARA,QAH2B;AAI3BC,QAAAA,QAAQ,EAARA;AAJ2B,OAA7B;AAMAvJ,MAAAA,IAAI;AACL;AACF;;AACD,SAAO;AAACZ,IAAAA,MAAM,EAANA,MAAD;AAAS4G,IAAAA,MAAM,EAANA,MAAT;AAAiBhG,IAAAA,IAAI,EAAJA;AAAjB,GAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from './codecs';\nimport * as Compression from './compression';\nimport {\n  ParquetBuffer,\n  ParquetCodec,\n  ParquetCompression,\n  ParquetData,\n  ParquetField,\n  ParquetRecord,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from './schema/declare';\nimport {ParquetSchema} from './schema/schema';\nimport * as Shred from './schema/shred';\n// tslint:disable-next-line:max-line-length\nimport {\n  ColumnChunk,\n  CompressionCodec,\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  FileMetaData,\n  PageHeader,\n  PageType,\n  RowGroup,\n  SchemaElement,\n  Type\n} from './parquet-thrift';\nimport * as Util from './util';\n// import Fs = require('fs');\n\n/**\n * Parquet File Magic String\n */\nconst PARQUET_MAGIC = 'PAR1';\n\n/**\n * Parquet File Format Version\n */\nconst PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\n/**\n * A parquet cursor is used to retrieve rows from a parquet file in order\n */\nexport class ParquetCursor<T> implements AsyncIterable<T> {\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n  public columnList: string[][];\n  public rowGroup: ParquetRecord[];\n  public rowGroupIndex: number;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is usually not recommended to call this constructor directly except for\n   * advanced and internal use cases. Consider using getCursor() on the\n   * ParquetReader instead\n   */\n  constructor(\n    metadata: FileMetaData,\n    envelopeReader: ParquetEnvelopeReader,\n    schema: ParquetSchema,\n    columnList: string[][]\n  ) {\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    this.schema = schema;\n    this.columnList = columnList;\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Retrieve the next row from the cursor. Returns a row or NULL if the end\n   * of the file was reached\n   */\n  async next<T = any>(): Promise<T> {\n    if (this.rowGroup.length === 0) {\n      if (this.rowGroupIndex >= this.metadata.row_groups.length) {\n        // @ts-ignore\n        return null;\n      }\n      const rowBuffer = await this.envelopeReader.readRowGroup(\n        this.schema,\n        this.metadata.row_groups[this.rowGroupIndex],\n        this.columnList\n      );\n      this.rowGroup = Shred.materializeRecords(this.schema, rowBuffer);\n      this.rowGroupIndex++;\n    }\n    return this.rowGroup.shift() as any;\n  }\n\n  /**\n   * Rewind the cursor the the beginning of the file\n   */\n  rewind(): void {\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    let done = false;\n    return {\n      next: async () => {\n        if (done) {\n          return {done, value: null};\n        }\n        const value = await this.next();\n        if (value === null) {\n          return {done: true, value};\n        }\n        return {done: false, value};\n      },\n      return: async () => {\n        done = true;\n        return {done, value: null};\n      },\n      throw: async () => {\n        done = true;\n        return {done: true, value: null};\n      }\n    };\n  }\n}\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * Open the parquet file pointed to by the specified path and return a new\n   * parquet reader\n   */\n  static async openFile<T>(filePath: string): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n\n  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {\n    const fileStat = await Util.fstat(filePath);\n    const fileDescriptor = await Util.fopen(filePath);\n\n    const readFn = Util.fread.bind(undefined, fileDescriptor);\n    const closeFn = Util.fclose.bind(undefined, fileDescriptor);\n\n    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);\n  }\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n  }\n\n  async readHeader(): Promise<void> {\n    const buf = await this.read(0, PARQUET_MAGIC.length);\n\n    if (buf.toString() !== PARQUET_MAGIC) {\n      throw new Error('not valid parquet file');\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && Util.fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = Util.getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n    if (type !== field.primitiveType) throw new Error(`chunk type not matching schema: ${type}`);\n\n    const compression: ParquetCompression = Util.getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    const pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n\n    return decodeDataPages(pagesBuf, field, compression);\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    if (trailerBuf.slice(4).toString() !== PARQUET_MAGIC) {\n      throw new Error('not a valid parquet file');\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error('invalid metadata size');\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = Util.decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\nfunction decodeDataPages(\n  buffer: Buffer,\n  column: ParquetField,\n  compression: ParquetCompression\n): ParquetData {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetData = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    count: 0\n  };\n\n  // @ts-ignore size can be undefined\n  while (cursor.offset < cursor.size) {\n    // const pageHeader = new parquet_thrift.PageHeader();\n    // cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer);\n\n    const {pageHeader, length} = Util.decodePageHeader(cursor.buffer);\n    cursor.offset += length;\n\n    const pageType = Util.getThriftEnum(PageType, pageHeader.type);\n\n    let pageData: ParquetData | null = null;\n    switch (pageType) {\n      case 'DATA_PAGE':\n        pageData = decodeDataPage(cursor, pageHeader, column, compression);\n        break;\n      case 'DATA_PAGE_V2':\n        pageData = decodeDataPageV2(cursor, pageHeader, column, compression);\n        break;\n      default:\n        throw new Error(`invalid page type: ${pageType}`);\n    }\n\n    Array.prototype.push.apply(data.rlevels, pageData.rlevels);\n    Array.prototype.push.apply(data.dlevels, pageData.dlevels);\n    Array.prototype.push.apply(data.values, pageData.values);\n    data.count += pageData.count;\n  }\n\n  return data;\n}\n\nfunction decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  column: ParquetField,\n  compression: ParquetCompression\n): ParquetData {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  // const info = {\n  //   path: opts.column.path.join('.'),\n  //   valueEncoding,\n  //   dLevelEncoding,\n  //   rLevelEncoding,\n  //   cursorOffset: cursor.offset,\n  //   cursorEnd,\n  //   cusrorSize: cursor.size,\n  //   header,\n  //   opts,\n  //   buffer: cursor.buffer.toJSON(),\n  //   values: null as any[],\n  //   valBuf: null as any\n  // };\n  // Fs.writeFileSync(`dump/${info.path}.ts.json`, JSON.stringify(info, null, 2));\n\n  /* uncompress page */\n  let dataCursor = cursor;\n  if (compression !== 'UNCOMPRESSED') {\n    const valuesBuf = Compression.inflate(\n      compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header?.encoding!\n  ) as ParquetCodec;\n  const values = decodeValues(column.primitiveType!, valueEncoding, dataCursor, valueCountNonNull, {\n    typeLength: column.typeLength,\n    bitWidth: column.typeLength\n  });\n\n  // info.valBuf = uncursor.buffer.toJSON();\n  // info.values = values;\n  // Fs.writeFileSync(`dump/${info.path}.ts.json`, JSON.stringify(info, null, 2));\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!\n  };\n}\n\nfunction decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  column: ParquetField,\n  compression: ParquetCompression\n): ParquetData {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = Util.getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: Util.getBitWidth(column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = Compression.inflate(\n      compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const values = decodeValues(\n    column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    {\n      typeLength: column.typeLength,\n      bitWidth: column.typeLength\n    }\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!\n  };\n}\n\nfunction decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? Util.getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      let logicalType = Util.getThriftEnum(Type, schemaElement.type!);\n\n      if (schemaElement.converted_type) {\n        logicalType = Util.getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n"],"file":"reader.js"}

package/dist/es5/parquetjs/schema/declare.js

@@ -7,17 +7,19 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ParquetBuffer = void 0;
 
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
 
-class ParquetBuffer {
-  constructor(rowCount = 0, columnData = {}) {
-    (0, _defineProperty2.default)(this, "rowCount", void 0);
-    (0, _defineProperty2.default)(this, "columnData", void 0);
-    this.rowCount = rowCount;
-    this.columnData = columnData;
-  }
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
-}
+var ParquetBuffer = function ParquetBuffer() {
+  var rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
+  var columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+  (0, _classCallCheck2.default)(this, ParquetBuffer);
+  (0, _defineProperty2.default)(this, "rowCount", void 0);
+  (0, _defineProperty2.default)(this, "columnData", void 0);
+  this.rowCount = rowCount;
+  this.columnData = columnData;
+};
 
 exports.ParquetBuffer = ParquetBuffer;
 //# sourceMappingURL=declare.js.map

package/dist/es5/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/schema/declare.ts"],"names":["ParquetBuffer","constructor","rowCount","columnData"],"mappings":";;;;;;;;;;;AAqFO,MAAMA,aAAN,CAAoB;AAGzBC,EAAAA,WAAW,CAACC,QAAgB,GAAG,CAApB,EAAuBC,UAAuC,GAAG,EAAjE,EAAqE;AAAA;AAAA;AAC9E,SAAKD,QAAL,GAAgBA,QAAhB;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACD;;AANwB","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nexport type ParquetCodec = 'PLAIN' | 'RLE';\nexport type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"file":"declare.js"}
+{"version":3,"sources":["../../../../src/parquetjs/schema/declare.ts"],"names":["ParquetBuffer","rowCount","columnData"],"mappings":";;;;;;;;;;;;;IAqFaA,a,GAGX,yBAAgF;AAAA,MAApEC,QAAoE,uEAAjD,CAAiD;AAAA,MAA9CC,UAA8C,uEAAJ,EAAI;AAAA;AAAA;AAAA;AAC9E,OAAKD,QAAL,GAAgBA,QAAhB;AACA,OAAKC,UAAL,GAAkBA,UAAlB;AACD,C","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nexport type ParquetCodec = 'PLAIN' | 'RLE';\nexport type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"file":"declare.js"}

package/dist/es5/parquetjs/schema/schema.js

@@ -7,6 +7,10 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ParquetSchema = void 0;
 
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
 var _codecs = require("../codecs");
@@ -17,8 +21,9 @@ var _shred = require("./shred");
 
 var _types = require("./types");
 
-class ParquetSchema {
-  constructor(schema) {
+var ParquetSchema = function () {
+  function ParquetSchema(schema) {
+    (0, _classCallCheck2.default)(this, ParquetSchema);
     (0, _defineProperty2.default)(this, "schema", void 0);
     (0, _defineProperty2.default)(this, "fields", void 0);
     (0, _defineProperty2.default)(this, "fieldList", void 0);
@@ -27,66 +32,74 @@ class ParquetSchema {
     this.fieldList = listFields(this.fields);
   }
 
-  findField(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
-    } else {
-      path = path.slice(0);
-    }
-
-    let n = this.fields;
-
-    for (; path.length > 1; path.shift()) {
-      n = n[path[0]].fields;
-    }
-
-    return n[path[0]];
-  }
-
-  findFieldBranch(path) {
-    if (typeof path === 'string') {
-      path = path.split(',');
-    }
-
-    const branch = [];
-    let n = this.fields;
+  (0, _createClass2.default)(ParquetSchema, [{
+    key: "findField",
+    value: function findField(path) {
+      if (typeof path === 'string') {
+        path = path.split(',');
+      } else {
+        path = path.slice(0);
+      }
 
-    for (; path.length > 0; path.shift()) {
-      branch.push(n[path[0]]);
+      var n = this.fields;
 
-      if (path.length > 1) {
+      for (; path.length > 1; path.shift()) {
         n = n[path[0]].fields;
       }
-    }
-
-    return branch;
-  }
 
-  shredRecord(record, buffer) {
-    (0, _shred.shredRecord)(this, record, buffer);
-  }
+      return n[path[0]];
+    }
+  }, {
+    key: "findFieldBranch",
+    value: function findFieldBranch(path) {
+      if (typeof path === 'string') {
+        path = path.split(',');
+      }
 
-  materializeRecords(buffer) {
-    return (0, _shred.materializeRecords)(this, buffer);
-  }
+      var branch = [];
+      var n = this.fields;
 
-  compress(type) {
-    setCompress(this.schema, type);
-    setCompress(this.fields, type);
-    return this;
-  }
+      for (; path.length > 0; path.shift()) {
+        branch.push(n[path[0]]);
 
-  buffer() {
-    return (0, _shred.shredBuffer)(this);
-  }
+        if (path.length > 1) {
+          n = n[path[0]].fields;
+        }
+      }
 
-}
+      return branch;
+    }
+  }, {
+    key: "shredRecord",
+    value: function shredRecord(record, buffer) {
+      (0, _shred.shredRecord)(this, record, buffer);
+    }
+  }, {
+    key: "materializeRecords",
+    value: function materializeRecords(buffer) {
+      return (0, _shred.materializeRecords)(this, buffer);
+    }
+  }, {
+    key: "compress",
+    value: function compress(type) {
+      setCompress(this.schema, type);
+      setCompress(this.fields, type);
+      return this;
+    }
+  }, {
+    key: "buffer",
+    value: function buffer() {
+      return (0, _shred.shredBuffer)(this);
+    }
+  }]);
+  return ParquetSchema;
+}();
 
 exports.ParquetSchema = ParquetSchema;
 
 function setCompress(schema, type) {
-  for (const name in schema) {
-    const node = schema[name];
+  for (var name in schema) {
+    var node = schema[name];
 
     if (node.fields) {
       setCompress(node.fields, type);
@@ -97,15 +110,15 @@ function setCompress(schema, type) {
 }
 
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
-  const fieldList = {};
+  var fieldList = {};
 
-  for (const name in schema) {
-    const opts = schema[name];
-    const required = !opts.optional;
-    const repeated = Boolean(opts.repeated);
-    let rLevelMax = rLevelParentMax;
-    let dLevelMax = dLevelParentMax;
-    let repetitionType = 'REQUIRED';
+  for (var name in schema) {
+    var opts = schema[name];
+    var required = !opts.optional;
+    var repeated = Boolean(opts.repeated);
+    var rLevelMax = rLevelParentMax;
+    var dLevelMax = dLevelParentMax;
+    var repetitionType = 'REQUIRED';
 
     if (!required) {
       repetitionType = 'OPTIONAL';
@@ -119,22 +132,23 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
     }
 
     if (opts.fields) {
-      const cpath = path.concat([name]);
+      var _cpath = path.concat([name]);
+
       fieldList[name] = {
-        name,
-        path: cpath,
-        key: cpath.join(),
-        repetitionType,
-        rLevelMax,
-        dLevelMax,
+        name: name,
+        path: _cpath,
+        key: _cpath.join(),
+        repetitionType: repetitionType,
+        rLevelMax: rLevelMax,
+        dLevelMax: dLevelMax,
         isNested: true,
         fieldCount: Object.keys(opts.fields).length,
-        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+        fields: buildFields(opts.fields, rLevelMax, dLevelMax, _cpath)
       };
       continue;
     }
 
-    const typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
+    var typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
 
     if (!typeDef) {
       throw new Error("invalid parquet type: ".concat(opts.type));
@@ -152,19 +166,19 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       throw new Error("unsupported compression method: ".concat(opts.compression));
     }
 
-    const cpath = path.concat([name]);
+    var cpath = path.concat([name]);
     fieldList[name] = {
-      name,
+      name: name,
       primitiveType: typeDef.primitiveType,
       originalType: typeDef.originalType,
       path: cpath,
       key: cpath.join(),
-      repetitionType,
+      repetitionType: repetitionType,
       encoding: opts.encoding,
       compression: opts.compression,
       typeLength: opts.typeLength || typeDef.typeLength,
-      rLevelMax,
-      dLevelMax
+      rLevelMax: rLevelMax,
+      dLevelMax: dLevelMax
     };
   }
 
@@ -172,9 +186,9 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
 }
 
 function listFields(fields) {
-  let list = [];
+  var list = [];
 
-  for (const k in fields) {
+  for (var k in fields) {
     list.push(fields[k]);
 
     if (fields[k].isNested) {

package/dist/es5/parquetjs/schema/schema.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/schema/schema.ts"],"names":["ParquetSchema","constructor","schema","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","record","buffer","materializeRecords","compress","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","list","k"],"mappings":";;;;;;;;;;;AAEA;;AACA;;AAUA;;AACA;;AAKO,MAAMA,aAAN,CAAoB;AAQzBC,EAAAA,WAAW,CAACC,MAAD,EAA2B;AAAA;AAAA;AAAA;AACpC,SAAKA,MAAL,GAAcA,MAAd;AACA,SAAKC,MAAL,GAAcC,WAAW,CAACF,MAAD,EAAS,CAAT,EAAY,CAAZ,EAAe,EAAf,CAAzB;AACA,SAAKG,SAAL,GAAiBC,UAAU,CAAC,KAAKH,MAAN,CAA3B;AACD;;AAKDI,EAAAA,SAAS,CAACC,IAAD,EAAwC;AAC/C,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,MAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD,KAHD,MAGO;AAELD,MAAAA,IAAI,GAAGA,IAAI,CAACE,KAAL,CAAW,CAAX,CAAP;AACD;;AAED,QAAIC,CAAC,GAAG,KAAKR,MAAb;;AACA,WAAOK,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCF,MAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWL,MAAf;AACD;;AAED,WAAOQ,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAR;AACD;;AAKDM,EAAAA,eAAe,CAACN,IAAD,EAA0C;AACvD,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,MAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD;;AACD,UAAMM,MAAsB,GAAG,EAA/B;AACA,QAAIJ,CAAC,GAAG,KAAKR,MAAb;;AACA,WAAOK,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCE,MAAAA,MAAM,CAACC,IAAP,CAAYL,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAb;;AACA,UAAIA,IAAI,CAACI,MAAL,GAAc,CAAlB,EAAqB;AACnBD,QAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWL,MAAf;AACD;AACF;;AACD,WAAOY,MAAP;AACD;;AAEDE,EAAAA,WAAW,CAACC,MAAD,EAAwBC,MAAxB,EAAqD;AAC9D,4BAAY,IAAZ,EAAkBD,MAAlB,EAA0BC,MAA1B;AACD;;AAEDC,EAAAA,kBAAkB,CAACD,MAAD,EAAyC;AACzD,WAAO,+BAAmB,IAAnB,EAAyBA,MAAzB,CAAP;AACD;;AAEDE,EAAAA,QAAQ,CAACC,IAAD,EAAiC;AACvCC,IAAAA,WAAW,CAAC,KAAKrB,MAAN,EAAcoB,IAAd,CAAX;AACAC,IAAAA,WAAW,CAAC,KAAKpB,MAAN,EAAcmB,IAAd,CAAX;AACA,WAAO,IAAP;AACD;;AAEDH,EAAAA,MAAM,GAAkB;AACtB,WAAO,wBAAY,IAAZ,CAAP;AACD;;AArEwB;;;;AAwE3B,SAASI,WAAT,CAAqBrB,MAArB,EAAkCoB,IAAlC,EAA4D;AAC1D,OAAK,MAAME,IAAX,IAAmBtB,MAAnB,EAA2B;AACzB,UAAMuB,IAAI,GAAGvB,MAAM,CAACsB,IAAD,CAAnB;;AACA,QAAIC,IAAI,CAACtB,MAAT,EAAiB;AACfoB,MAAAA,WAAW,CAACE,IAAI,CAACtB,MAAN,EAAcmB,IAAd,CAAX;AACD,KAFD,MAEO;AACLG,MAAAA,IAAI,CAACC,WAAL,GAAmBJ,IAAnB;AACD;AACF;AACF;;AAGD,SAASlB,WAAT,CACEF,MADF,EAEEyB,eAFF,EAGEC,eAHF,EAIEpB,IAJF,EAKgC;AAC9B,QAAMH,SAAuC,GAAG,EAAhD;;AAEA,OAAK,MAAMmB,IAAX,IAAmBtB,MAAnB,EAA2B;AACzB,UAAM2B,IAAI,GAAG3B,MAAM,CAACsB,IAAD,CAAnB;AAGA,UAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAvB;AACA,UAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAN,CAAxB;AACA,QAAIE,SAAS,GAAGP,eAAhB;AACA,QAAIQ,SAAS,GAAGP,eAAhB;AAEA,QAAIQ,cAA8B,GAAG,UAArC;;AACA,QAAI,CAACN,QAAL,EAAe;AACbM,MAAAA,cAAc,GAAG,UAAjB;AACAD,MAAAA,SAAS;AACV;;AACD,QAAIH,QAAJ,EAAc;AACZI,MAAAA,cAAc,GAAG,UAAjB;AACAF,MAAAA,SAAS;AACT,UAAIJ,QAAJ,EAAcK,SAAS;AACxB;;AAGD,QAAIN,IAAI,CAAC1B,MAAT,EAAiB;AACf,YAAMkC,KAAK,GAAG7B,IAAI,CAAC8B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAnB,MAAAA,SAAS,CAACmB,IAAD,CAAT,GAAkB;AAChBA,QAAAA,IADgB;AAEhBhB,QAAAA,IAAI,EAAE6B,KAFU;AAGhBE,QAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EAHW;AAIhBJ,QAAAA,cAJgB;AAKhBF,QAAAA,SALgB;AAMhBC,QAAAA,SANgB;AAOhBM,QAAAA,QAAQ,EAAE,IAPM;AAQhBC,QAAAA,UAAU,EAAEC,MAAM,CAACC,IAAP,CAAYf,IAAI,CAAC1B,MAAjB,EAAyBS,MARrB;AAShBT,QAAAA,MAAM,EAAEC,WAAW,CAACyB,IAAI,CAAC1B,MAAN,EAAc+B,SAAd,EAAyBC,SAAzB,EAAoCE,KAApC;AATH,OAAlB;AAWA;AACD;;AAED,UAAMQ,OAAY,GAAGC,6BAAsBjB,IAAI,CAACP,IAA3B,CAArB;;AACA,QAAI,CAACuB,OAAL,EAAc;AACZ,YAAM,IAAIE,KAAJ,iCAAmClB,IAAI,CAACP,IAAxC,EAAN;AACD;;AAEDO,IAAAA,IAAI,CAACmB,QAAL,GAAgBnB,IAAI,CAACmB,QAAL,IAAiB,OAAjC;;AACA,QAAI,EAAEnB,IAAI,CAACmB,QAAL,IAAiBC,sBAAnB,CAAJ,EAAwC;AACtC,YAAM,IAAIF,KAAJ,yCAA2ClB,IAAI,CAACmB,QAAhD,EAAN;AACD;;AAEDnB,IAAAA,IAAI,CAACH,WAAL,GAAmBG,IAAI,CAACH,WAAL,IAAoB,cAAvC;;AACA,QAAI,EAAEG,IAAI,CAACH,WAAL,IAAoBwB,wCAAtB,CAAJ,EAAwD;AACtD,YAAM,IAAIH,KAAJ,2CAA6ClB,IAAI,CAACH,WAAlD,EAAN;AACD;;AAGD,UAAMW,KAAK,GAAG7B,IAAI,CAAC8B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAnB,IAAAA,SAAS,CAACmB,IAAD,CAAT,GAAkB;AAChBA,MAAAA,IADgB;AAEhB2B,MAAAA,aAAa,EAAEN,OAAO,CAACM,aAFP;AAGhBC,MAAAA,YAAY,EAAEP,OAAO,CAACO,YAHN;AAIhB5C,MAAAA,IAAI,EAAE6B,KAJU;AAKhBE,MAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EALW;AAMhBJ,MAAAA,cANgB;AAOhBY,MAAAA,QAAQ,EAAEnB,IAAI,CAACmB,QAPC;AAQhBtB,MAAAA,WAAW,EAAEG,IAAI,CAACH,WARF;AAShB2B,MAAAA,UAAU,EAAExB,IAAI,CAACwB,UAAL,IAAmBR,OAAO,CAACQ,UATvB;AAUhBnB,MAAAA,SAVgB;AAWhBC,MAAAA;AAXgB,KAAlB;AAaD;;AACD,SAAO9B,SAAP;AACD;;AAED,SAASC,UAAT,CAAoBH,MAApB,EAA0E;AACxE,MAAImD,IAAoB,GAAG,EAA3B;;AACA,OAAK,MAAMC,CAAX,IAAgBpD,MAAhB,EAAwB;AACtBmD,IAAAA,IAAI,CAACtC,IAAL,CAAUb,MAAM,CAACoD,CAAD,CAAhB;;AACA,QAAIpD,MAAM,CAACoD,CAAD,CAAN,CAAUd,QAAd,EAAwB;AACtBa,MAAAA,IAAI,GAAGA,IAAI,CAAChB,MAAL,CAAYhC,UAAU,CAACH,MAAM,CAACoD,CAAD,CAAN,CAAUpD,MAAX,CAAtB,CAAP;AACD;AACF;;AACD,SAAOmD,IAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"file":"schema.js"}
+{"version":3,"sources":["../../../../src/parquetjs/schema/schema.ts"],"names":["ParquetSchema","schema","fields","buildFields","fieldList","listFields","path","split","slice","n","length","shift","branch","push","record","buffer","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","list","k"],"mappings":";;;;;;;;;;;;;;;AAEA;;AACA;;AAUA;;AACA;;IAKaA,a;AAQX,yBAAYC,MAAZ,EAAsC;AAAA;AAAA;AAAA;AAAA;AACpC,SAAKA,MAAL,GAAcA,MAAd;AACA,SAAKC,MAAL,GAAcC,WAAW,CAACF,MAAD,EAAS,CAAT,EAAY,CAAZ,EAAe,EAAf,CAAzB;AACA,SAAKG,SAAL,GAAiBC,UAAU,CAAC,KAAKH,MAAN,CAA3B;AACD;;;;WAKD,mBAAUI,IAAV,EAAiD;AAC/C,UAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,QAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD,OAHD,MAGO;AAELD,QAAAA,IAAI,GAAGA,IAAI,CAACE,KAAL,CAAW,CAAX,CAAP;AACD;;AAED,UAAIC,CAAC,GAAG,KAAKP,MAAb;;AACA,aAAOI,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCF,QAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWJ,MAAf;AACD;;AAED,aAAOO,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAR;AACD;;;WAKD,yBAAgBA,IAAhB,EAAyD;AACvD,UAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAE5BA,QAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,GAAX,CAAP;AACD;;AACD,UAAMK,MAAsB,GAAG,EAA/B;AACA,UAAIH,CAAC,GAAG,KAAKP,MAAb;;AACA,aAAOI,IAAI,CAACI,MAAL,GAAc,CAArB,EAAwBJ,IAAI,CAACK,KAAL,EAAxB,EAAsC;AACpCC,QAAAA,MAAM,CAACC,IAAP,CAAYJ,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAb;;AACA,YAAIA,IAAI,CAACI,MAAL,GAAc,CAAlB,EAAqB;AACnBD,UAAAA,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAD,CAAL,CAAD,CAAWJ,MAAf;AACD;AACF;;AACD,aAAOU,MAAP;AACD;;;WAED,qBAAYE,MAAZ,EAAmCC,MAAnC,EAAgE;AAC9D,8BAAY,IAAZ,EAAkBD,MAAlB,EAA0BC,MAA1B;AACD;;;WAED,4BAAmBA,MAAnB,EAA2D;AACzD,aAAO,+BAAmB,IAAnB,EAAyBA,MAAzB,CAAP;AACD;;;WAED,kBAASC,IAAT,EAAyC;AACvCC,MAAAA,WAAW,CAAC,KAAKhB,MAAN,EAAce,IAAd,CAAX;AACAC,MAAAA,WAAW,CAAC,KAAKf,MAAN,EAAcc,IAAd,CAAX;AACA,aAAO,IAAP;AACD;;;WAED,kBAAwB;AACtB,aAAO,wBAAY,IAAZ,CAAP;AACD;;;;;;;AAGH,SAASC,WAAT,CAAqBhB,MAArB,EAAkCe,IAAlC,EAA4D;AAC1D,OAAK,IAAME,IAAX,IAAmBjB,MAAnB,EAA2B;AACzB,QAAMkB,IAAI,GAAGlB,MAAM,CAACiB,IAAD,CAAnB;;AACA,QAAIC,IAAI,CAACjB,MAAT,EAAiB;AACfe,MAAAA,WAAW,CAACE,IAAI,CAACjB,MAAN,EAAcc,IAAd,CAAX;AACD,KAFD,MAEO;AACLG,MAAAA,IAAI,CAACC,WAAL,GAAmBJ,IAAnB;AACD;AACF;AACF;;AAGD,SAASb,WAAT,CACEF,MADF,EAEEoB,eAFF,EAGEC,eAHF,EAIEhB,IAJF,EAKgC;AAC9B,MAAMF,SAAuC,GAAG,EAAhD;;AAEA,OAAK,IAAMc,IAAX,IAAmBjB,MAAnB,EAA2B;AACzB,QAAMsB,IAAI,GAAGtB,MAAM,CAACiB,IAAD,CAAnB;AAGA,QAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAvB;AACA,QAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAN,CAAxB;AACA,QAAIE,SAAS,GAAGP,eAAhB;AACA,QAAIQ,SAAS,GAAGP,eAAhB;AAEA,QAAIQ,cAA8B,GAAG,UAArC;;AACA,QAAI,CAACN,QAAL,EAAe;AACbM,MAAAA,cAAc,GAAG,UAAjB;AACAD,MAAAA,SAAS;AACV;;AACD,QAAIH,QAAJ,EAAc;AACZI,MAAAA,cAAc,GAAG,UAAjB;AACAF,MAAAA,SAAS;AACT,UAAIJ,QAAJ,EAAcK,SAAS;AACxB;;AAGD,QAAIN,IAAI,CAACrB,MAAT,EAAiB;AACf,UAAM6B,MAAK,GAAGzB,IAAI,CAAC0B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;;AACAd,MAAAA,SAAS,CAACc,IAAD,CAAT,GAAkB;AAChBA,QAAAA,IAAI,EAAJA,IADgB;AAEhBZ,QAAAA,IAAI,EAAEyB,MAFU;AAGhBE,QAAAA,GAAG,EAAEF,MAAK,CAACG,IAAN,EAHW;AAIhBJ,QAAAA,cAAc,EAAdA,cAJgB;AAKhBF,QAAAA,SAAS,EAATA,SALgB;AAMhBC,QAAAA,SAAS,EAATA,SANgB;AAOhBM,QAAAA,QAAQ,EAAE,IAPM;AAQhBC,QAAAA,UAAU,EAAEC,MAAM,CAACC,IAAP,CAAYf,IAAI,CAACrB,MAAjB,EAAyBQ,MARrB;AAShBR,QAAAA,MAAM,EAAEC,WAAW,CAACoB,IAAI,CAACrB,MAAN,EAAc0B,SAAd,EAAyBC,SAAzB,EAAoCE,MAApC;AATH,OAAlB;AAWA;AACD;;AAED,QAAMQ,OAAY,GAAGC,6BAAsBjB,IAAI,CAACP,IAA3B,CAArB;;AACA,QAAI,CAACuB,OAAL,EAAc;AACZ,YAAM,IAAIE,KAAJ,iCAAmClB,IAAI,CAACP,IAAxC,EAAN;AACD;;AAEDO,IAAAA,IAAI,CAACmB,QAAL,GAAgBnB,IAAI,CAACmB,QAAL,IAAiB,OAAjC;;AACA,QAAI,EAAEnB,IAAI,CAACmB,QAAL,IAAiBC,sBAAnB,CAAJ,EAAwC;AACtC,YAAM,IAAIF,KAAJ,yCAA2ClB,IAAI,CAACmB,QAAhD,EAAN;AACD;;AAEDnB,IAAAA,IAAI,CAACH,WAAL,GAAmBG,IAAI,CAACH,WAAL,IAAoB,cAAvC;;AACA,QAAI,EAAEG,IAAI,CAACH,WAAL,IAAoBwB,wCAAtB,CAAJ,EAAwD;AACtD,YAAM,IAAIH,KAAJ,2CAA6ClB,IAAI,CAACH,WAAlD,EAAN;AACD;;AAGD,QAAMW,KAAK,GAAGzB,IAAI,CAAC0B,MAAL,CAAY,CAACd,IAAD,CAAZ,CAAd;AACAd,IAAAA,SAAS,CAACc,IAAD,CAAT,GAAkB;AAChBA,MAAAA,IAAI,EAAJA,IADgB;AAEhB2B,MAAAA,aAAa,EAAEN,OAAO,CAACM,aAFP;AAGhBC,MAAAA,YAAY,EAAEP,OAAO,CAACO,YAHN;AAIhBxC,MAAAA,IAAI,EAAEyB,KAJU;AAKhBE,MAAAA,GAAG,EAAEF,KAAK,CAACG,IAAN,EALW;AAMhBJ,MAAAA,cAAc,EAAdA,cANgB;AAOhBY,MAAAA,QAAQ,EAAEnB,IAAI,CAACmB,QAPC;AAQhBtB,MAAAA,WAAW,EAAEG,IAAI,CAACH,WARF;AAShB2B,MAAAA,UAAU,EAAExB,IAAI,CAACwB,UAAL,IAAmBR,OAAO,CAACQ,UATvB;AAUhBnB,MAAAA,SAAS,EAATA,SAVgB;AAWhBC,MAAAA,SAAS,EAATA;AAXgB,KAAlB;AAaD;;AACD,SAAOzB,SAAP;AACD;;AAED,SAASC,UAAT,CAAoBH,MAApB,EAA0E;AACxE,MAAI8C,IAAoB,GAAG,EAA3B;;AACA,OAAK,IAAMC,CAAX,IAAgB/C,MAAhB,EAAwB;AACtB8C,IAAAA,IAAI,CAACnC,IAAL,CAAUX,MAAM,CAAC+C,CAAD,CAAhB;;AACA,QAAI/C,MAAM,CAAC+C,CAAD,CAAN,CAAUd,QAAd,EAAwB;AACtBa,MAAAA,IAAI,GAAGA,IAAI,CAAChB,MAAL,CAAY3B,UAAU,CAACH,MAAM,CAAC+C,CAAD,CAAN,CAAU/C,MAAX,CAAtB,CAAP;AACD;AACF;;AACD,SAAO8C,IAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"file":"schema.js"}