npm - @loaders.gl/parquet - Versions diffs - 3.0.10 → 3.0.14 - Mend

@loaders.gl/parquet 3.0.10 → 3.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (25) hide show

package/dist/dist.es5.min.js +1 -1
package/dist/dist.min.js +1 -1
package/dist/es5/parquet-loader.js +1 -1
package/dist/es5/parquet-writer.js +1 -1
package/dist/esm/parquet-loader.js +1 -1
package/dist/esm/parquet-writer.js +1 -1
package/dist/esm/parquetjs/codecs/plain.js +3 -3
package/dist/esm/parquetjs/codecs/plain.js.map +1 -1
package/dist/esm/parquetjs/codecs/rle.js +1 -1
package/dist/esm/parquetjs/codecs/rle.js.map +1 -1
package/dist/esm/parquetjs/compression.js +2 -2
package/dist/esm/parquetjs/compression.js.map +1 -1
package/dist/esm/parquetjs/reader.js +3 -3
package/dist/esm/parquetjs/reader.js.map +1 -1
package/dist/esm/parquetjs/schema/schema.js +3 -3
package/dist/esm/parquetjs/schema/schema.js.map +1 -1
package/dist/esm/parquetjs/schema/shred.js +2 -2
package/dist/esm/parquetjs/schema/shred.js.map +1 -1
package/dist/esm/parquetjs/schema/types.js +18 -18
package/dist/esm/parquetjs/schema/types.js.map +1 -1
package/dist/esm/parquetjs/writer.js +1 -1
package/dist/esm/parquetjs/writer.js.map +1 -1
package/dist/parquet-worker.js +1 -1
package/dist/parquet-worker.js.map +1 -1
package/package.json +4 -4

package/dist/esm/parquetjs/schema/types.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../../../src/parquetjs/schema/types.ts"],"names":["BSON","PARQUET_LOGICAL_TYPES","BOOLEAN","primitiveType","toPrimitive","toPrimitive_BOOLEAN","fromPrimitive","fromPrimitive_BOOLEAN","INT32","toPrimitive_INT32","INT64","toPrimitive_INT64","INT96","toPrimitive_INT96","FLOAT","toPrimitive_FLOAT","DOUBLE","toPrimitive_DOUBLE","BYTE_ARRAY","toPrimitive_BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","originalType","toPrimitive_UTF8","fromPrimitive_UTF8","TIME_MILLIS","toPrimitive_TIME_MILLIS","TIME_MICROS","toPrimitive_TIME_MICROS","DATE","toPrimitive_DATE","fromPrimitive_DATE","TIMESTAMP_MILLIS","toPrimitive_TIMESTAMP_MILLIS","fromPrimitive_TIMESTAMP_MILLIS","TIMESTAMP_MICROS","toPrimitive_TIMESTAMP_MICROS","fromPrimitive_TIMESTAMP_MICROS","UINT_8","toPrimitive_UINT8","UINT_16","toPrimitive_UINT16","UINT_32","toPrimitive_UINT32","UINT_64","toPrimitive_UINT64","INT_8","toPrimitive_INT8","INT_16","toPrimitive_INT16","INT_32","INT_64","JSON","toPrimitive_JSON","fromPrimitive_JSON","toPrimitive_BSON","fromPrimitive_BSON","INTERVAL","typeLength","toPrimitive_INTERVAL","fromPrimitive_INTERVAL","type","value","Error","Boolean","v","parseFloat","isNaN","parseInt","Buffer","from","toString","stringify","parse","serialize","deserialize","kMillisPerDay","Date","getTime","months","days","milliseconds","buf","alloc","writeUInt32LE","readUInt32LE","millis"],"mappings":"AAEA,OAAOA,IAAP,MAAiB,MAAjB;AAWA,OAAO,MAAMC,qBAA0D,GAAG;AACxEC,EAAAA,OAAO,EAAE;AACPC,IAAAA,aAAa,EAAE,SADR;AAEPC,IAAAA,WAAW,EAAEC,mBAFN;AAGPC,IAAAA,aAAa,EAAEC;AAHR,GAD+D;AAMxEC,EAAAA,KAAK,EAAE;AACLL,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEK;AAFR,GANiE;AAUxEC,EAAAA,KAAK,EAAE;AACLP,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEO;AAFR,GAViE;AAcxEC,EAAAA,KAAK,EAAE;AACLT,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAES;AAFR,GAdiE;AAkBxEC,EAAAA,KAAK,EAAE;AACLX,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEW;AAFR,GAlBiE;AAsBxEC,EAAAA,MAAM,EAAE;AACNb,IAAAA,aAAa,EAAE,QADT;AAENC,IAAAA,WAAW,EAAEa;AAFP,GAtBgE;AA0BxEC,EAAAA,UAAU,EAAE;AACVf,IAAAA,aAAa,EAAE,YADL;AAEVC,IAAAA,WAAW,EAAEe;AAFH,GA1B4D;AA8BxEC,EAAAA,oBAAoB,EAAE;AACpBjB,IAAAA,aAAa,EAAE,sBADK;AAEpBC,IAAAA,WAAW,EAAEe;AAFO,GA9BkD;AAkCxEE,EAAAA,IAAI,EAAE;AACJlB,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEmB,gBAHT;AAIJjB,IAAAA,aAAa,EAAEkB;AAJX,GAlCkE;AAwCxEC,EAAAA,WAAW,EAAE;AACXtB,IAAAA,aAAa,EAAE,OADJ;AAEXmB,IAAAA,YAAY,EAAE,aAFH;AAGXlB,IAAAA,WAAW,EAAEsB;AAHF,GAxC2D;AA6CxEC,EAAAA,WAAW,EAAE;AACXxB,IAAAA,aAAa,EAAE,OADJ;AAEXmB,IAAAA,YAAY,EAAE,aAFH;AAGXlB,IAAAA,WAAW,EAAEwB;AAHF,GA7C2D;AAkDxEC,EAAAA,IAAI,EAAE;AACJ1B,IAAAA,aAAa,EAAE,OADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAE0B,gBAHT;AAIJxB,IAAAA,aAAa,EAAEyB;AAJX,GAlDkE;AAwDxEC,EAAAA,gBAAgB,EAAE;AAChB7B,IAAAA,aAAa,EAAE,OADC;AAEhBmB,IAAAA,YAAY,EAAE,kBAFE;AAGhBlB,IAAAA,WAAW,EAAE6B,4BAHG;AAIhB3B,IAAAA,aAAa,EAAE4B;AAJC,GAxDsD;AA8DxEC,EAAAA,gBAAgB,EAAE;AAChBhC,IAAAA,aAAa,EAAE,OADC;AAEhBmB,IAAAA,YAAY,EAAE,kBAFE;AAGhBlB,IAAAA,WAAW,EAAEgC,4BAHG;AAIhB9B,IAAAA,aAAa,EAAE+B;AAJC,GA9DsD;AAoExEC,EAAAA,MAAM,EAAE;AACNnC,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEmC;AAHP,GApEgE;AAyExEC,EAAAA,OAAO,EAAE;AACPrC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEqC;AAHN,GAzE+D;AA8ExEC,EAAAA,OAAO,EAAE;AACPvC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEuC;AAHN,GA9E+D;AAmFxEC,EAAAA,OAAO,EAAE;AACPzC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEyC;AAHN,GAnF+D;AAwFxEC,EAAAA,KAAK,EAAE;AACL3C,IAAAA,aAAa,EAAE,OADV;AAELmB,IAAAA,YAAY,EAAE,OAFT;AAGLlB,IAAAA,WAAW,EAAE2C;AAHR,GAxFiE;AA6FxEC,EAAAA,MAAM,EAAE;AACN7C,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAE6C;AAHP,GA7FgE;AAkGxEC,EAAAA,MAAM,EAAE;AACN/C,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEK;AAHP,GAlGgE;AAuGxE0C,EAAAA,MAAM,EAAE;AACNhD,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEO;AAHP,GAvGgE;AA4GxEyC,EAAAA,IAAI,EAAE;AACJjD,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEiD,gBAHT;AAIJ/C,IAAAA,aAAa,EAAEgD;AAJX,GA5GkE;AAkHxEtD,EAAAA,IAAI,EAAE;AACJG,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEmD,gBAHT;AAIJjD,IAAAA,aAAa,EAAEkD;AAJX,GAlHkE;AAwHxEC,EAAAA,QAAQ,EAAE;AACRtD,IAAAA,aAAa,EAAE,sBADP;AAERmB,IAAAA,YAAY,EAAE,UAFN;AAGRoC,IAAAA,UAAU,EAAE,EAHJ;AAIRtD,IAAAA,WAAW,EAAEuD,oBAJL;AAKRrD,IAAAA,aAAa,EAAEsD;AALP;AAxH8D,CAAnE;AAqIP,OAAO,SAASxD,WAAT,CAAqByD,IAArB,EAAwCC,KAAxC,EAAoD;AACzD,MAAI,EAAED,IAAI,IAAI5D,qBAAV,CAAJ,EAAsC;AACpC,UAAM,IAAI8D,KAAJ,CAAW,iBAAgBF,IAAK,EAAhC,CAAN;AACD;;AAED,SAAO5D,qBAAqB,CAAC4D,IAAD,CAArB,CAA4BzD,WAA5B,CAAwC0D,KAAxC,CAAP;AACD;AAMD,OAAO,SAASxD,aAAT,CAAuBuD,IAAvB,EAA0CC,KAA1C,EAAsD;AAC3D,MAAI,EAAED,IAAI,IAAI5D,qBAAV,CAAJ,EAAsC;AACpC,UAAM,IAAI8D,KAAJ,CAAW,iBAAgBF,IAAK,EAAhC,CAAN;AACD;;AAED,MAAI,mBAAmB5D,qBAAqB,CAAC4D,IAAD,CAA5C,EAAoD;AAAA;;AAClD,oCAAO,0BAAA5D,qBAAqB,CAAC4D,IAAD,CAArB,EAA4BvD,aAAnC,0DAAO,mDAA4CwD,KAA5C,CAAP;AAED;;AACD,SAAOA,KAAP;AACD;;AAED,SAASzD,mBAAT,CAA6ByD,KAA7B,EAAyC;AACvC,SAAOE,OAAO,CAACF,KAAD,CAAd;AACD;;AAED,SAASvD,qBAAT,CAA+BuD,KAA/B,EAA2C;AACzC,SAAOE,OAAO,CAACF,KAAD,CAAd;AACD;;AAED,SAAS/C,iBAAT,CAA2B+C,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGC,UAAU,CAACJ,KAAD,CAApB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,CAAW,4BAA2BD,KAAM,EAA5C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAShD,kBAAT,CAA4B6C,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGC,UAAU,CAACJ,KAAD,CAApB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,CAAW,6BAA4BD,KAAM,EAA7C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASlB,gBAAT,CAA0Be,KAA1B,EAAsC;AACpC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAC,IAAL,IAAaA,CAAC,GAAG,IAAjB,IAAyBE,KAAK,CAACF,CAAD,CAAlC,EAAuC;AACrC,UAAM,IAAIF,KAAJ,CAAW,2BAA0BD,KAAM,EAA3C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAS1B,iBAAT,CAA2BuB,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,IAAb,IAAqBE,KAAK,CAACF,CAAD,CAA9B,EAAmC;AACjC,UAAM,IAAIF,KAAJ,CAAW,4BAA2BD,KAAM,EAA5C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAShB,iBAAT,CAA2Ba,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAC,MAAL,IAAeA,CAAC,GAAG,MAAnB,IAA6BE,KAAK,CAACF,CAAD,CAAtC,EAA2C;AACzC,UAAM,IAAIF,KAAJ,CAAW,4BAA2BD,KAAM,EAA5C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASxB,kBAAT,CAA4BqB,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,MAAb,IAAuBE,KAAK,CAACF,CAAD,CAAhC,EAAqC;AACnC,UAAM,IAAIF,KAAJ,CAAW,6BAA4BD,KAAM,EAA7C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASxD,iBAAT,CAA2BqD,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAC,UAAL,IAAmBA,CAAC,GAAG,UAAvB,IAAqCE,KAAK,CAACF,CAAD,CAA9C,EAAmD;AACjD,UAAM,IAAIF,KAAJ,CAAW,4BAA2BD,KAAM,EAA5C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAStB,kBAAT,CAA4BmB,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,cAAb,IAA+BE,KAAK,CAACF,CAAD,CAAxC,EAA6C;AAC3C,UAAM,IAAIF,KAAJ,CAAW,6BAA4BD,KAAM,EAA7C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAStD,iBAAT,CAA2BmD,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,CAAW,4BAA2BD,KAAM,EAA5C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASpB,kBAAT,CAA4BiB,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,UAAM,IAAIF,KAAJ,CAAW,6BAA4BD,KAAM,EAA7C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASpD,iBAAT,CAA2BiD,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,CAAW,4BAA2BD,KAAM,EAA5C,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAS9C,sBAAT,CAAgC2C,KAAhC,EAA4C;AAC1C,SAAOO,MAAM,CAACC,IAAP,CAAYR,KAAZ,CAAP;AACD;;AAED,SAASvC,gBAAT,CAA0BuC,KAA1B,EAAsC;AACpC,SAAOO,MAAM,CAACC,IAAP,CAAYR,KAAZ,EAAmB,MAAnB,CAAP;AACD;;AAED,SAAStC,kBAAT,CAA4BsC,KAA5B,EAAwC;AACtC,SAAOA,KAAK,CAACS,QAAN,EAAP;AACD;;AAED,SAASlB,gBAAT,CAA0BS,KAA1B,EAAsC;AACpC,SAAOO,MAAM,CAACC,IAAP,CAAYlB,IAAI,CAACoB,SAAL,CAAeV,KAAf,CAAZ,CAAP;AACD;;AAED,SAASR,kBAAT,CAA4BQ,KAA5B,EAAwC;AACtC,SAAOV,IAAI,CAACqB,KAAL,CAAWX,KAAX,CAAP;AACD;;AAED,SAASP,gBAAT,CAA0BO,KAA1B,EAAsC;AACpC,SAAOO,MAAM,CAACC,IAAP,CAAYtE,IAAI,CAAC0E,SAAL,CAAeZ,KAAf,CAAZ,CAAP;AACD;;AAED,SAASN,kBAAT,CAA4BM,KAA5B,EAAwC;AACtC,SAAO9D,IAAI,CAAC2E,WAAL,CAAiBb,KAAjB,CAAP;AACD;;AAED,SAASpC,uBAAT,CAAiCoC,KAAjC,EAA6C;AAC3C,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,kBAAb,IAAmCE,KAAK,CAACF,CAAD,CAA5C,EAAiD;AAC/C,UAAM,IAAIF,KAAJ,CAAW,kCAAiCD,KAAM,EAAlD,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASrC,uBAAT,CAAiCkC,KAAjC,EAA6C;AAC3C,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,UAAM,IAAIF,KAAJ,CAAW,kCAAiCD,KAAM,EAAlD,CAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,MAAMW,aAAa,GAAG,QAAtB;;AAEA,SAAS9C,gBAAT,CAA0BgC,KAA1B,EAAsC;AAEpC,MAAIA,KAAK,YAAYe,IAArB,EAA2B;AACzB,WAAOf,KAAK,CAACgB,OAAN,KAAkBF,aAAzB;AACD;;AAGD;AACE,UAAMX,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,CAAW,2BAA0BD,KAAM,EAA3C,CAAN;AACD;;AAED,WAAOG,CAAP;AACD;AACF;;AAED,SAASlC,kBAAT,CAA4B+B,KAA5B,EAAwC;AACtC,SAAO,IAAIe,IAAJ,CAASf,KAAK,GAAGc,aAAjB,CAAP;AACD;;AAED,SAAS3C,4BAAT,CAAsC6B,KAAtC,EAAkD;AAEhD,MAAIA,KAAK,YAAYe,IAArB,EAA2B;AACzB,WAAOf,KAAK,CAACgB,OAAN,EAAP;AACD;;AAGD;AACE,UAAMb,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,CAAW,uCAAsCD,KAAM,EAAvD,CAAN;AACD;;AAED,WAAOG,CAAP;AACD;AACF;;AAED,SAAS/B,8BAAT,CAAwC4B,KAAxC,EAAoD;AAClD,SAAO,IAAIe,IAAJ,CAASf,KAAT,CAAP;AACD;;AAED,SAAS1B,4BAAT,CAAsC0B,KAAtC,EAAkD;AAEhD,MAAIA,KAAK,YAAYe,IAArB,EAA2B;AACzB,WAAOf,KAAK,CAACgB,OAAN,KAAkB,IAAzB;AACD;;AAGD;AACE,UAAMb,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,CAAW,uCAAsCD,KAAM,EAAvD,CAAN;AACD;;AAED,WAAOG,CAAP;AACD;AACF;;AAED,SAAS5B,8BAAT,CAAwCyB,KAAxC,EAAoD;AAClD,SAAO,IAAIe,IAAJ,CAASf,KAAK,GAAG,IAAjB,CAAP;AACD;;AAED,SAASH,oBAAT,CAA8BG,KAA9B,EAA0C;AACxC,MAAI,CAACA,KAAK,CAACiB,MAAP,IAAiB,CAACjB,KAAK,CAACkB,IAAxB,IAAgC,CAAClB,KAAK,CAACmB,YAA3C,EAAyD;AACvD,UAAM,IAAIlB,KAAJ,CACJ,iFADI,CAAN;AAGD;;AAED,QAAMmB,GAAG,GAAGb,MAAM,CAACc,KAAP,CAAa,EAAb,CAAZ;AAEAD,EAAAA,GAAG,CAACE,aAAJ,CAAkBtB,KAAK,CAACiB,MAAxB,EAAgC,CAAhC;AACAG,EAAAA,GAAG,CAACE,aAAJ,CAAkBtB,KAAK,CAACkB,IAAxB,EAA8B,CAA9B;AACAE,EAAAA,GAAG,CAACE,aAAJ,CAAkBtB,KAAK,CAACmB,YAAxB,EAAsC,CAAtC;AACA,SAAOC,GAAP;AACD;;AAED,SAAStB,sBAAT,CAAgCE,KAAhC,EAA4C;AAC1C,QAAMoB,GAAG,GAAGb,MAAM,CAACC,IAAP,CAAYR,KAAZ,CAAZ;AACA,QAAMiB,MAAM,GAAGG,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAf;AACA,QAAML,IAAI,GAAGE,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAb;AACA,QAAMC,MAAM,GAAGJ,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAf;AAEA,SAAO;AAACN,IAAAA,MAAD;AAASC,IAAAA,IAAT;AAAeC,IAAAA,YAAY,EAAEK;AAA7B,GAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase /\nimport BSON from 'bson';\nimport {OriginalType, ParquetType, PrimitiveType} from './declare';\n\nexport interface ParquetTypeKit {\n primitiveType: PrimitiveType;\n originalType?: OriginalType;\n typeLength?: number;\n toPrimitive: Function;\n fromPrimitive?: Function;\n}\n\nexport const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {\n BOOLEAN: {\n primitiveType: 'BOOLEAN',\n toPrimitive: toPrimitive_BOOLEAN,\n fromPrimitive: fromPrimitive_BOOLEAN\n },\n INT32: {\n primitiveType: 'INT32',\n toPrimitive: toPrimitive_INT32\n },\n INT64: {\n primitiveType: 'INT64',\n toPrimitive: toPrimitive_INT64\n },\n INT96: {\n primitiveType: 'INT96',\n toPrimitive: toPrimitive_INT96\n },\n FLOAT: {\n primitiveType: 'FLOAT',\n toPrimitive: toPrimitive_FLOAT\n },\n DOUBLE: {\n primitiveType: 'DOUBLE',\n toPrimitive: toPrimitive_DOUBLE\n },\n BYTE_ARRAY: {\n primitiveType: 'BYTE_ARRAY',\n toPrimitive: toPrimitive_BYTE_ARRAY\n },\n FIXED_LEN_BYTE_ARRAY: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n toPrimitive: toPrimitive_BYTE_ARRAY\n },\n UTF8: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'UTF8',\n toPrimitive: toPrimitive_UTF8,\n fromPrimitive: fromPrimitive_UTF8\n },\n TIME_MILLIS: {\n primitiveType: 'INT32',\n originalType: 'TIME_MILLIS',\n toPrimitive: toPrimitive_TIME_MILLIS\n },\n TIME_MICROS: {\n primitiveType: 'INT64',\n originalType: 'TIME_MICROS',\n toPrimitive: toPrimitive_TIME_MICROS\n },\n DATE: {\n primitiveType: 'INT32',\n originalType: 'DATE',\n toPrimitive: toPrimitive_DATE,\n fromPrimitive: fromPrimitive_DATE\n },\n TIMESTAMP_MILLIS: {\n primitiveType: 'INT64',\n originalType: 'TIMESTAMP_MILLIS',\n toPrimitive: toPrimitive_TIMESTAMP_MILLIS,\n fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS\n },\n TIMESTAMP_MICROS: {\n primitiveType: 'INT64',\n originalType: 'TIMESTAMP_MICROS',\n toPrimitive: toPrimitive_TIMESTAMP_MICROS,\n fromPrimitive: fromPrimitive_TIMESTAMP_MICROS\n },\n UINT_8: {\n primitiveType: 'INT32',\n originalType: 'UINT_8',\n toPrimitive: toPrimitive_UINT8\n },\n UINT_16: {\n primitiveType: 'INT32',\n originalType: 'UINT_16',\n toPrimitive: toPrimitive_UINT16\n },\n UINT_32: {\n primitiveType: 'INT32',\n originalType: 'UINT_32',\n toPrimitive: toPrimitive_UINT32\n },\n UINT_64: {\n primitiveType: 'INT64',\n originalType: 'UINT_64',\n toPrimitive: toPrimitive_UINT64\n },\n INT_8: {\n primitiveType: 'INT32',\n originalType: 'INT_8',\n toPrimitive: toPrimitive_INT8\n },\n INT_16: {\n primitiveType: 'INT32',\n originalType: 'INT_16',\n toPrimitive: toPrimitive_INT16\n },\n INT_32: {\n primitiveType: 'INT32',\n originalType: 'INT_32',\n toPrimitive: toPrimitive_INT32\n },\n INT_64: {\n primitiveType: 'INT64',\n originalType: 'INT_64',\n toPrimitive: toPrimitive_INT64\n },\n JSON: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'JSON',\n toPrimitive: toPrimitive_JSON,\n fromPrimitive: fromPrimitive_JSON\n },\n BSON: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'BSON',\n toPrimitive: toPrimitive_BSON,\n fromPrimitive: fromPrimitive_BSON\n },\n INTERVAL: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n originalType: 'INTERVAL',\n typeLength: 12,\n toPrimitive: toPrimitive_INTERVAL,\n fromPrimitive: fromPrimitive_INTERVAL\n }\n};\n\n/\n Convert a value from it's native representation to the internal/underlying\n * primitive type\n /\nexport function toPrimitive(type: ParquetType, value: any) {\n if (!(type in PARQUET_LOGICAL_TYPES)) {\n throw new Error(`invalid type: ${type}`);\n }\n\n return PARQUET_LOGICAL_TYPES[type].toPrimitive(value);\n}\n\n/\n Convert a value from it's internal/underlying primitive representation to\n * the native representation\n /\nexport function fromPrimitive(type: ParquetType, value: any) {\n if (!(type in PARQUET_LOGICAL_TYPES)) {\n throw new Error(`invalid type: ${type}`);\n }\n\n if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {\n return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value);\n // tslint:disable-next-line:no-else-after-return\n }\n return value;\n}\n\nfunction toPrimitive_BOOLEAN(value: any) {\n return Boolean(value);\n}\n\nfunction fromPrimitive_BOOLEAN(value: any) {\n return Boolean(value);\n}\n\nfunction toPrimitive_FLOAT(value: any) {\n const v = parseFloat(value);\n if (isNaN(v)) {\n throw new Error(`invalid value for FLOAT: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_DOUBLE(value: any) {\n const v = parseFloat(value);\n if (isNaN(v)) {\n throw new Error(`invalid value for DOUBLE: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT8(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x80 \|\| v > 0x7f \|\| isNaN(v)) {\n throw new Error(`invalid value for INT8: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT8(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xff \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT8: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT16(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x8000 \|\| v > 0x7fff \|\| isNaN(v)) {\n throw new Error(`invalid value for INT16: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT16(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xffff \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT16: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT32(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x80000000 \|\| v > 0x7fffffff \|\| isNaN(v)) {\n throw new Error(`invalid value for INT32: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT32(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xffffffffffff \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT32: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT64(value: any) {\n const v = parseInt(value, 10);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT64: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT64(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT64: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT96(value: any) {\n const v = parseInt(value, 10);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT96: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_BYTE_ARRAY(value: any) {\n return Buffer.from(value);\n}\n\nfunction toPrimitive_UTF8(value: any) {\n return Buffer.from(value, 'utf8');\n}\n\nfunction fromPrimitive_UTF8(value: any) {\n return value.toString();\n}\n\nfunction toPrimitive_JSON(value: any) {\n return Buffer.from(JSON.stringify(value));\n}\n\nfunction fromPrimitive_JSON(value: any) {\n return JSON.parse(value);\n}\n\nfunction toPrimitive_BSON(value: any) {\n return Buffer.from(BSON.serialize(value));\n}\n\nfunction fromPrimitive_BSON(value: any) {\n return BSON.deserialize(value);\n}\n\nfunction toPrimitive_TIME_MILLIS(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xffffffffffffffff \|\| isNaN(v)) {\n throw new Error(`invalid value for TIME_MILLIS: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_TIME_MICROS(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for TIME_MICROS: ${value}`);\n }\n\n return v;\n}\n\nconst kMillisPerDay = 86400000;\n\nfunction toPrimitive_DATE(value: any) {\n / convert from date /\n if (value instanceof Date) {\n return value.getTime() / kMillisPerDay;\n }\n\n / convert from integer /\n {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for DATE: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_DATE(value: any) {\n return new Date(value kMillisPerDay);\n}\n\nfunction toPrimitive_TIMESTAMP_MILLIS(value: any) {\n /* convert from date /\n if (value instanceof Date) {\n return value.getTime();\n }\n\n / convert from integer /\n {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_TIMESTAMP_MILLIS(value: any) {\n return new Date(value);\n}\n\nfunction toPrimitive_TIMESTAMP_MICROS(value: any) {\n / convert from date /\n if (value instanceof Date) {\n return value.getTime() 1000;\n }\n\n /* convert from integer */\n {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_TIMESTAMP_MICROS(value: any) {\n return new Date(value / 1000);\n}\n\nfunction toPrimitive_INTERVAL(value: any) {\n if (!value.months \|\| !value.days \|\| !value.milliseconds) {\n throw new Error(\n 'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }'\n );\n }\n\n const buf = Buffer.alloc(12);\n\n buf.writeUInt32LE(value.months, 0);\n buf.writeUInt32LE(value.days, 4);\n buf.writeUInt32LE(value.milliseconds, 8);\n return buf;\n}\n\nfunction fromPrimitive_INTERVAL(value: any) {\n const buf = Buffer.from(value);\n const months = buf.readUInt32LE(0);\n const days = buf.readUInt32LE(4);\n const millis = buf.readUInt32LE(8);\n\n return {months, days, milliseconds: millis};\n}\n"],"file":"types.js"}
1	+ {"version":3,"sources":["../../../../src/parquetjs/schema/types.ts"],"names":["BSON","PARQUET_LOGICAL_TYPES","BOOLEAN","primitiveType","toPrimitive","toPrimitive_BOOLEAN","fromPrimitive","fromPrimitive_BOOLEAN","INT32","toPrimitive_INT32","INT64","toPrimitive_INT64","INT96","toPrimitive_INT96","FLOAT","toPrimitive_FLOAT","DOUBLE","toPrimitive_DOUBLE","BYTE_ARRAY","toPrimitive_BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","originalType","toPrimitive_UTF8","fromPrimitive_UTF8","TIME_MILLIS","toPrimitive_TIME_MILLIS","TIME_MICROS","toPrimitive_TIME_MICROS","DATE","toPrimitive_DATE","fromPrimitive_DATE","TIMESTAMP_MILLIS","toPrimitive_TIMESTAMP_MILLIS","fromPrimitive_TIMESTAMP_MILLIS","TIMESTAMP_MICROS","toPrimitive_TIMESTAMP_MICROS","fromPrimitive_TIMESTAMP_MICROS","UINT_8","toPrimitive_UINT8","UINT_16","toPrimitive_UINT16","UINT_32","toPrimitive_UINT32","UINT_64","toPrimitive_UINT64","INT_8","toPrimitive_INT8","INT_16","toPrimitive_INT16","INT_32","INT_64","JSON","toPrimitive_JSON","fromPrimitive_JSON","toPrimitive_BSON","fromPrimitive_BSON","INTERVAL","typeLength","toPrimitive_INTERVAL","fromPrimitive_INTERVAL","type","value","Error","Boolean","v","parseFloat","isNaN","parseInt","Buffer","from","toString","stringify","parse","serialize","deserialize","kMillisPerDay","Date","getTime","months","days","milliseconds","buf","alloc","writeUInt32LE","readUInt32LE","millis"],"mappings":"AAEA,OAAOA,IAAP,MAAiB,MAAjB;AAWA,OAAO,MAAMC,qBAA0D,GAAG;AACxEC,EAAAA,OAAO,EAAE;AACPC,IAAAA,aAAa,EAAE,SADR;AAEPC,IAAAA,WAAW,EAAEC,mBAFN;AAGPC,IAAAA,aAAa,EAAEC;AAHR,GAD+D;AAMxEC,EAAAA,KAAK,EAAE;AACLL,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEK;AAFR,GANiE;AAUxEC,EAAAA,KAAK,EAAE;AACLP,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEO;AAFR,GAViE;AAcxEC,EAAAA,KAAK,EAAE;AACLT,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAES;AAFR,GAdiE;AAkBxEC,EAAAA,KAAK,EAAE;AACLX,IAAAA,aAAa,EAAE,OADV;AAELC,IAAAA,WAAW,EAAEW;AAFR,GAlBiE;AAsBxEC,EAAAA,MAAM,EAAE;AACNb,IAAAA,aAAa,EAAE,QADT;AAENC,IAAAA,WAAW,EAAEa;AAFP,GAtBgE;AA0BxEC,EAAAA,UAAU,EAAE;AACVf,IAAAA,aAAa,EAAE,YADL;AAEVC,IAAAA,WAAW,EAAEe;AAFH,GA1B4D;AA8BxEC,EAAAA,oBAAoB,EAAE;AACpBjB,IAAAA,aAAa,EAAE,sBADK;AAEpBC,IAAAA,WAAW,EAAEe;AAFO,GA9BkD;AAkCxEE,EAAAA,IAAI,EAAE;AACJlB,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEmB,gBAHT;AAIJjB,IAAAA,aAAa,EAAEkB;AAJX,GAlCkE;AAwCxEC,EAAAA,WAAW,EAAE;AACXtB,IAAAA,aAAa,EAAE,OADJ;AAEXmB,IAAAA,YAAY,EAAE,aAFH;AAGXlB,IAAAA,WAAW,EAAEsB;AAHF,GAxC2D;AA6CxEC,EAAAA,WAAW,EAAE;AACXxB,IAAAA,aAAa,EAAE,OADJ;AAEXmB,IAAAA,YAAY,EAAE,aAFH;AAGXlB,IAAAA,WAAW,EAAEwB;AAHF,GA7C2D;AAkDxEC,EAAAA,IAAI,EAAE;AACJ1B,IAAAA,aAAa,EAAE,OADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAE0B,gBAHT;AAIJxB,IAAAA,aAAa,EAAEyB;AAJX,GAlDkE;AAwDxEC,EAAAA,gBAAgB,EAAE;AAChB7B,IAAAA,aAAa,EAAE,OADC;AAEhBmB,IAAAA,YAAY,EAAE,kBAFE;AAGhBlB,IAAAA,WAAW,EAAE6B,4BAHG;AAIhB3B,IAAAA,aAAa,EAAE4B;AAJC,GAxDsD;AA8DxEC,EAAAA,gBAAgB,EAAE;AAChBhC,IAAAA,aAAa,EAAE,OADC;AAEhBmB,IAAAA,YAAY,EAAE,kBAFE;AAGhBlB,IAAAA,WAAW,EAAEgC,4BAHG;AAIhB9B,IAAAA,aAAa,EAAE+B;AAJC,GA9DsD;AAoExEC,EAAAA,MAAM,EAAE;AACNnC,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEmC;AAHP,GApEgE;AAyExEC,EAAAA,OAAO,EAAE;AACPrC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEqC;AAHN,GAzE+D;AA8ExEC,EAAAA,OAAO,EAAE;AACPvC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEuC;AAHN,GA9E+D;AAmFxEC,EAAAA,OAAO,EAAE;AACPzC,IAAAA,aAAa,EAAE,OADR;AAEPmB,IAAAA,YAAY,EAAE,SAFP;AAGPlB,IAAAA,WAAW,EAAEyC;AAHN,GAnF+D;AAwFxEC,EAAAA,KAAK,EAAE;AACL3C,IAAAA,aAAa,EAAE,OADV;AAELmB,IAAAA,YAAY,EAAE,OAFT;AAGLlB,IAAAA,WAAW,EAAE2C;AAHR,GAxFiE;AA6FxEC,EAAAA,MAAM,EAAE;AACN7C,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAE6C;AAHP,GA7FgE;AAkGxEC,EAAAA,MAAM,EAAE;AACN/C,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEK;AAHP,GAlGgE;AAuGxE0C,EAAAA,MAAM,EAAE;AACNhD,IAAAA,aAAa,EAAE,OADT;AAENmB,IAAAA,YAAY,EAAE,QAFR;AAGNlB,IAAAA,WAAW,EAAEO;AAHP,GAvGgE;AA4GxEyC,EAAAA,IAAI,EAAE;AACJjD,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEiD,gBAHT;AAIJ/C,IAAAA,aAAa,EAAEgD;AAJX,GA5GkE;AAkHxEtD,EAAAA,IAAI,EAAE;AACJG,IAAAA,aAAa,EAAE,YADX;AAEJmB,IAAAA,YAAY,EAAE,MAFV;AAGJlB,IAAAA,WAAW,EAAEmD,gBAHT;AAIJjD,IAAAA,aAAa,EAAEkD;AAJX,GAlHkE;AAwHxEC,EAAAA,QAAQ,EAAE;AACRtD,IAAAA,aAAa,EAAE,sBADP;AAERmB,IAAAA,YAAY,EAAE,UAFN;AAGRoC,IAAAA,UAAU,EAAE,EAHJ;AAIRtD,IAAAA,WAAW,EAAEuD,oBAJL;AAKRrD,IAAAA,aAAa,EAAEsD;AALP;AAxH8D,CAAnE;AAqIP,OAAO,SAASxD,WAAT,CAAqByD,IAArB,EAAwCC,KAAxC,EAAoD;AACzD,MAAI,EAAED,IAAI,IAAI5D,qBAAV,CAAJ,EAAsC;AACpC,UAAM,IAAI8D,KAAJ,yBAA2BF,IAA3B,EAAN;AACD;;AAED,SAAO5D,qBAAqB,CAAC4D,IAAD,CAArB,CAA4BzD,WAA5B,CAAwC0D,KAAxC,CAAP;AACD;AAMD,OAAO,SAASxD,aAAT,CAAuBuD,IAAvB,EAA0CC,KAA1C,EAAsD;AAC3D,MAAI,EAAED,IAAI,IAAI5D,qBAAV,CAAJ,EAAsC;AACpC,UAAM,IAAI8D,KAAJ,yBAA2BF,IAA3B,EAAN;AACD;;AAED,MAAI,mBAAmB5D,qBAAqB,CAAC4D,IAAD,CAA5C,EAAoD;AAAA;;AAClD,oCAAO,0BAAA5D,qBAAqB,CAAC4D,IAAD,CAArB,EAA4BvD,aAAnC,0DAAO,mDAA4CwD,KAA5C,CAAP;AAED;;AACD,SAAOA,KAAP;AACD;;AAED,SAASzD,mBAAT,CAA6ByD,KAA7B,EAAyC;AACvC,SAAOE,OAAO,CAACF,KAAD,CAAd;AACD;;AAED,SAASvD,qBAAT,CAA+BuD,KAA/B,EAA2C;AACzC,SAAOE,OAAO,CAACF,KAAD,CAAd;AACD;;AAED,SAAS/C,iBAAT,CAA2B+C,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGC,UAAU,CAACJ,KAAD,CAApB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCD,KAAtC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAShD,kBAAT,CAA4B6C,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGC,UAAU,CAACJ,KAAD,CAApB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,qCAAuCD,KAAvC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASlB,gBAAT,CAA0Be,KAA1B,EAAsC;AACpC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAC,IAAL,IAAaA,CAAC,GAAG,IAAjB,IAAyBE,KAAK,CAACF,CAAD,CAAlC,EAAuC;AACrC,UAAM,IAAIF,KAAJ,mCAAqCD,KAArC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAS1B,iBAAT,CAA2BuB,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,IAAb,IAAqBE,KAAK,CAACF,CAAD,CAA9B,EAAmC;AACjC,UAAM,IAAIF,KAAJ,oCAAsCD,KAAtC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAShB,iBAAT,CAA2Ba,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAC,MAAL,IAAeA,CAAC,GAAG,MAAnB,IAA6BE,KAAK,CAACF,CAAD,CAAtC,EAA2C;AACzC,UAAM,IAAIF,KAAJ,oCAAsCD,KAAtC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASxB,kBAAT,CAA4BqB,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,MAAb,IAAuBE,KAAK,CAACF,CAAD,CAAhC,EAAqC;AACnC,UAAM,IAAIF,KAAJ,qCAAuCD,KAAvC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASxD,iBAAT,CAA2BqD,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAC,UAAL,IAAmBA,CAAC,GAAG,UAAvB,IAAqCE,KAAK,CAACF,CAAD,CAA9C,EAAmD;AACjD,UAAM,IAAIF,KAAJ,oCAAsCD,KAAtC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAStB,kBAAT,CAA4BmB,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,cAAb,IAA+BE,KAAK,CAACF,CAAD,CAAxC,EAA6C;AAC3C,UAAM,IAAIF,KAAJ,qCAAuCD,KAAvC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAStD,iBAAT,CAA2BmD,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCD,KAAtC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASpB,kBAAT,CAA4BiB,KAA5B,EAAwC;AACtC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,UAAM,IAAIF,KAAJ,qCAAuCD,KAAvC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASpD,iBAAT,CAA2BiD,KAA3B,EAAuC;AACrC,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIK,KAAK,CAACF,CAAD,CAAT,EAAc;AACZ,UAAM,IAAIF,KAAJ,oCAAsCD,KAAtC,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAAS9C,sBAAT,CAAgC2C,KAAhC,EAA4C;AAC1C,SAAOO,MAAM,CAACC,IAAP,CAAYR,KAAZ,CAAP;AACD;;AAED,SAASvC,gBAAT,CAA0BuC,KAA1B,EAAsC;AACpC,SAAOO,MAAM,CAACC,IAAP,CAAYR,KAAZ,EAAmB,MAAnB,CAAP;AACD;;AAED,SAAStC,kBAAT,CAA4BsC,KAA5B,EAAwC;AACtC,SAAOA,KAAK,CAACS,QAAN,EAAP;AACD;;AAED,SAASlB,gBAAT,CAA0BS,KAA1B,EAAsC;AACpC,SAAOO,MAAM,CAACC,IAAP,CAAYlB,IAAI,CAACoB,SAAL,CAAeV,KAAf,CAAZ,CAAP;AACD;;AAED,SAASR,kBAAT,CAA4BQ,KAA5B,EAAwC;AACtC,SAAOV,IAAI,CAACqB,KAAL,CAAWX,KAAX,CAAP;AACD;;AAED,SAASP,gBAAT,CAA0BO,KAA1B,EAAsC;AACpC,SAAOO,MAAM,CAACC,IAAP,CAAYtE,IAAI,CAAC0E,SAAL,CAAeZ,KAAf,CAAZ,CAAP;AACD;;AAED,SAASN,kBAAT,CAA4BM,KAA5B,EAAwC;AACtC,SAAO9D,IAAI,CAAC2E,WAAL,CAAiBb,KAAjB,CAAP;AACD;;AAED,SAASpC,uBAAT,CAAiCoC,KAAjC,EAA6C;AAC3C,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASA,CAAC,GAAG,kBAAb,IAAmCE,KAAK,CAACF,CAAD,CAA5C,EAAiD;AAC/C,UAAM,IAAIF,KAAJ,0CAA4CD,KAA5C,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,SAASrC,uBAAT,CAAiCkC,KAAjC,EAA6C;AAC3C,QAAMG,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,MAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,UAAM,IAAIF,KAAJ,0CAA4CD,KAA5C,EAAN;AACD;;AAED,SAAOG,CAAP;AACD;;AAED,MAAMW,aAAa,GAAG,QAAtB;;AAEA,SAAS9C,gBAAT,CAA0BgC,KAA1B,EAAsC;AAEpC,MAAIA,KAAK,YAAYe,IAArB,EAA2B;AACzB,WAAOf,KAAK,CAACgB,OAAN,KAAkBF,aAAzB;AACD;;AAGD;AACE,UAAMX,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,mCAAqCD,KAArC,EAAN;AACD;;AAED,WAAOG,CAAP;AACD;AACF;;AAED,SAASlC,kBAAT,CAA4B+B,KAA5B,EAAwC;AACtC,SAAO,IAAIe,IAAJ,CAASf,KAAK,GAAGc,aAAjB,CAAP;AACD;;AAED,SAAS3C,4BAAT,CAAsC6B,KAAtC,EAAkD;AAEhD,MAAIA,KAAK,YAAYe,IAArB,EAA2B;AACzB,WAAOf,KAAK,CAACgB,OAAN,EAAP;AACD;;AAGD;AACE,UAAMb,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,+CAAiDD,KAAjD,EAAN;AACD;;AAED,WAAOG,CAAP;AACD;AACF;;AAED,SAAS/B,8BAAT,CAAwC4B,KAAxC,EAAoD;AAClD,SAAO,IAAIe,IAAJ,CAASf,KAAT,CAAP;AACD;;AAED,SAAS1B,4BAAT,CAAsC0B,KAAtC,EAAkD;AAEhD,MAAIA,KAAK,YAAYe,IAArB,EAA2B;AACzB,WAAOf,KAAK,CAACgB,OAAN,KAAkB,IAAzB;AACD;;AAGD;AACE,UAAMb,CAAC,GAAGG,QAAQ,CAACN,KAAD,EAAQ,EAAR,CAAlB;;AACA,QAAIG,CAAC,GAAG,CAAJ,IAASE,KAAK,CAACF,CAAD,CAAlB,EAAuB;AACrB,YAAM,IAAIF,KAAJ,+CAAiDD,KAAjD,EAAN;AACD;;AAED,WAAOG,CAAP;AACD;AACF;;AAED,SAAS5B,8BAAT,CAAwCyB,KAAxC,EAAoD;AAClD,SAAO,IAAIe,IAAJ,CAASf,KAAK,GAAG,IAAjB,CAAP;AACD;;AAED,SAASH,oBAAT,CAA8BG,KAA9B,EAA0C;AACxC,MAAI,CAACA,KAAK,CAACiB,MAAP,IAAiB,CAACjB,KAAK,CAACkB,IAAxB,IAAgC,CAAClB,KAAK,CAACmB,YAA3C,EAAyD;AACvD,UAAM,IAAIlB,KAAJ,CACJ,iFADI,CAAN;AAGD;;AAED,QAAMmB,GAAG,GAAGb,MAAM,CAACc,KAAP,CAAa,EAAb,CAAZ;AAEAD,EAAAA,GAAG,CAACE,aAAJ,CAAkBtB,KAAK,CAACiB,MAAxB,EAAgC,CAAhC;AACAG,EAAAA,GAAG,CAACE,aAAJ,CAAkBtB,KAAK,CAACkB,IAAxB,EAA8B,CAA9B;AACAE,EAAAA,GAAG,CAACE,aAAJ,CAAkBtB,KAAK,CAACmB,YAAxB,EAAsC,CAAtC;AACA,SAAOC,GAAP;AACD;;AAED,SAAStB,sBAAT,CAAgCE,KAAhC,EAA4C;AAC1C,QAAMoB,GAAG,GAAGb,MAAM,CAACC,IAAP,CAAYR,KAAZ,CAAZ;AACA,QAAMiB,MAAM,GAAGG,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAf;AACA,QAAML,IAAI,GAAGE,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAb;AACA,QAAMC,MAAM,GAAGJ,GAAG,CAACG,YAAJ,CAAiB,CAAjB,CAAf;AAEA,SAAO;AAACN,IAAAA,MAAD;AAASC,IAAAA,IAAT;AAAeC,IAAAA,YAAY,EAAEK;AAA7B,GAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase /\nimport BSON from 'bson';\nimport {OriginalType, ParquetType, PrimitiveType} from './declare';\n\nexport interface ParquetTypeKit {\n primitiveType: PrimitiveType;\n originalType?: OriginalType;\n typeLength?: number;\n toPrimitive: Function;\n fromPrimitive?: Function;\n}\n\nexport const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {\n BOOLEAN: {\n primitiveType: 'BOOLEAN',\n toPrimitive: toPrimitive_BOOLEAN,\n fromPrimitive: fromPrimitive_BOOLEAN\n },\n INT32: {\n primitiveType: 'INT32',\n toPrimitive: toPrimitive_INT32\n },\n INT64: {\n primitiveType: 'INT64',\n toPrimitive: toPrimitive_INT64\n },\n INT96: {\n primitiveType: 'INT96',\n toPrimitive: toPrimitive_INT96\n },\n FLOAT: {\n primitiveType: 'FLOAT',\n toPrimitive: toPrimitive_FLOAT\n },\n DOUBLE: {\n primitiveType: 'DOUBLE',\n toPrimitive: toPrimitive_DOUBLE\n },\n BYTE_ARRAY: {\n primitiveType: 'BYTE_ARRAY',\n toPrimitive: toPrimitive_BYTE_ARRAY\n },\n FIXED_LEN_BYTE_ARRAY: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n toPrimitive: toPrimitive_BYTE_ARRAY\n },\n UTF8: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'UTF8',\n toPrimitive: toPrimitive_UTF8,\n fromPrimitive: fromPrimitive_UTF8\n },\n TIME_MILLIS: {\n primitiveType: 'INT32',\n originalType: 'TIME_MILLIS',\n toPrimitive: toPrimitive_TIME_MILLIS\n },\n TIME_MICROS: {\n primitiveType: 'INT64',\n originalType: 'TIME_MICROS',\n toPrimitive: toPrimitive_TIME_MICROS\n },\n DATE: {\n primitiveType: 'INT32',\n originalType: 'DATE',\n toPrimitive: toPrimitive_DATE,\n fromPrimitive: fromPrimitive_DATE\n },\n TIMESTAMP_MILLIS: {\n primitiveType: 'INT64',\n originalType: 'TIMESTAMP_MILLIS',\n toPrimitive: toPrimitive_TIMESTAMP_MILLIS,\n fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS\n },\n TIMESTAMP_MICROS: {\n primitiveType: 'INT64',\n originalType: 'TIMESTAMP_MICROS',\n toPrimitive: toPrimitive_TIMESTAMP_MICROS,\n fromPrimitive: fromPrimitive_TIMESTAMP_MICROS\n },\n UINT_8: {\n primitiveType: 'INT32',\n originalType: 'UINT_8',\n toPrimitive: toPrimitive_UINT8\n },\n UINT_16: {\n primitiveType: 'INT32',\n originalType: 'UINT_16',\n toPrimitive: toPrimitive_UINT16\n },\n UINT_32: {\n primitiveType: 'INT32',\n originalType: 'UINT_32',\n toPrimitive: toPrimitive_UINT32\n },\n UINT_64: {\n primitiveType: 'INT64',\n originalType: 'UINT_64',\n toPrimitive: toPrimitive_UINT64\n },\n INT_8: {\n primitiveType: 'INT32',\n originalType: 'INT_8',\n toPrimitive: toPrimitive_INT8\n },\n INT_16: {\n primitiveType: 'INT32',\n originalType: 'INT_16',\n toPrimitive: toPrimitive_INT16\n },\n INT_32: {\n primitiveType: 'INT32',\n originalType: 'INT_32',\n toPrimitive: toPrimitive_INT32\n },\n INT_64: {\n primitiveType: 'INT64',\n originalType: 'INT_64',\n toPrimitive: toPrimitive_INT64\n },\n JSON: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'JSON',\n toPrimitive: toPrimitive_JSON,\n fromPrimitive: fromPrimitive_JSON\n },\n BSON: {\n primitiveType: 'BYTE_ARRAY',\n originalType: 'BSON',\n toPrimitive: toPrimitive_BSON,\n fromPrimitive: fromPrimitive_BSON\n },\n INTERVAL: {\n primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n originalType: 'INTERVAL',\n typeLength: 12,\n toPrimitive: toPrimitive_INTERVAL,\n fromPrimitive: fromPrimitive_INTERVAL\n }\n};\n\n/\n Convert a value from it's native representation to the internal/underlying\n * primitive type\n /\nexport function toPrimitive(type: ParquetType, value: any) {\n if (!(type in PARQUET_LOGICAL_TYPES)) {\n throw new Error(`invalid type: ${type}`);\n }\n\n return PARQUET_LOGICAL_TYPES[type].toPrimitive(value);\n}\n\n/\n Convert a value from it's internal/underlying primitive representation to\n * the native representation\n /\nexport function fromPrimitive(type: ParquetType, value: any) {\n if (!(type in PARQUET_LOGICAL_TYPES)) {\n throw new Error(`invalid type: ${type}`);\n }\n\n if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {\n return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value);\n // tslint:disable-next-line:no-else-after-return\n }\n return value;\n}\n\nfunction toPrimitive_BOOLEAN(value: any) {\n return Boolean(value);\n}\n\nfunction fromPrimitive_BOOLEAN(value: any) {\n return Boolean(value);\n}\n\nfunction toPrimitive_FLOAT(value: any) {\n const v = parseFloat(value);\n if (isNaN(v)) {\n throw new Error(`invalid value for FLOAT: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_DOUBLE(value: any) {\n const v = parseFloat(value);\n if (isNaN(v)) {\n throw new Error(`invalid value for DOUBLE: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT8(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x80 \|\| v > 0x7f \|\| isNaN(v)) {\n throw new Error(`invalid value for INT8: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT8(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xff \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT8: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT16(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x8000 \|\| v > 0x7fff \|\| isNaN(v)) {\n throw new Error(`invalid value for INT16: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT16(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xffff \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT16: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT32(value: any) {\n const v = parseInt(value, 10);\n if (v < -0x80000000 \|\| v > 0x7fffffff \|\| isNaN(v)) {\n throw new Error(`invalid value for INT32: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT32(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xffffffffffff \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT32: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT64(value: any) {\n const v = parseInt(value, 10);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT64: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_UINT64(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for UINT64: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_INT96(value: any) {\n const v = parseInt(value, 10);\n if (isNaN(v)) {\n throw new Error(`invalid value for INT96: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_BYTE_ARRAY(value: any) {\n return Buffer.from(value);\n}\n\nfunction toPrimitive_UTF8(value: any) {\n return Buffer.from(value, 'utf8');\n}\n\nfunction fromPrimitive_UTF8(value: any) {\n return value.toString();\n}\n\nfunction toPrimitive_JSON(value: any) {\n return Buffer.from(JSON.stringify(value));\n}\n\nfunction fromPrimitive_JSON(value: any) {\n return JSON.parse(value);\n}\n\nfunction toPrimitive_BSON(value: any) {\n return Buffer.from(BSON.serialize(value));\n}\n\nfunction fromPrimitive_BSON(value: any) {\n return BSON.deserialize(value);\n}\n\nfunction toPrimitive_TIME_MILLIS(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| v > 0xffffffffffffffff \|\| isNaN(v)) {\n throw new Error(`invalid value for TIME_MILLIS: ${value}`);\n }\n\n return v;\n}\n\nfunction toPrimitive_TIME_MICROS(value: any) {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for TIME_MICROS: ${value}`);\n }\n\n return v;\n}\n\nconst kMillisPerDay = 86400000;\n\nfunction toPrimitive_DATE(value: any) {\n / convert from date /\n if (value instanceof Date) {\n return value.getTime() / kMillisPerDay;\n }\n\n / convert from integer /\n {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for DATE: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_DATE(value: any) {\n return new Date(value kMillisPerDay);\n}\n\nfunction toPrimitive_TIMESTAMP_MILLIS(value: any) {\n /* convert from date /\n if (value instanceof Date) {\n return value.getTime();\n }\n\n / convert from integer /\n {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_TIMESTAMP_MILLIS(value: any) {\n return new Date(value);\n}\n\nfunction toPrimitive_TIMESTAMP_MICROS(value: any) {\n / convert from date /\n if (value instanceof Date) {\n return value.getTime() 1000;\n }\n\n /* convert from integer */\n {\n const v = parseInt(value, 10);\n if (v < 0 \|\| isNaN(v)) {\n throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);\n }\n\n return v;\n }\n}\n\nfunction fromPrimitive_TIMESTAMP_MICROS(value: any) {\n return new Date(value / 1000);\n}\n\nfunction toPrimitive_INTERVAL(value: any) {\n if (!value.months \|\| !value.days \|\| !value.milliseconds) {\n throw new Error(\n 'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }'\n );\n }\n\n const buf = Buffer.alloc(12);\n\n buf.writeUInt32LE(value.months, 0);\n buf.writeUInt32LE(value.days, 4);\n buf.writeUInt32LE(value.milliseconds, 8);\n return buf;\n}\n\nfunction fromPrimitive_INTERVAL(value: any) {\n const buf = Buffer.from(value);\n const months = buf.readUInt32LE(0);\n const days = buf.readUInt32LE(4);\n const millis = buf.readUInt32LE(8);\n\n return {months, days, milliseconds: millis};\n}\n"],"file":"types.js"}

package/dist/esm/parquetjs/writer.js CHANGED Viewed

@@ -200,7 +200,7 @@ export class ParquetTransformer extends Transform {
 function encodeValues(type, encoding, values, opts) {
   if (!(encoding in PARQUET_CODECS)) {
-    throw new Error(`invalid encoding: ${encoding}`);
+    throw new Error("invalid encoding: ".concat(encoding));
   }
   return PARQUET_CODECS[encoding].encodeValues(type, values, opts);

package/dist/esm/parquetjs/writer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../../src/parquetjs/writer.ts"],"names":["Transform","PARQUET_CODECS","Compression","Shred","ColumnChunk","ColumnMetaData","CompressionCodec","ConvertedType","DataPageHeader","DataPageHeaderV2","Encoding","FieldRepetitionType","FileMetaData","KeyValue","PageHeader","PageType","RowGroup","SchemaElement","Type","Util","Int64","PARQUET_MAGIC","PARQUET_VERSION","PARQUET_DEFAULT_PAGE_SIZE","PARQUET_DEFAULT_ROW_GROUP_SIZE","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","ParquetWriter","openFile","schema","path","opts","outputStream","osopen","openStream","envelopeWriter","ParquetEnvelopeWriter","constructor","rowBuffer","rowGroupSize","closed","userMetadata","writeHeader","err","close","appendRow","row","Error","shredRecord","rowCount","callback","writeFooter","setMetadata","key","value","String","setRowGroupSize","cnt","setPageSize","writeFn","oswrite","bind","undefined","closeFn","osclose","fileOffset","write","offset","rowGroups","pageSize","useDataPageV2","Boolean","writeSection","buf","length","Buffer","from","writeRowGroup","records","rgroup","encodeRowGroup","baseOffset","push","metadata","body","encodeFooter","ParquetTransformer","objectMode","writeProxy","t","b","writer","_transform","encoding","then","Promise","resolve","_flush","encodeValues","type","values","encodeDataPage","column","data","rLevelsBuf","alloc","rLevelMax","rlevels","bitWidth","getBitWidth","dLevelsBuf","dLevelMax","dlevels","valuesBuf","primitiveType","typeLength","dataBuf","concat","compressedBuf","deflate","compression","header","DATA_PAGE","data_page_header","num_values","count","definition_level_encoding","repetition_level_encoding","uncompressed_page_size","compressed_page_size","headerBuf","serializeThrift","page","headerSize","encodeDataPageV2","disableEnvelope","DATA_PAGE_V2","data_page_header_v2","num_nulls","num_rows","definition_levels_byte_length","repetition_levels_byte_length","is_compressed","encodeColumnChunk","buffer","columnData","join","pageBuf","total_uncompressed_size","total_compressed_size","result","path_in_schema","data_page_offset","encodings","codec","metadataOffset","columns","total_byte_size","field","fieldList","isNested","cchunkData","cchunk","file_offset","meta_data","Number","version","created_by","row_groups","key_value_metadata","kv","schemaRoot","name","num_children","Object","keys","fields","relt","repetitionType","schemaElem","repetition_type","fieldCount","originalType","converted_type","type_length","metadataEncoded","footerEncoded","copy","writeUInt32LE"],"mappings":";AAEA,SAAQA,SAAR,QAAkC,QAAlC;AACA,SAA6BC,cAA7B,QAAkD,UAAlD;AACA,OAAO,KAAKC,WAAZ,MAA6B,eAA7B;AASA,OAAO,KAAKC,KAAZ,MAAuB,gBAAvB;AACA,SACEC,WADF,EAEEC,cAFF,EAGEC,gBAHF,EAIEC,aAJF,EAKEC,cALF,EAMEC,gBANF,EAOEC,QAPF,EAQEC,mBARF,EASEC,YATF,EAUEC,QAVF,EAWEC,UAXF,EAYEC,QAZF,EAaEC,QAbF,EAcEC,aAdF,EAeEC,IAfF,QAgBO,kBAhBP;AAiBA,OAAO,KAAKC,IAAZ,MAAsB,QAAtB;AACA,OAAOC,KAAP,MAAkB,YAAlB;AAKA,MAAMC,aAAa,GAAG,MAAtB;AAKA,MAAMC,eAAe,GAAG,CAAxB;AAKA,MAAMC,yBAAyB,GAAG,IAAlC;AACA,MAAMC,8BAA8B,GAAG,IAAvC;AAKA,MAAMC,kBAAkB,GAAG,OAA3B;AACA,MAAMC,sBAAsB,GAAG,KAA/B;AAuBA,OAAO,MAAMC,aAAN,CAAuB;AAKP,eAARC,QAAQ,CACnBC,MADmB,EAEnBC,IAFmB,EAGnBC,IAHmB,EAIQ;AAC3B,UAAMC,YAAY,GAAG,MAAMb,IAAI,CAACc,MAAL,CAAYH,IAAZ,EAAkBC,IAAlB,CAA3B;AACA,WAAOJ,aAAa,CAACO,UAAd,CAAyBL,MAAzB,EAAiCG,YAAjC,EAA+CD,IAA/C,CAAP;AACD;;AAMsB,eAAVG,UAAU,CACrBL,MADqB,EAErBG,YAFqB,EAGrBD,IAHqB,EAIM;AAC3B,QAAI,CAACA,IAAL,EAAW;AAETA,MAAAA,IAAI,GAAG,EAAP;AACD;;AAED,UAAMI,cAAc,GAAG,MAAMC,qBAAqB,CAACF,UAAtB,CAAiCL,MAAjC,EAAyCG,YAAzC,EAAuDD,IAAvD,CAA7B;AAEA,WAAO,IAAIJ,aAAJ,CAAkBE,MAAlB,EAA0BM,cAA1B,EAA0CJ,IAA1C,CAAP;AACD;;AAYDM,EAAAA,WAAW,CACTR,MADS,EAETM,cAFS,EAGTJ,IAHS,EAIT;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AACA,SAAKF,MAAL,GAAcA,MAAd;AACA,SAAKM,cAAL,GAAsBA,cAAtB;AAEA,SAAKG,SAAL,GAAiB,EAAjB;AACA,SAAKC,YAAL,GAAoBR,IAAI,CAACQ,YAAL,IAAqBf,8BAAzC;AACA,SAAKgB,MAAL,GAAc,KAAd;AACA,SAAKC,YAAL,GAAoB,EAApB;;AAGA,QAAI;AAEFN,MAAAA,cAAc,CAACO,WAAf;AACD,KAHD,CAGE,OAAOC,GAAP,EAAY;AAEZR,MAAAA,cAAc,CAACS,KAAf;AACA,YAAMD,GAAN;AACD;AACF;;AAMc,QAATE,SAAS,CAAIC,GAAJ,EAA2B;AACxC,QAAI,KAAKN,MAAT,EAAiB;AACf,YAAM,IAAIO,KAAJ,CAAU,mBAAV,CAAN;AACD;;AACD5C,IAAAA,KAAK,CAAC6C,WAAN,CAAkB,KAAKnB,MAAvB,EAA+BiB,GAA/B,EAAoC,KAAKR,SAAzC;;AACA,QAAI,KAAKA,SAAL,CAAeW,QAAf,IAA2B,KAAKV,YAApC,EAAkD;AAEhD,WAAKD,SAAL,GAAiB,EAAjB;AACD;AACF;;AAQU,QAALM,KAAK,CAACM,QAAD,EAAuC;AAChD,QAAI,KAAKV,MAAT,EAAiB;AACf,YAAM,IAAIO,KAAJ,CAAU,mBAAV,CAAN;AACD;;AAED,SAAKP,MAAL,GAAc,IAAd;;AAEA,QAAI,KAAKF,SAAL,CAAeW,QAAf,GAA0B,CAA1B,IAA+B,KAAKX,SAAL,CAAeW,QAAf,IAA2B,KAAKV,YAAnE,EAAiF;AAE/E,WAAKD,SAAL,GAAiB,EAAjB;AACD;;AAED,UAAM,KAAKH,cAAL,CAAoBgB,WAApB,CAAgC,KAAKV,YAArC,CAAN;AACA,UAAM,KAAKN,cAAL,CAAoBS,KAApB,EAAN;;AAGA,QAAIM,QAAJ,EAAc;AACZA,MAAAA,QAAQ;AACT;AACF;;AAKDE,EAAAA,WAAW,CAACC,GAAD,EAAcC,KAAd,EAAmC;AAE5C,SAAKb,YAAL,CAAkBc,MAAM,CAACF,GAAD,CAAxB,IAAiCE,MAAM,CAACD,KAAD,CAAvC;AACD;;AAQDE,EAAAA,eAAe,CAACC,GAAD,EAAoB;AACjC,SAAKlB,YAAL,GAAoBkB,GAApB;AACD;;AAMDC,EAAAA,WAAW,CAACD,GAAD,EAAoB;AAC7B,SAAKtB,cAAL,CAAoBuB,WAApB,CAAgCD,GAAhC;AACD;;AArI2B;AA8I9B,OAAO,MAAMrB,qBAAN,CAA4B;AAIV,eAAVF,UAAU,CACrBL,MADqB,EAErBG,YAFqB,EAGrBD,IAHqB,EAIW;AAChC,UAAM4B,OAAO,GAAGxC,IAAI,CAACyC,OAAL,CAAaC,IAAb,CAAkBC,SAAlB,EAA6B9B,YAA7B,CAAhB;AACA,UAAM+B,OAAO,GAAG5C,IAAI,CAAC6C,OAAL,CAAaH,IAAb,CAAkBC,SAAlB,EAA6B9B,YAA7B,CAAhB;AACA,WAAO,IAAII,qBAAJ,CAA0BP,MAA1B,EAAkC8B,OAAlC,EAA2CI,OAA3C,EAAoD,CAApD,EAAuDhC,IAAvD,CAAP;AACD;;AAWDM,EAAAA,WAAW,CACTR,MADS,EAET8B,OAFS,EAGTI,OAHS,EAITE,UAJS,EAKTlC,IALS,EAMT;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AACA,SAAKF,MAAL,GAAcA,MAAd;AACA,SAAKqC,KAAL,GAAaP,OAAb;AACA,SAAKf,KAAL,GAAamB,OAAb;AACA,SAAKI,MAAL,GAAcF,UAAd;AACA,SAAKhB,QAAL,GAAgB,CAAhB;AACA,SAAKmB,SAAL,GAAiB,EAAjB;AACA,SAAKC,QAAL,GAAgBtC,IAAI,CAACsC,QAAL,IAAiB9C,yBAAjC;AACA,SAAK+C,aAAL,GAAqB,mBAAmBvC,IAAnB,GAA0BwC,OAAO,CAACxC,IAAI,CAACuC,aAAN,CAAjC,GAAwD,KAA7E;AACD;;AAEDE,EAAAA,YAAY,CAACC,GAAD,EAA6B;AACvC,SAAKN,MAAL,IAAeM,GAAG,CAACC,MAAnB;AACA,WAAO,KAAKR,KAAL,CAAWO,GAAX,CAAP;AACD;;AAKD/B,EAAAA,WAAW,GAAkB;AAC3B,WAAO,KAAK8B,YAAL,CAAkBG,MAAM,CAACC,IAAP,CAAYvD,aAAZ,CAAlB,CAAP;AACD;;AAMkB,QAAbwD,aAAa,CAACC,OAAD,EAAwC;AACzD,UAAMC,MAAM,GAAGC,cAAc,CAAC,KAAKnD,MAAN,EAAciD,OAAd,EAAuB;AAClDG,MAAAA,UAAU,EAAE,KAAKd,MADiC;AAElDE,MAAAA,QAAQ,EAAE,KAAKA,QAFmC;AAGlDC,MAAAA,aAAa,EAAE,KAAKA;AAH8B,KAAvB,CAA7B;AAMA,SAAKrB,QAAL,IAAiB6B,OAAO,CAAC7B,QAAzB;AACA,SAAKmB,SAAL,CAAec,IAAf,CAAoBH,MAAM,CAACI,QAA3B;AACA,WAAO,KAAKX,YAAL,CAAkBO,MAAM,CAACK,IAAzB,CAAP;AACD;;AAKDjC,EAAAA,WAAW,CAACV,YAAD,EAAsD;AAC/D,QAAI,CAACA,YAAL,EAAmB;AAEjBA,MAAAA,YAAY,GAAG,EAAf;AACD;;AAED,WAAO,KAAK+B,YAAL,CACLa,YAAY,CAAC,KAAKxD,MAAN,EAAc,KAAKoB,QAAnB,EAA6B,KAAKmB,SAAlC,EAA6C3B,YAA7C,CADP,CAAP;AAGD;;AAMDiB,EAAAA,WAAW,CAACD,GAAD,EAAoB;AAC7B,SAAKY,QAAL,GAAgBZ,GAAhB;AACD;;AAxFgC;AA8FnC,OAAO,MAAM6B,kBAAN,SAAoCtF,SAApC,CAA8C;AAGnDqC,EAAAA,WAAW,CAACR,MAAD,EAAwBE,IAA0B,GAAG,EAArD,EAAyD;AAClE,UAAM;AAACwD,MAAAA,UAAU,EAAE;AAAb,KAAN;;AADkE;;AAGlE,UAAMC,UAAU,GAAI,UAAUC,CAAV,EAAsC;AACxD,aAAO,gBAAgBC,CAAhB,EAAuC;AAC5CD,QAAAA,CAAC,CAACP,IAAF,CAAOQ,CAAP;AACD,OAFD;AAGD,KAJkB,CAIhB,IAJgB,CAAnB;;AAMA,SAAKC,MAAL,GAAc,IAAIhE,aAAJ,CACZE,MADY,EAEZ,IAAIO,qBAAJ,CAA0BP,MAA1B,EAAkC2D,UAAlC,EAA8C,YAAY,CAAE,CAA5D,EAA8D,CAA9D,EAAiEzD,IAAjE,CAFY,EAGZA,IAHY,CAAd;AAKD;;AAGD6D,EAAAA,UAAU,CAAC9C,GAAD,EAAW+C,QAAX,EAA6B3C,QAA7B,EAA2E;AACnF,QAAIJ,GAAJ,EAAS;AACP,aAAO,KAAK6C,MAAL,CAAY9C,SAAZ,CAAsBC,GAAtB,EAA2BgD,IAA3B,CAAgC5C,QAAhC,CAAP;AACD;;AACDA,IAAAA,QAAQ;AACR,WAAO6C,OAAO,CAACC,OAAR,EAAP;AACD;;AAGW,QAANC,MAAM,CAAC/C,QAAD,EAAgC;AAC1C,UAAM,KAAKyC,MAAL,CAAY/C,KAAZ,CAAkBM,QAAlB,CAAN;AACD;;AA/BkD;;AAqCrD,SAASgD,YAAT,CACEC,IADF,EAEEN,QAFF,EAGEO,MAHF,EAIErE,IAJF,EAKE;AACA,MAAI,EAAE8D,QAAQ,IAAI5F,cAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI8C,KAAJ,CAAW,qBAAoB8C,QAAS,EAAxC,CAAN;AACD;;AACD,SAAO5F,cAAc,CAAC4F,QAAD,CAAd,CAAyBK,YAAzB,CAAsCC,IAAtC,EAA4CC,MAA5C,EAAoDrE,IAApD,CAAP;AACD;;AAKD,SAASsE,cAAT,CACEC,MADF,EAEEC,IAFF,EAOE;AAEA,MAAIC,UAAU,GAAG7B,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACI,SAAP,GAAmB,CAAvB,EAA0B;AACxBF,IAAAA,UAAU,GAAGN,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACI,OAAlD,EAA2D;AAClFC,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACI,SAAxB;AADwE,KAA3D,CAAzB;AAID;;AAED,MAAII,UAAU,GAAGnC,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACS,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,UAAU,GAAGZ,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACS,OAAlD,EAA2D;AAClFJ,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACS,SAAxB;AADwE,KAA3D,CAAzB;AAID;;AAGD,QAAME,SAAS,GAAGf,YAAY,CAACI,MAAM,CAACY,aAAR,EAAwBZ,MAAM,CAACT,QAA/B,EAA0CU,IAAI,CAACH,MAA/C,EAAuD;AACnFe,IAAAA,UAAU,EAAEb,MAAM,CAACa,UADgE;AAEnFP,IAAAA,QAAQ,EAAEN,MAAM,CAACa;AAFkE,GAAvD,CAA9B;AAKA,QAAMC,OAAO,GAAGzC,MAAM,CAAC0C,MAAP,CAAc,CAACb,UAAD,EAAaM,UAAb,EAAyBG,SAAzB,CAAd,CAAhB;AAGA,QAAMK,aAAa,GAAGpH,WAAW,CAACqH,OAAZ,CAAoBjB,MAAM,CAACkB,WAA3B,EAAyCJ,OAAzC,CAAtB;AAGA,QAAMK,MAAM,GAAG,IAAI3G,UAAJ,CAAe;AAC5BqF,IAAAA,IAAI,EAAEpF,QAAQ,CAAC2G,SADa;AAE5BC,IAAAA,gBAAgB,EAAE,IAAInH,cAAJ,CAAmB;AACnCoH,MAAAA,UAAU,EAAErB,IAAI,CAACsB,KADkB;AAEnChC,MAAAA,QAAQ,EAAEnF,QAAQ,CAAC4F,MAAM,CAACT,QAAR,CAFiB;AAGnCiC,MAAAA,yBAAyB,EAAEpH,QAAQ,CAACgB,sBAAD,CAHA;AAInCqG,MAAAA,yBAAyB,EAAErH,QAAQ,CAACgB,sBAAD;AAJA,KAAnB,CAFU;AAQ5BsG,IAAAA,sBAAsB,EAAEZ,OAAO,CAAC1C,MARJ;AAS5BuD,IAAAA,oBAAoB,EAAEX,aAAa,CAAC5C;AATR,GAAf,CAAf;AAaA,QAAMwD,SAAS,GAAG/G,IAAI,CAACgH,eAAL,CAAqBV,MAArB,CAAlB;AACA,QAAMW,IAAI,GAAGzD,MAAM,CAAC0C,MAAP,CAAc,CAACa,SAAD,EAAYZ,aAAZ,CAAd,CAAb;AAEA,SAAO;AAACG,IAAAA,MAAD;AAASY,IAAAA,UAAU,EAAEH,SAAS,CAACxD,MAA/B;AAAuC0D,IAAAA;AAAvC,GAAP;AACD;;AAKD,SAASE,gBAAT,CACEhC,MADF,EAEEC,IAFF,EAGEtD,QAHF,EAQE;AAEA,QAAMgE,SAAS,GAAGf,YAAY,CAACI,MAAM,CAACY,aAAR,EAAwBZ,MAAM,CAACT,QAA/B,EAA0CU,IAAI,CAACH,MAA/C,EAAuD;AACnFe,IAAAA,UAAU,EAAEb,MAAM,CAACa,UADgE;AAEnFP,IAAAA,QAAQ,EAAEN,MAAM,CAACa;AAFkE,GAAvD,CAA9B;AAMA,QAAMG,aAAa,GAAGpH,WAAW,CAACqH,OAAZ,CAAoBjB,MAAM,CAACkB,WAA3B,EAAyCP,SAAzC,CAAtB;AAGA,MAAIT,UAAU,GAAG7B,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACI,SAAP,GAAmB,CAAvB,EAA0B;AACxBF,IAAAA,UAAU,GAAGN,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACI,OAAlD,EAA2D;AAClFC,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACI,SAAxB,CADwE;AAElF6B,MAAAA,eAAe,EAAE;AAFiE,KAA3D,CAAzB;AAID;;AAED,MAAIzB,UAAU,GAAGnC,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACS,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,UAAU,GAAGZ,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACS,OAAlD,EAA2D;AAClFJ,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACS,SAAxB,CADwE;AAElFwB,MAAAA,eAAe,EAAE;AAFiE,KAA3D,CAAzB;AAID;;AAGD,QAAMd,MAAM,GAAG,IAAI3G,UAAJ,CAAe;AAC5BqF,IAAAA,IAAI,EAAEpF,QAAQ,CAACyH,YADa;AAE5BC,IAAAA,mBAAmB,EAAE,IAAIhI,gBAAJ,CAAqB;AACxCmH,MAAAA,UAAU,EAAErB,IAAI,CAACsB,KADuB;AAExCa,MAAAA,SAAS,EAAEnC,IAAI,CAACsB,KAAL,GAAatB,IAAI,CAACH,MAAL,CAAY1B,MAFI;AAGxCiE,MAAAA,QAAQ,EAAE1F,QAH8B;AAIxC4C,MAAAA,QAAQ,EAAEnF,QAAQ,CAAC4F,MAAM,CAACT,QAAR,CAJsB;AAKxC+C,MAAAA,6BAA6B,EAAE9B,UAAU,CAACpC,MALF;AAMxCmE,MAAAA,6BAA6B,EAAErC,UAAU,CAAC9B,MANF;AAOxCoE,MAAAA,aAAa,EAAExC,MAAM,CAACkB,WAAP,KAAuB;AAPE,KAArB,CAFO;AAW5BQ,IAAAA,sBAAsB,EAAExB,UAAU,CAAC9B,MAAX,GAAoBoC,UAAU,CAACpC,MAA/B,GAAwCuC,SAAS,CAACvC,MAX9C;AAY5BuD,IAAAA,oBAAoB,EAAEzB,UAAU,CAAC9B,MAAX,GAAoBoC,UAAU,CAACpC,MAA/B,GAAwC4C,aAAa,CAAC5C;AAZhD,GAAf,CAAf;AAgBA,QAAMwD,SAAS,GAAG/G,IAAI,CAACgH,eAAL,CAAqBV,MAArB,CAAlB;AACA,QAAMW,IAAI,GAAGzD,MAAM,CAAC0C,MAAP,CAAc,CAACa,SAAD,EAAY1B,UAAZ,EAAwBM,UAAxB,EAAoCQ,aAApC,CAAd,CAAb;AACA,SAAO;AAACG,IAAAA,MAAD;AAASY,IAAAA,UAAU,EAAEH,SAAS,CAACxD,MAA/B;AAAuC0D,IAAAA;AAAvC,GAAP;AACD;;AAKD,SAASW,iBAAT,CACEzC,MADF,EAEE0C,MAFF,EAGE7E,MAHF,EAIEpC,IAJF,EASE;AACA,QAAMwE,IAAI,GAAGyC,MAAM,CAACC,UAAP,CAAkB3C,MAAM,CAACxE,IAAP,CAAYoH,IAAZ,EAAlB,CAAb;AACA,QAAMjE,UAAU,GAAG,CAAClD,IAAI,CAACkD,UAAL,IAAmB,CAApB,IAAyBd,MAA5C;AAGA,MAAIgF,OAAJ;AAEA,MAAIC,uBAAuB,GAAG,CAA9B;AAEA,MAAIC,qBAAqB,GAAG,CAA5B;AACA;AACE,QAAIC,MAAJ;;AACA,QAAIvH,IAAI,CAACuC,aAAT,EAAwB;AACtBgF,MAAAA,MAAM,GAAGhB,gBAAgB,CAAChC,MAAD,EAASC,IAAT,EAAeyC,MAAM,CAAC/F,QAAtB,CAAzB;AACD,KAFD,MAEO;AACLqG,MAAAA,MAAM,GAAGjD,cAAc,CAACC,MAAD,EAASC,IAAT,CAAvB;AACD;;AAED4C,IAAAA,OAAO,GAAGG,MAAM,CAAClB,IAAjB;AACAgB,IAAAA,uBAAuB,IAAIE,MAAM,CAAC7B,MAAP,CAAcO,sBAAd,GAAuCsB,MAAM,CAACjB,UAAzE;AACAgB,IAAAA,qBAAqB,IAAIC,MAAM,CAAC7B,MAAP,CAAcQ,oBAAd,GAAqCqB,MAAM,CAACjB,UAArE;AACD;AAMD,QAAMlD,QAAQ,GAAG,IAAI9E,cAAJ,CAAmB;AAClCkJ,IAAAA,cAAc,EAAEjD,MAAM,CAACxE,IADW;AAElC8F,IAAAA,UAAU,EAAErB,IAAI,CAACsB,KAFiB;AAGlC2B,IAAAA,gBAAgB,EAAEvE,UAHgB;AAIlCwE,IAAAA,SAAS,EAAE,EAJuB;AAKlCL,IAAAA,uBALkC;AAMlCC,IAAAA,qBANkC;AAOlClD,IAAAA,IAAI,EAAEjF,IAAI,CAACoF,MAAM,CAACY,aAAR,CAPwB;AAQlCwC,IAAAA,KAAK,EAAEpJ,gBAAgB,CAACgG,MAAM,CAACkB,WAAR;AARW,GAAnB,CAAjB;AAYArC,EAAAA,QAAQ,CAACsE,SAAT,CAAmBvE,IAAnB,CAAwBxE,QAAQ,CAACgB,sBAAD,CAAhC;AACAyD,EAAAA,QAAQ,CAACsE,SAAT,CAAmBvE,IAAnB,CAAwBxE,QAAQ,CAAC4F,MAAM,CAACT,QAAR,CAAhC;AAGA,QAAM8D,cAAc,GAAG1E,UAAU,GAAGkE,OAAO,CAACzE,MAA5C;AACA,QAAMU,IAAI,GAAGT,MAAM,CAAC0C,MAAP,CAAc,CAAC8B,OAAD,EAAUhI,IAAI,CAACgH,eAAL,CAAqBhD,QAArB,CAAV,CAAd,CAAb;AACA,SAAO;AAACC,IAAAA,IAAD;AAAOD,IAAAA,QAAP;AAAiBwE,IAAAA;AAAjB,GAAP;AACD;;AAKD,SAAS3E,cAAT,CACEnD,MADF,EAEE0E,IAFF,EAGExE,IAHF,EAOE;AACA,QAAMoD,QAAQ,GAAG,IAAInE,QAAJ,CAAa;AAC5B2H,IAAAA,QAAQ,EAAEpC,IAAI,CAACtD,QADa;AAE5B2G,IAAAA,OAAO,EAAE,EAFmB;AAG5BC,IAAAA,eAAe,EAAE;AAHW,GAAb,CAAjB;AAMA,MAAIzE,IAAI,GAAGT,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAX;;AACA,OAAK,MAAMqD,KAAX,IAAoBjI,MAAM,CAACkI,SAA3B,EAAsC;AACpC,QAAID,KAAK,CAACE,QAAV,EAAoB;AAClB;AACD;;AAED,UAAMC,UAAU,GAAGlB,iBAAiB,CAACe,KAAD,EAAQvD,IAAR,EAAcnB,IAAI,CAACV,MAAnB,EAA2B3C,IAA3B,CAApC;AAEA,UAAMmI,MAAM,GAAG,IAAI9J,WAAJ,CAAgB;AAC7B+J,MAAAA,WAAW,EAAEF,UAAU,CAACN,cADK;AAE7BS,MAAAA,SAAS,EAAEH,UAAU,CAAC9E;AAFO,KAAhB,CAAf;AAKAA,IAAAA,QAAQ,CAACyE,OAAT,CAAiB1E,IAAjB,CAAsBgF,MAAtB;AACA/E,IAAAA,QAAQ,CAAC0E,eAAT,GAA2B,IAAIzI,KAAJ,CAAUiJ,MAAM,CAAClF,QAAQ,CAAC0E,eAAV,CAAN,GAAmCI,UAAU,CAAC7E,IAAX,CAAgBV,MAA7D,CAA3B;AAEAU,IAAAA,IAAI,GAAGT,MAAM,CAAC0C,MAAP,CAAc,CAACjC,IAAD,EAAO6E,UAAU,CAAC7E,IAAlB,CAAd,CAAP;AACD;;AAED,SAAO;AAACA,IAAAA,IAAD;AAAOD,IAAAA;AAAP,GAAP;AACD;;AAKD,SAASE,YAAT,CACExD,MADF,EAEEoB,QAFF,EAGEmB,SAHF,EAIE3B,YAJF,EAKU;AACR,QAAM0C,QAAQ,GAAG,IAAIvE,YAAJ,CAAiB;AAChC0J,IAAAA,OAAO,EAAEhJ,eADuB;AAEhCiJ,IAAAA,UAAU,EAAE,UAFoB;AAGhC5B,IAAAA,QAAQ,EAAE1F,QAHsB;AAIhCuH,IAAAA,UAAU,EAAEpG,SAJoB;AAKhCvC,IAAAA,MAAM,EAAE,EALwB;AAMhC4I,IAAAA,kBAAkB,EAAE;AANY,GAAjB,CAAjB;;AASA,OAAK,MAAMpH,GAAX,IAAkBZ,YAAlB,EAAgC;AAAA;;AAC9B,UAAMiI,EAAE,GAAG,IAAI7J,QAAJ,CAAa;AACtBwC,MAAAA,GADsB;AAEtBC,MAAAA,KAAK,EAAEb,YAAY,CAACY,GAAD;AAFG,KAAb,CAAX;AAIA,6BAAA8B,QAAQ,CAACsF,kBAAT,0GAA6BvF,IAA7B,8GAAoCwF,EAApC;AACD;;AAED;AACE,UAAMC,UAAU,GAAG,IAAI1J,aAAJ,CAAkB;AACnC2J,MAAAA,IAAI,EAAE,MAD6B;AAEnCC,MAAAA,YAAY,EAAEC,MAAM,CAACC,IAAP,CAAYlJ,MAAM,CAACmJ,MAAnB,EAA2BtG;AAFN,KAAlB,CAAnB;AAIAS,IAAAA,QAAQ,CAACtD,MAAT,CAAgBqD,IAAhB,CAAqByF,UAArB;AACD;;AAED,OAAK,MAAMb,KAAX,IAAoBjI,MAAM,CAACkI,SAA3B,EAAsC;AACpC,UAAMkB,IAAI,GAAGtK,mBAAmB,CAACmJ,KAAK,CAACoB,cAAP,CAAhC;AACA,UAAMC,UAAU,GAAG,IAAIlK,aAAJ,CAAkB;AACnC2J,MAAAA,IAAI,EAAEd,KAAK,CAACc,IADuB;AAEnCQ,MAAAA,eAAe,EAAEH;AAFkB,KAAlB,CAAnB;;AAKA,QAAInB,KAAK,CAACE,QAAV,EAAoB;AAClBmB,MAAAA,UAAU,CAACN,YAAX,GAA0Bf,KAAK,CAACuB,UAAhC;AACD,KAFD,MAEO;AACLF,MAAAA,UAAU,CAAChF,IAAX,GAAkBjF,IAAI,CAAC4I,KAAK,CAAC5C,aAAP,CAAtB;AACD;;AAED,QAAI4C,KAAK,CAACwB,YAAV,EAAwB;AACtBH,MAAAA,UAAU,CAACI,cAAX,GAA4BhL,aAAa,CAACuJ,KAAK,CAACwB,YAAP,CAAzC;AACD;;AAEDH,IAAAA,UAAU,CAACK,WAAX,GAAyB1B,KAAK,CAAC3C,UAA/B;AAEAhC,IAAAA,QAAQ,CAACtD,MAAT,CAAgBqD,IAAhB,CAAqBiG,UAArB;AACD;;AAED,QAAMM,eAAe,GAAGtK,IAAI,CAACgH,eAAL,CAAqBhD,QAArB,CAAxB;AACA,QAAMuG,aAAa,GAAG/G,MAAM,CAAC8B,KAAP,CAAagF,eAAe,CAAC/G,MAAhB,GAAyB,CAAtC,CAAtB;AACA+G,EAAAA,eAAe,CAACE,IAAhB,CAAqBD,aAArB;AACAA,EAAAA,aAAa,CAACE,aAAd,CAA4BH,eAAe,CAAC/G,MAA5C,EAAoD+G,eAAe,CAAC/G,MAApE;AACAgH,EAAAA,aAAa,CAACxH,KAAd,CAAoB7C,aAApB,EAAmCoK,eAAe,CAAC/G,MAAhB,GAAyB,CAA5D;AACA,SAAOgH,aAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase /\nimport {Transform, Writable} from 'stream';\nimport {ParquetCodecOptions, PARQUET_CODECS} from './codecs';\nimport as Compression from './compression';\nimport {\n ParquetBuffer,\n ParquetCodec,\n ParquetData,\n ParquetField,\n PrimitiveType\n} from './schema/declare';\nimport {ParquetSchema} from './schema/schema';\nimport * as Shred from './schema/shred';\nimport {\n ColumnChunk,\n ColumnMetaData,\n CompressionCodec,\n ConvertedType,\n DataPageHeader,\n DataPageHeaderV2,\n Encoding,\n FieldRepetitionType,\n FileMetaData,\n KeyValue,\n PageHeader,\n PageType,\n RowGroup,\n SchemaElement,\n Type\n} from './parquet-thrift';\nimport * as Util from './util';\nimport Int64 from 'node-int64';\n\n/*\n Parquet File Magic String\n /\nconst PARQUET_MAGIC = 'PAR1';\n\n/\n Parquet File Format Version\n /\nconst PARQUET_VERSION = 1;\n\n/\n Default Page and Row Group sizes\n /\nconst PARQUET_DEFAULT_PAGE_SIZE = 8192;\nconst PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;\n\n/\n Repetition and Definition Level Encoding\n /\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\nexport interface ParquetWriterOptions {\n baseOffset?: number;\n rowGroupSize?: number;\n pageSize?: number;\n useDataPageV2?: boolean;\n\n // Write Stream Options\n flags?: string;\n encoding?: string;\n fd?: number;\n mode?: number;\n autoClose?: boolean;\n start?: number;\n}\n\n/\n Write a parquet file to an output stream. The ParquetWriter will perform\n * buffering/batching for performance, so close() must be called after all rows\n * are written.\n /\n// eslint-disable-next-line @typescript-eslint/no-unused-vars\nexport class ParquetWriter<T> {\n /\n Convenience method to create a new buffered parquet writer that writes to\n * the specified file\n /\n static async openFile<T>(\n schema: ParquetSchema,\n path: string,\n opts?: ParquetWriterOptions\n ): Promise<ParquetWriter<T>> {\n const outputStream = await Util.osopen(path, opts);\n return ParquetWriter.openStream(schema, outputStream, opts);\n }\n\n /\n Convenience method to create a new buffered parquet writer that writes to\n * the specified stream\n /\n static async openStream<T>(\n schema: ParquetSchema,\n outputStream: Writable,\n opts?: ParquetWriterOptions\n ): Promise<ParquetWriter<T>> {\n if (!opts) {\n // tslint:disable-next-line:no-parameter-reassignment\n opts = {};\n }\n\n const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);\n\n return new ParquetWriter(schema, envelopeWriter, opts);\n }\n\n public schema: ParquetSchema;\n public envelopeWriter: ParquetEnvelopeWriter;\n public rowBuffer: ParquetBuffer;\n public rowGroupSize: number;\n public closed: boolean;\n public userMetadata: Record<string, string>;\n\n /\n Create a new buffered parquet writer for a given envelope writer\n /\n constructor(\n schema: ParquetSchema,\n envelopeWriter: ParquetEnvelopeWriter,\n opts: ParquetWriterOptions\n ) {\n this.schema = schema;\n this.envelopeWriter = envelopeWriter;\n // @ts-ignore Row buffer typings...\n this.rowBuffer = {};\n this.rowGroupSize = opts.rowGroupSize \|\| PARQUET_DEFAULT_ROW_GROUP_SIZE;\n this.closed = false;\n this.userMetadata = {};\n\n // TODO - better not mess with promises in the constructor\n try {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n envelopeWriter.writeHeader();\n } catch (err) {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n envelopeWriter.close();\n throw err;\n }\n }\n\n /\n Append a single row to the parquet file. Rows are buffered in memory until\n * rowGroupSize rows are in the buffer or close() is called\n /\n async appendRow<T>(row: T): Promise<void> {\n if (this.closed) {\n throw new Error('writer was closed');\n }\n Shred.shredRecord(this.schema, row, this.rowBuffer);\n if (this.rowBuffer.rowCount >= this.rowGroupSize) {\n // @ts-ignore\n this.rowBuffer = {};\n }\n }\n\n /\n Finish writing the parquet file and commit the footer to disk. This method\n * MUST be called after you are finished adding rows. You must not call this\n * method twice on the same object or add any rows after the close() method has\n * been called\n /\n async close(callback?: () => void): Promise<void> {\n if (this.closed) {\n throw new Error('writer was closed');\n }\n\n this.closed = true;\n\n if (this.rowBuffer.rowCount > 0 \|\| this.rowBuffer.rowCount >= this.rowGroupSize) {\n // @ts-ignore\n this.rowBuffer = {};\n }\n\n await this.envelopeWriter.writeFooter(this.userMetadata);\n await this.envelopeWriter.close();\n // this.envelopeWriter = null;\n\n if (callback) {\n callback();\n }\n }\n\n /\n Add key<>value metadata to the file\n /\n setMetadata(key: string, value: string): void {\n // TODO: value to be any, obj -> JSON\n this.userMetadata[String(key)] = String(value);\n }\n\n /\n Set the parquet row group size. This values controls the maximum number\n * of rows that are buffered in memory at any given time as well as the number\n * of rows that are co-located on disk. A higher value is generally better for\n * read-time I/O performance at the tradeoff of write-time memory usage.\n /\n setRowGroupSize(cnt: number): void {\n this.rowGroupSize = cnt;\n }\n\n /\n Set the parquet data page size. The data page size controls the maximum\n * number of column values that are written to disk as a consecutive array\n /\n setPageSize(cnt: number): void {\n this.envelopeWriter.setPageSize(cnt);\n }\n}\n\n/\n Create a parquet file from a schema and a number of row groups. This class\n * performs direct, unbuffered writes to the underlying output stream and is\n * intendend for advanced and internal users; the writeXXX methods must be\n * called in the correct order to produce a valid file.\n /\nexport class ParquetEnvelopeWriter {\n /\n Create a new parquet envelope writer that writes to the specified stream\n /\n static async openStream(\n schema: ParquetSchema,\n outputStream: Writable,\n opts: ParquetWriterOptions\n ): Promise<ParquetEnvelopeWriter> {\n const writeFn = Util.oswrite.bind(undefined, outputStream);\n const closeFn = Util.osclose.bind(undefined, outputStream);\n return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);\n }\n\n public schema: ParquetSchema;\n public write: (buf: Buffer) => Promise<void>;\n public close: () => Promise<void>;\n public offset: number;\n public rowCount: number;\n public rowGroups: RowGroup[];\n public pageSize: number;\n public useDataPageV2: boolean;\n\n constructor(\n schema: ParquetSchema,\n writeFn: (buf: Buffer) => Promise<void>,\n closeFn: () => Promise<void>,\n fileOffset: number,\n opts: ParquetWriterOptions\n ) {\n this.schema = schema;\n this.write = writeFn;\n this.close = closeFn;\n this.offset = fileOffset;\n this.rowCount = 0;\n this.rowGroups = [];\n this.pageSize = opts.pageSize \|\| PARQUET_DEFAULT_PAGE_SIZE;\n this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;\n }\n\n writeSection(buf: Buffer): Promise<void> {\n this.offset += buf.length;\n return this.write(buf);\n }\n\n /\n Encode the parquet file header\n /\n writeHeader(): Promise<void> {\n return this.writeSection(Buffer.from(PARQUET_MAGIC));\n }\n\n /\n Encode a parquet row group. The records object should be created using the\n * shredRecord method\n /\n async writeRowGroup(records: ParquetBuffer): Promise<void> {\n const rgroup = encodeRowGroup(this.schema, records, {\n baseOffset: this.offset,\n pageSize: this.pageSize,\n useDataPageV2: this.useDataPageV2\n });\n\n this.rowCount += records.rowCount;\n this.rowGroups.push(rgroup.metadata);\n return this.writeSection(rgroup.body);\n }\n\n /\n Write the parquet file footer\n /\n writeFooter(userMetadata: Record<string, string>): Promise<void> {\n if (!userMetadata) {\n // tslint:disable-next-line:no-parameter-reassignment\n userMetadata = {};\n }\n\n return this.writeSection(\n encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata)\n );\n }\n\n /\n Set the parquet data page size. The data page size controls the maximum\n * number of column values that are written to disk as a consecutive array\n /\n setPageSize(cnt: number): void {\n this.pageSize = cnt;\n }\n}\n\n/\n Create a parquet transform stream\n /\nexport class ParquetTransformer<T> extends Transform {\n public writer: ParquetWriter<T>;\n\n constructor(schema: ParquetSchema, opts: ParquetWriterOptions = {}) {\n super({objectMode: true});\n\n const writeProxy = (function (t: ParquetTransformer<any>) {\n return async function (b: any): Promise<void> {\n t.push(b);\n };\n })(this);\n\n this.writer = new ParquetWriter(\n schema,\n new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),\n opts\n );\n }\n\n // tslint:disable-next-line:function-name\n _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {\n if (row) {\n return this.writer.appendRow(row).then(callback);\n }\n callback();\n return Promise.resolve();\n }\n\n // tslint:disable-next-line:function-name\n async _flush(callback: (val?: any) => void) {\n await this.writer.close(callback);\n }\n}\n\n/\n Encode a consecutive array of data using one of the parquet encodings\n /\nfunction encodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n values: any[],\n opts: ParquetCodecOptions\n) {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].encodeValues(type, values, opts);\n}\n\n/\n Encode a parquet data page\n /\nfunction encodeDataPage(\n column: ParquetField,\n data: ParquetData\n): {\n header: PageHeader;\n headerSize: number;\n page: Buffer;\n} {\n / encode repetition and definition levels /\n let rLevelsBuf = Buffer.alloc(0);\n if (column.rLevelMax > 0) {\n rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n bitWidth: Util.getBitWidth(column.rLevelMax)\n // disableEnvelope: false\n });\n }\n\n let dLevelsBuf = Buffer.alloc(0);\n if (column.dLevelMax > 0) {\n dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n bitWidth: Util.getBitWidth(column.dLevelMax)\n // disableEnvelope: false\n });\n }\n\n / encode values /\n const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n typeLength: column.typeLength,\n bitWidth: column.typeLength\n });\n\n const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);\n\n // compression = column.compression === 'UNCOMPRESSED' ? (compression \|\| 'UNCOMPRESSED') : column.compression;\n const compressedBuf = Compression.deflate(column.compression!, dataBuf);\n\n / build page header /\n const header = new PageHeader({\n type: PageType.DATA_PAGE,\n data_page_header: new DataPageHeader({\n num_values: data.count,\n encoding: Encoding[column.encoding!] as any,\n definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],\n repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]\n }),\n uncompressed_page_size: dataBuf.length,\n compressed_page_size: compressedBuf.length\n });\n\n / concat page header, repetition and definition levels and values /\n const headerBuf = Util.serializeThrift(header);\n const page = Buffer.concat([headerBuf, compressedBuf]);\n\n return {header, headerSize: headerBuf.length, page};\n}\n\n/\n Encode a parquet data page (v2)\n /\nfunction encodeDataPageV2(\n column: ParquetField,\n data: ParquetData,\n rowCount: number\n): {\n header: PageHeader;\n headerSize: number;\n page: Buffer;\n} {\n / encode values /\n const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n typeLength: column.typeLength,\n bitWidth: column.typeLength\n });\n\n // compression = column.compression === 'UNCOMPRESSED' ? (compression \|\| 'UNCOMPRESSED') : column.compression;\n const compressedBuf = Compression.deflate(column.compression!, valuesBuf);\n\n / encode repetition and definition levels /\n let rLevelsBuf = Buffer.alloc(0);\n if (column.rLevelMax > 0) {\n rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n bitWidth: Util.getBitWidth(column.rLevelMax),\n disableEnvelope: true\n });\n }\n\n let dLevelsBuf = Buffer.alloc(0);\n if (column.dLevelMax > 0) {\n dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n bitWidth: Util.getBitWidth(column.dLevelMax),\n disableEnvelope: true\n });\n }\n\n / build page header /\n const header = new PageHeader({\n type: PageType.DATA_PAGE_V2,\n data_page_header_v2: new DataPageHeaderV2({\n num_values: data.count,\n num_nulls: data.count - data.values.length,\n num_rows: rowCount,\n encoding: Encoding[column.encoding!] as any,\n definition_levels_byte_length: dLevelsBuf.length,\n repetition_levels_byte_length: rLevelsBuf.length,\n is_compressed: column.compression !== 'UNCOMPRESSED'\n }),\n uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,\n compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length\n });\n\n / concat page header, repetition and definition levels and values /\n const headerBuf = Util.serializeThrift(header);\n const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);\n return {header, headerSize: headerBuf.length, page};\n}\n\n/\n Encode an array of values into a parquet column chunk\n /\nfunction encodeColumnChunk(\n column: ParquetField,\n buffer: ParquetBuffer,\n offset: number,\n opts: ParquetWriterOptions\n): {\n body: Buffer;\n metadata: ColumnMetaData;\n metadataOffset: number;\n} {\n const data = buffer.columnData[column.path.join()];\n const baseOffset = (opts.baseOffset \|\| 0) + offset;\n / encode data page(s) /\n // const pages: Buffer[] = [];\n let pageBuf: Buffer;\n // tslint:disable-next-line:variable-name\n let total_uncompressed_size = 0;\n // tslint:disable-next-line:variable-name\n let total_compressed_size = 0;\n {\n let result: any;\n if (opts.useDataPageV2) {\n result = encodeDataPageV2(column, data, buffer.rowCount);\n } else {\n result = encodeDataPage(column, data);\n }\n // pages.push(result.page);\n pageBuf = result.page;\n total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;\n total_compressed_size += result.header.compressed_page_size + result.headerSize;\n }\n\n // const pagesBuf = Buffer.concat(pages);\n // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression \|\| 'UNCOMPRESSED') : column.compression;\n\n / prepare metadata header /\n const metadata = new ColumnMetaData({\n path_in_schema: column.path,\n num_values: data.count,\n data_page_offset: baseOffset,\n encodings: [],\n total_uncompressed_size, // : pagesBuf.length,\n total_compressed_size,\n type: Type[column.primitiveType!],\n codec: CompressionCodec[column.compression!]\n });\n\n / list encodings /\n metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);\n metadata.encodings.push(Encoding[column.encoding!]);\n\n / concat metadata header and data pages /\n const metadataOffset = baseOffset + pageBuf.length;\n const body = Buffer.concat([pageBuf, Util.serializeThrift(metadata)]);\n return {body, metadata, metadataOffset};\n}\n\n/\n Encode a list of column values into a parquet row group\n /\nfunction encodeRowGroup(\n schema: ParquetSchema,\n data: ParquetBuffer,\n opts: ParquetWriterOptions\n): {\n body: Buffer;\n metadata: RowGroup;\n} {\n const metadata = new RowGroup({\n num_rows: data.rowCount,\n columns: [],\n total_byte_size: 0\n });\n\n let body = Buffer.alloc(0);\n for (const field of schema.fieldList) {\n if (field.isNested) {\n continue; // eslint-disable-line no-continue\n }\n\n const cchunkData = encodeColumnChunk(field, data, body.length, opts);\n\n const cchunk = new ColumnChunk({\n file_offset: cchunkData.metadataOffset,\n meta_data: cchunkData.metadata\n });\n\n metadata.columns.push(cchunk);\n metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);\n\n body = Buffer.concat([body, cchunkData.body]);\n }\n\n return {body, metadata};\n}\n\n/\n Encode a parquet file metadata footer\n */\nfunction encodeFooter(\n schema: ParquetSchema,\n rowCount: number,\n rowGroups: RowGroup[],\n userMetadata: Record<string, string>\n): Buffer {\n const metadata = new FileMetaData({\n version: PARQUET_VERSION,\n created_by: 'parquets',\n num_rows: rowCount,\n row_groups: rowGroups,\n schema: [],\n key_value_metadata: []\n });\n\n for (const key in userMetadata) {\n const kv = new KeyValue({\n key,\n value: userMetadata[key]\n });\n metadata.key_value_metadata?.push?.(kv);\n }\n\n {\n const schemaRoot = new SchemaElement({\n name: 'root',\n num_children: Object.keys(schema.fields).length\n });\n metadata.schema.push(schemaRoot);\n }\n\n for (const field of schema.fieldList) {\n const relt = FieldRepetitionType[field.repetitionType];\n const schemaElem = new SchemaElement({\n name: field.name,\n repetition_type: relt as any\n });\n\n if (field.isNested) {\n schemaElem.num_children = field.fieldCount;\n } else {\n schemaElem.type = Type[field.primitiveType!] as Type;\n }\n\n if (field.originalType) {\n schemaElem.converted_type = ConvertedType[field.originalType] as ConvertedType;\n }\n\n schemaElem.type_length = field.typeLength;\n\n metadata.schema.push(schemaElem);\n }\n\n const metadataEncoded = Util.serializeThrift(metadata);\n const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);\n metadataEncoded.copy(footerEncoded);\n footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);\n footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);\n return footerEncoded;\n}\n"],"file":"writer.js"}
1	+ {"version":3,"sources":["../../../src/parquetjs/writer.ts"],"names":["Transform","PARQUET_CODECS","Compression","Shred","ColumnChunk","ColumnMetaData","CompressionCodec","ConvertedType","DataPageHeader","DataPageHeaderV2","Encoding","FieldRepetitionType","FileMetaData","KeyValue","PageHeader","PageType","RowGroup","SchemaElement","Type","Util","Int64","PARQUET_MAGIC","PARQUET_VERSION","PARQUET_DEFAULT_PAGE_SIZE","PARQUET_DEFAULT_ROW_GROUP_SIZE","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","ParquetWriter","openFile","schema","path","opts","outputStream","osopen","openStream","envelopeWriter","ParquetEnvelopeWriter","constructor","rowBuffer","rowGroupSize","closed","userMetadata","writeHeader","err","close","appendRow","row","Error","shredRecord","rowCount","callback","writeFooter","setMetadata","key","value","String","setRowGroupSize","cnt","setPageSize","writeFn","oswrite","bind","undefined","closeFn","osclose","fileOffset","write","offset","rowGroups","pageSize","useDataPageV2","Boolean","writeSection","buf","length","Buffer","from","writeRowGroup","records","rgroup","encodeRowGroup","baseOffset","push","metadata","body","encodeFooter","ParquetTransformer","objectMode","writeProxy","t","b","writer","_transform","encoding","then","Promise","resolve","_flush","encodeValues","type","values","encodeDataPage","column","data","rLevelsBuf","alloc","rLevelMax","rlevels","bitWidth","getBitWidth","dLevelsBuf","dLevelMax","dlevels","valuesBuf","primitiveType","typeLength","dataBuf","concat","compressedBuf","deflate","compression","header","DATA_PAGE","data_page_header","num_values","count","definition_level_encoding","repetition_level_encoding","uncompressed_page_size","compressed_page_size","headerBuf","serializeThrift","page","headerSize","encodeDataPageV2","disableEnvelope","DATA_PAGE_V2","data_page_header_v2","num_nulls","num_rows","definition_levels_byte_length","repetition_levels_byte_length","is_compressed","encodeColumnChunk","buffer","columnData","join","pageBuf","total_uncompressed_size","total_compressed_size","result","path_in_schema","data_page_offset","encodings","codec","metadataOffset","columns","total_byte_size","field","fieldList","isNested","cchunkData","cchunk","file_offset","meta_data","Number","version","created_by","row_groups","key_value_metadata","kv","schemaRoot","name","num_children","Object","keys","fields","relt","repetitionType","schemaElem","repetition_type","fieldCount","originalType","converted_type","type_length","metadataEncoded","footerEncoded","copy","writeUInt32LE"],"mappings":";AAEA,SAAQA,SAAR,QAAkC,QAAlC;AACA,SAA6BC,cAA7B,QAAkD,UAAlD;AACA,OAAO,KAAKC,WAAZ,MAA6B,eAA7B;AASA,OAAO,KAAKC,KAAZ,MAAuB,gBAAvB;AACA,SACEC,WADF,EAEEC,cAFF,EAGEC,gBAHF,EAIEC,aAJF,EAKEC,cALF,EAMEC,gBANF,EAOEC,QAPF,EAQEC,mBARF,EASEC,YATF,EAUEC,QAVF,EAWEC,UAXF,EAYEC,QAZF,EAaEC,QAbF,EAcEC,aAdF,EAeEC,IAfF,QAgBO,kBAhBP;AAiBA,OAAO,KAAKC,IAAZ,MAAsB,QAAtB;AACA,OAAOC,KAAP,MAAkB,YAAlB;AAKA,MAAMC,aAAa,GAAG,MAAtB;AAKA,MAAMC,eAAe,GAAG,CAAxB;AAKA,MAAMC,yBAAyB,GAAG,IAAlC;AACA,MAAMC,8BAA8B,GAAG,IAAvC;AAKA,MAAMC,kBAAkB,GAAG,OAA3B;AACA,MAAMC,sBAAsB,GAAG,KAA/B;AAuBA,OAAO,MAAMC,aAAN,CAAuB;AAKP,eAARC,QAAQ,CACnBC,MADmB,EAEnBC,IAFmB,EAGnBC,IAHmB,EAIQ;AAC3B,UAAMC,YAAY,GAAG,MAAMb,IAAI,CAACc,MAAL,CAAYH,IAAZ,EAAkBC,IAAlB,CAA3B;AACA,WAAOJ,aAAa,CAACO,UAAd,CAAyBL,MAAzB,EAAiCG,YAAjC,EAA+CD,IAA/C,CAAP;AACD;;AAMsB,eAAVG,UAAU,CACrBL,MADqB,EAErBG,YAFqB,EAGrBD,IAHqB,EAIM;AAC3B,QAAI,CAACA,IAAL,EAAW;AAETA,MAAAA,IAAI,GAAG,EAAP;AACD;;AAED,UAAMI,cAAc,GAAG,MAAMC,qBAAqB,CAACF,UAAtB,CAAiCL,MAAjC,EAAyCG,YAAzC,EAAuDD,IAAvD,CAA7B;AAEA,WAAO,IAAIJ,aAAJ,CAAkBE,MAAlB,EAA0BM,cAA1B,EAA0CJ,IAA1C,CAAP;AACD;;AAYDM,EAAAA,WAAW,CACTR,MADS,EAETM,cAFS,EAGTJ,IAHS,EAIT;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AACA,SAAKF,MAAL,GAAcA,MAAd;AACA,SAAKM,cAAL,GAAsBA,cAAtB;AAEA,SAAKG,SAAL,GAAiB,EAAjB;AACA,SAAKC,YAAL,GAAoBR,IAAI,CAACQ,YAAL,IAAqBf,8BAAzC;AACA,SAAKgB,MAAL,GAAc,KAAd;AACA,SAAKC,YAAL,GAAoB,EAApB;;AAGA,QAAI;AAEFN,MAAAA,cAAc,CAACO,WAAf;AACD,KAHD,CAGE,OAAOC,GAAP,EAAY;AAEZR,MAAAA,cAAc,CAACS,KAAf;AACA,YAAMD,GAAN;AACD;AACF;;AAMc,QAATE,SAAS,CAAIC,GAAJ,EAA2B;AACxC,QAAI,KAAKN,MAAT,EAAiB;AACf,YAAM,IAAIO,KAAJ,CAAU,mBAAV,CAAN;AACD;;AACD5C,IAAAA,KAAK,CAAC6C,WAAN,CAAkB,KAAKnB,MAAvB,EAA+BiB,GAA/B,EAAoC,KAAKR,SAAzC;;AACA,QAAI,KAAKA,SAAL,CAAeW,QAAf,IAA2B,KAAKV,YAApC,EAAkD;AAEhD,WAAKD,SAAL,GAAiB,EAAjB;AACD;AACF;;AAQU,QAALM,KAAK,CAACM,QAAD,EAAuC;AAChD,QAAI,KAAKV,MAAT,EAAiB;AACf,YAAM,IAAIO,KAAJ,CAAU,mBAAV,CAAN;AACD;;AAED,SAAKP,MAAL,GAAc,IAAd;;AAEA,QAAI,KAAKF,SAAL,CAAeW,QAAf,GAA0B,CAA1B,IAA+B,KAAKX,SAAL,CAAeW,QAAf,IAA2B,KAAKV,YAAnE,EAAiF;AAE/E,WAAKD,SAAL,GAAiB,EAAjB;AACD;;AAED,UAAM,KAAKH,cAAL,CAAoBgB,WAApB,CAAgC,KAAKV,YAArC,CAAN;AACA,UAAM,KAAKN,cAAL,CAAoBS,KAApB,EAAN;;AAGA,QAAIM,QAAJ,EAAc;AACZA,MAAAA,QAAQ;AACT;AACF;;AAKDE,EAAAA,WAAW,CAACC,GAAD,EAAcC,KAAd,EAAmC;AAE5C,SAAKb,YAAL,CAAkBc,MAAM,CAACF,GAAD,CAAxB,IAAiCE,MAAM,CAACD,KAAD,CAAvC;AACD;;AAQDE,EAAAA,eAAe,CAACC,GAAD,EAAoB;AACjC,SAAKlB,YAAL,GAAoBkB,GAApB;AACD;;AAMDC,EAAAA,WAAW,CAACD,GAAD,EAAoB;AAC7B,SAAKtB,cAAL,CAAoBuB,WAApB,CAAgCD,GAAhC;AACD;;AArI2B;AA8I9B,OAAO,MAAMrB,qBAAN,CAA4B;AAIV,eAAVF,UAAU,CACrBL,MADqB,EAErBG,YAFqB,EAGrBD,IAHqB,EAIW;AAChC,UAAM4B,OAAO,GAAGxC,IAAI,CAACyC,OAAL,CAAaC,IAAb,CAAkBC,SAAlB,EAA6B9B,YAA7B,CAAhB;AACA,UAAM+B,OAAO,GAAG5C,IAAI,CAAC6C,OAAL,CAAaH,IAAb,CAAkBC,SAAlB,EAA6B9B,YAA7B,CAAhB;AACA,WAAO,IAAII,qBAAJ,CAA0BP,MAA1B,EAAkC8B,OAAlC,EAA2CI,OAA3C,EAAoD,CAApD,EAAuDhC,IAAvD,CAAP;AACD;;AAWDM,EAAAA,WAAW,CACTR,MADS,EAET8B,OAFS,EAGTI,OAHS,EAITE,UAJS,EAKTlC,IALS,EAMT;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AAAA;;AACA,SAAKF,MAAL,GAAcA,MAAd;AACA,SAAKqC,KAAL,GAAaP,OAAb;AACA,SAAKf,KAAL,GAAamB,OAAb;AACA,SAAKI,MAAL,GAAcF,UAAd;AACA,SAAKhB,QAAL,GAAgB,CAAhB;AACA,SAAKmB,SAAL,GAAiB,EAAjB;AACA,SAAKC,QAAL,GAAgBtC,IAAI,CAACsC,QAAL,IAAiB9C,yBAAjC;AACA,SAAK+C,aAAL,GAAqB,mBAAmBvC,IAAnB,GAA0BwC,OAAO,CAACxC,IAAI,CAACuC,aAAN,CAAjC,GAAwD,KAA7E;AACD;;AAEDE,EAAAA,YAAY,CAACC,GAAD,EAA6B;AACvC,SAAKN,MAAL,IAAeM,GAAG,CAACC,MAAnB;AACA,WAAO,KAAKR,KAAL,CAAWO,GAAX,CAAP;AACD;;AAKD/B,EAAAA,WAAW,GAAkB;AAC3B,WAAO,KAAK8B,YAAL,CAAkBG,MAAM,CAACC,IAAP,CAAYvD,aAAZ,CAAlB,CAAP;AACD;;AAMkB,QAAbwD,aAAa,CAACC,OAAD,EAAwC;AACzD,UAAMC,MAAM,GAAGC,cAAc,CAAC,KAAKnD,MAAN,EAAciD,OAAd,EAAuB;AAClDG,MAAAA,UAAU,EAAE,KAAKd,MADiC;AAElDE,MAAAA,QAAQ,EAAE,KAAKA,QAFmC;AAGlDC,MAAAA,aAAa,EAAE,KAAKA;AAH8B,KAAvB,CAA7B;AAMA,SAAKrB,QAAL,IAAiB6B,OAAO,CAAC7B,QAAzB;AACA,SAAKmB,SAAL,CAAec,IAAf,CAAoBH,MAAM,CAACI,QAA3B;AACA,WAAO,KAAKX,YAAL,CAAkBO,MAAM,CAACK,IAAzB,CAAP;AACD;;AAKDjC,EAAAA,WAAW,CAACV,YAAD,EAAsD;AAC/D,QAAI,CAACA,YAAL,EAAmB;AAEjBA,MAAAA,YAAY,GAAG,EAAf;AACD;;AAED,WAAO,KAAK+B,YAAL,CACLa,YAAY,CAAC,KAAKxD,MAAN,EAAc,KAAKoB,QAAnB,EAA6B,KAAKmB,SAAlC,EAA6C3B,YAA7C,CADP,CAAP;AAGD;;AAMDiB,EAAAA,WAAW,CAACD,GAAD,EAAoB;AAC7B,SAAKY,QAAL,GAAgBZ,GAAhB;AACD;;AAxFgC;AA8FnC,OAAO,MAAM6B,kBAAN,SAAoCtF,SAApC,CAA8C;AAGnDqC,EAAAA,WAAW,CAACR,MAAD,EAAwBE,IAA0B,GAAG,EAArD,EAAyD;AAClE,UAAM;AAACwD,MAAAA,UAAU,EAAE;AAAb,KAAN;;AADkE;;AAGlE,UAAMC,UAAU,GAAI,UAAUC,CAAV,EAAsC;AACxD,aAAO,gBAAgBC,CAAhB,EAAuC;AAC5CD,QAAAA,CAAC,CAACP,IAAF,CAAOQ,CAAP;AACD,OAFD;AAGD,KAJkB,CAIhB,IAJgB,CAAnB;;AAMA,SAAKC,MAAL,GAAc,IAAIhE,aAAJ,CACZE,MADY,EAEZ,IAAIO,qBAAJ,CAA0BP,MAA1B,EAAkC2D,UAAlC,EAA8C,YAAY,CAAE,CAA5D,EAA8D,CAA9D,EAAiEzD,IAAjE,CAFY,EAGZA,IAHY,CAAd;AAKD;;AAGD6D,EAAAA,UAAU,CAAC9C,GAAD,EAAW+C,QAAX,EAA6B3C,QAA7B,EAA2E;AACnF,QAAIJ,GAAJ,EAAS;AACP,aAAO,KAAK6C,MAAL,CAAY9C,SAAZ,CAAsBC,GAAtB,EAA2BgD,IAA3B,CAAgC5C,QAAhC,CAAP;AACD;;AACDA,IAAAA,QAAQ;AACR,WAAO6C,OAAO,CAACC,OAAR,EAAP;AACD;;AAGW,QAANC,MAAM,CAAC/C,QAAD,EAAgC;AAC1C,UAAM,KAAKyC,MAAL,CAAY/C,KAAZ,CAAkBM,QAAlB,CAAN;AACD;;AA/BkD;;AAqCrD,SAASgD,YAAT,CACEC,IADF,EAEEN,QAFF,EAGEO,MAHF,EAIErE,IAJF,EAKE;AACA,MAAI,EAAE8D,QAAQ,IAAI5F,cAAd,CAAJ,EAAmC;AACjC,UAAM,IAAI8C,KAAJ,6BAA+B8C,QAA/B,EAAN;AACD;;AACD,SAAO5F,cAAc,CAAC4F,QAAD,CAAd,CAAyBK,YAAzB,CAAsCC,IAAtC,EAA4CC,MAA5C,EAAoDrE,IAApD,CAAP;AACD;;AAKD,SAASsE,cAAT,CACEC,MADF,EAEEC,IAFF,EAOE;AAEA,MAAIC,UAAU,GAAG7B,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACI,SAAP,GAAmB,CAAvB,EAA0B;AACxBF,IAAAA,UAAU,GAAGN,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACI,OAAlD,EAA2D;AAClFC,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACI,SAAxB;AADwE,KAA3D,CAAzB;AAID;;AAED,MAAII,UAAU,GAAGnC,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACS,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,UAAU,GAAGZ,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACS,OAAlD,EAA2D;AAClFJ,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACS,SAAxB;AADwE,KAA3D,CAAzB;AAID;;AAGD,QAAME,SAAS,GAAGf,YAAY,CAACI,MAAM,CAACY,aAAR,EAAwBZ,MAAM,CAACT,QAA/B,EAA0CU,IAAI,CAACH,MAA/C,EAAuD;AACnFe,IAAAA,UAAU,EAAEb,MAAM,CAACa,UADgE;AAEnFP,IAAAA,QAAQ,EAAEN,MAAM,CAACa;AAFkE,GAAvD,CAA9B;AAKA,QAAMC,OAAO,GAAGzC,MAAM,CAAC0C,MAAP,CAAc,CAACb,UAAD,EAAaM,UAAb,EAAyBG,SAAzB,CAAd,CAAhB;AAGA,QAAMK,aAAa,GAAGpH,WAAW,CAACqH,OAAZ,CAAoBjB,MAAM,CAACkB,WAA3B,EAAyCJ,OAAzC,CAAtB;AAGA,QAAMK,MAAM,GAAG,IAAI3G,UAAJ,CAAe;AAC5BqF,IAAAA,IAAI,EAAEpF,QAAQ,CAAC2G,SADa;AAE5BC,IAAAA,gBAAgB,EAAE,IAAInH,cAAJ,CAAmB;AACnCoH,MAAAA,UAAU,EAAErB,IAAI,CAACsB,KADkB;AAEnChC,MAAAA,QAAQ,EAAEnF,QAAQ,CAAC4F,MAAM,CAACT,QAAR,CAFiB;AAGnCiC,MAAAA,yBAAyB,EAAEpH,QAAQ,CAACgB,sBAAD,CAHA;AAInCqG,MAAAA,yBAAyB,EAAErH,QAAQ,CAACgB,sBAAD;AAJA,KAAnB,CAFU;AAQ5BsG,IAAAA,sBAAsB,EAAEZ,OAAO,CAAC1C,MARJ;AAS5BuD,IAAAA,oBAAoB,EAAEX,aAAa,CAAC5C;AATR,GAAf,CAAf;AAaA,QAAMwD,SAAS,GAAG/G,IAAI,CAACgH,eAAL,CAAqBV,MAArB,CAAlB;AACA,QAAMW,IAAI,GAAGzD,MAAM,CAAC0C,MAAP,CAAc,CAACa,SAAD,EAAYZ,aAAZ,CAAd,CAAb;AAEA,SAAO;AAACG,IAAAA,MAAD;AAASY,IAAAA,UAAU,EAAEH,SAAS,CAACxD,MAA/B;AAAuC0D,IAAAA;AAAvC,GAAP;AACD;;AAKD,SAASE,gBAAT,CACEhC,MADF,EAEEC,IAFF,EAGEtD,QAHF,EAQE;AAEA,QAAMgE,SAAS,GAAGf,YAAY,CAACI,MAAM,CAACY,aAAR,EAAwBZ,MAAM,CAACT,QAA/B,EAA0CU,IAAI,CAACH,MAA/C,EAAuD;AACnFe,IAAAA,UAAU,EAAEb,MAAM,CAACa,UADgE;AAEnFP,IAAAA,QAAQ,EAAEN,MAAM,CAACa;AAFkE,GAAvD,CAA9B;AAMA,QAAMG,aAAa,GAAGpH,WAAW,CAACqH,OAAZ,CAAoBjB,MAAM,CAACkB,WAA3B,EAAyCP,SAAzC,CAAtB;AAGA,MAAIT,UAAU,GAAG7B,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACI,SAAP,GAAmB,CAAvB,EAA0B;AACxBF,IAAAA,UAAU,GAAGN,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACI,OAAlD,EAA2D;AAClFC,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACI,SAAxB,CADwE;AAElF6B,MAAAA,eAAe,EAAE;AAFiE,KAA3D,CAAzB;AAID;;AAED,MAAIzB,UAAU,GAAGnC,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAjB;;AACA,MAAIH,MAAM,CAACS,SAAP,GAAmB,CAAvB,EAA0B;AACxBD,IAAAA,UAAU,GAAGZ,YAAY,CAACzE,kBAAD,EAAqBC,sBAArB,EAA6C6E,IAAI,CAACS,OAAlD,EAA2D;AAClFJ,MAAAA,QAAQ,EAAEzF,IAAI,CAAC0F,WAAL,CAAiBP,MAAM,CAACS,SAAxB,CADwE;AAElFwB,MAAAA,eAAe,EAAE;AAFiE,KAA3D,CAAzB;AAID;;AAGD,QAAMd,MAAM,GAAG,IAAI3G,UAAJ,CAAe;AAC5BqF,IAAAA,IAAI,EAAEpF,QAAQ,CAACyH,YADa;AAE5BC,IAAAA,mBAAmB,EAAE,IAAIhI,gBAAJ,CAAqB;AACxCmH,MAAAA,UAAU,EAAErB,IAAI,CAACsB,KADuB;AAExCa,MAAAA,SAAS,EAAEnC,IAAI,CAACsB,KAAL,GAAatB,IAAI,CAACH,MAAL,CAAY1B,MAFI;AAGxCiE,MAAAA,QAAQ,EAAE1F,QAH8B;AAIxC4C,MAAAA,QAAQ,EAAEnF,QAAQ,CAAC4F,MAAM,CAACT,QAAR,CAJsB;AAKxC+C,MAAAA,6BAA6B,EAAE9B,UAAU,CAACpC,MALF;AAMxCmE,MAAAA,6BAA6B,EAAErC,UAAU,CAAC9B,MANF;AAOxCoE,MAAAA,aAAa,EAAExC,MAAM,CAACkB,WAAP,KAAuB;AAPE,KAArB,CAFO;AAW5BQ,IAAAA,sBAAsB,EAAExB,UAAU,CAAC9B,MAAX,GAAoBoC,UAAU,CAACpC,MAA/B,GAAwCuC,SAAS,CAACvC,MAX9C;AAY5BuD,IAAAA,oBAAoB,EAAEzB,UAAU,CAAC9B,MAAX,GAAoBoC,UAAU,CAACpC,MAA/B,GAAwC4C,aAAa,CAAC5C;AAZhD,GAAf,CAAf;AAgBA,QAAMwD,SAAS,GAAG/G,IAAI,CAACgH,eAAL,CAAqBV,MAArB,CAAlB;AACA,QAAMW,IAAI,GAAGzD,MAAM,CAAC0C,MAAP,CAAc,CAACa,SAAD,EAAY1B,UAAZ,EAAwBM,UAAxB,EAAoCQ,aAApC,CAAd,CAAb;AACA,SAAO;AAACG,IAAAA,MAAD;AAASY,IAAAA,UAAU,EAAEH,SAAS,CAACxD,MAA/B;AAAuC0D,IAAAA;AAAvC,GAAP;AACD;;AAKD,SAASW,iBAAT,CACEzC,MADF,EAEE0C,MAFF,EAGE7E,MAHF,EAIEpC,IAJF,EASE;AACA,QAAMwE,IAAI,GAAGyC,MAAM,CAACC,UAAP,CAAkB3C,MAAM,CAACxE,IAAP,CAAYoH,IAAZ,EAAlB,CAAb;AACA,QAAMjE,UAAU,GAAG,CAAClD,IAAI,CAACkD,UAAL,IAAmB,CAApB,IAAyBd,MAA5C;AAGA,MAAIgF,OAAJ;AAEA,MAAIC,uBAAuB,GAAG,CAA9B;AAEA,MAAIC,qBAAqB,GAAG,CAA5B;AACA;AACE,QAAIC,MAAJ;;AACA,QAAIvH,IAAI,CAACuC,aAAT,EAAwB;AACtBgF,MAAAA,MAAM,GAAGhB,gBAAgB,CAAChC,MAAD,EAASC,IAAT,EAAeyC,MAAM,CAAC/F,QAAtB,CAAzB;AACD,KAFD,MAEO;AACLqG,MAAAA,MAAM,GAAGjD,cAAc,CAACC,MAAD,EAASC,IAAT,CAAvB;AACD;;AAED4C,IAAAA,OAAO,GAAGG,MAAM,CAAClB,IAAjB;AACAgB,IAAAA,uBAAuB,IAAIE,MAAM,CAAC7B,MAAP,CAAcO,sBAAd,GAAuCsB,MAAM,CAACjB,UAAzE;AACAgB,IAAAA,qBAAqB,IAAIC,MAAM,CAAC7B,MAAP,CAAcQ,oBAAd,GAAqCqB,MAAM,CAACjB,UAArE;AACD;AAMD,QAAMlD,QAAQ,GAAG,IAAI9E,cAAJ,CAAmB;AAClCkJ,IAAAA,cAAc,EAAEjD,MAAM,CAACxE,IADW;AAElC8F,IAAAA,UAAU,EAAErB,IAAI,CAACsB,KAFiB;AAGlC2B,IAAAA,gBAAgB,EAAEvE,UAHgB;AAIlCwE,IAAAA,SAAS,EAAE,EAJuB;AAKlCL,IAAAA,uBALkC;AAMlCC,IAAAA,qBANkC;AAOlClD,IAAAA,IAAI,EAAEjF,IAAI,CAACoF,MAAM,CAACY,aAAR,CAPwB;AAQlCwC,IAAAA,KAAK,EAAEpJ,gBAAgB,CAACgG,MAAM,CAACkB,WAAR;AARW,GAAnB,CAAjB;AAYArC,EAAAA,QAAQ,CAACsE,SAAT,CAAmBvE,IAAnB,CAAwBxE,QAAQ,CAACgB,sBAAD,CAAhC;AACAyD,EAAAA,QAAQ,CAACsE,SAAT,CAAmBvE,IAAnB,CAAwBxE,QAAQ,CAAC4F,MAAM,CAACT,QAAR,CAAhC;AAGA,QAAM8D,cAAc,GAAG1E,UAAU,GAAGkE,OAAO,CAACzE,MAA5C;AACA,QAAMU,IAAI,GAAGT,MAAM,CAAC0C,MAAP,CAAc,CAAC8B,OAAD,EAAUhI,IAAI,CAACgH,eAAL,CAAqBhD,QAArB,CAAV,CAAd,CAAb;AACA,SAAO;AAACC,IAAAA,IAAD;AAAOD,IAAAA,QAAP;AAAiBwE,IAAAA;AAAjB,GAAP;AACD;;AAKD,SAAS3E,cAAT,CACEnD,MADF,EAEE0E,IAFF,EAGExE,IAHF,EAOE;AACA,QAAMoD,QAAQ,GAAG,IAAInE,QAAJ,CAAa;AAC5B2H,IAAAA,QAAQ,EAAEpC,IAAI,CAACtD,QADa;AAE5B2G,IAAAA,OAAO,EAAE,EAFmB;AAG5BC,IAAAA,eAAe,EAAE;AAHW,GAAb,CAAjB;AAMA,MAAIzE,IAAI,GAAGT,MAAM,CAAC8B,KAAP,CAAa,CAAb,CAAX;;AACA,OAAK,MAAMqD,KAAX,IAAoBjI,MAAM,CAACkI,SAA3B,EAAsC;AACpC,QAAID,KAAK,CAACE,QAAV,EAAoB;AAClB;AACD;;AAED,UAAMC,UAAU,GAAGlB,iBAAiB,CAACe,KAAD,EAAQvD,IAAR,EAAcnB,IAAI,CAACV,MAAnB,EAA2B3C,IAA3B,CAApC;AAEA,UAAMmI,MAAM,GAAG,IAAI9J,WAAJ,CAAgB;AAC7B+J,MAAAA,WAAW,EAAEF,UAAU,CAACN,cADK;AAE7BS,MAAAA,SAAS,EAAEH,UAAU,CAAC9E;AAFO,KAAhB,CAAf;AAKAA,IAAAA,QAAQ,CAACyE,OAAT,CAAiB1E,IAAjB,CAAsBgF,MAAtB;AACA/E,IAAAA,QAAQ,CAAC0E,eAAT,GAA2B,IAAIzI,KAAJ,CAAUiJ,MAAM,CAAClF,QAAQ,CAAC0E,eAAV,CAAN,GAAmCI,UAAU,CAAC7E,IAAX,CAAgBV,MAA7D,CAA3B;AAEAU,IAAAA,IAAI,GAAGT,MAAM,CAAC0C,MAAP,CAAc,CAACjC,IAAD,EAAO6E,UAAU,CAAC7E,IAAlB,CAAd,CAAP;AACD;;AAED,SAAO;AAACA,IAAAA,IAAD;AAAOD,IAAAA;AAAP,GAAP;AACD;;AAKD,SAASE,YAAT,CACExD,MADF,EAEEoB,QAFF,EAGEmB,SAHF,EAIE3B,YAJF,EAKU;AACR,QAAM0C,QAAQ,GAAG,IAAIvE,YAAJ,CAAiB;AAChC0J,IAAAA,OAAO,EAAEhJ,eADuB;AAEhCiJ,IAAAA,UAAU,EAAE,UAFoB;AAGhC5B,IAAAA,QAAQ,EAAE1F,QAHsB;AAIhCuH,IAAAA,UAAU,EAAEpG,SAJoB;AAKhCvC,IAAAA,MAAM,EAAE,EALwB;AAMhC4I,IAAAA,kBAAkB,EAAE;AANY,GAAjB,CAAjB;;AASA,OAAK,MAAMpH,GAAX,IAAkBZ,YAAlB,EAAgC;AAAA;;AAC9B,UAAMiI,EAAE,GAAG,IAAI7J,QAAJ,CAAa;AACtBwC,MAAAA,GADsB;AAEtBC,MAAAA,KAAK,EAAEb,YAAY,CAACY,GAAD;AAFG,KAAb,CAAX;AAIA,6BAAA8B,QAAQ,CAACsF,kBAAT,0GAA6BvF,IAA7B,8GAAoCwF,EAApC;AACD;;AAED;AACE,UAAMC,UAAU,GAAG,IAAI1J,aAAJ,CAAkB;AACnC2J,MAAAA,IAAI,EAAE,MAD6B;AAEnCC,MAAAA,YAAY,EAAEC,MAAM,CAACC,IAAP,CAAYlJ,MAAM,CAACmJ,MAAnB,EAA2BtG;AAFN,KAAlB,CAAnB;AAIAS,IAAAA,QAAQ,CAACtD,MAAT,CAAgBqD,IAAhB,CAAqByF,UAArB;AACD;;AAED,OAAK,MAAMb,KAAX,IAAoBjI,MAAM,CAACkI,SAA3B,EAAsC;AACpC,UAAMkB,IAAI,GAAGtK,mBAAmB,CAACmJ,KAAK,CAACoB,cAAP,CAAhC;AACA,UAAMC,UAAU,GAAG,IAAIlK,aAAJ,CAAkB;AACnC2J,MAAAA,IAAI,EAAEd,KAAK,CAACc,IADuB;AAEnCQ,MAAAA,eAAe,EAAEH;AAFkB,KAAlB,CAAnB;;AAKA,QAAInB,KAAK,CAACE,QAAV,EAAoB;AAClBmB,MAAAA,UAAU,CAACN,YAAX,GAA0Bf,KAAK,CAACuB,UAAhC;AACD,KAFD,MAEO;AACLF,MAAAA,UAAU,CAAChF,IAAX,GAAkBjF,IAAI,CAAC4I,KAAK,CAAC5C,aAAP,CAAtB;AACD;;AAED,QAAI4C,KAAK,CAACwB,YAAV,EAAwB;AACtBH,MAAAA,UAAU,CAACI,cAAX,GAA4BhL,aAAa,CAACuJ,KAAK,CAACwB,YAAP,CAAzC;AACD;;AAEDH,IAAAA,UAAU,CAACK,WAAX,GAAyB1B,KAAK,CAAC3C,UAA/B;AAEAhC,IAAAA,QAAQ,CAACtD,MAAT,CAAgBqD,IAAhB,CAAqBiG,UAArB;AACD;;AAED,QAAMM,eAAe,GAAGtK,IAAI,CAACgH,eAAL,CAAqBhD,QAArB,CAAxB;AACA,QAAMuG,aAAa,GAAG/G,MAAM,CAAC8B,KAAP,CAAagF,eAAe,CAAC/G,MAAhB,GAAyB,CAAtC,CAAtB;AACA+G,EAAAA,eAAe,CAACE,IAAhB,CAAqBD,aAArB;AACAA,EAAAA,aAAa,CAACE,aAAd,CAA4BH,eAAe,CAAC/G,MAA5C,EAAoD+G,eAAe,CAAC/G,MAApE;AACAgH,EAAAA,aAAa,CAACxH,KAAd,CAAoB7C,aAApB,EAAmCoK,eAAe,CAAC/G,MAAhB,GAAyB,CAA5D;AACA,SAAOgH,aAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase /\nimport {Transform, Writable} from 'stream';\nimport {ParquetCodecOptions, PARQUET_CODECS} from './codecs';\nimport as Compression from './compression';\nimport {\n ParquetBuffer,\n ParquetCodec,\n ParquetData,\n ParquetField,\n PrimitiveType\n} from './schema/declare';\nimport {ParquetSchema} from './schema/schema';\nimport * as Shred from './schema/shred';\nimport {\n ColumnChunk,\n ColumnMetaData,\n CompressionCodec,\n ConvertedType,\n DataPageHeader,\n DataPageHeaderV2,\n Encoding,\n FieldRepetitionType,\n FileMetaData,\n KeyValue,\n PageHeader,\n PageType,\n RowGroup,\n SchemaElement,\n Type\n} from './parquet-thrift';\nimport * as Util from './util';\nimport Int64 from 'node-int64';\n\n/*\n Parquet File Magic String\n /\nconst PARQUET_MAGIC = 'PAR1';\n\n/\n Parquet File Format Version\n /\nconst PARQUET_VERSION = 1;\n\n/\n Default Page and Row Group sizes\n /\nconst PARQUET_DEFAULT_PAGE_SIZE = 8192;\nconst PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;\n\n/\n Repetition and Definition Level Encoding\n /\nconst PARQUET_RDLVL_TYPE = 'INT32';\nconst PARQUET_RDLVL_ENCODING = 'RLE';\n\nexport interface ParquetWriterOptions {\n baseOffset?: number;\n rowGroupSize?: number;\n pageSize?: number;\n useDataPageV2?: boolean;\n\n // Write Stream Options\n flags?: string;\n encoding?: string;\n fd?: number;\n mode?: number;\n autoClose?: boolean;\n start?: number;\n}\n\n/\n Write a parquet file to an output stream. The ParquetWriter will perform\n * buffering/batching for performance, so close() must be called after all rows\n * are written.\n /\n// eslint-disable-next-line @typescript-eslint/no-unused-vars\nexport class ParquetWriter<T> {\n /\n Convenience method to create a new buffered parquet writer that writes to\n * the specified file\n /\n static async openFile<T>(\n schema: ParquetSchema,\n path: string,\n opts?: ParquetWriterOptions\n ): Promise<ParquetWriter<T>> {\n const outputStream = await Util.osopen(path, opts);\n return ParquetWriter.openStream(schema, outputStream, opts);\n }\n\n /\n Convenience method to create a new buffered parquet writer that writes to\n * the specified stream\n /\n static async openStream<T>(\n schema: ParquetSchema,\n outputStream: Writable,\n opts?: ParquetWriterOptions\n ): Promise<ParquetWriter<T>> {\n if (!opts) {\n // tslint:disable-next-line:no-parameter-reassignment\n opts = {};\n }\n\n const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);\n\n return new ParquetWriter(schema, envelopeWriter, opts);\n }\n\n public schema: ParquetSchema;\n public envelopeWriter: ParquetEnvelopeWriter;\n public rowBuffer: ParquetBuffer;\n public rowGroupSize: number;\n public closed: boolean;\n public userMetadata: Record<string, string>;\n\n /\n Create a new buffered parquet writer for a given envelope writer\n /\n constructor(\n schema: ParquetSchema,\n envelopeWriter: ParquetEnvelopeWriter,\n opts: ParquetWriterOptions\n ) {\n this.schema = schema;\n this.envelopeWriter = envelopeWriter;\n // @ts-ignore Row buffer typings...\n this.rowBuffer = {};\n this.rowGroupSize = opts.rowGroupSize \|\| PARQUET_DEFAULT_ROW_GROUP_SIZE;\n this.closed = false;\n this.userMetadata = {};\n\n // TODO - better not mess with promises in the constructor\n try {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n envelopeWriter.writeHeader();\n } catch (err) {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n envelopeWriter.close();\n throw err;\n }\n }\n\n /\n Append a single row to the parquet file. Rows are buffered in memory until\n * rowGroupSize rows are in the buffer or close() is called\n /\n async appendRow<T>(row: T): Promise<void> {\n if (this.closed) {\n throw new Error('writer was closed');\n }\n Shred.shredRecord(this.schema, row, this.rowBuffer);\n if (this.rowBuffer.rowCount >= this.rowGroupSize) {\n // @ts-ignore\n this.rowBuffer = {};\n }\n }\n\n /\n Finish writing the parquet file and commit the footer to disk. This method\n * MUST be called after you are finished adding rows. You must not call this\n * method twice on the same object or add any rows after the close() method has\n * been called\n /\n async close(callback?: () => void): Promise<void> {\n if (this.closed) {\n throw new Error('writer was closed');\n }\n\n this.closed = true;\n\n if (this.rowBuffer.rowCount > 0 \|\| this.rowBuffer.rowCount >= this.rowGroupSize) {\n // @ts-ignore\n this.rowBuffer = {};\n }\n\n await this.envelopeWriter.writeFooter(this.userMetadata);\n await this.envelopeWriter.close();\n // this.envelopeWriter = null;\n\n if (callback) {\n callback();\n }\n }\n\n /\n Add key<>value metadata to the file\n /\n setMetadata(key: string, value: string): void {\n // TODO: value to be any, obj -> JSON\n this.userMetadata[String(key)] = String(value);\n }\n\n /\n Set the parquet row group size. This values controls the maximum number\n * of rows that are buffered in memory at any given time as well as the number\n * of rows that are co-located on disk. A higher value is generally better for\n * read-time I/O performance at the tradeoff of write-time memory usage.\n /\n setRowGroupSize(cnt: number): void {\n this.rowGroupSize = cnt;\n }\n\n /\n Set the parquet data page size. The data page size controls the maximum\n * number of column values that are written to disk as a consecutive array\n /\n setPageSize(cnt: number): void {\n this.envelopeWriter.setPageSize(cnt);\n }\n}\n\n/\n Create a parquet file from a schema and a number of row groups. This class\n * performs direct, unbuffered writes to the underlying output stream and is\n * intendend for advanced and internal users; the writeXXX methods must be\n * called in the correct order to produce a valid file.\n /\nexport class ParquetEnvelopeWriter {\n /\n Create a new parquet envelope writer that writes to the specified stream\n /\n static async openStream(\n schema: ParquetSchema,\n outputStream: Writable,\n opts: ParquetWriterOptions\n ): Promise<ParquetEnvelopeWriter> {\n const writeFn = Util.oswrite.bind(undefined, outputStream);\n const closeFn = Util.osclose.bind(undefined, outputStream);\n return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);\n }\n\n public schema: ParquetSchema;\n public write: (buf: Buffer) => Promise<void>;\n public close: () => Promise<void>;\n public offset: number;\n public rowCount: number;\n public rowGroups: RowGroup[];\n public pageSize: number;\n public useDataPageV2: boolean;\n\n constructor(\n schema: ParquetSchema,\n writeFn: (buf: Buffer) => Promise<void>,\n closeFn: () => Promise<void>,\n fileOffset: number,\n opts: ParquetWriterOptions\n ) {\n this.schema = schema;\n this.write = writeFn;\n this.close = closeFn;\n this.offset = fileOffset;\n this.rowCount = 0;\n this.rowGroups = [];\n this.pageSize = opts.pageSize \|\| PARQUET_DEFAULT_PAGE_SIZE;\n this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;\n }\n\n writeSection(buf: Buffer): Promise<void> {\n this.offset += buf.length;\n return this.write(buf);\n }\n\n /\n Encode the parquet file header\n /\n writeHeader(): Promise<void> {\n return this.writeSection(Buffer.from(PARQUET_MAGIC));\n }\n\n /\n Encode a parquet row group. The records object should be created using the\n * shredRecord method\n /\n async writeRowGroup(records: ParquetBuffer): Promise<void> {\n const rgroup = encodeRowGroup(this.schema, records, {\n baseOffset: this.offset,\n pageSize: this.pageSize,\n useDataPageV2: this.useDataPageV2\n });\n\n this.rowCount += records.rowCount;\n this.rowGroups.push(rgroup.metadata);\n return this.writeSection(rgroup.body);\n }\n\n /\n Write the parquet file footer\n /\n writeFooter(userMetadata: Record<string, string>): Promise<void> {\n if (!userMetadata) {\n // tslint:disable-next-line:no-parameter-reassignment\n userMetadata = {};\n }\n\n return this.writeSection(\n encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata)\n );\n }\n\n /\n Set the parquet data page size. The data page size controls the maximum\n * number of column values that are written to disk as a consecutive array\n /\n setPageSize(cnt: number): void {\n this.pageSize = cnt;\n }\n}\n\n/\n Create a parquet transform stream\n /\nexport class ParquetTransformer<T> extends Transform {\n public writer: ParquetWriter<T>;\n\n constructor(schema: ParquetSchema, opts: ParquetWriterOptions = {}) {\n super({objectMode: true});\n\n const writeProxy = (function (t: ParquetTransformer<any>) {\n return async function (b: any): Promise<void> {\n t.push(b);\n };\n })(this);\n\n this.writer = new ParquetWriter(\n schema,\n new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),\n opts\n );\n }\n\n // tslint:disable-next-line:function-name\n _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {\n if (row) {\n return this.writer.appendRow(row).then(callback);\n }\n callback();\n return Promise.resolve();\n }\n\n // tslint:disable-next-line:function-name\n async _flush(callback: (val?: any) => void) {\n await this.writer.close(callback);\n }\n}\n\n/\n Encode a consecutive array of data using one of the parquet encodings\n /\nfunction encodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n values: any[],\n opts: ParquetCodecOptions\n) {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].encodeValues(type, values, opts);\n}\n\n/\n Encode a parquet data page\n /\nfunction encodeDataPage(\n column: ParquetField,\n data: ParquetData\n): {\n header: PageHeader;\n headerSize: number;\n page: Buffer;\n} {\n / encode repetition and definition levels /\n let rLevelsBuf = Buffer.alloc(0);\n if (column.rLevelMax > 0) {\n rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n bitWidth: Util.getBitWidth(column.rLevelMax)\n // disableEnvelope: false\n });\n }\n\n let dLevelsBuf = Buffer.alloc(0);\n if (column.dLevelMax > 0) {\n dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n bitWidth: Util.getBitWidth(column.dLevelMax)\n // disableEnvelope: false\n });\n }\n\n / encode values /\n const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n typeLength: column.typeLength,\n bitWidth: column.typeLength\n });\n\n const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);\n\n // compression = column.compression === 'UNCOMPRESSED' ? (compression \|\| 'UNCOMPRESSED') : column.compression;\n const compressedBuf = Compression.deflate(column.compression!, dataBuf);\n\n / build page header /\n const header = new PageHeader({\n type: PageType.DATA_PAGE,\n data_page_header: new DataPageHeader({\n num_values: data.count,\n encoding: Encoding[column.encoding!] as any,\n definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],\n repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]\n }),\n uncompressed_page_size: dataBuf.length,\n compressed_page_size: compressedBuf.length\n });\n\n / concat page header, repetition and definition levels and values /\n const headerBuf = Util.serializeThrift(header);\n const page = Buffer.concat([headerBuf, compressedBuf]);\n\n return {header, headerSize: headerBuf.length, page};\n}\n\n/\n Encode a parquet data page (v2)\n /\nfunction encodeDataPageV2(\n column: ParquetField,\n data: ParquetData,\n rowCount: number\n): {\n header: PageHeader;\n headerSize: number;\n page: Buffer;\n} {\n / encode values /\n const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {\n typeLength: column.typeLength,\n bitWidth: column.typeLength\n });\n\n // compression = column.compression === 'UNCOMPRESSED' ? (compression \|\| 'UNCOMPRESSED') : column.compression;\n const compressedBuf = Compression.deflate(column.compression!, valuesBuf);\n\n / encode repetition and definition levels /\n let rLevelsBuf = Buffer.alloc(0);\n if (column.rLevelMax > 0) {\n rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {\n bitWidth: Util.getBitWidth(column.rLevelMax),\n disableEnvelope: true\n });\n }\n\n let dLevelsBuf = Buffer.alloc(0);\n if (column.dLevelMax > 0) {\n dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {\n bitWidth: Util.getBitWidth(column.dLevelMax),\n disableEnvelope: true\n });\n }\n\n / build page header /\n const header = new PageHeader({\n type: PageType.DATA_PAGE_V2,\n data_page_header_v2: new DataPageHeaderV2({\n num_values: data.count,\n num_nulls: data.count - data.values.length,\n num_rows: rowCount,\n encoding: Encoding[column.encoding!] as any,\n definition_levels_byte_length: dLevelsBuf.length,\n repetition_levels_byte_length: rLevelsBuf.length,\n is_compressed: column.compression !== 'UNCOMPRESSED'\n }),\n uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,\n compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length\n });\n\n / concat page header, repetition and definition levels and values /\n const headerBuf = Util.serializeThrift(header);\n const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);\n return {header, headerSize: headerBuf.length, page};\n}\n\n/\n Encode an array of values into a parquet column chunk\n /\nfunction encodeColumnChunk(\n column: ParquetField,\n buffer: ParquetBuffer,\n offset: number,\n opts: ParquetWriterOptions\n): {\n body: Buffer;\n metadata: ColumnMetaData;\n metadataOffset: number;\n} {\n const data = buffer.columnData[column.path.join()];\n const baseOffset = (opts.baseOffset \|\| 0) + offset;\n / encode data page(s) /\n // const pages: Buffer[] = [];\n let pageBuf: Buffer;\n // tslint:disable-next-line:variable-name\n let total_uncompressed_size = 0;\n // tslint:disable-next-line:variable-name\n let total_compressed_size = 0;\n {\n let result: any;\n if (opts.useDataPageV2) {\n result = encodeDataPageV2(column, data, buffer.rowCount);\n } else {\n result = encodeDataPage(column, data);\n }\n // pages.push(result.page);\n pageBuf = result.page;\n total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;\n total_compressed_size += result.header.compressed_page_size + result.headerSize;\n }\n\n // const pagesBuf = Buffer.concat(pages);\n // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression \|\| 'UNCOMPRESSED') : column.compression;\n\n / prepare metadata header /\n const metadata = new ColumnMetaData({\n path_in_schema: column.path,\n num_values: data.count,\n data_page_offset: baseOffset,\n encodings: [],\n total_uncompressed_size, // : pagesBuf.length,\n total_compressed_size,\n type: Type[column.primitiveType!],\n codec: CompressionCodec[column.compression!]\n });\n\n / list encodings /\n metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);\n metadata.encodings.push(Encoding[column.encoding!]);\n\n / concat metadata header and data pages /\n const metadataOffset = baseOffset + pageBuf.length;\n const body = Buffer.concat([pageBuf, Util.serializeThrift(metadata)]);\n return {body, metadata, metadataOffset};\n}\n\n/\n Encode a list of column values into a parquet row group\n /\nfunction encodeRowGroup(\n schema: ParquetSchema,\n data: ParquetBuffer,\n opts: ParquetWriterOptions\n): {\n body: Buffer;\n metadata: RowGroup;\n} {\n const metadata = new RowGroup({\n num_rows: data.rowCount,\n columns: [],\n total_byte_size: 0\n });\n\n let body = Buffer.alloc(0);\n for (const field of schema.fieldList) {\n if (field.isNested) {\n continue; // eslint-disable-line no-continue\n }\n\n const cchunkData = encodeColumnChunk(field, data, body.length, opts);\n\n const cchunk = new ColumnChunk({\n file_offset: cchunkData.metadataOffset,\n meta_data: cchunkData.metadata\n });\n\n metadata.columns.push(cchunk);\n metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);\n\n body = Buffer.concat([body, cchunkData.body]);\n }\n\n return {body, metadata};\n}\n\n/\n Encode a parquet file metadata footer\n */\nfunction encodeFooter(\n schema: ParquetSchema,\n rowCount: number,\n rowGroups: RowGroup[],\n userMetadata: Record<string, string>\n): Buffer {\n const metadata = new FileMetaData({\n version: PARQUET_VERSION,\n created_by: 'parquets',\n num_rows: rowCount,\n row_groups: rowGroups,\n schema: [],\n key_value_metadata: []\n });\n\n for (const key in userMetadata) {\n const kv = new KeyValue({\n key,\n value: userMetadata[key]\n });\n metadata.key_value_metadata?.push?.(kv);\n }\n\n {\n const schemaRoot = new SchemaElement({\n name: 'root',\n num_children: Object.keys(schema.fields).length\n });\n metadata.schema.push(schemaRoot);\n }\n\n for (const field of schema.fieldList) {\n const relt = FieldRepetitionType[field.repetitionType];\n const schemaElem = new SchemaElement({\n name: field.name,\n repetition_type: relt as any\n });\n\n if (field.isNested) {\n schemaElem.num_children = field.fieldCount;\n } else {\n schemaElem.type = Type[field.primitiveType!] as Type;\n }\n\n if (field.originalType) {\n schemaElem.converted_type = ConvertedType[field.originalType] as ConvertedType;\n }\n\n schemaElem.type_length = field.typeLength;\n\n metadata.schema.push(schemaElem);\n }\n\n const metadataEncoded = Util.serializeThrift(metadata);\n const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);\n metadataEncoded.copy(footerEncoded);\n footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);\n footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);\n return footerEncoded;\n}\n"],"file":"writer.js"}