@mailwoman/corpus 5.9.0 → 5.10.0

package/out/src/tools/ingest-csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../../src/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,8EAA8E;AAC9E,wCAAwC;AACxC,8EAA8E;AAE9E,MAAM,KAAK,GAAG,EAAE,CAAA;AAChB,MAAM,YAAY,GAAG,EAAE,CAAA;AAEvB,+FAA+F;AAC/F,kGAAkG;AAClG,mGAAmG;AACnG,mGAAmG;AACnG,+FAA+F;AAC/F,oEAAoE;AACpE,SAAS,YAAY,CAAC,IAAY,EAAE,YAAoB,KAAK;IAC5D,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,QAAQ,GAAG,KAAK,CAAA;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAE7B,IAAI,EAAE,KAAK,YAAY,EAAE,CAAC;YACzB,QAAQ,GAAG,CAAC,QAAQ,CAAA;QACrB,CAAC;aAAM,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;YACjC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IAE9B,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAE5B,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7E,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACpC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACvC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAChB,SAAS,EAAE,CAAA;YACX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;IAExC,+DAA+D;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,iGAAiG;IACjG,qGAAqG;IACrG,qGAAqG;IACrG,sGAAsG;IACtG,8BAA8B;IAC9B,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,2BAA2B;QAC3B,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,UAAU,GAAG,IAAI,CAAA;YACjB,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,iCAAiC;IACjC,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,UAAU,EAAE,CAAC;QAClC,UAAU,GAAG,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5D,CAAC;SAAM,CAAC;QACP,gDAAgD;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,6BAA6B;IAC7B,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,uBAAuB;IACvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,kGAAkG;IAClG,sGAAsG;IACtG,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,SAAS,GAAG,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,QAAQ,eAAe,IAAI,CAAC,SAAS,WAAW,CAAA;IAE3H,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,mCAAmC;IACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IACpD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAExD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC5C,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,mFAAmF;IACnF,wCAAwC;IACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAC5C,MAAM,SAAS,GAAG;QACjB,sBAAsB,IAAI,CAAC,SAAS,aAAa,OAAO,IAAI;QAC5D,WAAW;QACX,eAAe,IAAI,CAAC,SAAS,GAAG;QAChC,YAAY,WAAW,YAAY,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,SAAS,SAAS;QACrH,SAAS;QACT,oBAAoB,IAAI,CAAC,SAAS,WAAW;KAC7C,CAAA;IAED,mFAAmF;IACnF,yDAAyD;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,KAAK,CAAA;IAExB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YACpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QACvD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAExE,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAA;YAEzE,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,kCAAkC;QAClC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QACD,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,wEAAwE;IACxE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QACrE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,EAAE,CAAC,KAAK,EAAE,CAAA;IAEV,iBAAiB;IACjB,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAA;IACxD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;KACnB,CAAA;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IAErE,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAC7G,CAAA;AACF,CAAC;AAcD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;QAClE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;QAC3E,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC"}

package/out/src/tools/jsonl-to-parquet.d.ts

@@ -0,0 +1,49 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Convert a JSONL of LabeledRow objects to a Parquet shard matching the v0.5.0 schema.
+ *
+ *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
+ *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
+ *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
+ *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the shard. DuckDB reproduces the exact
+ *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
+ *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
+ *   Verified field-for-field against the PyArrow original (same column order, same logical types,
+ *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
+ *   table. The trainer in any case reads shards by column name (`pq.read_table(...).to_pylist()`),
+ *   which is blind to physical layout. INT32 matches the corpus's native TS writer
+ *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base shards this overlay
+ *   rides alongside.
+ *
+ *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
+ *   source_id, corpus_version, license, synth_method, synth_base_id.
+ *
+ *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
+ *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
+ *   writing it would silently drop the v0.5.0 labels from the shard. Loud failure, naming the row
+ *   number, instead.
+ *
+ *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
+ *   /tmp/part-po-box.parquet
+ */
+/** Options for {@linkcode jsonlToParquet}. */
+export interface JsonlToParquetOptions {
+    /** The labeled-row JSONL to convert. */
+    input: string;
+    /** The parquet shard to write. */
+    output: string;
+    /** Parquet row-group size. Default 50000. */
+    rowGroupSize?: number;
+}
+/** Summary returned by {@linkcode jsonlToParquet}. */
+export interface JsonlToParquetSummary {
+    read: number;
+    written: number;
+    outPath: string;
+}
+/** Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard. */
+export declare function jsonlToParquet(options: JsonlToParquetOptions, report?: (line: string) => void): Promise<JsonlToParquetSummary>;
+//# sourceMappingURL=jsonl-to-parquet.d.ts.map

package/out/src/tools/jsonl-to-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAsDH,8CAA8C;AAC9C,MAAM,WAAW,qBAAqB;IACrC,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAA;IACb,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,6CAA6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,sDAAsD;AACtD,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AAkCD,oEAAoE;AACpE,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA8DhC"}

package/out/src/tools/jsonl-to-parquet.js

@@ -0,0 +1,157 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Convert a JSONL of LabeledRow objects to a Parquet shard matching the v0.5.0 schema.
+ *
+ *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
+ *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
+ *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
+ *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the shard. DuckDB reproduces the exact
+ *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
+ *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
+ *   Verified field-for-field against the PyArrow original (same column order, same logical types,
+ *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
+ *   table. The trainer in any case reads shards by column name (`pq.read_table(...).to_pylist()`),
+ *   which is blind to physical layout. INT32 matches the corpus's native TS writer
+ *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base shards this overlay
+ *   rides alongside.
+ *
+ *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
+ *   source_id, corpus_version, license, synth_method, synth_base_id.
+ *
+ *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
+ *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
+ *   writing it would silently drop the v0.5.0 labels from the shard. Loud failure, naming the row
+ *   number, instead.
+ *
+ *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
+ *   /tmp/part-po-box.parquet
+ */
+import { randomUUID } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { TextSpliterator } from "spliterator";
+const REQUIRED_COLUMNS = [
+    "raw",
+    "tokens",
+    "labels",
+    "span_starts",
+    "span_ends",
+    "span_tags",
+    "country",
+    "locale",
+    "source",
+    "source_id",
+    "corpus_version",
+    "license",
+    "synth_method",
+    "synth_base_id",
+];
+const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
+/**
+ * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
+ *
+ * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
+ * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
+ * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
+ * round-trips as a plain integer where INT64 would surface as bigint.
+ */
+const COLUMN_TYPES = {
+    raw: "VARCHAR",
+    tokens: "VARCHAR[]",
+    labels: "VARCHAR[]",
+    span_starts: "INTEGER[]",
+    span_ends: "INTEGER[]",
+    span_tags: "VARCHAR[]",
+    country: "VARCHAR",
+    locale: "VARCHAR",
+    source: "VARCHAR",
+    source_id: "VARCHAR",
+    corpus_version: "VARCHAR",
+    license: "VARCHAR",
+    synth_method: "VARCHAR",
+    synth_base_id: "VARCHAR",
+};
+/**
+ * Enforce the #519 span contract per row: all three present, parallel lengths.
+ *
+ * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
+ */
+function assertSpanTriple(row, lineNo) {
+    const present = SPAN_COLUMNS.filter((c) => row[c] != null);
+    if (present.length !== SPAN_COLUMNS.length) {
+        const missing = SPAN_COLUMNS.filter((c) => row[c] == null);
+        throw new Error(`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
+            `missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
+            "must carry span_starts/span_ends/span_tags; re-emit this shard through alignRow.");
+    }
+    const n = row.span_starts.length;
+    if (row.span_ends.length !== n || row.span_tags.length !== n) {
+        throw new Error(`line ${lineNo}: span triple arrays are not parallel — ` +
+            `starts=${row.span_starts.length} ends=${row.span_ends.length} ` +
+            `tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
+    }
+}
+/** Escape a path for single-quoted SQL string literals. */
+function sqlString(value) {
+    return value.replace(/'/g, "''");
+}
+/** Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard. */
+export async function jsonlToParquet(options, report) {
+    const rowGroupSize = options.rowGroupSize ?? 50000;
+    if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
+        throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
+    }
+    // Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
+    // memory O(1) on the Node side (the Python original buffered every column into memory first). The
+    // `finally` covers the validation pass too, so a mid-stream span-triple failure leaves no orphan.
+    const stagePath = join(tmpdir(), `mw-jsonl-to-parquet-${randomUUID()}.ndjson`);
+    const stage = createWriteStream(stagePath, { encoding: "utf8" });
+    try {
+        let rows = 0;
+        let lineNo = 0;
+        // TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
+        // DuckDB verbatim (JSON.parse here only validates), so a re-serialized JSONSpliterator row would
+        // defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
+        // same as readline's crlfDelay:Infinity did.
+        for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
+            lineNo++;
+            const line = rawLine.trim();
+            if (!line)
+                continue;
+            const row = JSON.parse(line);
+            assertSpanTriple(row, lineNo);
+            // Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
+            // map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
+            stage.write(line + "\n");
+            rows++;
+        }
+        await new Promise((resolve, reject) => stage.end((err) => (err ? reject(err) : resolve())));
+        report?.(`Read ${rows} rows from ${options.input}`);
+        const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
+        const selectList = REQUIRED_COLUMNS.join(", ");
+        // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
+        const { DuckDBInstance } = await import("@duckdb/node-api");
+        const instance = await DuckDBInstance.create();
+        const db = await instance.connect();
+        // Row order is load-bearing: the overlay-manifest assembler records first/last source_id from
+        // shard order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
+        await db.run("SET preserve_insertion_order=true");
+        await db.run(`COPY (SELECT ${selectList} FROM read_json('${sqlString(stagePath)}', ` +
+            `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
+            `TO '${sqlString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
+        const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${sqlString(options.output)}')`);
+        const written = Number(counted.getRowObjects()[0].n);
+        report?.(`Wrote ${written} rows to ${options.output}`);
+        return { read: rows, written, outPath: options.output };
+    }
+    finally {
+        stage.destroy();
+        await unlink(stagePath).catch(() => { });
+    }
+}
+//# sourceMappingURL=jsonl-to-parquet.js.map

package/out/src/tools/jsonl-to-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AAmBD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IACD,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED,2DAA2D;AAC3D,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;AACjC,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,KAAK,CAAA;IAElD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,kGAAkG;IAClG,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,UAAU,EAAE,SAAS,CAAC,CAAA;IAC9E,MAAM,KAAK,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhE,IAAI,CAAC;QACJ,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,8FAA8F;QAC9F,iGAAiG;QACjG,6FAA6F;QAC7F,6CAA6C;QAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,CAAA;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;YAE3B,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAA;YACvD,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAC7B,6FAA6F;YAC7F,yFAAyF;YACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;YACxB,IAAI,EAAE,CAAA;QACP,CAAC;QACD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAA;QAEhH,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;QACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE9C,gFAAgF;QAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;QAC9C,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;QACnC,8FAA8F;QAC9F,6FAA6F;QAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;QACjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,SAAS,CAAC,SAAS,CAAC,KAAK;YACtE,aAAa,cAAc,mCAAmC;YAC9D,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CACzG,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAChH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;IACxD,CAAC;YAAS,CAAC;QACV,KAAK,CAAC,OAAO,EAAE,CAAA;QACf,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACxC,CAAC;AACF,CAAC"}

package/out/src/tools/lint-shard-vocab.d.ts

@@ -0,0 +1,66 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic shard vs the base.
+ *
+ *   Ported from scripts/lint-shard-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
+ *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
+ *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
+ *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
+ *   the Python default.
+ *
+ *   The #511 lesson: a synthetic shard must not label a token a tag the BASE dominantly labels
+ *   something else, or training gets conflicting gradients on the same token and the minority (the
+ *   shard) loses. This reads a shard's own (token -> tag) and checks each token against the base.
+ *
+ *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
+ *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
+ *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
+ *
+ *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
+ *        retry both false-flagged FR cities as "street" from US street-contexts).
+ *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
+ *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
+ *        its true 219700:29 LOCALITY). The fix: tally each shard token's base tag SCOPED to the
+ *        country the shard uses it in (the base has a `country` column), over a LARGE/FULL scan
+ *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
+ *        context-determined). An affix-split flag (shard street_suffix/_prefix vs base "street") is
+ *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
+ *
+ *   Usage: mailwoman dev lint shard-vocab --shard <shard.parquet>
+ *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
+ *   50]
+ */
+/** Options for {@linkcode lintShardVocab}. */
+export interface LintShardVocabOptions {
+    /** The shard parquet to lint. */
+    shard: string;
+    /** Base corpus version. Default `v0.5.0`. */
+    baseVersion?: string;
+    /** Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`. */
+    baseRoot?: string;
+    /** Base-majority confidence floor for a contradiction. Default 0.7. */
+    threshold?: number;
+    /** Minimum base support to judge a token. Default 50. */
+    minCount?: number;
+    /** Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0. */
+    fraction?: number;
+}
+/** One contradiction row: token, shard tag, base tag, base fraction, base total. */
+export type ShardVocabRow = [token: string, shardTag: string, baseTag: string, baseFrac: number, baseTotal: number];
+/** Findings summary returned by {@linkcode lintShardVocab}. */
+export interface LintShardVocabSummary {
+    /** Real contradictions — the command exits 1 when nonzero. */
+    errors: number;
+    /** Affix-split rows (EXPECTED — the loader's affix-relabel handles them). */
+    warnings: number;
+    findings: {
+        contradictions: ShardVocabRow[];
+        affixSplits: ShardVocabRow[];
+    };
+}
+/** Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped. */
+export declare function lintShardVocab(options: LintShardVocabOptions): Promise<LintShardVocabSummary>;
+//# sourceMappingURL=lint-shard-vocab.d.ts.map

package/out/src/tools/lint-shard-vocab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lint-shard-vocab.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AA0JH,8CAA8C;AAC9C,MAAM,WAAW,qBAAqB;IACrC,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,6CAA6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,yDAAyD;IACzD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,oFAAoF;AACpF,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAEnH,+DAA+D;AAC/D,MAAM,WAAW,qBAAqB;IACrC,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAA;IACd,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,aAAa,EAAE,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAA;CAC3E;AAED,iGAAiG;AACjG,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA2InG"}

package/out/src/tools/lint-shard-vocab.js

@@ -0,0 +1,266 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic shard vs the base.
+ *
+ *   Ported from scripts/lint-shard-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
+ *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
+ *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
+ *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
+ *   the Python default.
+ *
+ *   The #511 lesson: a synthetic shard must not label a token a tag the BASE dominantly labels
+ *   something else, or training gets conflicting gradients on the same token and the minority (the
+ *   shard) loses. This reads a shard's own (token -> tag) and checks each token against the base.
+ *
+ *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
+ *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
+ *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
+ *
+ *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
+ *        retry both false-flagged FR cities as "street" from US street-contexts).
+ *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
+ *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
+ *        its true 219700:29 LOCALITY). The fix: tally each shard token's base tag SCOPED to the
+ *        country the shard uses it in (the base has a `country` column), over a LARGE/FULL scan
+ *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
+ *        context-determined). An affix-split flag (shard street_suffix/_prefix vs base "street") is
+ *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
+ *
+ *   Usage: mailwoman dev lint shard-vocab --shard <shard.parquet>
+ *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
+ *   50]
+ */
+import { readdirSync } from "node:fs";
+import { join } from "node:path";
+import { dataRootPath } from "@mailwoman/core/utils";
+/** Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`. */
+function stripBIO(label) {
+    const head = label.slice(0, 2);
+    return head === "B-" || head === "I-" ? label.slice(2) : label;
+}
+/**
+ * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
+ * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
+ * corpora actually contain.
+ */
+function isDigit(token) {
+    return token.length > 0 && /^\p{Nd}+$/u.test(token);
+}
+/**
+ * Round half to even (banker's rounding) — Python's built-in `round()` and `format(..., ".0%")` both use it, so percent
+ * strings and the proportional `fraction` slice match the Python output exactly.
+ */
+function pyRound(x) {
+    const floor = Math.floor(x);
+    const diff = x - floor;
+    if (diff < 0.5)
+        return floor;
+    if (diff > 0.5)
+        return floor + 1;
+    return floor % 2 === 0 ? floor : floor + 1;
+}
+/** Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%". */
+function pct(frac) {
+    return `${pyRound(frac * 100)}%`;
+}
+/**
+ * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
+ * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
+ * print.
+ */
+function pyFloat(n) {
+    return Number.isInteger(n) ? n.toFixed(1) : String(n);
+}
+/** Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field. */
+function pad(value, width) {
+    return value.padEnd(width);
+}
+/**
+ * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
+ * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
+ */
+function dominant(counter) {
+    let total = 0;
+    let bestTag = "";
+    let bestCount = -1;
+    for (const [tag, count] of counter) {
+        total += count;
+        if (count > bestCount) {
+            bestCount = count;
+            bestTag = tag;
+        }
+    }
+    if (total === 0)
+        return ["", 0, 0.0];
+    return [bestTag, total, bestCount / total];
+}
+/** Bump a (key -> count) tally, creating the inner counter on first sight. */
+function bump(table, key, sub) {
+    let counter = table.get(key);
+    if (!counter) {
+        counter = new Map();
+        table.set(key, counter);
+    }
+    counter.set(sub, (counter.get(sub) ?? 0) + 1);
+}
+/**
+ * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
+ * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
+ * here.
+ */
+async function readRows(con, path) {
+    const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
+    const raw = result.getRowObjects();
+    const rows = [];
+    for (const r of raw) {
+        const tokens = JSON.parse(String(r.tokens));
+        const labels = JSON.parse(String(r.labels));
+        if (!Array.isArray(tokens) || !Array.isArray(labels))
+            continue;
+        rows.push({
+            tokens: tokens,
+            labels: labels,
+            country: r.country == null ? null : String(r.country),
+        });
+    }
+    return rows;
+}
+/** Read just the first row's `source` value — used to group base parts for a proportional slice. */
+async function readSource(con, path) {
+    const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
+    const rows = result.getRowObjects();
+    return rows.length ? String(rows[0].source) : "";
+}
+/** Non-recursive `*.parquet` glob, sorted lexicographically — the Python `sorted(glob.glob(...))`. */
+function globParquet(dir) {
+    let names;
+    try {
+        names = readdirSync(dir);
+    }
+    catch {
+        return [];
+    }
+    return names
+        .filter((f) => f.endsWith(".parquet"))
+        .map((f) => join(dir, f))
+        .sort();
+}
+/** Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped. */
+export async function lintShardVocab(options) {
+    const baseVersion = options.baseVersion ?? "v0.5.0";
+    const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
+    const threshold = options.threshold ?? 0.7;
+    const minCount = options.minCount ?? 50;
+    const fraction = options.fraction ?? 1.0;
+    // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
+    const { DuckDBInstance } = await import("@duckdb/node-api");
+    const instance = await DuckDBInstance.create();
+    const con = await instance.connect();
+    // 1. the shard's own (token -> dominant tag) + the COUNTRIES it uses each token in
+    const shardRows = await readRows(con, options.shard);
+    const shardTags = new Map();
+    const shardCountries = new Map();
+    for (const { tokens, labels, country } of shardRows) {
+        const n = Math.min(tokens.length, labels.length);
+        for (let i = 0; i < n; i++) {
+            const w = tokens[i];
+            const l = labels[i];
+            if (isDigit(w))
+                continue; // numbers are context-determined (house_number/postcode), not lexical vocab
+            bump(shardTags, w, stripBIO(l));
+            let set = shardCountries.get(w);
+            if (!set) {
+                set = new Set();
+                shardCountries.set(w, set);
+            }
+            set.add(country);
+        }
+    }
+    const shardVocab = new Set(shardTags.keys());
+    console.log(`shard: ${shardRows.length} rows, ${shardVocab.size} unique tokens`);
+    // 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
+    const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
+    let parts = globParquet(trainDir);
+    if (!parts.length) {
+        throw new Error("no base parts found");
+    }
+    if (fraction < 1.0) {
+        const bysrc = new Map();
+        for (const p of parts) {
+            const src = await readSource(con, p);
+            let list = bysrc.get(src);
+            if (!list) {
+                list = [];
+                bysrc.set(src, list);
+            }
+            list.push(p);
+        }
+        const sliced = [];
+        for (const ps of bysrc.values()) {
+            const take = Math.max(2, pyRound(ps.length * fraction));
+            for (const p of ps.slice(0, take)) {
+                sliced.push(p);
+            }
+        }
+        parts = sliced;
+    }
+    console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${pyFloat(fraction)}), COUNTRY-scoped`);
+    // 3. tally each shard token's base tag, SCOPED to the country the shard uses it in
+    const baseTags = new Map();
+    for (let i = 0; i < parts.length; i++) {
+        const rows = await readRows(con, parts[i]);
+        for (const { tokens, labels, country } of rows) {
+            const n = Math.min(tokens.length, labels.length);
+            for (let j = 0; j < n; j++) {
+                const w = tokens[j];
+                if (shardVocab.has(w) && shardCountries.get(w).has(country)) {
+                    bump(baseTags, w, stripBIO(labels[j]));
+                }
+            }
+        }
+        if ((i + 1) % 100 === 0) {
+            console.log(`  ...${i + 1}/${parts.length} parts`);
+        }
+    }
+    // 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
+    const flagged = [];
+    const affix = [];
+    for (const w of shardVocab) {
+        const [sTag] = dominant(shardTags.get(w));
+        const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
+        if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
+            continue;
+        const row = [w, sTag, bTag, bFrac, bTotal];
+        if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
+            affix.push(row);
+        }
+        else {
+            flagged.push(row);
+        }
+    }
+    const sections = [
+        ["CONTRADICTION", flagged],
+        ["affix-split (EXPECTED — affix-relabel handles)", affix],
+    ];
+    for (const [label, rows] of sections) {
+        if (!rows.length)
+            continue;
+        rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
+        console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
+        for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
+            console.log(`  ${pad(w, 18)} shard=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
+        }
+    }
+    if (!flagged.length) {
+        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — shard base-consistent`);
+    }
+    return {
+        errors: flagged.length,
+        warnings: affix.length,
+        findings: { contradictions: flagged, affixSplits: affix },
+    };
+}
+//# sourceMappingURL=lint-shard-vocab.js.map

package/out/src/tools/lint-shard-vocab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lint-shard-vocab.js","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AASpD,mFAAmF;AACnF,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAA;IAEtB,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,CAAA;IAE5B,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,GAAG,CAAC,CAAA;IAEhC,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAA;AAC3C,CAAC;AAED,oGAAoG;AACpG,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,+FAA+F;AAC/F,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;IAEpC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED,8EAA8E;AAC9E,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAOD;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAY,CAAA;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAY,CAAA;QAEtD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAC9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,oGAAoG;AACpG,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAED,sGAAsG;AACtG,SAAS,WAAW,CAAC,GAAW;IAC/B,IAAI,KAAe,CAAA;IAEnB,IAAI,CAAC;QACJ,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,KAAK;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;SACxB,IAAI,EAAE,CAAA;AACT,CAAC;AA8BD,iGAAiG;AACjG,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAA;IAExC,gFAAgF;IAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAC9C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;IAEpC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAC5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9E,IAAI,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QACD,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QACD,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;IAElH,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}