@mailwoman/corpus 9.2.0 → 9.3.0

package/out/tools/golden-relabel-street.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden-relabel-street.js","sourceRoot":"","sources":["../../lib/tools/golden-relabel-street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAEH,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAuB,MAAM,qBAAqB,CAAA;AACxG,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AACjH,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AACpH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAA;AAC7D,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAkFhE;;;;GAIG;AACH,SAAS,aAAa,CAAC,CAAS;IAC/B,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAE1C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,CAAA;AAC5D,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,CAAS;IAChC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAE1C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,CAAA;AAC5D,CAAC;AAED,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,eAAe,CACvB,UAAkC,EAClC,KAA2D;IAE3D,MAAM,GAAG,GAA2B,EAAE,CAAA;IAEtC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;YACtB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAClB,GAAG,CAAC,aAAa,GAAG,KAAK,CAAC,MAAM,CAAA;YACjC,CAAC;YAED,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;YAEzB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAClB,GAAG,CAAC,aAAa,GAAG,KAAK,CAAC,MAAM,CAAA;YACjC,CAAC;YAED,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,KAAK,eAAe,IAAI,GAAG,KAAK,eAAe;YAAE,SAAQ;QAChE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IACjB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAiBD;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACrC,GAAoB,EACpB,UAAmC,EAAE;IAErC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAA;IAE/C,MAAM,IAAI,GAAG,CAAC,QAA4B,EAAuB,EAAE,CAAC,CAAC;QACpE,GAAG;QACH,OAAO,EAAE,KAAK;QACd,QAAQ;QACR,KAAK,EAAE,EAAE;QACT,WAAW,EAAE,KAAK;KAClB,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrE,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,WAAW,CAAC,CAAA;IAErC,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC,eAAe,CAAC,CAAA;IAE9D,IAAI,MAAM,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,kBAAkB,CAAC,CAAA;IAE7D,MAAM,KAAK,GAAwB,EAAE,CAAA;IACrC,IAAI,QAA4B,CAAA;IAChC,IAAI,IAAI,GAAG,MAAM,CAAA;IACjB,IAAI,MAA0B,CAAA;IAC9B,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,IAAI,SAAqC,CAAA;IAEzC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAEnC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAA;IAC/E,CAAC;SAAM,IAAI,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,kGAAkG;QAClG,uEAAuE;QACvE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAEhE,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,QAAQ,GAAG,2BAA2B,CAAA;QACvC,CAAC;aAAM,CAAC;YACP,QAAQ,GAAG,8BAA8B,CAAA;YACzC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;YACjB,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;YACjD,SAAS,GAAG,KAAK,CAAC,GAAG,CAAA;YACrB,SAAS,GAAG,SAAS,CAAC,SAAS,CAAA;QAChC,CAAC;IACF,CAAC;SAAM,CAAC;QACP,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,QAAQ,GAAG,iBAAiB,CAAA;QAC7B,CAAC;aAAM,CAAC;YACP,QAAQ,GAAG,cAAc,CAAA;YACzB,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;YACjB,MAAM,GAAG,KAAK,CAAC,IAAI,CAAA;YACnB,SAAS,GAAG,KAAK,CAAC,GAAG,CAAA;YACrB,SAAS,GAAG,SAAS,CAAC,SAAS,CAAA;QAChC,CAAC;IACF,CAAC;IAED,qGAAqG;IACrG,kEAAkE;IAClE,IAAI,MAA0B,CAAA;IAC9B,IAAI,SAAS,GAAG,EAAE,CAAA;IAElB,IAAI,WAAW,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;QAEjC,IAAI,IAAI,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,GAAG,IAAI,CAAC,IAAI,CAAA;YAClB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAA;YACpB,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACjB,CAAC;IACF,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAA;IAE7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,QAAQ,GAAG,mBAAmB,CAAA;IAC/B,CAAC;IAED,sGAAsG;IACtG,kEAAkE;IAClE,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IAE/F,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,kDAAkD,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,SAAS,IAAI,sBAAsB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,mBAAmB;YACzB,MAAM,EAAE,GAAG,SAAS,8CAA8C;SAClE,CAAC,CAAA;IACH,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,CAAA;IAElC,IAAI,MAAM,IAAI,KAAK,IAAI,IAAI,MAAM,CAAC,UAAU,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7F,KAAK,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;SAC/E,CAAC,CAAA;IACH,CAAC;IAED,qGAAqG;IACrG,sGAAsG;IACtG,uGAAuG;IACvG,2BAA2B;IAC3B,IAAI,wBAAwB,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,oBAAoB;YAC1B,MAAM,EAAE,4CAA4C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;SAC1E,CAAC,CAAA;IACH,CAAC;IAED,OAAO;QACN,GAAG,EAAE;YACJ,GAAG,GAAG;YACN,UAAU,EAAE,eAAe,CAAC,GAAG,CAAC,UAAU,EAAE;gBAC3C,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvG,MAAM,EAAE,IAAI;gBACZ,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7B,CAAC;SACF;QACD,OAAO,EAAE,IAAI;QACb,QAAQ;QACR,KAAK;QACL,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM;KACpB,CAAA;AACF,CAAC;AAkED,MAAM,YAAY,GAAG,GAAwB,EAAE,CAAC,CAAC;IAChD,cAAc,EAAE,CAAC;IACjB,8BAA8B,EAAE,CAAC;IACjC,mBAAmB,EAAE,CAAC;IACtB,eAAe,EAAE,CAAC;IAClB,cAAc,EAAE,CAAC;IACjB,oBAAoB,EAAE,CAAC;IACvB,2BAA2B,EAAE,CAAC;IAC9B,iBAAiB,EAAE,CAAC;IACpB,WAAW,EAAE,CAAC;IACd,QAAQ,EAAE,CAAC;IACX,kBAAkB,EAAE,CAAC;CACrB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAoC,IAAI,GAAG,CAAC;IACtE,oBAAoB;IACpB,2BAA2B;IAC3B,kBAAkB;CAClB,CAAC,CAAA;AAEF;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC3C,OAA6B,EAC7B,SAAiC,OAAO,CAAC,GAAG;IAE5C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAA;IAClE,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAE7B,MAAM,IAAI,GAA6B,EAAE,CAAA;IACzC,MAAM,KAAK,GAAiC,EAAE,CAAA;IAE9C,MAAM,IAAI,GAAG,KAAK,EAAE,KAAa,EAAE,MAAc,EAAE,MAAc,EAAiB,EAAE;QACnF,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;QAE7B,KAAK,MAAM,IAAI,IAAI,MAAM,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACnC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,CAAA;gBAE9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,uFAAuF;gBACvF,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;oBACnC,MAAM,cAAc,CAAC,MAAM,eAAe,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;gBACtD,CAAC;gBAED,SAAQ;YACT,CAAC;YAED,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;YAC7B,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,WAAW,GAAG,CAAC,CAAA;YACnB,MAAM,GAAG,GAAa,EAAE,CAAA;YACxB,IAAI,UAAU,GAAG,CAAC,CAAA;YAElB,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACxE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAQ;gBAE1B,UAAU,EAAE,CAAA;gBACZ,2FAA2F;gBAC3F,yEAAyE;gBACzE,MAAM,GAAG,GAAG,eAAe,CAAkB,IAAI,CAAC,CAAA;gBAClD,MAAM,MAAM,GAAG,sBAAsB,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC,CAAA;gBAExF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;gBAEzB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpB,OAAO,EAAE,CAAA;gBACV,CAAC;gBAED,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;oBACxB,WAAW,EAAE,CAAA;gBACd,CAAC;gBAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,CAAA;gBACV,CAAC;gBAED,IAAI,MAAM,CAAC,OAAO,IAAI,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAClE,IAAI,CAAC,IAAI,CAAC;wBACT,IAAI,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE;wBAC7B,IAAI,EAAE,UAAU;wBAChB,GAAG,EAAE,GAAG,CAAC,GAAG;wBACZ,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,MAAM,EAAE,GAAG,CAAC,UAAU;wBACtB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU;wBAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;qBACnB,CAAC,CAAA;gBACH,CAAC;gBAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,CAAC;YAED,MAAM,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,CAAA;YACnD,KAAK,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAA;YAE/F,MAAM,CACL,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,KAAK,UAAU,UAAU,OAAO,aAAa,WAAW,yBAAyB,OAAO,UAAU,CACzH,CAAA;QACF,CAAC;IACF,CAAC,CAAA;IAED,MAAM,CAAC,WAAW,KAAK,MAAM,MAAM,EAAE,CAAC,CAAA;IACtC,MAAM,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAA;IAE7B,MAAM,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC,CAAA;IAChG,MAAM,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAA;IAEtH,MAAM,aAAa,GAGf,EAAE,CAAA;IAEN,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,aAAa,CAAC,IAAI,CAAC,GAAG;YACrB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAC5C,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,YAAY,EAAE,KAAK,CAAC,WAAW;SAC/B,CAAA;IACF,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC5E,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAE5E,MAAM,QAAQ,GAAG;QAChB,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAC,KAAK,CAAC;QAC9C,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtC,IAAI,EAAE,6EAA6E;QACnF,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,UAAU,EAAE;YACX,iBAAiB,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE;YACjD,QAAQ,EACP,uGAAuG;gBACvG,0GAA0G;gBAC1G,qGAAqG;gBACrG,oGAAoG;gBACpG,aAAa;YACd,UAAU,EACT,mFAAmF;gBACnF,+CAA+C;YAChD,WAAW,EACV,wGAAwG;gBACxG,uDAAuD;YACxD,YAAY,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI;SACzC;QACD,MAAM,EAAE;YACP,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,YAAY;YACrB,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACzE,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;SAChG;QACD,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC;KAC/B,CAAA;IAED,MAAM,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAA;IACjE,MAAM,CAAC,KAAK,YAAY,gBAAgB,YAAY,sBAAsB,QAAQ,EAAE,CAAC,CAAA;IAErF,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,CAAA;AAC1E,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,IAA8B,EAAE,MAAc,EAAE,OAAe;IAC1F,MAAM,IAAI,GAAG,CAAC,UAAkC,EAAU,EAAE,CAC3D,CAAC,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,aAAa,CAAC;SACrE,MAAM,CAAC,SAAS,CAAC;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;SAC7B,IAAI,CAAC,KAAK,CAAC,CAAA;IAEd,mGAAmG;IACnG,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7D,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACzD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAA;IAClE,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAA;IAEzF,MAAM,IAAI,GAAG,CAAC,OAAiC,EAAU,EAAE,CAC1D,OAAO;SACL,GAAG,CACH,CAAC,KAAK,EAAE,EAAE,CACT,KAAK,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CACtK;SACA,IAAI,CAAC,IAAI,CAAC,CAAA;IAEb,MAAM,MAAM,GAAG,+DAA+D,CAAA;IAE9E,OAAO;QACN,gDAAgD,MAAM,MAAM,OAAO,EAAE;QACrE,EAAE;QACF,mFAAmF;QACnF,sFAAsF;QACtF,EAAE;QACF,eAAe,OAAO,CAAC,MAAM,oBAAoB;QACjD,EAAE;QACF,MAAM;QACN,IAAI,CAAC,OAAO,CAAC;QACb,EAAE;QACF,mBAAmB,MAAM,CAAC,MAAM,sCAAsC;QACtE,EAAE;QACF,MAAM;QACN,IAAI,CAAC,MAAM,CAAC;QACZ,EAAE;QACF,4BAA4B,KAAK,CAAC,MAAM,6CAA6C;QACrF,EAAE;QACF,MAAM;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACzB,EAAE;KACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,QAA4B;IACxD,OAAO,QAAQ,KAAK,cAAc,IAAI,QAAQ,KAAK,8BAA8B,IAAI,QAAQ,KAAK,mBAAmB,CAAA;AACtH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,GAAW;IAC3D,KAAK,MAAM,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;QACxF,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;YAAE,SAAQ;QAE5C,MAAM,QAAQ,GAAG,cAAc,CAC9B,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAClC,CAAA;QAED,IAAI,QAAQ,EAAE,UAAU,EAAE,iBAAiB,EAAE,EAAE,KAAK,OAAO;YAAE,OAAO,IAAI,CAAA;IACzE,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC"}

package/out/tools/index.d.ts

@@ -0,0 +1,28 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus operator tools — the `run()`-style modules behind `mailwoman corpus …` commands. No argv,
+ *   no `process.exit`: commands own parsing, rendering, and exit codes.
+ */
+export * from "#tools/align-slice";
+export * from "#tools/audit";
+export * from "#tools/corpus-stats";
+export * from "#tools/ingest-csv";
+export * from "#tools/overlay-manifest";
+export * from "#tools/fetch/download";
+export * from "#tools/fetch/index";
+export * from "#tools/golden-expand";
+export * from "#tools/golden-promote";
+export * from "#tools/golden-relabel-street";
+export * from "#tools/jsonl-to-parquet";
+export * from "#tools/lint-slice";
+export * from "#tools/lint-slice-vocab";
+export * from "#tools/slice-kryptonite";
+export * from "#tools/overture-subvenue";
+export * from "#tools/postcode-triples";
+export * from "#tools/sub-venue-lexicon";
+export * from "#tools/sub-venue-promotions";
+export * from "#tools/slice-translit";
+//# sourceMappingURL=index.d.ts.map

package/out/tools/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/tools/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,cAAc,oBAAoB,CAAA;AAClC,cAAc,cAAc,CAAA;AAC5B,cAAc,qBAAqB,CAAA;AACnC,cAAc,mBAAmB,CAAA;AACjC,cAAc,yBAAyB,CAAA;AACvC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,8BAA8B,CAAA;AAC5C,cAAc,yBAAyB,CAAA;AACvC,cAAc,mBAAmB,CAAA;AACjC,cAAc,yBAAyB,CAAA;AACvC,cAAc,yBAAyB,CAAA;AACvC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,0BAA0B,CAAA;AACxC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,uBAAuB,CAAA"}

package/out/tools/index.js

@@ -0,0 +1,28 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus operator tools — the `run()`-style modules behind `mailwoman corpus …` commands. No argv,
+ *   no `process.exit`: commands own parsing, rendering, and exit codes.
+ */
+export * from "#tools/align-slice";
+export * from "#tools/audit";
+export * from "#tools/corpus-stats";
+export * from "#tools/ingest-csv";
+export * from "#tools/overlay-manifest";
+export * from "#tools/fetch/download";
+export * from "#tools/fetch/index";
+export * from "#tools/golden-expand";
+export * from "#tools/golden-promote";
+export * from "#tools/golden-relabel-street";
+export * from "#tools/jsonl-to-parquet";
+export * from "#tools/lint-slice";
+export * from "#tools/lint-slice-vocab";
+export * from "#tools/slice-kryptonite";
+export * from "#tools/overture-subvenue";
+export * from "#tools/postcode-triples";
+export * from "#tools/sub-venue-lexicon";
+export * from "#tools/sub-venue-promotions";
+export * from "#tools/slice-translit";
+//# sourceMappingURL=index.js.map

package/out/tools/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/tools/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,cAAc,oBAAoB,CAAA;AAClC,cAAc,cAAc,CAAA;AAC5B,cAAc,qBAAqB,CAAA;AACnC,cAAc,mBAAmB,CAAA;AACjC,cAAc,yBAAyB,CAAA;AACvC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,8BAA8B,CAAA;AAC5C,cAAc,yBAAyB,CAAA;AACvC,cAAc,mBAAmB,CAAA;AACjC,cAAc,yBAAyB,CAAA;AACvC,cAAc,yBAAyB,CAAA;AACvC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,0BAA0B,CAAA;AACxC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,uBAAuB,CAAA"}

package/out/tools/ingest-csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest-csv.d.ts","sourceRoot":"","sources":["../../lib/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAyTH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAED;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBxE"}

package/out/tools/ingest-csv.js

@@ -0,0 +1,265 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ *
+ *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
+ *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus ingest-csv \
+ *   --input /data/corpus/sources/usgov-nppes/npidata_pfile.csv \
+ *   --table nppes_providers \
+ *   --output /data/corpus/sources/usgov-nppes/nppes.db
+ * ```
+ *
+ *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
+ *   derived from input filename) --output <path> SQLite database path (default: input dir /
+ *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
+ *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
+ *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
+ *   CREATE TABLE, but don't import
+ */
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { basename, dirname, extname, join } from "path-ts";
+import { CSVSpliterator } from "spliterator";
+//#region Column name normalization
+/**
+ * @deprecated belongs in core, if really needed. spliterator should handle this.
+ */
+function normalizeColumnName(raw) {
+    return (raw
+        .trim()
+        .replaceAll(/^"+|"+$/g, "")
+        .toLowerCase()
+        .replaceAll(/[^a-z0-9]+/g, "_")
+        .replaceAll(/^_|_$/g, "") || "unnamed");
+}
+function dedupColumns(names) {
+    const seen = new Map();
+    return names.map((name) => {
+        const count = seen.get(name) ?? 0;
+        seen.set(name, count + 1);
+        return count === 0 ? name : `${name}_${count + 1}`;
+    });
+}
+function normalizeField(raw) {
+    let s = raw.trim();
+    if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
+        s = s.slice(1, -1).replaceAll('""', '"');
+    }
+    // Normalize common null-like values
+    if (!s || s === "null" || s === "NULL" || s === "N/A" || s === "n/a" || s === "-" || s === "<UNAVAIL>") {
+        return null;
+    }
+    return s;
+}
+function inferColumnType(samples) {
+    let nullCount = 0;
+    let intCount = 0;
+    let realCount = 0;
+    let textCount = 0;
+    for (const s of samples) {
+        if (s === null) {
+            nullCount++;
+            continue;
+        }
+        if (/^-?\d+$/.test(s)) {
+            intCount++;
+        }
+        else if (/^-?\d+\.?\d+$/.test(s)) {
+            realCount++;
+        }
+        else {
+            textCount++;
+        }
+    }
+    const total = samples.length;
+    const type = realCount / total >= 0.5 ? "REAL" : intCount / total >= 0.5 ? "INTEGER" : "TEXT";
+    return { name: "", type, nullable: nullCount / total >= 0.5 };
+}
+async function runIngest(opts) {
+    // `header: false` with `drop` is what expresses `--skip N`: the spliterator's own header handling consumes the
+    // FIRST row as the header, and `drop` counts from the row after it, so a preamble before the header has to be
+    // dropped here and the header row taken by hand. Quote handling is end-to-end (quoted delimiters, doubled quotes);
+    // `skipEmpty` drops blank lines that readline would have turned into all-null rows. The early `break` closes the
+    // file descriptor.
+    const rows = () => CSVSpliterator.fromAsync(opts.inputPath, {
+        mode: "array",
+        header: false,
+        columnDelimiter: opts.separator,
+        drop: opts.skipLines,
+        enableQuoteHandling: true,
+    });
+    // --- Pass 1: read header + sample rows for type inference ---
+    process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`);
+    let headerRow = null;
+    const sampleRows = [];
+    for await (const row of rows()) {
+        if (!headerRow && opts.hasHeader) {
+            headerRow = row;
+            continue;
+        }
+        if (sampleRows.length < opts.sampleSize) {
+            sampleRows.push(row);
+        }
+        else {
+            break;
+        }
+    }
+    if (!headerRow && opts.hasHeader) {
+        throw new Error("No header line found in CSV");
+    }
+    // --- Determine column names ---
+    let rawHeaders;
+    if (opts.hasHeader && headerRow) {
+        rawHeaders = headerRow;
+    }
+    else {
+        // Auto-generate column names: col_0, col_1, ...
+        const numCols = sampleRows[0]?.length ?? 0;
+        rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`);
+    }
+    const colNames = dedupColumns(rawHeaders.map(normalizeColumnName));
+    // --- Infer column types ---
+    const columns = colNames.map((name, i) => {
+        const samples = sampleRows.map((row) => {
+            const raw = row[i] ?? "";
+            return normalizeField(raw);
+        });
+        const info = inferColumnType(samples);
+        info.name = name;
+        return info;
+    });
+    // --- Generate SQL ---
+    const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ");
+    // Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
+    // so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
+    const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`;
+    const tempCols = columns.map((c) => `"${c.name}"`).join(", ");
+    process.stderr.write(`\nSchema inferred from ${sampleRows.length} sample rows:\n`);
+    process.stderr.write(`${createTableSQL}\n\n`);
+    if (opts.dryRun) {
+        process.stderr.write("--dry-run: stopping before import\n");
+        return;
+    }
+    // --- Create database + import ---
+    const { DatabaseClient } = await import("@mailwoman/sqlite/client");
+    await makeDirectories(dirname(opts.outputPath));
+    // `Database` — the EMPTY schema — deliberately, not by default: `createTableSQL` is built from the columns and
+    // types inferred from the CSV at runtime, so there is no table this file could name at compile time. Every write
+    // below goes through `exec`/`prepare` for the same reason.
+    using db = new DatabaseClient(opts.outputPath);
+    db.exec("PRAGMA journal_mode = OFF"); // faster for bulk import
+    db.exec("PRAGMA synchronous = OFF");
+    db.exec(createTableSQL);
+    // Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
+    // NULL normalization and type coercion.
+    // better-sqlite3 doesn't support .import natively, so we use a different approach:
+    // Read the CSV line-by-line and INSERT in a transaction.
+    process.stderr.write(`Importing rows...\n`);
+    const insertStmt = db.prepare(`INSERT INTO "${opts.tableName}" (${tempCols}) VALUES (${columns.map(() => "?").join(", ")})`);
+    let imported = 0;
+    let headerSkipped = false;
+    // node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
+    const doInsert = () => {
+        db.exec("BEGIN");
+        try {
+            for (const row of batch) {
+                insertStmt.run(...row);
+            }
+            db.exec("COMMIT");
+        }
+        catch (error) {
+            db.exec("ROLLBACK");
+            throw error;
+        }
+    };
+    const batch = [];
+    const BATCH_SIZE = 10_000;
+    for await (const fields of rows()) {
+        if (opts.hasHeader && !headerSkipped) {
+            headerSkipped = true;
+            continue;
+        }
+        const values = fields.map((f, i) => {
+            const v = normalizeField(f);
+            if (v === null)
+                return null;
+            const col = columns[i];
+            if (col?.type === "INTEGER" && /^-?\d+$/.test(v))
+                return Number.parseInt(v, 10);
+            if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v))
+                return Number.parseFloat(v);
+            return v;
+        });
+        // Pad or truncate to column count
+        while (values.length < columns.length) {
+            values.push(null);
+        }
+        values.length = columns.length;
+        batch.push(values);
+        if (batch.length >= BATCH_SIZE) {
+            doInsert();
+            imported += batch.length;
+            batch.length = 0;
+            if (imported % 100_000 === 0) {
+                process.stderr.write(`  ${(imported / 1_000_000).toFixed(1)}M rows...\n`);
+            }
+        }
+    }
+    // Flush remaining
+    if (batch.length) {
+        doInsert();
+        imported += batch.length;
+    }
+    process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`);
+    // Build a basic index on the first TEXT column (likely the primary key)
+    const firstTextCol = columns.find((c) => c.type === "TEXT");
+    if (firstTextCol) {
+        process.stderr.write(`Building index on "${firstTextCol.name}"...\n`);
+        db.exec(`CREATE INDEX IF NOT EXISTS idx_${opts.tableName}_${firstTextCol.name} ON "${opts.tableName}"("${firstTextCol.name}");`);
+    }
+    const stat = await tryStat(opts.outputPath);
+    const manifest = {
+        ingested_at: new Date().toISOString(),
+        source_csv: basename(opts.inputPath),
+        table_name: opts.tableName,
+        columns: columns.map((c) => ({ name: c.name, type: c.type, nullable: c.nullable })),
+        row_count: imported,
+        db_bytes: stat.size,
+    };
+    const manifestPath = opts.outputPath.replace(/\.db$/, ".manifest.json");
+    await writeLocalJSONFile(manifest, manifestPath);
+    process.stderr.write(`Done. ${imported.toLocaleString()} rows → ${opts.outputPath} (${ByteFormatter.formatIEC(stat.size)})\n`);
+}
+/**
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
+ * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
+ * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
+ * it.
+ */
+export async function ingestCSV(options) {
+    if (!(await pathExists(options.input))) {
+        throw new Error(`File not found: ${options.input}`);
+    }
+    const csvName = basename(options.input, extname(options.input));
+    await runIngest({
+        inputPath: options.input,
+        tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
+        outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
+        sampleSize: options.sample ?? 100,
+        separator: options.separator ?? ",",
+        skipLines: options.skip ?? 0,
+        hasHeader: !options.noHeader,
+        dryRun: options.dryRun ?? false,
+    });
+}
+//#endregion
+//# sourceMappingURL=ingest-csv.js.map

package/out/tools/ingest-csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../lib/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAGhF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,mCAAmC;AAEnC;;GAEG;AACH,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;SAC1B,WAAW,EAAE;SACb,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACvC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACzC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAChB,SAAS,EAAE,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,+GAA+G;IAC/G,8GAA8G;IAC9G,mHAAmH;IACnH,iHAAiH;IACjH,mBAAmB;IACnB,MAAM,IAAI,GAAG,GAA4B,EAAE,CAC1C,cAAc,CAAC,SAAS,CAAW,IAAI,CAAC,SAAS,EAAE;QAClD,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,KAAK;QACb,eAAe,EAAE,IAAI,CAAC,SAAS;QAC/B,IAAI,EAAE,IAAI,CAAC,SAAS;QACpB,mBAAmB,EAAE,IAAI;KACzB,CAAC,CAAA;IAEH,+DAA+D;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,IAAI,SAAS,GAAoB,IAAI,CAAA;IACrC,MAAM,UAAU,GAAe,EAAE,CAAA;IAEjC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAClC,SAAS,GAAG,GAAG,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrB,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,iCAAiC;IACjC,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;QACjC,UAAU,GAAG,SAAS,CAAA;IACvB,CAAC;SAAM,CAAC;QACP,gDAAgD;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,6BAA6B;IAC7B,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,uBAAuB;IACvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,kGAAkG;IAClG,sGAAsG;IACtG,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,mCAAmC;IACnC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAA;IACnE,MAAM,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAA;IAE/C,+GAA+G;IAC/G,iHAAiH;IACjH,2DAA2D;IAC3D,MAAM,EAAE,GAAG,IAAI,cAAc,CAAW,IAAI,CAAC,UAAU,CAAC,CAAA;IACxD,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,mFAAmF;IACnF,wCAAwC;IACxC,mFAAmF;IACnF,yDAAyD;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,MAAM,CAAA;IAEzB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YAEpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAE/E,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAEhF,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,kCAAkC;QAClC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QAED,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,wEAAwE;IACxE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QAErE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE3C,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAK,CAAC,IAAI;KACpB,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAEhD,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,SAAS,CAAC,IAAK,CAAC,IAAI,CAAC,KAAK,CACzG,CAAA;AACF,CAAC;AAgBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,GAAG,CAAC;QACrE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;QAC3E,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC;AAED,YAAY"}

package/out/tools/jsonl-to-parquet.d.ts

@@ -0,0 +1,61 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ *
+ *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
+ *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
+ *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
+ *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
+ *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
+ *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
+ *   Verified field-for-field against the PyArrow original (same column order, same logical types,
+ *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
+ *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
+ *   which is blind to physical layout. INT32 matches the corpus's native TS writer
+ *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
+ *   rides alongside.
+ *
+ *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
+ *   source_id, corpus_version, license, synth_method, synth_base_id.
+ *
+ *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
+ *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
+ *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
+ *   number, instead.
+ *
+ *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
+ *   /tmp/part-po-box.parquet
+ */
+/**
+ * Options for {@linkcode jsonlToParquet}.
+ */
+export interface JSONLToParquetOptions {
+    /**
+     * The labeled-row JSONL to convert.
+     */
+    input: string;
+    /**
+     * The parquet slice to write.
+     */
+    output: string;
+    /**
+     * Parquet row-group size. Default 50000.
+     */
+    rowGroupSize?: number;
+}
+/**
+ * Summary returned by {@linkcode jsonlToParquet}.
+ */
+export interface JSONLToParquetSummary {
+    read: number;
+    written: number;
+    outPath: string;
+}
+/**
+ * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ */
+export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
+//# sourceMappingURL=jsonl-to-parquet.d.ts.map

package/out/tools/jsonl-to-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAsDH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AA8BD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA6DhC"}

package/out/tools/jsonl-to-parquet.js

@@ -0,0 +1,150 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ *
+ *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
+ *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
+ *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
+ *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
+ *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
+ *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
+ *   Verified field-for-field against the PyArrow original (same column order, same logical types,
+ *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
+ *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
+ *   which is blind to physical layout. INT32 matches the corpus's native TS writer
+ *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
+ *   rides alongside.
+ *
+ *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
+ *   source_id, corpus_version, license, synth_method, synth_base_id.
+ *
+ *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
+ *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
+ *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
+ *   number, instead.
+ *
+ *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
+ *   /tmp/part-po-box.parquet
+ */
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
+import { parseJSONStrict } from "@mailwoman/core/json";
+import { join } from "path-ts";
+import { TextSpliterator } from "spliterator";
+import { connectDuckDB, escapeSQLString } from "#utils/parquet";
+const REQUIRED_COLUMNS = [
+    "raw",
+    "tokens",
+    "labels",
+    "span_starts",
+    "span_ends",
+    "span_tags",
+    "country",
+    "locale",
+    "source",
+    "source_id",
+    "corpus_version",
+    "license",
+    "synth_method",
+    "synth_base_id",
+];
+const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
+/**
+ * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
+ *
+ * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
+ * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
+ * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
+ * round-trips as a plain integer where INT64 would surface as bigint.
+ */
+const COLUMN_TYPES = {
+    raw: "VARCHAR",
+    tokens: "VARCHAR[]",
+    labels: "VARCHAR[]",
+    span_starts: "INTEGER[]",
+    span_ends: "INTEGER[]",
+    span_tags: "VARCHAR[]",
+    country: "VARCHAR",
+    locale: "VARCHAR",
+    source: "VARCHAR",
+    source_id: "VARCHAR",
+    corpus_version: "VARCHAR",
+    license: "VARCHAR",
+    synth_method: "VARCHAR",
+    synth_base_id: "VARCHAR",
+};
+/**
+ * Enforce the #519 span contract per row: all three present, parallel lengths.
+ *
+ * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
+ */
+function assertSpanTriple(row, lineNo) {
+    const present = SPAN_COLUMNS.filter((c) => row[c] != null);
+    if (present.length !== SPAN_COLUMNS.length) {
+        const missing = SPAN_COLUMNS.filter((c) => row[c] == null);
+        throw new Error(`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
+            `missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
+            "must carry span_starts/span_ends/span_tags; re-emit this slice through alignRow.");
+    }
+    const n = row.span_starts.length;
+    if (row.span_ends.length !== n || row.span_tags.length !== n) {
+        throw new Error(`line ${lineNo}: span triple arrays are not parallel — ` +
+            `starts=${row.span_starts.length} ends=${row.span_ends.length} ` +
+            `tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
+    }
+}
+/**
+ * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ */
+export async function jsonlToParquet(options, report) {
+    const rowGroupSize = options.rowGroupSize ?? 50_000;
+    if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
+        throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
+    }
+    // Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
+    // memory O(1) on the Node side (the Python original buffered every column into memory first). The
+    // staging directory owns the write stream, so it is closed before the directory is removed — and a
+    // mid-stream span-triple failure leaves no orphan.
+    await using staging = await temporaryDirectory("mw-jsonl-to-parquet-");
+    const stagePath = join(staging.path, "rows.ndjson");
+    const stage = staging.use(openWriteStream(stagePath, { encoding: "utf8" }));
+    let rows = 0;
+    let lineNo = 0;
+    // TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
+    // DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
+    // defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
+    // same as readline's crlfDelay:Infinity did.
+    for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
+        lineNo++;
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        const row = parseJSONStrict(line);
+        assertSpanTriple(row, lineNo);
+        // Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
+        // map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
+        stage.write(line + "\n");
+        rows++;
+    }
+    await new Promise((resolve, reject) => {
+        stage.end((err) => (err ? reject(err) : resolve()));
+    });
+    report?.(`Read ${rows} rows from ${options.input}`);
+    const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
+    const selectList = REQUIRED_COLUMNS.join(", ");
+    const db = await connectDuckDB();
+    // Row order is required: the overlay-manifest assembler records first/last source_id from
+    // slice order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
+    await db.run("SET preserve_insertion_order=true");
+    await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+        `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
+        `TO '${escapeSQLString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
+    const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(options.output)}')`);
+    const written = Number(counted.getRowObjects()[0].n);
+    report?.(`Wrote ${written} rows to ${options.output}`);
+    return { read: rows, written, outPath: options.output };
+}
+//# sourceMappingURL=jsonl-to-parquet.js.map

package/out/tools/jsonl-to-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAE/D,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AA6BD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,mGAAmG;IACnG,mDAAmD;IACnD,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IACtE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IAE3E,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,8FAA8F;IAC9F,gGAAgG;IAChG,6FAA6F;IAC7F,6CAA6C;IAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,EAAE,CAAA;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;QAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC7B,6FAA6F;QAC7F,yFAAyF;QACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;QAExB,IAAI,EAAE,CAAA;IACP,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnE,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9C,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,0FAA0F;IAC1F,6FAA6F;IAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,KAAK;QAC5E,aAAa,cAAc,mCAAmC;QAC9D,OAAO,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CAC/G,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACtH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;AACxD,CAAC"}