@mailwoman/corpus 4.10.0 → 4.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +79 -0
  2. package/out/src/adapters/ban/adapter.d.ts +6 -2
  3. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  4. package/out/src/adapters/ban/adapter.js +8 -4
  5. package/out/src/adapters/ban/adapter.js.map +1 -1
  6. package/out/src/adapters/index.d.ts.map +1 -1
  7. package/out/src/adapters/index.js +2 -0
  8. package/out/src/adapters/index.js.map +1 -1
  9. package/out/src/adapters/openaddresses/adapter.d.ts +5 -2
  10. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  11. package/out/src/adapters/openaddresses/adapter.js +2 -7
  12. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  13. package/out/src/adapters/overture/adapter.d.ts +37 -0
  14. package/out/src/adapters/overture/adapter.d.ts.map +1 -0
  15. package/out/src/adapters/overture/adapter.js +119 -0
  16. package/out/src/adapters/overture/adapter.js.map +1 -0
  17. package/out/src/adapters/synth-po-box/adapter.d.ts +8 -0
  18. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  19. package/out/src/adapters/synth-po-box/adapter.js +37 -2
  20. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  21. package/out/src/adapters/tiger/adapter.js +1 -1
  22. package/out/src/adapters/tiger/adapter.js.map +1 -1
  23. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
  24. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +1 -1
  25. package/out/src/align.js +1 -1
  26. package/out/src/align.js.map +1 -1
  27. package/out/src/build.d.ts +14 -0
  28. package/out/src/build.d.ts.map +1 -1
  29. package/out/src/build.js +25 -0
  30. package/out/src/build.js.map +1 -1
  31. package/out/src/license.d.ts +37 -0
  32. package/out/src/license.d.ts.map +1 -0
  33. package/out/src/license.js +46 -0
  34. package/out/src/license.js.map +1 -0
  35. package/out/src/synthesize-boundary-stress.d.ts +77 -0
  36. package/out/src/synthesize-boundary-stress.d.ts.map +1 -0
  37. package/out/src/synthesize-boundary-stress.js +449 -0
  38. package/out/src/synthesize-boundary-stress.js.map +1 -0
  39. package/out/src/synthesize-no-street.d.ts +1 -1
  40. package/out/src/synthesize-no-street.js +2 -2
  41. package/out/src/synthesize-no-street.js.map +1 -1
  42. package/out/src/synthesize-po-box.d.ts +10 -1
  43. package/out/src/synthesize-po-box.d.ts.map +1 -1
  44. package/out/src/synthesize-po-box.js +63 -3
  45. package/out/src/synthesize-po-box.js.map +1 -1
  46. package/out/src/synthesize.d.ts +15 -4
  47. package/out/src/synthesize.d.ts.map +1 -1
  48. package/out/src/synthesize.js +124 -4
  49. package/out/src/synthesize.js.map +1 -1
  50. package/out/src/wof-json.d.ts +1 -1
  51. package/out/src/wof-json.js +1 -1
  52. package/package.json +4 -4
package/out/src/align.js CHANGED
@@ -150,7 +150,7 @@ function locateSpan(args) {
150
150
  // Pass 1: verbatim substring. Word-boundary-aligned matches are PREFERRED over intra-word ones
151
151
  // — leftmost-substring alone let a short value claim the inside of an earlier word (region "AK"
152
152
  // matched inside "Umak"/"Lake", scrambling every later span; caught by the v0.5.0 pilot build).
153
- // Intra-word matches stay allowed as the fallback because they are load-bearing for affix
153
+ // Intra-word matches stay allowed as the fallback because they are essential for affix
154
154
  // supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence —
155
155
  // sub-word spans are the point of the char-offset format).
156
156
  let intraWord;
@@ -1 +1 @@
1
- {"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;AAsCnF,0BAA0B;AAC1B,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,OAAqB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,qGAAqG;IACrG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QACpD,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,4BAA4B;IAC5B,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,IAAI,CAC7F,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,+FAA+F;QAC/F,8FAA8F;QAC9F,yFAAyF;QACzF,8FAA8F;QAC9F,4FAA4F;QAC5F,oDAAoD;QACpD,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,uFAAuF;QACvF,mFAAmF;QACnF,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAEzC,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,0FAA0F;IAC1F,gGAAgG;IAChG,2DAA2D;IAC3D,IAAI,SAAqD,CAAA;IACzD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,IAAI,GAAG,GAAG,CAAC;YAAE,MAAK;QAClB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;YACrE,SAAS,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QAClC,CAAC;QACD,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IACD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAA;IAE/B,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,sFAAsF;IACtF,sFAAsF;IACtF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QACzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA,CAAC,+BAA+B;QACxF,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,SAAS,GAAG,eAAe,CAAA;AAEjC,mGAAmG;AACnG,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAa,EAAE,GAAW;IACtE,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC,CAAA;IACxE,OAAO,MAAM,IAAI,KAAK,CAAA;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAA4B,EAAE,KAA+B;IACjF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA,CAAC,qBAAqB;IAExD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YACnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBACD,MAAK;YACN,CAAC;QACF,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
1
+ {"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;AAsCnF,0BAA0B;AAC1B,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,OAAqB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,qGAAqG;IACrG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QACpD,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,4BAA4B;IAC5B,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,IAAI,CAC7F,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,+FAA+F;QAC/F,8FAA8F;QAC9F,yFAAyF;QACzF,8FAA8F;QAC9F,4FAA4F;QAC5F,oDAAoD;QACpD,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,uFAAuF;QACvF,mFAAmF;QACnF,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAEzC,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,uFAAuF;IACvF,gGAAgG;IAChG,2DAA2D;IAC3D,IAAI,SAAqD,CAAA;IACzD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,IAAI,GAAG,GAAG,CAAC;YAAE,MAAK;QAClB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;YACrE,SAAS,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QAClC,CAAC;QACD,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IACD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAA;IAE/B,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,sFAAsF;IACtF,sFAAsF;IACtF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QACzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA,CAAC,+BAA+B;QACxF,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,SAAS,GAAG,eAAe,CAAA;AAEjC,mGAAmG;AACnG,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAa,EAAE,GAAW;IACtE,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC,CAAA;IACxE,OAAO,MAAM,IAAI,KAAK,CAAA;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAA4B,EAAE,KAA+B;IACjF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA,CAAC,qBAAqB;IAExD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YACnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBACD,MAAK;YACN,CAAC;QACF,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
@@ -74,6 +74,14 @@ export interface BuildCorpusOptions {
74
74
  rowsPerShard?: number;
75
75
  /** Progress hook. Errors thrown abort the build. */
76
76
  onProgress?: (stage: BuildStage, message: string) => void;
77
+ /**
78
+ * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see
79
+ * `compileLicenseExcludes` / `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license`
80
+ * matches any is dropped at ingest. Default (omitted) includes EVERYTHING — exclusion is a
81
+ * deliberate act, not a silent default. A proprietary-weights build passes the share-alike set
82
+ * (`--exclude-share-alike`).
83
+ */
84
+ excludeLicenses?: readonly RegExp[];
77
85
  }
78
86
  /** Top-level manifest tying every stage together. */
79
87
  export interface BuildCorpusManifest {
@@ -91,6 +99,12 @@ export interface BuildCorpusManifest {
91
99
  };
92
100
  quarantine_count: number;
93
101
  total_aligned_rows: number;
102
+ /**
103
+ * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped
104
+ * by `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
105
+ */
106
+ licenses: Record<string, number>;
107
+ excluded_by_license: number;
94
108
  }
95
109
  /**
96
110
  * Drive the full corpus build to completion.
@@ -1 +1 @@
1
- {"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAQH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACzD;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;CAC1B;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA4JxF"}
1
+ {"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AASH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAwLxF"}
package/out/src/build.js CHANGED
@@ -52,6 +52,7 @@ import { join } from "node:path";
52
52
  import { createInterface } from "node:readline";
53
53
  import { defaultAdapterRegistry } from "./adapter.js";
54
54
  import { alignRow } from "./align.js";
55
+ import { licenseExcluded } from "./license.js";
55
56
  import { writeShards } from "./parquet.js";
56
57
  import { runAdapter } from "./runner.js";
57
58
  import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "./split.js";
@@ -127,12 +128,27 @@ export async function buildCorpus(opts) {
127
128
  let quarantined = 0;
128
129
  const counts = { train: 0, val: 0, test: 0 };
129
130
  const holdouts = defaultHoldouts();
131
+ // License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
132
+ // license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
133
+ // everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
134
+ const excludeLicenses = opts.excludeLicenses ?? [];
135
+ const licenseCounts = new Map();
136
+ let excludedByLicense = 0;
130
137
  const writeQuarantine = (row, reason) => {
131
138
  quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
132
139
  };
133
140
  for (const adapterRun of adapterRuns) {
134
141
  opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
135
142
  for await (const row of streamJsonl(adapterRun.jsonl_path)) {
143
+ licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
144
+ // Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
145
+ // kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
146
+ // purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
147
+ // set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
148
+ if (licenseExcluded(row.license, excludeLicenses)) {
149
+ excludedByLicense++;
150
+ continue;
151
+ }
136
152
  const fanned = [row];
137
153
  if (synthesize) {
138
154
  for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
@@ -194,6 +210,13 @@ export async function buildCorpus(opts) {
194
210
  corpusVersion: opts.corpusVersion,
195
211
  rowsPerShard,
196
212
  });
213
+ // License-set visibility (#26): loudly report the resolved license set so a build is an obvious
214
+ // deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
215
+ const licenseSummary = [...licenseCounts.entries()].sort((a, b) => b[1] - a[1]);
216
+ opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
217
+ (excludedByLicense > 0
218
+ ? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
219
+ : " | NO license exclusion applied (all rows kept)"));
197
220
  // 6. Top-level manifest.
198
221
  opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
199
222
  const manifest = {
@@ -205,6 +228,8 @@ export async function buildCorpus(opts) {
205
228
  shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
206
229
  quarantine_count: quarantined,
207
230
  total_aligned_rows: aligned,
231
+ licenses: Object.fromEntries(licenseSummary),
232
+ excluded_by_license: excludedByLicense,
208
233
  };
209
234
  await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
210
235
  return manifest;
@@ -1 +1 @@
1
- {"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAgD/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QACxD,IACC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG;YACpC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAElC,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBACvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACzE,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACT,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;KAC3B,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACxG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA+D/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QACxD,IACC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG;YACpC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBACnB,SAAQ;YACT,CAAC;YACD,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBACvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACzE,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACT,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACxG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
@@ -0,0 +1,37 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Corpus licensing — the single source of truth for the training-data license policy (#26).
7
+ *
8
+ * Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
9
+ * build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
10
+ * license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
11
+ * inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
12
+ * })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
13
+ * string unless the operator named it. This avoids the trap of silently dropping allowed data
14
+ * mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
15
+ * ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
16
+ * dropped 48M allowed rows).
17
+ *
18
+ * Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
19
+ * — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
20
+ * proprietary-weights build via `--exclude-share-alike`.
21
+ */
22
+ /**
23
+ * Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
24
+ * `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use
25
+ * it.
26
+ */
27
+ export declare const SHARE_ALIKE_PATTERN: RegExp;
28
+ /**
29
+ * Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
30
+ * case-insensitive prefix patterns. Each entry matches a license string that STARTS with it, so
31
+ * `CC-BY-SA` catches `CC-BY-SA-3.0`, `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the
32
+ * spec is a literal license prefix, not a user-supplied regex.
33
+ */
34
+ export declare function compileLicenseExcludes(spec: string): RegExp[];
35
+ /** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
36
+ export declare function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean;
37
+ //# sourceMappingURL=license.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAAmD,CAAA;AAEnF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM7D;AAED,kGAAkG;AAClG,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAGjG"}
@@ -0,0 +1,46 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Corpus licensing — the single source of truth for the training-data license policy (#26).
7
+ *
8
+ * Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
9
+ * build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
10
+ * license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
11
+ * inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
12
+ * })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
13
+ * string unless the operator named it. This avoids the trap of silently dropping allowed data
14
+ * mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
15
+ * ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
16
+ * dropped 48M allowed rows).
17
+ *
18
+ * Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
19
+ * — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
20
+ * proprietary-weights build via `--exclude-share-alike`.
21
+ */
22
+ /**
23
+ * Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
24
+ * `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use
25
+ * it.
26
+ */
27
+ export const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i;
28
+ /**
29
+ * Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
30
+ * case-insensitive prefix patterns. Each entry matches a license string that STARTS with it, so
31
+ * `CC-BY-SA` catches `CC-BY-SA-3.0`, `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the
32
+ * spec is a literal license prefix, not a user-supplied regex.
33
+ */
34
+ export function compileLicenseExcludes(spec) {
35
+ return spec
36
+ .split(",")
37
+ .map((s) => s.trim())
38
+ .filter(Boolean)
39
+ .map((s) => new RegExp("^" + s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"));
40
+ }
41
+ /** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
42
+ export function licenseExcluded(license, patterns) {
43
+ const l = license ?? "";
44
+ return patterns.some((p) => p.test(l));
45
+ }
46
+ //# sourceMappingURL=license.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AAC9E,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IACvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC"}
@@ -0,0 +1,77 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Boundary-instability synthesizer (#375 — the highest-impact parser lever). The failure taxonomy
7
+ *
8
+ * The within-token-punctuation decomposition (#702) found one failure FAMILY surfacing under many
9
+ * names: the model mis-places token boundaries between adjacent components when the boundary is
10
+ * ambiguous or unmarked. This generator emits diverse BIO-labeled rows that put the gold boundary
11
+ * exactly where the model wobbles, so a retrain learns the boundary from context, not the
12
+ * lexeme.
13
+ *
14
+ * The four token-aligned stress shapes, all in BASE LOCALES (US/FR/DE) so the shard never
15
+ * introduces tokens the base corpus lacks (the #511 base-consistency lint flagged an earlier
16
+ * AU-bearing draft: AU 4-digit postcodes collide with US house numbers, and AU localities are
17
+ * absent from the US/FR/DE base — a real contradiction). Each component is a whitespace-separated
18
+ * token run, so `alignRow` labels it cleanly:
19
+ *
20
+ * 1. `street-eats-affix` — multi-word street + suffix (`Country Club Rd` → street + street_suffix),
21
+ * the #1 wobble: the model keeps the suffix in the street.
22
+ * 2. `comma-less-city-state` — no comma between street / locality / region (`100 Main St Springfield
23
+ * IL 62701`), the #694 family: concatenated input loses the segmentation cue. US-only (US
24
+ * zips are base-consistent; the boundary is locale-agnostic).
25
+ * 3. `fr-prefix` — FR street-type prefix split from the name (`Rue Jean-Baptiste Lebas` →
26
+ * street_prefix
27
+ *
28
+ * - Street), postcode-first order.
29
+ * 4. `house-number-after-street` — FR/DE number-follows-street (`Neuve-des-Capucines 5` → street +
30
+ * house_number), the model absorbs the number into the street.
31
+ *
32
+ * Two BALANCING shapes (added 2026-06-18 after the v1.6.0 probes). The first pass at weight 1.0
33
+ * lifted the boundaries but over-fit a NARROW distribution — every row was a clean, full,
34
+ * structured address — so the model regressed on out-of-distribution real rows (held-out US
35
+ * locality 66.3→58.2%). These two widen the distribution the shard teaches over, per the
36
+ * diagnosis (scripts/eval/locality-regression-probe): 5. `bare-locality` — locality with NO
37
+ * street (`Public Library, Lisbon ND`, `75003 Paris`), the ship-blocker: 84% of the v1.6.0
38
+ * locality regression was DROPPED locality on bare "City, STATE" rows, because every other shape
39
+ * placed a street before the city. Bare / comma-less / postcode'd / venue-prefixed forms, US +
40
+ * FR. 6. `house-number-before-street` — the confounding mirror of #4 (same FR vocab, number
41
+ * BEFORE the street). A balanced before:after mix breaks the positional shortcut behind the #4
42
+ * order-bias.
43
+ *
44
+ * EXCLUDED: the region+postcode glue (`NY14201` — sub-token, no punctuation to split) and the
45
+ * AU/NZ/UK slash unit-convention (`4/2A` → unit+house_number). The slash labels cleanly (the
46
+ * tokenizer splits `/`) and is the worst within-token class — but it inherently requires non-base
47
+ * AU/NZ/UK locales, which contradict the US/FR/DE base (the lint catch). It belongs in a
48
+ * separately-scoped AU/NZ/UK boundary-coverage shard that ALSO adds AU base coverage, not in this
49
+ * base-locale shard. `synthesize-boundary-stress.test.ts` proves the alignments.
50
+ */
51
+ import type { CanonicalRow } from "./types.js";
52
+ export type BoundaryStressTemplate = "street-eats-affix" | "comma-less-city-state" | "fr-prefix" | "house-number-after-street" | "bare-locality" | "house-number-before-street";
53
+ export interface BoundaryStressBaseTuple {
54
+ locality: string;
55
+ region: string;
56
+ postcode: string;
57
+ country: string;
58
+ }
59
+ export interface BoundaryStressSynthesisOpts {
60
+ random?: () => number;
61
+ /** Force a specific shape (tests + balanced shard composition). */
62
+ forceTemplate?: BoundaryStressTemplate;
63
+ }
64
+ export interface SynthesizedBoundaryStressRow {
65
+ raw: string;
66
+ components: CanonicalRow["components"];
67
+ locale: string;
68
+ template: BoundaryStressTemplate;
69
+ }
70
+ /**
71
+ * Synthesize one boundary-stress row. `base` is optional — when omitted, a locale-appropriate tuple
72
+ * is drawn from the internal pools (so the generator is self-contained; a build script can pass
73
+ * real tuples for scale + diversity). Every component value is a verbatim substring of `raw`, so
74
+ * `alignRow` locates + BIO-labels it.
75
+ */
76
+ export declare function synthesizeBoundaryStressRow(base: BoundaryStressBaseTuple | undefined, opts?: BoundaryStressSynthesisOpts): SynthesizedBoundaryStressRow;
77
+ //# sourceMappingURL=synthesize-boundary-stress.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"synthesize-boundary-stress.d.ts","sourceRoot":"","sources":["../../src/synthesize-boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC/B,mBAAmB,GACnB,uBAAuB,GACvB,WAAW,GACX,2BAA2B,GAG3B,eAAe,GACf,4BAA4B,CAAA;AAE/B,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,2BAA2B;IAC3C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,mEAAmE;IACnE,aAAa,CAAC,EAAE,sBAAsB,CAAA;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,sBAAsB,CAAA;CAChC;AAyRD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAC1C,IAAI,EAAE,uBAAuB,GAAG,SAAS,EACzC,IAAI,GAAE,2BAAgC,GACpC,4BAA4B,CA4H9B"}