@mailwoman/corpus 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/out/src/adapters/ban/adapter.d.ts +6 -2
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +8 -4
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +2 -0
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +5 -2
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -7
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +37 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -0
- package/out/src/adapters/overture/adapter.js +119 -0
- package/out/src/adapters/overture/adapter.js.map +1 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts +8 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +37 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.js +1 -1
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +1 -1
- package/out/src/align.js +1 -1
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +14 -0
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +25 -0
- package/out/src/build.js.map +1 -1
- package/out/src/license.d.ts +37 -0
- package/out/src/license.d.ts.map +1 -0
- package/out/src/license.js +46 -0
- package/out/src/license.js.map +1 -0
- package/out/src/synthesize-boundary-stress.d.ts +77 -0
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -0
- package/out/src/synthesize-boundary-stress.js +449 -0
- package/out/src/synthesize-boundary-stress.js.map +1 -0
- package/out/src/synthesize-no-street.d.ts +1 -1
- package/out/src/synthesize-no-street.js +2 -2
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +10 -1
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +63 -3
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize.d.ts +15 -4
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +124 -4
- package/out/src/synthesize.js.map +1 -1
- package/out/src/wof-json.d.ts +1 -1
- package/out/src/wof-json.js +1 -1
- package/package.json +4 -4
package/out/src/align.js
CHANGED
|
@@ -150,7 +150,7 @@ function locateSpan(args) {
|
|
|
150
150
|
// Pass 1: verbatim substring. Word-boundary-aligned matches are PREFERRED over intra-word ones
|
|
151
151
|
// — leftmost-substring alone let a short value claim the inside of an earlier word (region "AK"
|
|
152
152
|
// matched inside "Umak"/"Lake", scrambling every later span; caught by the v0.5.0 pilot build).
|
|
153
|
-
// Intra-word matches stay allowed as the fallback because they are
|
|
153
|
+
// Intra-word matches stay allowed as the fallback because they are essential for affix
|
|
154
154
|
// supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence —
|
|
155
155
|
// sub-word spans are the point of the char-offset format).
|
|
156
156
|
let intraWord;
|
package/out/src/align.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;AAsCnF,0BAA0B;AAC1B,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,OAAqB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,qGAAqG;IACrG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QACpD,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,4BAA4B;IAC5B,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,IAAI,CAC7F,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,+FAA+F;QAC/F,8FAA8F;QAC9F,yFAAyF;QACzF,8FAA8F;QAC9F,4FAA4F;QAC5F,oDAAoD;QACpD,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,uFAAuF;QACvF,mFAAmF;QACnF,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAEzC,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,
|
|
1
|
+
{"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;AAsCnF,0BAA0B;AAC1B,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,OAAqB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,qGAAqG;IACrG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QACpD,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,4BAA4B;IAC5B,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,IAAI,CAC7F,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,+FAA+F;QAC/F,8FAA8F;QAC9F,yFAAyF;QACzF,8FAA8F;QAC9F,4FAA4F;QAC5F,oDAAoD;QACpD,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,uFAAuF;QACvF,mFAAmF;QACnF,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAEzC,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,uFAAuF;IACvF,gGAAgG;IAChG,2DAA2D;IAC3D,IAAI,SAAqD,CAAA;IACzD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,IAAI,GAAG,GAAG,CAAC;YAAE,MAAK;QAClB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;YACrE,SAAS,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QAClC,CAAC;QACD,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IACD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAA;IAE/B,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,sFAAsF;IACtF,sFAAsF;IACtF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QACzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA,CAAC,+BAA+B;QACxF,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,SAAS,GAAG,eAAe,CAAA;AAEjC,mGAAmG;AACnG,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAa,EAAE,GAAW;IACtE,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC,CAAA;IACxE,OAAO,MAAM,IAAI,KAAK,CAAA;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAA4B,EAAE,KAA+B;IACjF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA,CAAC,qBAAqB;IAExD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YACnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBACD,MAAK;YACN,CAAC;QACF,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/src/build.d.ts
CHANGED
|
@@ -74,6 +74,14 @@ export interface BuildCorpusOptions {
|
|
|
74
74
|
rowsPerShard?: number;
|
|
75
75
|
/** Progress hook. Errors thrown abort the build. */
|
|
76
76
|
onProgress?: (stage: BuildStage, message: string) => void;
|
|
77
|
+
/**
|
|
78
|
+
* License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see
|
|
79
|
+
* `compileLicenseExcludes` / `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license`
|
|
80
|
+
* matches any is dropped at ingest. Default (omitted) includes EVERYTHING — exclusion is a
|
|
81
|
+
* deliberate act, not a silent default. A proprietary-weights build passes the share-alike set
|
|
82
|
+
* (`--exclude-share-alike`).
|
|
83
|
+
*/
|
|
84
|
+
excludeLicenses?: readonly RegExp[];
|
|
77
85
|
}
|
|
78
86
|
/** Top-level manifest tying every stage together. */
|
|
79
87
|
export interface BuildCorpusManifest {
|
|
@@ -91,6 +99,12 @@ export interface BuildCorpusManifest {
|
|
|
91
99
|
};
|
|
92
100
|
quarantine_count: number;
|
|
93
101
|
total_aligned_rows: number;
|
|
102
|
+
/**
|
|
103
|
+
* Resolved license set across all INCLUDED rows (license string → row count), + the count dropped
|
|
104
|
+
* by `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
|
|
105
|
+
*/
|
|
106
|
+
licenses: Record<string, number>;
|
|
107
|
+
excluded_by_license: number;
|
|
94
108
|
}
|
|
95
109
|
/**
|
|
96
110
|
* Drive the full corpus build to completion.
|
package/out/src/build.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AASH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAwLxF"}
|
package/out/src/build.js
CHANGED
|
@@ -52,6 +52,7 @@ import { join } from "node:path";
|
|
|
52
52
|
import { createInterface } from "node:readline";
|
|
53
53
|
import { defaultAdapterRegistry } from "./adapter.js";
|
|
54
54
|
import { alignRow } from "./align.js";
|
|
55
|
+
import { licenseExcluded } from "./license.js";
|
|
55
56
|
import { writeShards } from "./parquet.js";
|
|
56
57
|
import { runAdapter } from "./runner.js";
|
|
57
58
|
import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "./split.js";
|
|
@@ -127,12 +128,27 @@ export async function buildCorpus(opts) {
|
|
|
127
128
|
let quarantined = 0;
|
|
128
129
|
const counts = { train: 0, val: 0, test: 0 };
|
|
129
130
|
const holdouts = defaultHoldouts();
|
|
131
|
+
// License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
|
|
132
|
+
// license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
|
|
133
|
+
// everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
|
|
134
|
+
const excludeLicenses = opts.excludeLicenses ?? [];
|
|
135
|
+
const licenseCounts = new Map();
|
|
136
|
+
let excludedByLicense = 0;
|
|
130
137
|
const writeQuarantine = (row, reason) => {
|
|
131
138
|
quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
|
|
132
139
|
};
|
|
133
140
|
for (const adapterRun of adapterRuns) {
|
|
134
141
|
opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
|
|
135
142
|
for await (const row of streamJsonl(adapterRun.jsonl_path)) {
|
|
143
|
+
licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
|
|
144
|
+
// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
|
|
145
|
+
// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
|
|
146
|
+
// purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
|
|
147
|
+
// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
|
|
148
|
+
if (licenseExcluded(row.license, excludeLicenses)) {
|
|
149
|
+
excludedByLicense++;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
136
152
|
const fanned = [row];
|
|
137
153
|
if (synthesize) {
|
|
138
154
|
for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
|
|
@@ -194,6 +210,13 @@ export async function buildCorpus(opts) {
|
|
|
194
210
|
corpusVersion: opts.corpusVersion,
|
|
195
211
|
rowsPerShard,
|
|
196
212
|
});
|
|
213
|
+
// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
|
|
214
|
+
// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
|
|
215
|
+
const licenseSummary = [...licenseCounts.entries()].sort((a, b) => b[1] - a[1]);
|
|
216
|
+
opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
|
|
217
|
+
(excludedByLicense > 0
|
|
218
|
+
? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
|
|
219
|
+
: " | NO license exclusion applied (all rows kept)"));
|
|
197
220
|
// 6. Top-level manifest.
|
|
198
221
|
opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
|
|
199
222
|
const manifest = {
|
|
@@ -205,6 +228,8 @@ export async function buildCorpus(opts) {
|
|
|
205
228
|
shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
|
|
206
229
|
quarantine_count: quarantined,
|
|
207
230
|
total_aligned_rows: aligned,
|
|
231
|
+
licenses: Object.fromEntries(licenseSummary),
|
|
232
|
+
excluded_by_license: excludedByLicense,
|
|
208
233
|
};
|
|
209
234
|
await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
210
235
|
return manifest;
|
package/out/src/build.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA+D/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QACxD,IACC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG;YACpC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBACnB,SAAQ;YACT,CAAC;YACD,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBACvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACzE,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACT,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACxG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Corpus licensing — the single source of truth for the training-data license policy (#26).
|
|
7
|
+
*
|
|
8
|
+
* Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
|
|
9
|
+
* build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
|
|
10
|
+
* license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
|
|
11
|
+
* inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
|
|
12
|
+
* })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
|
|
13
|
+
* string unless the operator named it. This avoids the trap of silently dropping allowed data
|
|
14
|
+
* mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
|
|
15
|
+
* ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
|
|
16
|
+
* dropped 48M allowed rows).
|
|
17
|
+
*
|
|
18
|
+
* Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
|
|
19
|
+
* — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
|
|
20
|
+
* proprietary-weights build via `--exclude-share-alike`.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
|
|
24
|
+
* `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use
|
|
25
|
+
* it.
|
|
26
|
+
*/
|
|
27
|
+
export declare const SHARE_ALIKE_PATTERN: RegExp;
|
|
28
|
+
/**
|
|
29
|
+
* Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
|
|
30
|
+
* case-insensitive prefix patterns. Each entry matches a license string that STARTS with it, so
|
|
31
|
+
* `CC-BY-SA` catches `CC-BY-SA-3.0`, `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the
|
|
32
|
+
* spec is a literal license prefix, not a user-supplied regex.
|
|
33
|
+
*/
|
|
34
|
+
export declare function compileLicenseExcludes(spec: string): RegExp[];
|
|
35
|
+
/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
|
|
36
|
+
export declare function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean;
|
|
37
|
+
//# sourceMappingURL=license.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAAmD,CAAA;AAEnF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM7D;AAED,kGAAkG;AAClG,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAGjG"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Corpus licensing — the single source of truth for the training-data license policy (#26).
|
|
7
|
+
*
|
|
8
|
+
* Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
|
|
9
|
+
* build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
|
|
10
|
+
* license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
|
|
11
|
+
* inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
|
|
12
|
+
* })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
|
|
13
|
+
* string unless the operator named it. This avoids the trap of silently dropping allowed data
|
|
14
|
+
* mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
|
|
15
|
+
* ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
|
|
16
|
+
* dropped 48M allowed rows).
|
|
17
|
+
*
|
|
18
|
+
* Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
|
|
19
|
+
* — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
|
|
20
|
+
* proprietary-weights build via `--exclude-share-alike`.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
|
|
24
|
+
* `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use
|
|
25
|
+
* it.
|
|
26
|
+
*/
|
|
27
|
+
export const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i;
|
|
28
|
+
/**
|
|
29
|
+
* Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
|
|
30
|
+
* case-insensitive prefix patterns. Each entry matches a license string that STARTS with it, so
|
|
31
|
+
* `CC-BY-SA` catches `CC-BY-SA-3.0`, `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the
|
|
32
|
+
* spec is a literal license prefix, not a user-supplied regex.
|
|
33
|
+
*/
|
|
34
|
+
export function compileLicenseExcludes(spec) {
|
|
35
|
+
return spec
|
|
36
|
+
.split(",")
|
|
37
|
+
.map((s) => s.trim())
|
|
38
|
+
.filter(Boolean)
|
|
39
|
+
.map((s) => new RegExp("^" + s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"));
|
|
40
|
+
}
|
|
41
|
+
/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
|
|
42
|
+
export function licenseExcluded(license, patterns) {
|
|
43
|
+
const l = license ?? "";
|
|
44
|
+
return patterns.some((p) => p.test(l));
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=license.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AAC9E,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IACvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Boundary-instability synthesizer (#375 — the highest-impact parser lever). The failure taxonomy
|
|
7
|
+
*
|
|
8
|
+
* The within-token-punctuation decomposition (#702) found one failure FAMILY surfacing under many
|
|
9
|
+
* names: the model mis-places token boundaries between adjacent components when the boundary is
|
|
10
|
+
* ambiguous or unmarked. This generator emits diverse BIO-labeled rows that put the gold boundary
|
|
11
|
+
* exactly where the model wobbles, so a retrain learns the boundary from context, not the
|
|
12
|
+
* lexeme.
|
|
13
|
+
*
|
|
14
|
+
* The four token-aligned stress shapes, all in BASE LOCALES (US/FR/DE) so the shard never
|
|
15
|
+
* introduces tokens the base corpus lacks (the #511 base-consistency lint flagged an earlier
|
|
16
|
+
* AU-bearing draft: AU 4-digit postcodes collide with US house numbers, and AU localities are
|
|
17
|
+
* absent from the US/FR/DE base — a real contradiction). Each component is a whitespace-separated
|
|
18
|
+
* token run, so `alignRow` labels it cleanly:
|
|
19
|
+
*
|
|
20
|
+
* 1. `street-eats-affix` — multi-word street + suffix (`Country Club Rd` → street + street_suffix),
|
|
21
|
+
* the #1 wobble: the model keeps the suffix in the street.
|
|
22
|
+
* 2. `comma-less-city-state` — no comma between street / locality / region (`100 Main St Springfield
|
|
23
|
+
* IL 62701`), the #694 family: concatenated input loses the segmentation cue. US-only (US
|
|
24
|
+
* zips are base-consistent; the boundary is locale-agnostic).
|
|
25
|
+
* 3. `fr-prefix` — FR street-type prefix split from the name (`Rue Jean-Baptiste Lebas` →
|
|
26
|
+
* street_prefix
|
|
27
|
+
*
|
|
28
|
+
* - Street), postcode-first order.
|
|
29
|
+
* 4. `house-number-after-street` — FR/DE number-follows-street (`Neuve-des-Capucines 5` → street +
|
|
30
|
+
* house_number), the model absorbs the number into the street.
|
|
31
|
+
*
|
|
32
|
+
* Two BALANCING shapes (added 2026-06-18 after the v1.6.0 probes). The first pass at weight 1.0
|
|
33
|
+
* lifted the boundaries but over-fit a NARROW distribution — every row was a clean, full,
|
|
34
|
+
* structured address — so the model regressed on out-of-distribution real rows (held-out US
|
|
35
|
+
* locality 66.3→58.2%). These two widen the distribution the shard teaches over, per the
|
|
36
|
+
* diagnosis (scripts/eval/locality-regression-probe): 5. `bare-locality` — locality with NO
|
|
37
|
+
* street (`Public Library, Lisbon ND`, `75003 Paris`), the ship-blocker: 84% of the v1.6.0
|
|
38
|
+
* locality regression was DROPPED locality on bare "City, STATE" rows, because every other shape
|
|
39
|
+
* placed a street before the city. Bare / comma-less / postcode'd / venue-prefixed forms, US +
|
|
40
|
+
* FR. 6. `house-number-before-street` — the confounding mirror of #4 (same FR vocab, number
|
|
41
|
+
* BEFORE the street). A balanced before:after mix breaks the positional shortcut behind the #4
|
|
42
|
+
* order-bias.
|
|
43
|
+
*
|
|
44
|
+
* EXCLUDED: the region+postcode glue (`NY14201` — sub-token, no punctuation to split) and the
|
|
45
|
+
* AU/NZ/UK slash unit-convention (`4/2A` → unit+house_number). The slash labels cleanly (the
|
|
46
|
+
* tokenizer splits `/`) and is the worst within-token class — but it inherently requires non-base
|
|
47
|
+
* AU/NZ/UK locales, which contradict the US/FR/DE base (the lint catch). It belongs in a
|
|
48
|
+
* separately-scoped AU/NZ/UK boundary-coverage shard that ALSO adds AU base coverage, not in this
|
|
49
|
+
* base-locale shard. `synthesize-boundary-stress.test.ts` proves the alignments.
|
|
50
|
+
*/
|
|
51
|
+
import type { CanonicalRow } from "./types.js";
|
|
52
|
+
export type BoundaryStressTemplate = "street-eats-affix" | "comma-less-city-state" | "fr-prefix" | "house-number-after-street" | "bare-locality" | "house-number-before-street";
|
|
53
|
+
export interface BoundaryStressBaseTuple {
|
|
54
|
+
locality: string;
|
|
55
|
+
region: string;
|
|
56
|
+
postcode: string;
|
|
57
|
+
country: string;
|
|
58
|
+
}
|
|
59
|
+
export interface BoundaryStressSynthesisOpts {
|
|
60
|
+
random?: () => number;
|
|
61
|
+
/** Force a specific shape (tests + balanced shard composition). */
|
|
62
|
+
forceTemplate?: BoundaryStressTemplate;
|
|
63
|
+
}
|
|
64
|
+
export interface SynthesizedBoundaryStressRow {
|
|
65
|
+
raw: string;
|
|
66
|
+
components: CanonicalRow["components"];
|
|
67
|
+
locale: string;
|
|
68
|
+
template: BoundaryStressTemplate;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Synthesize one boundary-stress row. `base` is optional — when omitted, a locale-appropriate tuple
|
|
72
|
+
* is drawn from the internal pools (so the generator is self-contained; a build script can pass
|
|
73
|
+
* real tuples for scale + diversity). Every component value is a verbatim substring of `raw`, so
|
|
74
|
+
* `alignRow` locates + BIO-labels it.
|
|
75
|
+
*/
|
|
76
|
+
export declare function synthesizeBoundaryStressRow(base: BoundaryStressBaseTuple | undefined, opts?: BoundaryStressSynthesisOpts): SynthesizedBoundaryStressRow;
|
|
77
|
+
//# sourceMappingURL=synthesize-boundary-stress.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"synthesize-boundary-stress.d.ts","sourceRoot":"","sources":["../../src/synthesize-boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC/B,mBAAmB,GACnB,uBAAuB,GACvB,WAAW,GACX,2BAA2B,GAG3B,eAAe,GACf,4BAA4B,CAAA;AAE/B,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,2BAA2B;IAC3C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,mEAAmE;IACnE,aAAa,CAAC,EAAE,sBAAsB,CAAA;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,sBAAsB,CAAA;CAChC;AAyRD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAC1C,IAAI,EAAE,uBAAuB,GAAG,SAAS,EACzC,IAAI,GAAE,2BAAgC,GACpC,4BAA4B,CA4H9B"}
|