@mailwoman/neural 4.16.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/address-system.d.ts +4 -5
- package/out/address-system.d.ts.map +1 -1
- package/out/address-system.js +4 -5
- package/out/address-system.js.map +1 -1
- package/out/anchor-inference.d.ts +16 -18
- package/out/anchor-inference.d.ts.map +1 -1
- package/out/anchor-inference.js +16 -18
- package/out/anchor-inference.js.map +1 -1
- package/out/browser.js +1 -1
- package/out/case-normalize.d.ts +13 -8
- package/out/case-normalize.d.ts.map +1 -1
- package/out/case-normalize.js +18 -11
- package/out/case-normalize.js.map +1 -1
- package/out/classifier.d.ts +125 -134
- package/out/classifier.d.ts.map +1 -1
- package/out/classifier.js +46 -49
- package/out/classifier.js.map +1 -1
- package/out/fst-prior.d.ts +19 -21
- package/out/fst-prior.d.ts.map +1 -1
- package/out/fst-prior.js +13 -14
- package/out/fst-prior.js.map +1 -1
- package/out/gazetteer-inference.d.ts +13 -16
- package/out/gazetteer-inference.d.ts.map +1 -1
- package/out/gazetteer-inference.js +13 -16
- package/out/gazetteer-inference.js.map +1 -1
- package/out/index.d.ts +1 -1
- package/out/index.js +1 -1
- package/out/labels.d.ts +9 -10
- package/out/labels.d.ts.map +1 -1
- package/out/labels.js +5 -6
- package/out/labels.js.map +1 -1
- package/out/onnx-runner.d.ts +33 -21
- package/out/onnx-runner.d.ts.map +1 -1
- package/out/onnx-runner.js +39 -19
- package/out/onnx-runner.js.map +1 -1
- package/out/postcode-anchor.d.ts +28 -34
- package/out/postcode-anchor.d.ts.map +1 -1
- package/out/postcode-anchor.js +48 -55
- package/out/postcode-anchor.js.map +1 -1
- package/out/postcode-binary-resolver.d.ts +10 -13
- package/out/postcode-binary-resolver.d.ts.map +1 -1
- package/out/postcode-binary-resolver.js +12 -15
- package/out/postcode-binary-resolver.js.map +1 -1
- package/out/postcode-repair.d.ts +5 -5
- package/out/postcode-repair.d.ts.map +1 -1
- package/out/postcode-repair.js +9 -10
- package/out/postcode-repair.js.map +1 -1
- package/out/proposal-classifier.d.ts +3 -3
- package/out/proposal-classifier.d.ts.map +1 -1
- package/out/proposal-classifier.js.map +1 -1
- package/out/query-shape-prior.d.ts +10 -11
- package/out/query-shape-prior.d.ts.map +1 -1
- package/out/query-shape-prior.js +12 -15
- package/out/query-shape-prior.js.map +1 -1
- package/out/scorer.d.ts +28 -31
- package/out/scorer.d.ts.map +1 -1
- package/out/scorer.js +39 -44
- package/out/scorer.js.map +1 -1
- package/out/soft-features.d.ts +12 -14
- package/out/soft-features.d.ts.map +1 -1
- package/out/soft-features.js +5 -6
- package/out/soft-features.js.map +1 -1
- package/out/span-bridge.d.ts +9 -11
- package/out/span-bridge.d.ts.map +1 -1
- package/out/span-bridge.js +8 -10
- package/out/span-bridge.js.map +1 -1
- package/out/span-proposal-prior.d.ts +13 -15
- package/out/span-proposal-prior.d.ts.map +1 -1
- package/out/span-proposal-prior.js +2 -2
- package/out/span-proposal-prior.js.map +1 -1
- package/out/span-proposer-lexicon.d.ts +2 -3
- package/out/span-proposer-lexicon.d.ts.map +1 -1
- package/out/span-proposer-lexicon.js +9 -10
- package/out/span-proposer-lexicon.js.map +1 -1
- package/out/street-morphology-prior.d.ts +11 -12
- package/out/street-morphology-prior.d.ts.map +1 -1
- package/out/street-morphology-prior.js +8 -9
- package/out/street-morphology-prior.js.map +1 -1
- package/out/tokenizer.d.ts +6 -7
- package/out/tokenizer.d.ts.map +1 -1
- package/out/tokenizer.js +6 -7
- package/out/tokenizer.js.map +1 -1
- package/out/unit-repair.d.ts +2 -2
- package/out/unit-repair.d.ts.map +1 -1
- package/out/unit-repair.js +11 -14
- package/out/unit-repair.js.map +1 -1
- package/out/viterbi.d.ts +13 -13
- package/out/viterbi.d.ts.map +1 -1
- package/out/viterbi.js +12 -12
- package/out/viterbi.js.map +1 -1
- package/out/vitest.config.d.ts.map +1 -1
- package/out/vitest.config.js.map +1 -1
- package/out/weights.d.ts +62 -71
- package/out/weights.d.ts.map +1 -1
- package/out/weights.js +33 -37
- package/out/weights.js.map +1 -1
- package/out/word-consistency.d.ts +4 -4
- package/out/word-consistency.d.ts.map +1 -1
- package/out/word-consistency.js +4 -4
- package/out/word-consistency.js.map +1 -1
- package/package.json +13 -13
package/out/scorer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAElD,OAAO,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,iBAAiB,EAAqB,MAAM,uBAAuB,CAAA;AAC5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AACzD,OAAO,EAAE,qBAAqB,EAAyB,MAAM,0BAA0B,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AACnD,OAAO,EACN,+BAA+B,EAC/B,mBAAmB,EACnB,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,cAAc,GAEd,MAAM,cAAc,CAAA;AAErB
|
|
1
|
+
{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAElD,OAAO,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,iBAAiB,EAAqB,MAAM,uBAAuB,CAAA;AAC5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AACzD,OAAO,EAAE,qBAAqB,EAAyB,MAAM,0BAA0B,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AACnD,OAAO,EACN,+BAA+B,EAC/B,mBAAmB,EACnB,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,cAAc,GAEd,MAAM,cAAc,CAAA;AAErB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAA;AAE9C,iGAAiG;AACjG,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAA;AAEvF,yEAAyE;AACzE,MAAM,CAAC,MAAM,yBAAyB,GAAG,uCAAuC,CAAA;AAEhF;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,MAA0B;IACtD,IAAI,UAAU,CAAC,qBAAqB,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IAE5F,IAAI,CAAC;QACJ,OAAO,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,gBAAgB,CAAA;IACnD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,MAA0B;IAC1D,IAAI,UAAU,CAAC,yBAAyB,CAAC;QAAE,OAAO,yBAAyB,CAAA;IAE3E,IAAI,CAAC;QACJ,OAAO,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,oBAAoB,CAAA;IACvD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAED,yFAAyF;AACzF,SAAS,gBAAgB,CAAC,MAAyC;IAClE,OAAO,MAAM,CAAC,MAAM;QACnB,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE;QACxF,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;AACpE,CAAC;AA6DD,8EAA8E;AAC9E,MAAM,iBAAkB,SAAQ,KAAK;IACpC,YAAY,OAAe;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAA;IAChC,CAAC;CACD;AAED;;;;GAIG;AACH,MAAM,wBAAyB,SAAQ,KAAK;IAC3C,YAAY,OAAe;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAA;IACvC,CAAC;CACD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,IAAI,oBAAoB,GAAG,KAAK,CAAA;AAEhC,SAAS,oCAAoC,CAAC,aAAqB,EAAE,IAAY,EAAE,MAAe;IACjG,MAAM,QAAQ,GAAG,sBAAsB,CAAC,aAAa,CAAC,CAAA;IAEtD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,gGAAgG;QAChG,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC3B,oBAAoB,GAAG,IAAI,CAAA;YAC3B,OAAO,CAAC,KAAK,CACZ,4FAA4F;gBAC3F,6FAA6F;gBAC7F,iDAAiD,CAClD,CAAA;QACF,CAAC;QAED,OAAM;IACP,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,EAAE,CAAC;QAChF,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,aAAa,IAAI,EAAE,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,mBAAmB,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;YAE5D,IAAI,CAAC,GAAG;gBAAE,SAAQ,CAAC,yEAAyE;YAC5F,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAA;YAEjD,IAAI,KAAK,GAAG,0BAA0B,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,wCAAwC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBAC3G,IAAI,CACH,MAAM,EACN,yBAAyB,GAAG,mBAAmB,MAAM,2CAA2C;oBAC/F,WAAW,IAAI,kBAAkB,GAAG,CAAC,SAAS,cAAc,MAAM,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;oBAC9F,GAAG,0BAA0B,4DAA4D;oBACzF,mBAAmB,GAAG,yCAAyC,MAAM,wBAAwB;oBAC7F,gEAAgE,0BAA0B,eAAe,EAC1G,wBAAwB,CACxB,CAAA;YACF,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAA;IAClC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAA;IAEtC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;IAE7G,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACrF,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACrF,CAAC;IAED,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC1D,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,kBAAkB,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC;QACnD,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;KACjC,CAAC,CAAA;IAEF,mGAAmG;IACnG,uGAAuG;IACvG,kGAAkG;IAClG,MAAM,QAAQ,GACb,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,+BAA+B,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvG,iGAAiG;IACjG,kGAAkG;IAClG,kGAAkG;IAClG,iGAAiG;IACjG,gGAAgG;IAChG,0FAA0F;IAC1F,oCAAoC,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,MAAM,CAAC,CAAA;IAEvF,iGAAiG;IACjG,2FAA2F;IAC3F,oGAAoG;IACpG,MAAM,YAAY,GAAkD,IAAI,CAAC,gBAAgB;QACxF,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE;QAChD,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACnC,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,IAAI,KAAK,CAAA;IACzD,IAAI,oBAA8C,CAAA;IAElD,IAAI,SAAS,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAChC,IAAI,cAAc,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CACZ,6FAA6F;gBAC5F,uFAAuF,CACxF,CAAA;QACF,CAAC;IACF,CAAC;SAAM,CAAC;QACP,oBAAoB,GAAG,YAAY,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAEjH,gGAAgG;QAChG,sFAAsF;QACtF,IAAI,cAAc,IAAI,CAAC,CAAC,oBAAoB,IAAI,oBAAoB,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;YAChF,MAAM,MAAM,GAAG,oBAAoB;gBAClC,CAAC,CAAC,iCAAiC;gBACnC,CAAC,CAAC,2BAA2B,YAAY,EAAE,IAAI,IAAI,qBAAqB,6BAA6B,CAAA;YACtG,IAAI,CACH,MAAM,EACN,4EAA4E,MAAM,IAAI;gBACrF,4FAA4F,CAC7F,CAAA;QACF,CAAC;IACF,CAAC;IAED,iGAAiG;IACjG,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,IAAI,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC9F,MAAM,iBAAiB,GAAG,QAAQ,CAAC,SAAS,EAAE,QAAQ,IAAI,KAAK,CAAA;IAC/D,IAAI,gBAA8C,CAAA;IAElD,IAAI,SAAS,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;QACnC,IAAI,iBAAiB,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CACZ,wFAAwF;gBACvF,kGAAkG,CACnG,CAAA;QACF,CAAC;IACF,CAAC;SAAM,CAAC;QACP,gBAAgB;YACf,oBAAoB,IAAI,UAAU,CAAC,oBAAoB,CAAC;gBACvD,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC/E,CAAC,CAAC,SAAS,CAAA;QAEb,IAAI,iBAAiB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC5C,IAAI,CACH,MAAM,EACN,8FAA8F;gBAC7F,MAAM,oBAAoB,IAAI,yBAAyB,iDAAiD;gBACxG,sDAAsD,CACvD,CAAA;QACF,CAAC;IACF,CAAC;IAED,iGAAiG;IACjG,MAAM,mBAAmB,GAAG,QAAQ,CAAC,WAAW,EAAE,QAAQ,IAAI,KAAK,CAAA;IACnE,MAAM,uBAAuB,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,IAAI,MAAM,CAAA;IACpE,IAAI,wBAAqD,CAAA;IAEzD,IAAI,SAAS,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACzC,IAAI,SAAS,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YACrC,wBAAwB,GAAG,SAAS,CAAA;YAEpC,IAAI,mBAAmB,EAAE,CAAC;gBACzB,OAAO,CAAC,KAAK,CACZ,qFAAqF;oBACpF,4CAA4C,uBAAuB,KAAK,CACzE,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,wBAAwB,GAAG,SAAS,CAAC,WAAW,CAAA;YAEhD,IAAI,SAAS,CAAC,WAAW,KAAK,uBAAuB,EAAE,CAAC;gBACvD,OAAO,CAAC,KAAK,CACZ,qDAAqD,SAAS,CAAC,WAAW,gBAAgB;oBACzF,aAAa,uBAAuB,KAAK,CAC1C,CAAA;YACF,CAAC;QACF,CAAC;IACF,CAAC;SAAM,CAAC;QACP,qGAAqG;QACrG,wBAAwB,GAAG,mBAAmB,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,CAAA;QAEpF,IAAI,mBAAmB,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACtD,IAAI,CAAC,MAAM,EAAE,oFAAoF,CAAC,CAAA;QACnG,CAAC;IACF,CAAC;IAED,iGAAiG;IACjG,MAAM,qBAAqB,GAAG,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,QAAQ,IAAI,KAAK,CAAA;IACpF,MAAM,6BAA6B,GAClC,SAAS,CAAC,6BAA6B,IAAI,QAAQ,CAAC,gCAAgC,IAAI,KAAK,CAAA;IAE9F,OAAO,IAAI,uBAAuB,CAAC;QAClC,SAAS;QACT,MAAM;QACN,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,6BAA6B;QAC7B,GAAG,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,wBAAwB,EAAE,wBAAkC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrG,qBAAqB;KACrB,CAAC,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,IAAI,CAAC,MAAe,EAAE,OAAe,EAAE,aAA6C,iBAAiB;IAC7G,MAAM,IAAI,GAAG,kBAAkB,OAAO,EAAE,CAAA;IAExC,IAAI,MAAM;QAAE,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,CAAA;IACtC,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,mEAAmE,CAAC,CAAA;AAC1F,CAAC"}
|
package/out/soft-features.d.ts
CHANGED
|
@@ -30,9 +30,9 @@ export interface SoftFeatures {
|
|
|
30
30
|
/** Postcode-anchor channel (#239/#240) — present iff `postcodeAnchorLookup` was supplied. */
|
|
31
31
|
anchor?: SoftFeatureChannel;
|
|
32
32
|
/**
|
|
33
|
-
* Gazetteer-anchor channel (#464) — present iff `gazetteerLexicon` was supplied. Already
|
|
34
|
-
*
|
|
35
|
-
*
|
|
33
|
+
* Gazetteer-anchor channel (#464) — present iff `gazetteerLexicon` was supplied. Already choreographed: when
|
|
34
|
+
* `suppressGazetteerNearPostcode` is set AND an anchor channel exists, the clue is zeroed adjacent to postcode-anchor
|
|
35
|
+
* hits before it's returned here.
|
|
36
36
|
*/
|
|
37
37
|
gazetteer?: SoftFeatureChannel;
|
|
38
38
|
}
|
|
@@ -43,25 +43,23 @@ export interface SoftFeatureSources {
|
|
|
43
43
|
/** Gazetteer-anchor lexicon (#464). Omit to skip the gazetteer channel. */
|
|
44
44
|
gazetteerLexicon?: GazetteerLexicon;
|
|
45
45
|
/**
|
|
46
|
-
* Channel choreography (#464, v0.9.13 postcode fix): zero the gazetteer clue on pieces adjacent
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* `suppressGazetteerNearPostcode` in `gazetteer-inference.ts`.
|
|
46
|
+
* Channel choreography (#464, v0.9.13 postcode fix): zero the gazetteer clue on pieces adjacent to a postcode-anchor
|
|
47
|
+
* hit. Needs BOTH a `gazetteerLexicon` and a `postcodeAnchorLookup` to take effect (the suppression is keyed off the
|
|
48
|
+
* anchor's confidence). PAIRING IS ESSENTIAL — enable this IFF the model was trained with the matching train-time
|
|
49
|
+
* choreography. See `suppressGazetteerNearPostcode` in `gazetteer-inference.ts`.
|
|
51
50
|
*/
|
|
52
51
|
suppressGazetteerNearPostcode?: boolean;
|
|
53
52
|
}
|
|
54
53
|
/**
|
|
55
|
-
* Build the soft-feature channels for `text`/`pieces` from the configured sources — the EXACT
|
|
56
|
-
*
|
|
54
|
+
* Build the soft-feature channels for `text`/`pieces` from the configured sources — the EXACT choreography previously
|
|
55
|
+
* inlined in `NeuralAddressClassifier.#decode`:
|
|
57
56
|
*
|
|
58
57
|
* 1. Anchor channel from `postcodeAnchorLookup` (no-op when unset).
|
|
59
58
|
* 2. Gazetteer channel from `gazetteerLexicon` (no-op when unset).
|
|
60
|
-
* 3. If both channels exist AND `suppressGazetteerNearPostcode`, zero the gazetteer clue adjacent to
|
|
61
|
-
* postcode-anchor hits.
|
|
59
|
+
* 3. If both channels exist AND `suppressGazetteerNearPostcode`, zero the gazetteer clue adjacent to postcode-anchor hits.
|
|
62
60
|
*
|
|
63
|
-
* Pure + byte-stable: the returned channels are identical to the pre-#718 inline path, so wiring
|
|
64
|
-
*
|
|
61
|
+
* Pure + byte-stable: the returned channels are identical to the pre-#718 inline path, so wiring this into `#decode` is
|
|
62
|
+
* a behavior-preserving refactor.
|
|
65
63
|
*/
|
|
66
64
|
export declare function buildSoftFeatures(text: string, pieces: ReadonlyArray<TokenizedPiece>, sources: SoftFeatureSources): SoftFeatures;
|
|
67
65
|
//# sourceMappingURL=soft-features.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"soft-features.d.ts","sourceRoot":"","sources":["../soft-features.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAuB,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAC9E,OAAO,EAAyD,KAAK,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AACvH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAEpD,mFAAmF;AACnF,MAAM,WAAW,kBAAkB;IAClC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAA;IACpB,UAAU,EAAE,MAAM,EAAE,CAAA;CACpB;AAED,uGAAuG;AACvG,MAAM,WAAW,YAAY;IAC5B,6FAA6F;IAC7F,MAAM,CAAC,EAAE,kBAAkB,CAAA;IAC3B;;;;OAIG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAA;CAC9B;AAED,oGAAoG;AACpG,MAAM,WAAW,kBAAkB;IAClC,2EAA2E;IAC3E,oBAAoB,CAAC,EAAE,YAAY,CAAA;IACnC,2EAA2E;IAC3E,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC
|
|
1
|
+
{"version":3,"file":"soft-features.d.ts","sourceRoot":"","sources":["../soft-features.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAuB,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAC9E,OAAO,EAAyD,KAAK,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AACvH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAEpD,mFAAmF;AACnF,MAAM,WAAW,kBAAkB;IAClC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAA;IACpB,UAAU,EAAE,MAAM,EAAE,CAAA;CACpB;AAED,uGAAuG;AACvG,MAAM,WAAW,YAAY;IAC5B,6FAA6F;IAC7F,MAAM,CAAC,EAAE,kBAAkB,CAAA;IAC3B;;;;OAIG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAA;CAC9B;AAED,oGAAoG;AACpG,MAAM,WAAW,kBAAkB;IAClC,2EAA2E;IAC3E,oBAAoB,CAAC,EAAE,YAAY,CAAA;IACnC,2EAA2E;IAC3E,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC;;;;;OAKG;IACH,6BAA6B,CAAC,EAAE,OAAO,CAAA;CACvC;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,EACrC,OAAO,EAAE,kBAAkB,GACzB,YAAY,CAgBd"}
|
package/out/soft-features.js
CHANGED
|
@@ -20,16 +20,15 @@
|
|
|
20
20
|
import { buildAnchorFeatures } from "./anchor-inference.js";
|
|
21
21
|
import { buildGazetteerFeatures, suppressGazetteerNearPostcode } from "./gazetteer-inference.js";
|
|
22
22
|
/**
|
|
23
|
-
* Build the soft-feature channels for `text`/`pieces` from the configured sources — the EXACT
|
|
24
|
-
*
|
|
23
|
+
* Build the soft-feature channels for `text`/`pieces` from the configured sources — the EXACT choreography previously
|
|
24
|
+
* inlined in `NeuralAddressClassifier.#decode`:
|
|
25
25
|
*
|
|
26
26
|
* 1. Anchor channel from `postcodeAnchorLookup` (no-op when unset).
|
|
27
27
|
* 2. Gazetteer channel from `gazetteerLexicon` (no-op when unset).
|
|
28
|
-
* 3. If both channels exist AND `suppressGazetteerNearPostcode`, zero the gazetteer clue adjacent to
|
|
29
|
-
* postcode-anchor hits.
|
|
28
|
+
* 3. If both channels exist AND `suppressGazetteerNearPostcode`, zero the gazetteer clue adjacent to postcode-anchor hits.
|
|
30
29
|
*
|
|
31
|
-
* Pure + byte-stable: the returned channels are identical to the pre-#718 inline path, so wiring
|
|
32
|
-
*
|
|
30
|
+
* Pure + byte-stable: the returned channels are identical to the pre-#718 inline path, so wiring this into `#decode` is
|
|
31
|
+
* a behavior-preserving refactor.
|
|
33
32
|
*/
|
|
34
33
|
export function buildSoftFeatures(text, pieces, sources) {
|
|
35
34
|
const anchor = sources.postcodeAnchorLookup
|
package/out/soft-features.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"soft-features.js","sourceRoot":"","sources":["../soft-features.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,mBAAmB,EAAqB,MAAM,uBAAuB,CAAA;AAC9E,OAAO,EAAE,sBAAsB,EAAE,6BAA6B,EAAyB,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"soft-features.js","sourceRoot":"","sources":["../soft-features.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,mBAAmB,EAAqB,MAAM,uBAAuB,CAAA;AAC9E,OAAO,EAAE,sBAAsB,EAAE,6BAA6B,EAAyB,MAAM,0BAA0B,CAAA;AAoCvH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,MAAqC,EACrC,OAA2B;IAE3B,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB;QAC1C,CAAC,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC;QACjE,CAAC,CAAC,SAAS,CAAA;IACZ,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB;QACzC,CAAC,CAAC,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC;QAChE,CAAC,CAAC,SAAS,CAAA;IACZ,MAAM,MAAM,GACX,SAAS,IAAI,MAAM,IAAI,OAAO,CAAC,6BAA6B;QAC3D,CAAC,CAAC,6BAA6B,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAA;IAEb,OAAO;QACN,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAA;AACF,CAAC"}
|
package/out/span-bridge.d.ts
CHANGED
|
@@ -26,13 +26,12 @@ import type { DecoderToken } from "@mailwoman/core/decoder";
|
|
|
26
26
|
/** Options for {@link bridgePunctuationGaps}. */
|
|
27
27
|
export interface BridgePunctuationOpts {
|
|
28
28
|
/**
|
|
29
|
-
* Structural spans (from the Stage 2.7 span proposer — ANNOTATION/QUOTED groups, delimiters
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* policed.
|
|
29
|
+
* Structural spans (from the Stage 2.7 span proposer — ANNOTATION/QUOTED groups, delimiters inclusive) whose
|
|
30
|
+
* boundaries no merge may straddle: M2's crossing constraint, the bridge's mirror image (the bridge merges across
|
|
31
|
+
* WEAK punctuation; this blocks merging across STRUCTURAL punctuation). A merge is blocked when either span boundary
|
|
32
|
+
* falls inside the gap being bridged — e.g. an apostrophe-quoted name whose closing quote sits in an
|
|
33
|
+
* otherwise-bridgeable gap. Boundaries already inside a labeled token are the model's call, not the bridge's; only
|
|
34
|
+
* gaps are policed.
|
|
36
35
|
*/
|
|
37
36
|
blockedSpans?: ReadonlyArray<{
|
|
38
37
|
start: number;
|
|
@@ -40,10 +39,9 @@ export interface BridgePunctuationOpts {
|
|
|
40
39
|
}>;
|
|
41
40
|
}
|
|
42
41
|
/**
|
|
43
|
-
* Merge same-label fragments separated only by punctuation gaps. Returns a new token array where
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* Labels, ordering, and all non-bridged tokens are untouched.
|
|
42
|
+
* Merge same-label fragments separated only by punctuation gaps. Returns a new token array where the first fragment of
|
|
43
|
+
* each bridged group is widened to the group's full char range (so span extraction reads the raw text straight through
|
|
44
|
+
* the punctuation), and later fragments are dropped. Labels, ordering, and all non-bridged tokens are untouched.
|
|
47
45
|
*/
|
|
48
46
|
export declare function bridgePunctuationGaps(text: string, input: readonly DecoderToken[], opts?: BridgePunctuationOpts): DecoderToken[];
|
|
49
47
|
//# sourceMappingURL=span-bridge.d.ts.map
|
package/out/span-bridge.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"span-bridge.d.ts","sourceRoot":"","sources":["../span-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;
|
|
1
|
+
{"version":3,"file":"span-bridge.d.ts","sourceRoot":"","sources":["../span-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AAkB3D,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IACrC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAC5D;AAoBD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,YAAY,EAAE,EAC9B,IAAI,CAAC,EAAE,qBAAqB,GAC1B,YAAY,EAAE,CAyChB"}
|
package/out/span-bridge.js
CHANGED
|
@@ -23,13 +23,12 @@
|
|
|
23
23
|
* Runs beside the postcode/unit repair passes in the classifier, before tree-building.
|
|
24
24
|
*/
|
|
25
25
|
/**
|
|
26
|
-
* Gap text qualifies when short, made only of INTRA-TOKEN punctuation (period/hyphen/slash/
|
|
27
|
-
*
|
|
28
|
-
*
|
|
26
|
+
* Gap text qualifies when short, made only of INTRA-TOKEN punctuation (period/hyphen/slash/ apostrophe) plus
|
|
27
|
+
* whitespace, with at least one non-space char. Separator punctuation (comma, semicolon) is EXCLUDED — measured
|
|
28
|
+
* 2026-06-11: the comma form merged "47110, 9016"-style postcode
|
|
29
29
|
*
|
|
30
|
-
* - House-number fragments on six FR golden rows (the model double-labels the number; the comma is
|
|
31
|
-
* the
|
|
32
|
-
* never the inside of a surface form.
|
|
30
|
+
* - House-number fragments on six FR golden rows (the model double-labels the number; the comma is the only thing keeping
|
|
31
|
+
* the spans honest). A comma between same-tag spans is a list/separator, never the inside of a surface form.
|
|
33
32
|
*/
|
|
34
33
|
function bridgeable(gap) {
|
|
35
34
|
if (gap.length === 0 || gap.length > 3)
|
|
@@ -52,10 +51,9 @@ function crossesBlockedBoundary(gapStart, gapEnd, blockedSpans) {
|
|
|
52
51
|
return false;
|
|
53
52
|
}
|
|
54
53
|
/**
|
|
55
|
-
* Merge same-label fragments separated only by punctuation gaps. Returns a new token array where
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* Labels, ordering, and all non-bridged tokens are untouched.
|
|
54
|
+
* Merge same-label fragments separated only by punctuation gaps. Returns a new token array where the first fragment of
|
|
55
|
+
* each bridged group is widened to the group's full char range (so span extraction reads the raw text straight through
|
|
56
|
+
* the punctuation), and later fragments are dropped. Labels, ordering, and all non-bridged tokens are untouched.
|
|
59
57
|
*/
|
|
60
58
|
export function bridgePunctuationGaps(text, input, opts) {
|
|
61
59
|
const out = [];
|
package/out/span-bridge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"span-bridge.js","sourceRoot":"","sources":["../span-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH
|
|
1
|
+
{"version":3,"file":"span-bridge.js","sourceRoot":"","sources":["../span-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,GAAW;IAC9B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAEpD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAEjD,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACzB,CAAC;AAeD,iGAAiG;AACjG,SAAS,sBAAsB,CAC9B,QAAgB,EAChB,MAAc,EACd,YAAuE;IAEvE,IAAI,CAAC,YAAY;QAAE,OAAO,KAAK,CAAA;IAE/B,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,mFAAmF;QACnF,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM;YAAE,OAAO,IAAI,CAAA;QAE/D,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,QAAQ,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAA;IACpE,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CACpC,IAAY,EACZ,KAA8B,EAC9B,IAA4B;IAE5B,MAAM,GAAG,GAAmB,EAAE,CAAA;IAE9B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,KAAK,KAAK,GAAG,EAAE,CAAC;YACzB,wFAAwF;YACxF,oEAAoE;YACpE,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;YAEzB,OAAO,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAE,CAAC,KAAK,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;gBAAE,IAAI,EAAE,CAAA;YACrG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC,CAAC,SAAS,CAAA;YAC/C,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YAC7C,MAAM,OAAO,GAAG,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAA;YACnC,MAAM,gBAAgB,GAAG,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;YAEhH,IACC,IAAI;gBACJ,IAAI,CAAC,KAAK,KAAK,GAAG;gBAClB,OAAO,KAAK,GAAG;gBACf,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG;gBACvB,gBAAgB;gBAChB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC7C,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,EACjE,CAAC;gBACF,oFAAoF;gBACpF,mFAAmF;gBACnF,GAAG,CAAC,MAAM,GAAG,IAAI,GAAG,CAAC,CAAA;gBACrB,GAAG,CAAC,IAAI,CAAC,GAAG;oBACX,GAAG,IAAI;oBACP,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;oBACxC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC;iBACvD,CAAA;gBACD,SAAQ;YACT,CAAC;QACF,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
|
@@ -29,31 +29,29 @@ import type { ProposedSpan } from "@mailwoman/core/pipeline";
|
|
|
29
29
|
import type { TokenLike } from "./query-shape-prior.js";
|
|
30
30
|
export interface SpanProposalPriorOpts {
|
|
31
31
|
/**
|
|
32
|
-
* Bias magnitude for tag-mapped proposals, in log-odds units. Confidence-scaled. Default 5.0 —
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
* where its logit gap is larger (the bare `3/45` row stays fused).
|
|
32
|
+
* Bias magnitude for tag-mapped proposals, in log-odds units. Confidence-scaled. Default 5.0 — measured on the
|
|
33
|
+
* punctuation-stress sweep (2026-06-12, v4.4.0 int8): the proposer's job is to flip CONFIDENTLY-wrong emissions
|
|
34
|
+
* (fused `2/14` → split), which 1-2-nat query-shape-style scales cannot reach; 5.0 moved slash +11.1 with every other
|
|
35
|
+
* class flat, and the model still vetoes where its logit gap is larger (the bare `3/45` row stays fused).
|
|
37
36
|
*/
|
|
38
37
|
biasScale?: number;
|
|
39
38
|
/**
|
|
40
|
-
* Bias magnitude for the annotation O-prior. Confidence-scaled. Default 12.0 — deliberately
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* component-shaped groups.
|
|
39
|
+
* Bias magnitude for the annotation O-prior. Confidence-scaled. Default 12.0 — deliberately near-mask strength
|
|
40
|
+
* (measured saturation point on the same sweep: bracketed +9.1, paren regressions zero): a BALANCED bracket pair with
|
|
41
|
+
* aside-shaped content is the strongest structural cue the proposer has, and the confidence floor (not the scale) is
|
|
42
|
+
* what protects the component-shaped groups.
|
|
45
43
|
*/
|
|
46
44
|
annotationBiasScale?: number;
|
|
47
45
|
/**
|
|
48
|
-
* Annotation proposals below this confidence contribute NO O-bias (their span still feeds the
|
|
49
|
-
*
|
|
50
|
-
*
|
|
46
|
+
* Annotation proposals below this confidence contribute NO O-bias (their span still feeds the decode-side crossing
|
|
47
|
+
* constraint). Default 0.6 — above the trailing-component shape (0.45), below the capitalized mid-string aside
|
|
48
|
+
* (0.75).
|
|
51
49
|
*/
|
|
52
50
|
annotationConfidenceFloor?: number;
|
|
53
51
|
}
|
|
54
52
|
/**
|
|
55
|
-
* Build the additive prior matrix for one parse. Returns all-zeros rows for pieces no proposal
|
|
56
|
-
*
|
|
53
|
+
* Build the additive prior matrix for one parse. Returns all-zeros rows for pieces no proposal covers — composes
|
|
54
|
+
* harmlessly via `addEmissionMatrix`.
|
|
57
55
|
*/
|
|
58
56
|
export declare function buildSpanProposalPriors(proposals: ReadonlyArray<ProposedSpan>, tokens: ReadonlyArray<TokenLike>, labels: ReadonlyArray<string>, opts?: SpanProposalPriorOpts): number[][];
|
|
59
57
|
//# sourceMappingURL=span-proposal-prior.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"span-proposal-prior.d.ts","sourceRoot":"","sources":["../span-proposal-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAE5D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAEvD,MAAM,WAAW,qBAAqB;IACrC
|
|
1
|
+
{"version":3,"file":"span-proposal-prior.d.ts","sourceRoot":"","sources":["../span-proposal-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAE5D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAEvD,MAAM,WAAW,qBAAqB;IACrC;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;;OAIG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAA;CAClC;AAWD;;;GAGG;AACH,wBAAgB,uBAAuB,CACtC,SAAS,EAAE,aAAa,CAAC,YAAY,CAAC,EACtC,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAChC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAC7B,IAAI,GAAE,qBAA0B,GAC9B,MAAM,EAAE,EAAE,CAsDZ"}
|
|
@@ -34,8 +34,8 @@ const KIND_TO_TAG = new Map([
|
|
|
34
34
|
["FUSED_NUMBER", "house_number"],
|
|
35
35
|
]);
|
|
36
36
|
/**
|
|
37
|
-
* Build the additive prior matrix for one parse. Returns all-zeros rows for pieces no proposal
|
|
38
|
-
*
|
|
37
|
+
* Build the additive prior matrix for one parse. Returns all-zeros rows for pieces no proposal covers — composes
|
|
38
|
+
* harmlessly via `addEmissionMatrix`.
|
|
39
39
|
*/
|
|
40
40
|
export function buildSpanProposalPriors(proposals, tokens, labels, opts = {}) {
|
|
41
41
|
const T = tokens.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"span-proposal-prior.js","sourceRoot":"","sources":["../span-proposal-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;
|
|
1
|
+
{"version":3,"file":"span-proposal-prior.js","sourceRoot":"","sources":["../span-proposal-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AA6BH,MAAM,WAAW,GAAgC,IAAI,GAAG,CAAC;IACxD,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC3B,CAAC,aAAa,EAAE,MAAM,CAAC;IACvB,CAAC,cAAc,EAAE,MAAM,CAAC;IACxB,CAAC,YAAY,EAAE,MAAM,CAAC;IACtB,CAAC,oBAAoB,EAAE,cAAc,CAAC;IACtC,CAAC,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACtC,SAAsC,EACtC,MAAgC,EAChC,MAA6B,EAC7B,OAA8B,EAAE;IAEhC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAA;IACvC,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAA;IAC5D,MAAM,eAAe,GAAG,IAAI,CAAC,yBAAyB,IAAI,GAAG,CAAA;IAE7D,MAAM,MAAM,GAAe,EAAE,CAAA;IAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAErE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IAEzC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAEhC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa;YAAE,SAAQ;QAE7C,IAAI,QAAQ,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YACzC,IAAI,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,UAAU,GAAG,eAAe;gBAAE,SAAQ;YACzE,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,GAAG,mBAAmB,CAAA;YAEtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAE,EAAE,IAAI,CAAC,CAAA;gBACrD,CAAC;YACF,CAAC;YACD,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAE1C,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAA;QACvC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAA;QAEvC,IAAI,IAAI,KAAK,SAAS;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAA;QAC5C,IAAI,KAAK,GAAG,IAAI,CAAA;QAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC;gBAAE,SAAQ;YAC7C,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;YAC/B,KAAK,GAAG,KAAK,CAAA;YAEb,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAQ;YAC/B,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAE,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,CAAiC,EAAE,CAAiC;IACrF,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAA;AAC1C,CAAC"}
|
|
@@ -19,9 +19,8 @@
|
|
|
19
19
|
import { type SystemCode } from "@mailwoman/codex";
|
|
20
20
|
import type { SpanProposerLexicon } from "@mailwoman/core/pipeline";
|
|
21
21
|
/**
|
|
22
|
-
* Build the span-proposer lexicon from the codex tables of the requested systems. Defaults to every
|
|
23
|
-
*
|
|
24
|
-
* parses.
|
|
22
|
+
* Build the span-proposer lexicon from the codex tables of the requested systems. Defaults to every system with
|
|
23
|
+
* designator tables in the codex today. The result is pure data — safe to share across parses.
|
|
25
24
|
*/
|
|
26
25
|
export declare function buildCodexSpanLexicon(systems?: readonly SystemCode[]): SpanProposerLexicon;
|
|
27
26
|
//# sourceMappingURL=span-proposer-lexicon.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"span-proposer-lexicon.d.ts","sourceRoot":"","sources":["../span-proposer-lexicon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAoDnE
|
|
1
|
+
{"version":3,"file":"span-proposer-lexicon.d.ts","sourceRoot":"","sources":["../span-proposer-lexicon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAoDnE;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,SAAS,UAAU,EAAuB,GAAG,mBAAmB,CAuD9G"}
|
|
@@ -18,9 +18,9 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { au, nz, us } from "@mailwoman/codex";
|
|
20
20
|
/**
|
|
21
|
-
* USPS Pub-28 C2 canonicals whose designator is DESCRIPTIVE rather than addressing ("Building A"
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* USPS Pub-28 C2 canonicals whose designator is DESCRIPTIVE rather than addressing ("Building A" describes the
|
|
22
|
+
* building; "Suite 9" addresses a unit). Inside a bracketed group, these read as annotation content (gold convention 2
|
|
23
|
+
* of the punctuation-stress eval).
|
|
24
24
|
*/
|
|
25
25
|
const WEAK_CANONICALS = new Set([
|
|
26
26
|
"BUILDING",
|
|
@@ -44,10 +44,10 @@ const SCAN_EXCLUDED_DELIVERY = new Set([
|
|
|
44
44
|
"Poste Restante",
|
|
45
45
|
]);
|
|
46
46
|
/**
|
|
47
|
-
* Convert one designator phrase from a codex table into a scan-pattern fragment. Short alphabetic
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
47
|
+
* Convert one designator phrase from a codex table into a scan-pattern fragment. Short alphabetic words (≤ 3 chars:
|
|
48
|
+
* "PO", "GPO", "RMB") are treated as initialisms with optional periods/spacing — the punctuation AMAS tells mailers to
|
|
49
|
+
* strip but deliverable mail still carries ("P.O. Box", "R.M.B 4600"). Longer words match literally with flexible
|
|
50
|
+
* whitespace.
|
|
51
51
|
*/
|
|
52
52
|
function phraseToPattern(phrase) {
|
|
53
53
|
return phrase
|
|
@@ -62,9 +62,8 @@ function phraseToPattern(phrase) {
|
|
|
62
62
|
.join("\\s+");
|
|
63
63
|
}
|
|
64
64
|
/**
|
|
65
|
-
* Build the span-proposer lexicon from the codex tables of the requested systems. Defaults to every
|
|
66
|
-
*
|
|
67
|
-
* parses.
|
|
65
|
+
* Build the span-proposer lexicon from the codex tables of the requested systems. Defaults to every system with
|
|
66
|
+
* designator tables in the codex today. The result is pure data — safe to share across parses.
|
|
68
67
|
*/
|
|
69
68
|
export function buildCodexSpanLexicon(systems = ["us", "au", "nz"]) {
|
|
70
69
|
const sys = new Set(systems);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"span-proposer-lexicon.js","sourceRoot":"","sources":["../span-proposer-lexicon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAmB,MAAM,kBAAkB,CAAA;AAG9D;;;;GAIG;AACH,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC;IACpD,UAAU;IACV,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,MAAM;CACN,CAAC,CAAA;AAEF,2FAA2F;AAC3F,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAA;AAElG,qGAAqG;AACrG,MAAM,sBAAsB,GAAwB,IAAI,GAAG,CAAC;IAC3D,IAAI;IACJ,SAAS;IACT,KAAK;IACL,KAAK;IACL,kBAAkB;IAClB,gBAAgB;CAChB,CAAC,CAAA;AAEF;;;;;GAKG;AACH,SAAS,eAAe,CAAC,MAAc;IACtC,OAAO,MAAM;SACX,IAAI,EAAE;SACN,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACb,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK;QAC3F,CAAC,CAAC,IAAI;aACH,KAAK,CAAC,EAAE,CAAC;aACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC;aACxB,IAAI,CAAC,MAAM,CAAC;QACf,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAC9C;SACA,IAAI,CAAC,MAAM,CAAC,CAAA;AACf,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"span-proposer-lexicon.js","sourceRoot":"","sources":["../span-proposer-lexicon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAmB,MAAM,kBAAkB,CAAA;AAG9D;;;;GAIG;AACH,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC;IACpD,UAAU;IACV,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,MAAM;CACN,CAAC,CAAA;AAEF,2FAA2F;AAC3F,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAA;AAElG,qGAAqG;AACrG,MAAM,sBAAsB,GAAwB,IAAI,GAAG,CAAC;IAC3D,IAAI;IACJ,SAAS;IACT,KAAK;IACL,KAAK;IACL,kBAAkB;IAClB,gBAAgB;CAChB,CAAC,CAAA;AAEF;;;;;GAKG;AACH,SAAS,eAAe,CAAC,MAAc;IACtC,OAAO,MAAM;SACX,IAAI,EAAE;SACN,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACb,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK;QAC3F,CAAC,CAAC,IAAI;aACH,KAAK,CAAC,EAAE,CAAC;aACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC;aACxB,IAAI,CAAC,MAAM,CAAC;QACf,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAC9C;SACA,IAAI,CAAC,MAAM,CAAC,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAAiC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAS,OAAO,CAAC,CAAA;IACpC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAA;IACzC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;IAC1C,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAA;IACzC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAA;IAEzC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,EAAE,CAAC;YACrE,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,2BAA2B,CAAC,SAAgC,CAAC,CAAC,CAAA;YACjG,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,eAAe,CAAA;YAEnF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;gBAE3B,IAAI,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;YACzE,CAAC;QACF,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,EAAE,CAAC,qBAAqB;YAAE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC3E,CAAC;IAED,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,+BAA+B,EAAE,CAAC;YACtD,IAAI,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;gBAAE,SAAQ;YAE1D,IAAI,CAAC,GAAG,CAAC,cAAc;gBAAE,SAAQ;YACjC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;YACrC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC;IAED,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,yBAAyB,EAAE,CAAC;YAChD,IAAI,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAElD,IAAI,GAAG,CAAC,UAAU,KAAK,UAAU;gBAAE,SAAQ;YAC3C,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC;IAED,qEAAqE;IACrE,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAClG,MAAM,eAAe,GACpB,YAAY,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,sCAAsC,EAAE,IAAI,CAAC;QAClG,CAAC,CAAC,SAAS,CAAA;IAEb,OAAO;QACN,OAAO,EAAE,GAAG;QACZ,eAAe;QACf,gBAAgB;QAChB,eAAe;QACf,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C,CAAA;AACF,CAAC"}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* Street-morphology emission bias — Layer 1 of the four-layer street-supplement architecture (see
|
|
7
7
|
* `docs/articles/concepts/street-supplement-architecture.md`).
|
|
8
8
|
*
|
|
9
|
-
* This module composes with {@linkcode
|
|
9
|
+
* This module composes with {@linkcode buildFSTEmissionPriors} (admin FST) and the QueryShape prior
|
|
10
10
|
* via {@linkcode addEmissionMatrix} — same shape, same additive semantics. Where the admin FST
|
|
11
11
|
* biases admin BIO labels (`B/I-locality`, `B/I-region`, ...), the morphology FST biases:
|
|
12
12
|
*
|
|
@@ -18,29 +18,28 @@
|
|
|
18
18
|
* (see [[project-v061-failure-mechanism]]).
|
|
19
19
|
*
|
|
20
20
|
* The morphology FST itself is built by `resolver-wof-sqlite/street-morphology-fst-builder.ts` and
|
|
21
|
-
* ships as a separate binary (`fst-street-morphology.bin`) loaded into a second `
|
|
21
|
+
* ships as a separate binary (`fst-street-morphology.bin`) loaded into a second `FSTMatcher`
|
|
22
22
|
* instance.
|
|
23
23
|
*/
|
|
24
|
-
import { type
|
|
24
|
+
import { type FSTMatcherLike } from "./fst-prior.js";
|
|
25
25
|
import type { TokenLike } from "./query-shape-prior.js";
|
|
26
26
|
export interface StreetMorphologyPriorOpts {
|
|
27
27
|
/** Multiplier on the base bias before {@linkcode maxBias} is applied. Default 1.0. */
|
|
28
28
|
biasScale?: number;
|
|
29
29
|
/**
|
|
30
|
-
* Maximum bias magnitude (logits) on the affix span itself. Default 3.0 — same as the admin FST.
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* Maximum bias magnitude (logits) on the affix span itself. Default 3.0 — same as the admin FST. The morphology
|
|
31
|
+
* signal is structurally less ambiguous than admin names (`Avenue` is almost never anything but street-typing), so
|
|
32
|
+
* equal magnitude is justified.
|
|
33
33
|
*/
|
|
34
34
|
maxAffixBias?: number;
|
|
35
35
|
/**
|
|
36
|
-
* Maximum bias magnitude (logits) on the adjacent (neighbour) tokens for the `street` label.
|
|
37
|
-
*
|
|
38
|
-
* adjacency, not direct match.
|
|
36
|
+
* Maximum bias magnitude (logits) on the adjacent (neighbour) tokens for the `street` label. Default 2.0 — a touch
|
|
37
|
+
* weaker than the affix bias because the neighbour is inferred from adjacency, not direct match.
|
|
39
38
|
*/
|
|
40
39
|
maxNeighbourStreetBias?: number;
|
|
41
40
|
/**
|
|
42
|
-
* Magnitude of the negative bias applied to `dependent_locality` BIO labels on the adjacent
|
|
43
|
-
*
|
|
41
|
+
* Magnitude of the negative bias applied to `dependent_locality` BIO labels on the adjacent tokens. Default 2.0. This
|
|
42
|
+
* is the essential piece.
|
|
44
43
|
*/
|
|
45
44
|
dependentLocalityPenalty?: number;
|
|
46
45
|
}
|
|
@@ -50,7 +49,7 @@ export interface StreetMorphologyPriorOpts {
|
|
|
50
49
|
* The output composes with the admin FST bias matrix via {@linkcode addEmissionMatrix} — same
|
|
51
50
|
* `addEmissionMatrix(emissions, fstBias) → biasedEmissions` pattern as the existing admin prior.
|
|
52
51
|
*/
|
|
53
|
-
export declare function buildStreetMorphologyEmissionPriors(fst:
|
|
52
|
+
export declare function buildStreetMorphologyEmissionPriors(fst: FSTMatcherLike, pieces: ReadonlyArray<TokenLike & {
|
|
54
53
|
piece: string;
|
|
55
54
|
}>, labels: ReadonlyArray<string>, opts?: StreetMorphologyPriorOpts): number[][];
|
|
56
55
|
//# sourceMappingURL=street-morphology-prior.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"street-morphology-prior.d.ts","sourceRoot":"","sources":["../street-morphology-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAwB,KAAK,cAAc,EAAkB,MAAM,gBAAgB,CAAA;AAC1F,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAEvD,MAAM,WAAW,yBAAyB;IACzC,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB
|
|
1
|
+
{"version":3,"file":"street-morphology-prior.d.ts","sourceRoot":"","sources":["../street-morphology-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAwB,KAAK,cAAc,EAAkB,MAAM,gBAAgB,CAAA;AAC1F,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAEvD,MAAM,WAAW,yBAAyB;IACzC,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAA;IAC/B;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAA;CACjC;AAED;;;;;GAKG;AACH,wBAAgB,mCAAmC,CAClD,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,aAAa,CAAC,SAAS,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EACpD,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAC7B,IAAI,GAAE,yBAA8B,GAClC,MAAM,EAAE,EAAE,CAiJZ"}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* Street-morphology emission bias — Layer 1 of the four-layer street-supplement architecture (see
|
|
7
7
|
* `docs/articles/concepts/street-supplement-architecture.md`).
|
|
8
8
|
*
|
|
9
|
-
* This module composes with {@linkcode
|
|
9
|
+
* This module composes with {@linkcode buildFSTEmissionPriors} (admin FST) and the QueryShape prior
|
|
10
10
|
* via {@linkcode addEmissionMatrix} — same shape, same additive semantics. Where the admin FST
|
|
11
11
|
* biases admin BIO labels (`B/I-locality`, `B/I-region`, ...), the morphology FST biases:
|
|
12
12
|
*
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* (see [[project-v061-failure-mechanism]]).
|
|
19
19
|
*
|
|
20
20
|
* The morphology FST itself is built by `resolver-wof-sqlite/street-morphology-fst-builder.ts` and
|
|
21
|
-
* ships as a separate binary (`fst-street-morphology.bin`) loaded into a second `
|
|
21
|
+
* ships as a separate binary (`fst-street-morphology.bin`) loaded into a second `FSTMatcher`
|
|
22
22
|
* instance.
|
|
23
23
|
*/
|
|
24
24
|
import { groupPiecesIntoWords } from "./fst-prior.js";
|
|
@@ -68,10 +68,10 @@ export function buildStreetMorphologyEmissionPriors(fst, pieces, labels, opts =
|
|
|
68
68
|
if (!initial)
|
|
69
69
|
continue;
|
|
70
70
|
let bestEnd = -1;
|
|
71
|
-
let
|
|
71
|
+
let bestStateID = -1;
|
|
72
72
|
if (initial.accepted) {
|
|
73
73
|
bestEnd = start;
|
|
74
|
-
|
|
74
|
+
bestStateID = initial.stateID;
|
|
75
75
|
}
|
|
76
76
|
let current = initial;
|
|
77
77
|
for (let end = start + 1; end < wordGroups.length; end++) {
|
|
@@ -83,7 +83,7 @@ export function buildStreetMorphologyEmissionPriors(fst, pieces, labels, opts =
|
|
|
83
83
|
break;
|
|
84
84
|
if (next.accepted) {
|
|
85
85
|
bestEnd = end;
|
|
86
|
-
|
|
86
|
+
bestStateID = next.stateID;
|
|
87
87
|
}
|
|
88
88
|
current = next;
|
|
89
89
|
}
|
|
@@ -91,7 +91,7 @@ export function buildStreetMorphologyEmissionPriors(fst, pieces, labels, opts =
|
|
|
91
91
|
continue;
|
|
92
92
|
// Verify the accepting entries are street_affix (the morphology FST may eventually contain
|
|
93
93
|
// other placetypes if the binary format is reused for related priors).
|
|
94
|
-
const entries = fst.accepting(
|
|
94
|
+
const entries = fst.accepting(bestStateID);
|
|
95
95
|
const hasAffix = entries.some((e) => e.placetype === "street_affix");
|
|
96
96
|
if (!hasAffix)
|
|
97
97
|
continue;
|
|
@@ -144,9 +144,8 @@ export function buildStreetMorphologyEmissionPriors(fst, pieces, labels, opts =
|
|
|
144
144
|
return matrix;
|
|
145
145
|
}
|
|
146
146
|
/**
|
|
147
|
-
* Walk word groups outward from `fromGroupIdx` in `direction` (+1 or -1), skipping empty groups
|
|
148
|
-
*
|
|
149
|
-
* such neighbour exists.
|
|
147
|
+
* Walk word groups outward from `fromGroupIdx` in `direction` (+1 or -1), skipping empty groups (whitespace /
|
|
148
|
+
* punctuation), and return the first non-empty group encountered — or `null` if no such neighbour exists.
|
|
150
149
|
*/
|
|
151
150
|
function findNeighbour(groups, fromGroupIdx, direction) {
|
|
152
151
|
for (let i = fromGroupIdx + direction; i >= 0 && i < groups.length; i += direction) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"street-morphology-prior.js","sourceRoot":"","sources":["../street-morphology-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,oBAAoB,EAAuC,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"street-morphology-prior.js","sourceRoot":"","sources":["../street-morphology-prior.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,oBAAoB,EAAuC,MAAM,gBAAgB,CAAA;AAwB1F;;;;;GAKG;AACH,MAAM,UAAU,mCAAmC,CAClD,GAAmB,EACnB,MAAoD,EACpD,MAA6B,EAC7B,OAAkC,EAAE;IAEpC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAA;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAA;IAC7C,MAAM,sBAAsB,GAAG,IAAI,CAAC,sBAAsB,IAAI,GAAG,CAAA;IACjE,MAAM,wBAAwB,GAAG,IAAI,CAAC,wBAAwB,IAAI,GAAG,CAAA;IAErE,MAAM,MAAM,GAAe,EAAE,CAAA;IAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAErE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC,CAAA;IAErE,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;IACvD,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;IACvD,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;IACvD,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;IACvD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;IAC1C,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;IAC1C,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IAEtD,6FAA6F;IAC7F,kFAAkF;IAClF,IAAI,OAAO,KAAK,SAAS,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACzF,OAAO,MAAM,CAAA;IACd,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IAE/C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IAQ1C,MAAM,YAAY,GAAiB,EAAE,CAAA;IAErC,0FAA0F;IAC1F,oCAAoC;IACpC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAE,CAAA;QAEhC,IAAI,KAAK,CAAC,QAAQ,KAAK,EAAE;YAAE,SAAQ;QAEnC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAA;QAE1C,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,OAAO,GAAG,CAAC,CAAC,CAAA;QAChB,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;QAEpB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,OAAO,GAAG,KAAK,CAAA;YACf,WAAW,GAAG,OAAO,CAAC,OAAO,CAAA;QAC9B,CAAC;QAED,IAAI,OAAO,GAAG,OAAO,CAAA;QAErB,KAAK,IAAI,GAAG,GAAG,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YAC1D,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAE,CAAA;YAElC,IAAI,SAAS,CAAC,QAAQ,KAAK,EAAE;gBAAE,SAAQ;YAEvC,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAA;YAEtD,IAAI,CAAC,IAAI;gBAAE,MAAK;YAEhB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,OAAO,GAAG,GAAG,CAAA;gBACb,WAAW,GAAG,IAAI,CAAC,OAAO,CAAA;YAC3B,CAAC;YACD,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,CAAC;YAAE,SAAQ;QAE5B,2FAA2F;QAC3F,uEAAuE;QACvE,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,cAAc,CAAC,CAAA;QAEpE,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAEvB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAA;QAEjE,8CAA8C;QAC9C,MAAM,iBAAiB,GAAa,EAAE,CAAA;QAEtC,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,UAAU,CAAC,CAAC,CAAE,CAAA;YAEzB,IAAI,EAAE,CAAC,QAAQ,KAAK,EAAE;gBAAE,SAAQ;YAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,YAAY;gBAAE,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,0FAA0F;QAC1F,yFAAyF;QACzF,yFAAyF;QACzF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAA;QAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAE,CAAA;YAChC,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,aAAa,CAAC,CAAA;YAC5E,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,aAAa,CAAC,CAAA;YAC5E,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAE,EAAE,SAAS,CAAC,CAAA;YACrE,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAE,EAAE,SAAS,CAAC,CAAA;QACtE,CAAC;IACF,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IAE5C,yFAAyF;IACzF,kFAAkF;IAClF,sBAAsB;IACtB,MAAM,mBAAmB,GAAG,SAAS,GAAG,sBAAsB,CAAA;IAE9D,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAA;QACjE,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;QAE9D,KAAK,MAAM,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,SAAS;gBAAE,SAAQ;YACxB,MAAM,OAAO,GAAG,SAAS,CAAC,YAAY,CAAA;YAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACtB,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,CAAA;gBAC1D,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAE,EAAE,mBAAmB,CAAC,CAAA;gBAE/E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC3B,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,CAAA;oBAC1D,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,SAAS,CAAE,EAAE,CAAC,wBAAwB,CAAC,CAAA;gBACtF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,MAAmB,EAAE,YAAoB,EAAE,SAAiB;IAClF,KAAK,IAAI,CAAC,GAAG,YAAY,GAAG,SAAS,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACpF,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,CAAC,CAAC,QAAQ,KAAK,EAAE;YAAE,OAAO,CAAC,CAAA;IAChC,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC"}
|
package/out/tokenizer.d.ts
CHANGED
|
@@ -52,18 +52,17 @@ export declare class MailwomanTokenizer {
|
|
|
52
52
|
/** Load from a base64-encoded `tokenizer.model`. Use for in-memory / test setups. */
|
|
53
53
|
static loadFromBase64(b64: string): Promise<MailwomanTokenizer>;
|
|
54
54
|
/**
|
|
55
|
-
* Load from a path to a `tokenizer.model` file on disk. **Node-only** — the dynamic `node:fs`
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
55
|
+
* Load from a path to a `tokenizer.model` file on disk. **Node-only** — the dynamic `node:fs` import keeps this
|
|
56
|
+
* method out of the static dependency graph so the rest of the tokenizer bundles cleanly for the browser. Calling it
|
|
57
|
+
* in a browser throws at runtime; use `loadFromBase64` (or the URL-fetching loaders in `@mailwoman/neural-web`)
|
|
58
|
+
* instead.
|
|
59
59
|
*/
|
|
60
60
|
static loadFromFile(modelPath: string): Promise<MailwomanTokenizer>;
|
|
61
61
|
/**
|
|
62
62
|
* Tokenize `text` to pieces + ids + realigned char offsets.
|
|
63
63
|
*
|
|
64
|
-
* The returned `pieces[i].piece` matches what the Python `sp.EncodeAsPieces(text)[i]` returns,
|
|
65
|
-
*
|
|
66
|
-
* file header for the algorithm.
|
|
64
|
+
* The returned `pieces[i].piece` matches what the Python `sp.EncodeAsPieces(text)[i]` returns, and `pieces[i].id`
|
|
65
|
+
* matches `sp.EncodeAsIds(text)[i]`. The offsets are reconstructed in TS — see file header for the algorithm.
|
|
67
66
|
*/
|
|
68
67
|
encode(text: string): EncodeResult;
|
|
69
68
|
/** Decode a list of ids back to a string. Delegates to the underlying processor. */
|
package/out/tokenizer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,4EAA4E;AAC5E,eAAO,MAAM,cAAc,WAAM,CAAA;AAEjC,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC9B,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;IACb,mCAAmC;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAA;IACb,uDAAuD;IACvD,GAAG,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,GAAG,EAAE,MAAM,EAAE,CAAA;CACb;AAED,qBAAa,kBAAkB;IACV,OAAO,CAAC,QAAQ,CAAC,SAAS;IAA9C,OAAO;IAEP,qFAAqF;WACxE,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,4EAA4E;AAC5E,eAAO,MAAM,cAAc,WAAM,CAAA;AAEjC,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC9B,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;IACb,mCAAmC;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAA;IACb,uDAAuD;IACvD,GAAG,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,GAAG,EAAE,MAAM,EAAE,CAAA;CACb;AAED,qBAAa,kBAAkB;IACV,OAAO,CAAC,QAAQ,CAAC,SAAS;IAA9C,OAAO;IAEP,qFAAqF;WACxE,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAOrE;;;;;OAKG;WACU,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAOzE;;;;;OAKG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY;IA2BlC,oFAAoF;IACpF,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,UAAU,GAAG,MAAM;CAK1C"}
|
package/out/tokenizer.js
CHANGED
|
@@ -44,10 +44,10 @@ export class MailwomanTokenizer {
|
|
|
44
44
|
return new MailwomanTokenizer(processor);
|
|
45
45
|
}
|
|
46
46
|
/**
|
|
47
|
-
* Load from a path to a `tokenizer.model` file on disk. **Node-only** — the dynamic `node:fs`
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
47
|
+
* Load from a path to a `tokenizer.model` file on disk. **Node-only** — the dynamic `node:fs` import keeps this
|
|
48
|
+
* method out of the static dependency graph so the rest of the tokenizer bundles cleanly for the browser. Calling it
|
|
49
|
+
* in a browser throws at runtime; use `loadFromBase64` (or the URL-fetching loaders in `@mailwoman/neural-web`)
|
|
50
|
+
* instead.
|
|
51
51
|
*/
|
|
52
52
|
static async loadFromFile(modelPath) {
|
|
53
53
|
const { readFile } = await import(/* webpackIgnore: true */ "node:fs/promises");
|
|
@@ -57,9 +57,8 @@ export class MailwomanTokenizer {
|
|
|
57
57
|
/**
|
|
58
58
|
* Tokenize `text` to pieces + ids + realigned char offsets.
|
|
59
59
|
*
|
|
60
|
-
* The returned `pieces[i].piece` matches what the Python `sp.EncodeAsPieces(text)[i]` returns,
|
|
61
|
-
*
|
|
62
|
-
* file header for the algorithm.
|
|
60
|
+
* The returned `pieces[i].piece` matches what the Python `sp.EncodeAsPieces(text)[i]` returns, and `pieces[i].id`
|
|
61
|
+
* matches `sp.EncodeAsIds(text)[i]`. The offsets are reconstructed in TS — see file header for the algorithm.
|
|
63
62
|
*/
|
|
64
63
|
encode(text) {
|
|
65
64
|
const pieces = this.processor.encodePieces(text);
|