@mailwoman/neural 4.10.0 → 4.12.0

package/out/word-consistency.d.ts

@@ -0,0 +1,60 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Per-word BIO tag-consistency repair (#727 + the fr.country / admin-token fragmentation class).
+ *
+ *   The model emits per-PIECE BIO labels. On rows where an admin token is adjacent to a non-latin
+ *   (byte-fallback) locality, carries diacritics, or is all-caps/reordered, the per-piece labels
+ *   can DISAGREE within a single word — `VERMONT` → `VER`[B-locality] + `MONT`[B-region], `Lozère`
+ *   → `Loz`[locality] + `ère`[B-region] — and the span decoder reads that as a tag-change
+ *   mid-token, fracturing the admin component. The model already KNOWS the word's tag (99.7% of
+ *   normal rows are unanimous); the defect is the lack of a word-level consistency constraint.
+ *
+ *   Fix (DeepSeek-Pro consult `contested-frag`, 2026-06-19): a SentencePiece word — a `▁`-started
+ *   piece + its non-`▁` continuations — must carry ONE tag. The tag is chosen by a
+ *   CONFIDENCE-WEIGHTED vote, NOT first-piece-wins: sum each piece's softmax mass per TAG TYPE
+ *   (B-X
+ *
+ *   - I-X collapsed; `O` included) across the word, argmax the type, and force `B-<type>` then
+ *       `I-<type>` (or all `O`). The near-certain `ère`→region then pulls the whole word to region,
+ *       healing both the fragment and the `Loz` bleed. Operates ONLY within a `▁`-delimited word,
+ *       so cross-word multi-token names ("Saint Paul" → two words) are untouched and the decoder's
+ *       existing cross-word merge still joins them.
+ *
+ *   Safety: a word whose pieces already agree is left byte-identical (no change). The vote includes
+ *   `O`, so an all-`O` word stays `O` (no spurious spans).
+ *
+ *   Gate outcome (2026-06-19, fr-admin-split-gate + per-locale-f1, MAILWOMAN_WORD_CONSISTENCY=1): NOT
+ *   a clean win, so this ships DEFAULT-OFF. It heals clean-latin fragments (`PRUNIÈRES LOZÈRE
+ *   FRANCE` → region=`LOZÈRE`), but on OOD byte-soup rows where the per-piece confidence is itself
+ *   unreliable the vote AMPLIFIES that noise — e.g. `VERMONT, ウェストミンスター` lets the region word
+ *   absorb the adjacent CJK locality — and it net-regressed street −12.6 on the adversarial golden.
+ *   The confidence-weighted vote out-votes a stray mis-tagged piece only when the surviving pieces
+ *   are themselves trustworthy; on byte-fallback pieces that premise breaks. A confidence-gated
+ *   variant (skip the heal when the word's mean p(bestType) is below a floor, or when any piece is
+ *   a raw byte-fallback piece) is the path to a clean win — tracked on #727.
+ */
+export interface WordConsistencyResult {
+    /** A new per-piece label-index array, word-consistent (input is not mutated). */
+    labelIndices: number[];
+    /** PieceIndex → mean p(chosen type) across the word, for pieces in a word that was HEALED. */
+    healedConfidence: Map<number, number>;
+    /** Count of words whose labels were rewritten (0 = byte-identical to the input). */
+    healedWords: number;
+}
+/**
+ * Rewrite per-piece label indices so every `▁`-delimited word carries one tag, chosen by a
+ * confidence-weighted vote over the post-prior `emissions`. See the module docstring.
+ *
+ * @param pieces SentencePiece pieces (the `▁`-marked surface is the word-boundary signal).
+ * @param emissions Per-piece × per-label scores AFTER all priors/masks (the distribution the argmax
+ *   would see). Softmaxed per piece for the vote so each piece's confidence carries its weight.
+ * @param labels The BIO label vocabulary (index ↔ label).
+ * @param labelIndices The current per-piece decision (viterbi path or argmax). Not mutated.
+ */
+export declare function enforceWordConsistency(pieces: ReadonlyArray<{
+    piece: string;
+}>, emissions: ReadonlyArray<ReadonlyArray<number>>, labels: readonly string[], labelIndices: readonly number[]): WordConsistencyResult;
+//# sourceMappingURL=word-consistency.d.ts.map

package/out/word-consistency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"word-consistency.d.ts","sourceRoot":"","sources":["../word-consistency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAKH,MAAM,WAAW,qBAAqB;IACrC,iFAAiF;IACjF,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,8FAA8F;IAC9F,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACrC,oFAAoF;IACpF,WAAW,EAAE,MAAM,CAAA;CACnB;AASD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACrC,MAAM,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EACxC,SAAS,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,EAC/C,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,YAAY,EAAE,SAAS,MAAM,EAAE,GAC7B,qBAAqB,CA+EvB"}

package/out/word-consistency.js

@@ -0,0 +1,140 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Per-word BIO tag-consistency repair (#727 + the fr.country / admin-token fragmentation class).
+ *
+ *   The model emits per-PIECE BIO labels. On rows where an admin token is adjacent to a non-latin
+ *   (byte-fallback) locality, carries diacritics, or is all-caps/reordered, the per-piece labels
+ *   can DISAGREE within a single word — `VERMONT` → `VER`[B-locality] + `MONT`[B-region], `Lozère`
+ *   → `Loz`[locality] + `ère`[B-region] — and the span decoder reads that as a tag-change
+ *   mid-token, fracturing the admin component. The model already KNOWS the word's tag (99.7% of
+ *   normal rows are unanimous); the defect is the lack of a word-level consistency constraint.
+ *
+ *   Fix (DeepSeek-Pro consult `contested-frag`, 2026-06-19): a SentencePiece word — a `▁`-started
+ *   piece + its non-`▁` continuations — must carry ONE tag. The tag is chosen by a
+ *   CONFIDENCE-WEIGHTED vote, NOT first-piece-wins: sum each piece's softmax mass per TAG TYPE
+ *   (B-X
+ *
+ *   - I-X collapsed; `O` included) across the word, argmax the type, and force `B-<type>` then
+ *       `I-<type>` (or all `O`). The near-certain `ère`→region then pulls the whole word to region,
+ *       healing both the fragment and the `Loz` bleed. Operates ONLY within a `▁`-delimited word,
+ *       so cross-word multi-token names ("Saint Paul" → two words) are untouched and the decoder's
+ *       existing cross-word merge still joins them.
+ *
+ *   Safety: a word whose pieces already agree is left byte-identical (no change). The vote includes
+ *   `O`, so an all-`O` word stays `O` (no spurious spans).
+ *
+ *   Gate outcome (2026-06-19, fr-admin-split-gate + per-locale-f1, MAILWOMAN_WORD_CONSISTENCY=1): NOT
+ *   a clean win, so this ships DEFAULT-OFF. It heals clean-latin fragments (`PRUNIÈRES LOZÈRE
+ *   FRANCE` → region=`LOZÈRE`), but on OOD byte-soup rows where the per-piece confidence is itself
+ *   unreliable the vote AMPLIFIES that noise — e.g. `VERMONT, ウェストミンスター` lets the region word
+ *   absorb the adjacent CJK locality — and it net-regressed street −12.6 on the adversarial golden.
+ *   The confidence-weighted vote out-votes a stray mis-tagged piece only when the surviving pieces
+ *   are themselves trustworthy; on byte-fallback pieces that premise breaks. A confidence-gated
+ *   variant (skip the heal when the word's mean p(bestType) is below a floor, or when any piece is
+ *   a raw byte-fallback piece) is the path to a clean win — tracked on #727.
+ */
+import { SPACE_SENTINEL } from "./tokenizer.js";
+import { softmax } from "./viterbi.js";
+/** The tag TYPE of a BIO label: `"region"` from `B-region`/`I-region`; `"O"` from `O`. */
+function labelType(label) {
+    if (label === "O")
+        return "O";
+    const dash = label.indexOf("-");
+    return dash >= 0 ? label.slice(dash + 1) : label;
+}
+/**
+ * Rewrite per-piece label indices so every `▁`-delimited word carries one tag, chosen by a
+ * confidence-weighted vote over the post-prior `emissions`. See the module docstring.
+ *
+ * @param pieces SentencePiece pieces (the `▁`-marked surface is the word-boundary signal).
+ * @param emissions Per-piece × per-label scores AFTER all priors/masks (the distribution the argmax
+ *   would see). Softmaxed per piece for the vote so each piece's confidence carries its weight.
+ * @param labels The BIO label vocabulary (index ↔ label).
+ * @param labelIndices The current per-piece decision (viterbi path or argmax). Not mutated.
+ */
+export function enforceWordConsistency(pieces, emissions, labels, labelIndices) {
+    // Type → {B index, I index}; the standalone O index; per-label-index → type.
+    const typeB = new Map();
+    const typeI = new Map();
+    const idxType = labels.map((l, idx) => {
+        const t = labelType(l);
+        if (l.startsWith("B-"))
+            typeB.set(t, idx);
+        else if (l.startsWith("I-"))
+            typeI.set(t, idx);
+        return t;
+    });
+    const oIdx = labels.indexOf("O");
+    const out = [...labelIndices];
+    const healedConfidence = new Map();
+    let healedWords = 0;
+    // Group pieces into words. A word = a `▁`-started piece + its non-`▁` continuations. A bare `▁`
+    // (whitespace-only) piece is a SEPARATOR — it ends the current word and joins no word (its label
+    // is left as-is, matching the decoder's "zero-width O is not a boundary" handling).
+    const words = [];
+    let cur = [];
+    const flush = () => {
+        if (cur.length)
+            words.push(cur);
+        cur = [];
+    };
+    for (let i = 0; i < pieces.length; i++) {
+        const pc = pieces[i].piece;
+        const isSentinel = pc.startsWith(SPACE_SENTINEL);
+        const content = isSentinel ? pc.slice(SPACE_SENTINEL.length) : pc;
+        if (content.trim() === "") {
+            // Separator (bare `▁` or whitespace) — ends the current word, belongs to none.
+            flush();
+            continue;
+        }
+        if (isSentinel) {
+            flush();
+            cur = [i];
+        }
+        else {
+            // Continuation. (An orphan continuation with no started word — shouldn't happen since the
+            // input's first piece is `▁`-marked — defensively starts its own word.)
+            cur.push(i);
+        }
+    }
+    flush();
+    for (const w of words) {
+        // Confidence-weighted vote: sum each piece's softmax mass per TYPE (B-X + I-X) across the word.
+        const score = new Map();
+        for (const pi of w) {
+            const probs = softmax([...emissions[pi]]);
+            for (let li = 0; li < probs.length; li++) {
+                const t = idxType[li];
+                score.set(t, (score.get(t) ?? 0) + probs[li]);
+            }
+        }
+        let bestType = "O";
+        let bestScore = -1;
+        for (const [t, s] of score) {
+            if (s > bestScore) {
+                bestScore = s;
+                bestType = t;
+            }
+        }
+        // Target label index per piece: B-<type> for the first piece, I-<type> for the rest (or O).
+        const targets = w.map((_pi, k) => {
+            if (bestType === "O")
+                return oIdx;
+            return k === 0 ? (typeB.get(bestType) ?? oIdx) : (typeI.get(bestType) ?? oIdx);
+        });
+        const changed = w.some((pi, k) => out[pi] !== targets[k]);
+        if (!changed)
+            continue; // word already consistent → byte-identical, leave it
+        healedWords++;
+        const meanConf = bestScore / w.length; // mean p(bestType) — length-invariant (DeepSeek t3)
+        w.forEach((pi, k) => {
+            out[pi] = targets[k];
+            healedConfidence.set(pi, meanConf);
+        });
+    }
+    return { labelIndices: out, healedConfidence, healedWords };
+}
+//# sourceMappingURL=word-consistency.js.map

package/out/word-consistency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"word-consistency.js","sourceRoot":"","sources":["../word-consistency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAWtC,0FAA0F;AAC1F,SAAS,SAAS,CAAC,KAAa;IAC/B,IAAI,KAAK,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC/B,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AACjD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACrC,MAAwC,EACxC,SAA+C,EAC/C,MAAyB,EACzB,YAA+B;IAE/B,6EAA6E;IAC7E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QACtB,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;aACpC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;QAC9C,OAAO,CAAC,CAAA;IACT,CAAC,CAAC,CAAA;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAEhC,MAAM,GAAG,GAAG,CAAC,GAAG,YAAY,CAAC,CAAA;IAC7B,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAA;IAClD,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,gGAAgG;IAChG,iGAAiG;IACjG,oFAAoF;IACpF,MAAM,KAAK,GAAe,EAAE,CAAA;IAC5B,IAAI,GAAG,GAAa,EAAE,CAAA;IACtB,MAAM,KAAK,GAAG,GAAS,EAAE;QACxB,IAAI,GAAG,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC/B,GAAG,GAAG,EAAE,CAAA;IACT,CAAC,CAAA;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAA;QAC3B,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACjE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3B,+EAA+E;YAC/E,KAAK,EAAE,CAAA;YACP,SAAQ;QACT,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YAChB,KAAK,EAAE,CAAA;YACP,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;QACV,CAAC;aAAM,CAAC;YACP,0FAA0F;YAC1F,wEAAwE;YACxE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;IACF,CAAC;IACD,KAAK,EAAE,CAAA;IAEP,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,gGAAgG;QAChG,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;QACvC,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,EAAE,CAAE,CAAC,CAAC,CAAA;YAC1C,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;gBAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,EAAE,CAAE,CAAA;gBACtB,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAE,CAAC,CAAA;YAC/C,CAAC;QACF,CAAC;QACD,IAAI,QAAQ,GAAG,GAAG,CAAA;QAClB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;QAClB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,SAAS,EAAE,CAAC;gBACnB,SAAS,GAAG,CAAC,CAAA;gBACb,QAAQ,GAAG,CAAC,CAAA;YACb,CAAC;QACF,CAAC;QACD,4FAA4F;QAC5F,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YAChC,IAAI,QAAQ,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAA;YACjC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAA;QAC/E,CAAC,CAAC,CAAA;QACF,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;QACzD,IAAI,CAAC,OAAO;YAAE,SAAQ,CAAC,qDAAqD;QAC5E,WAAW,EAAE,CAAA;QACb,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,MAAM,CAAA,CAAC,oDAAoD;QAC1F,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;YACnB,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;YACrB,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QACnC,CAAC,CAAC,CAAA;IACH,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAA;AAC5D,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/neural",
-  "version": "4.10.0",
+  "version": "4.12.0",
   "description": "Mailwoman neural classifier runtime: SentencePiece tokenizer + ONNX inference + decoder wiring.",
   "license": "AGPL-3.0-only",
   "repository": {
@@ -14,14 +14,16 @@
     ".": "./out/index.js",
     "./postcode-anchor": "./out/postcode-anchor.js",
     "./postcode-binary-resolver": "./out/postcode-binary-resolver.js",
+    "./soft-features": "./out/soft-features.js",
+    "./scorer": "./out/scorer.js",
     "./tokenizer": "./out/tokenizer.js",
     "./onnx-runner": "./out/onnx-runner.js",
     "./weights": "./out/weights.js",
     "./browser": "./out/browser.js"
   },
   "dependencies": {
-    "@mailwoman/codex": "4.10.0",
-    "@mailwoman/core": "4.10.0",
+    "@mailwoman/codex": "4.12.0",
+    "@mailwoman/core": "4.12.0",
     "@sctg/sentencepiece-js": "^1.3.3",
     "onnxruntime-node": "^1.26.0"
   },