@mailwoman/neural 4.16.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/address-system.d.ts +4 -5
- package/out/address-system.d.ts.map +1 -1
- package/out/address-system.js +4 -5
- package/out/address-system.js.map +1 -1
- package/out/anchor-inference.d.ts +16 -18
- package/out/anchor-inference.d.ts.map +1 -1
- package/out/anchor-inference.js +16 -18
- package/out/anchor-inference.js.map +1 -1
- package/out/browser.js +1 -1
- package/out/case-normalize.d.ts +13 -8
- package/out/case-normalize.d.ts.map +1 -1
- package/out/case-normalize.js +18 -11
- package/out/case-normalize.js.map +1 -1
- package/out/classifier.d.ts +125 -134
- package/out/classifier.d.ts.map +1 -1
- package/out/classifier.js +46 -49
- package/out/classifier.js.map +1 -1
- package/out/fst-prior.d.ts +19 -21
- package/out/fst-prior.d.ts.map +1 -1
- package/out/fst-prior.js +13 -14
- package/out/fst-prior.js.map +1 -1
- package/out/gazetteer-inference.d.ts +13 -16
- package/out/gazetteer-inference.d.ts.map +1 -1
- package/out/gazetteer-inference.js +13 -16
- package/out/gazetteer-inference.js.map +1 -1
- package/out/index.d.ts +1 -1
- package/out/index.js +1 -1
- package/out/labels.d.ts +9 -10
- package/out/labels.d.ts.map +1 -1
- package/out/labels.js +5 -6
- package/out/labels.js.map +1 -1
- package/out/onnx-runner.d.ts +33 -21
- package/out/onnx-runner.d.ts.map +1 -1
- package/out/onnx-runner.js +39 -19
- package/out/onnx-runner.js.map +1 -1
- package/out/postcode-anchor.d.ts +28 -34
- package/out/postcode-anchor.d.ts.map +1 -1
- package/out/postcode-anchor.js +48 -55
- package/out/postcode-anchor.js.map +1 -1
- package/out/postcode-binary-resolver.d.ts +10 -13
- package/out/postcode-binary-resolver.d.ts.map +1 -1
- package/out/postcode-binary-resolver.js +12 -15
- package/out/postcode-binary-resolver.js.map +1 -1
- package/out/postcode-repair.d.ts +5 -5
- package/out/postcode-repair.d.ts.map +1 -1
- package/out/postcode-repair.js +9 -10
- package/out/postcode-repair.js.map +1 -1
- package/out/proposal-classifier.d.ts +3 -3
- package/out/proposal-classifier.d.ts.map +1 -1
- package/out/proposal-classifier.js.map +1 -1
- package/out/query-shape-prior.d.ts +10 -11
- package/out/query-shape-prior.d.ts.map +1 -1
- package/out/query-shape-prior.js +12 -15
- package/out/query-shape-prior.js.map +1 -1
- package/out/scorer.d.ts +28 -31
- package/out/scorer.d.ts.map +1 -1
- package/out/scorer.js +39 -44
- package/out/scorer.js.map +1 -1
- package/out/soft-features.d.ts +12 -14
- package/out/soft-features.d.ts.map +1 -1
- package/out/soft-features.js +5 -6
- package/out/soft-features.js.map +1 -1
- package/out/span-bridge.d.ts +9 -11
- package/out/span-bridge.d.ts.map +1 -1
- package/out/span-bridge.js +8 -10
- package/out/span-bridge.js.map +1 -1
- package/out/span-proposal-prior.d.ts +13 -15
- package/out/span-proposal-prior.d.ts.map +1 -1
- package/out/span-proposal-prior.js +2 -2
- package/out/span-proposal-prior.js.map +1 -1
- package/out/span-proposer-lexicon.d.ts +2 -3
- package/out/span-proposer-lexicon.d.ts.map +1 -1
- package/out/span-proposer-lexicon.js +9 -10
- package/out/span-proposer-lexicon.js.map +1 -1
- package/out/street-morphology-prior.d.ts +11 -12
- package/out/street-morphology-prior.d.ts.map +1 -1
- package/out/street-morphology-prior.js +8 -9
- package/out/street-morphology-prior.js.map +1 -1
- package/out/tokenizer.d.ts +6 -7
- package/out/tokenizer.d.ts.map +1 -1
- package/out/tokenizer.js +6 -7
- package/out/tokenizer.js.map +1 -1
- package/out/unit-repair.d.ts +2 -2
- package/out/unit-repair.d.ts.map +1 -1
- package/out/unit-repair.js +11 -14
- package/out/unit-repair.js.map +1 -1
- package/out/viterbi.d.ts +13 -13
- package/out/viterbi.d.ts.map +1 -1
- package/out/viterbi.js +12 -12
- package/out/viterbi.js.map +1 -1
- package/out/vitest.config.d.ts.map +1 -1
- package/out/vitest.config.js.map +1 -1
- package/out/weights.d.ts +62 -71
- package/out/weights.d.ts.map +1 -1
- package/out/weights.js +33 -37
- package/out/weights.js.map +1 -1
- package/out/word-consistency.d.ts +4 -4
- package/out/word-consistency.d.ts.map +1 -1
- package/out/word-consistency.js +4 -4
- package/out/word-consistency.js.map +1 -1
- package/package.json +13 -13
package/out/classifier.d.ts
CHANGED
|
@@ -7,13 +7,13 @@
|
|
|
7
7
|
* `@mailwoman/core` decoder. Single user-facing entrypoint: `parse(text)` returns an
|
|
8
8
|
* `AddressTree` ready for projection into JSON / tuple / XML.
|
|
9
9
|
*
|
|
10
|
-
* Convenience wrappers `
|
|
10
|
+
* Convenience wrappers `parseJSON` / `parseTuples` / `parseXML` project the tree on the way out.
|
|
11
11
|
*/
|
|
12
12
|
import { type SystemCode } from "@mailwoman/codex";
|
|
13
|
-
import {
|
|
13
|
+
import { decodeAsXML, type AddressTree, type Calibrator, type ComponentTag, type SerializeJSONOpts, type SerializeTuplesOpts, type UnknownSpan } from "@mailwoman/core/decoder";
|
|
14
14
|
import { type SpanProposerLexicon } from "@mailwoman/core/pipeline";
|
|
15
15
|
import type { AnchorLookup } from "./anchor-inference.js";
|
|
16
|
-
import { type
|
|
16
|
+
import { type FSTMatcherLike } from "./fst-prior.js";
|
|
17
17
|
import type { GazetteerLexicon } from "./gazetteer-inference.js";
|
|
18
18
|
import type { InferResult } from "./onnx-runner.js";
|
|
19
19
|
import { type QueryShapeLike } from "./query-shape-prior.js";
|
|
@@ -22,9 +22,9 @@ import { type StreetMorphologyPriorOpts } from "./street-morphology-prior.js";
|
|
|
22
22
|
import { MailwomanTokenizer } from "./tokenizer.js";
|
|
23
23
|
import type { ResolveWeightsOpts } from "./weights.js";
|
|
24
24
|
/**
|
|
25
|
-
* Structural type the classifier needs from a runner. Lets callers swap the Node-side `
|
|
26
|
-
*
|
|
27
|
-
*
|
|
25
|
+
* Structural type the classifier needs from a runner. Lets callers swap the Node-side `ONNXRunner` for a browser-side
|
|
26
|
+
* runner (e.g. `@mailwoman/neural-web`'s `WebONNXRunner`) without inheritance — the classifier only ever calls
|
|
27
|
+
* `infer(ids)`.
|
|
28
28
|
*/
|
|
29
29
|
export interface NeuralRunner {
|
|
30
30
|
infer(tokenIds: number[], anchor?: {
|
|
@@ -39,24 +39,24 @@ export interface NeuralAddressClassifierConfig {
|
|
|
39
39
|
tokenizer: MailwomanTokenizer;
|
|
40
40
|
runner: NeuralRunner;
|
|
41
41
|
/**
|
|
42
|
-
* Label vocabulary in the order the model emits them. Defaults to Stage 2 (v0.3.0). Stage 2
|
|
43
|
-
*
|
|
44
|
-
*
|
|
42
|
+
* Label vocabulary in the order the model emits them. Defaults to Stage 2 (v0.3.0). Stage 2 strictly extends Stage 1
|
|
43
|
+
* at the same indices, so a v0.2.0 Stage 1 model loaded with this default still decodes correctly — its emissions
|
|
44
|
+
* only span the first 15 entries.
|
|
45
45
|
*/
|
|
46
46
|
labels?: readonly string[];
|
|
47
47
|
/**
|
|
48
48
|
* Decoding strategy:
|
|
49
49
|
*
|
|
50
|
-
* - `"viterbi"` (default) — linear-chain CRF Viterbi with the BIO structural mask. Prevents
|
|
51
|
-
*
|
|
52
|
-
* - `"argmax"` — per-token argmax. Faster but produces structurally invalid sequences. Use only for
|
|
53
|
-
*
|
|
50
|
+
* - `"viterbi"` (default) — linear-chain CRF Viterbi with the BIO structural mask. Prevents orphan-`I-*` sequences. If
|
|
51
|
+
* `transitions` is provided, uses learned scores on top.
|
|
52
|
+
* - `"argmax"` — per-token argmax. Faster but produces structurally invalid sequences. Use only for debugging /
|
|
53
|
+
* comparison.
|
|
54
54
|
*/
|
|
55
55
|
decode?: "viterbi" | "argmax";
|
|
56
56
|
/**
|
|
57
|
-
* Optional learned CRF transition scores. Square matrix of size `labels.length × labels.length`.
|
|
58
|
-
*
|
|
59
|
-
*
|
|
57
|
+
* Optional learned CRF transition scores. Square matrix of size `labels.length × labels.length`. Added on top of the
|
|
58
|
+
* structural BIO mask. Future weights releases ship this; today's v3.0.0 weights don't, so the structural mask alone
|
|
59
|
+
* is used.
|
|
60
60
|
*/
|
|
61
61
|
transitions?: number[][];
|
|
62
62
|
/** Optional learned start-of-sequence transition scores per label. */
|
|
@@ -64,72 +64,66 @@ export interface NeuralAddressClassifierConfig {
|
|
|
64
64
|
/** Optional learned end-of-sequence transition scores per label. */
|
|
65
65
|
endTransitions?: number[];
|
|
66
66
|
/**
|
|
67
|
-
* Optional postcode-anchor lookup (#239/#240). When set, `parse` builds per-piece anchor features
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
67
|
+
* Optional postcode-anchor lookup (#239/#240). When set, `parse` builds per-piece anchor features from the text +
|
|
68
|
+
* this lookup and feeds them to the runner — for models trained with the anchor channel (exported with the
|
|
69
|
+
* `anchor_features`/`anchor_confidence` ONNX inputs). Omit for plain models. Load via `loadAnchorLookup` from
|
|
70
|
+
* `./anchor-inference.js`.
|
|
71
71
|
*/
|
|
72
72
|
postcodeAnchorLookup?: AnchorLookup;
|
|
73
73
|
/**
|
|
74
|
-
* Optional gazetteer-anchor lexicon (#464, knowledge-ladder rung 3.2). When set, `parse` builds
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
74
|
+
* Optional gazetteer-anchor lexicon (#464, knowledge-ladder rung 3.2). When set, `parse` builds per-token
|
|
75
|
+
* candidate-tag-set clues (country/region/po_box/cedex/homograph) from the text + this lexicon and feeds them to the
|
|
76
|
+
* runner — for models trained with the gazetteer-anchor channel (exported with the
|
|
77
|
+
* `gazetteer_features`/`gazetteer_confidence` ONNX inputs). Omit for plain models. Load via `parseGazetteerLexicon`
|
|
78
|
+
* from `./gazetteer-inference.js`.
|
|
79
79
|
*/
|
|
80
80
|
gazetteerLexicon?: GazetteerLexicon;
|
|
81
81
|
/**
|
|
82
|
-
* Channel choreography (#464, v0.9.13 postcode fix): when true, zero the gazetteer clue on pieces
|
|
83
|
-
*
|
|
84
|
-
*
|
|
82
|
+
* Channel choreography (#464, v0.9.13 postcode fix): when true, zero the gazetteer clue on pieces adjacent to a
|
|
83
|
+
* postcode-anchor hit (needs both `gazetteerLexicon` and `postcodeAnchorLookup`). Targets the region-clue→postcode
|
|
84
|
+
* CRF interference (~3pp US postcode).
|
|
85
85
|
*
|
|
86
|
-
* PAIRING IS ESSENTIAL: set this IFF the model was TRAINED with the matching train-time
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
* trained with the train-time half.
|
|
86
|
+
* PAIRING IS ESSENTIAL: set this IFF the model was TRAINED with the matching train-time choreography
|
|
87
|
+
* (`data.gazetteer_choreography`). The 2026-06-10 diagnostic showed the harm is WEIGHT-BAKED — applying this at
|
|
88
|
+
* inference on a model trained _without_ train-choreography does NOT recover postcode and adds train/inference skew.
|
|
89
|
+
* Only enable for a consolidation-era model trained with the train-time half.
|
|
91
90
|
*/
|
|
92
91
|
suppressGazetteerNearPostcode?: boolean;
|
|
93
92
|
/**
|
|
94
|
-
* Default address-system conventions mode for every parse (see
|
|
95
|
-
* `
|
|
96
|
-
*
|
|
97
|
-
* (no detection, no mask).
|
|
93
|
+
* Default address-system conventions mode for every parse (see `ParseOpts.addressSystemConventions` for semantics —
|
|
94
|
+
* `"auto"` reads the model's locale head; a `SystemCode` pins it). Per-parse opts override this. Omit for the
|
|
95
|
+
* byte-stable pre-#511 default (no detection, no mask).
|
|
98
96
|
*/
|
|
99
97
|
addressSystemConventions?: "auto" | SystemCode;
|
|
100
98
|
/**
|
|
101
|
-
* Punctuation-gap span bridging (the v4.4.0 corrective; see `span-bridge.ts`). The corpus label
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
99
|
+
* Punctuation-gap span bridging (the v4.4.0 corrective; see `span-bridge.ts`). The corpus label format cannot express
|
|
100
|
+
* punctuation inside a span, so dotted surfaces ("P.O. Box", "C.P.") decode as fragments. When true, adjacent
|
|
101
|
+
* same-tag spans separated only by short punctuation gaps are merged after decode. Per-parse opts override. Omit for
|
|
102
|
+
* the byte-stable pre-v4.4.0 behavior.
|
|
105
103
|
*/
|
|
106
104
|
bridgePunctuationGaps?: boolean;
|
|
107
105
|
/**
|
|
108
|
-
* Stage 2.7 span proposer (M2+M3 from the punctuation survey, #518). When set, every parse runs
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* delimiter). Build the lexicon with `buildCodexSpanLexicon` (`./span-proposer-lexicon.js`).
|
|
106
|
+
* Stage 2.7 span proposer (M2+M3 from the punctuation survey, #518). When set, every parse runs `proposeSpans`
|
|
107
|
+
* (`@mailwoman/core/pipeline`) over the raw text and consumes the typed proposals two ways: (a) as additive emission
|
|
108
|
+
* priors — the phrase-prior path; the classifier conditions on the boundary hypotheses and can still disagree — and
|
|
109
|
+
* (b) ANNOTATION/QUOTED span boundaries feed the span bridge as merge-crossing constraints (no same-tag merge may
|
|
110
|
+
* straddle a structural delimiter). Build the lexicon with `buildCodexSpanLexicon` (`./span-proposer-lexicon.js`).
|
|
114
111
|
* Per-parse opts override.
|
|
115
112
|
*
|
|
116
|
-
* DEFAULT ON (operator ruling 2026-06-12, after the #518 measurement closed both v0-win quadrants
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* (the pre-2026-06-12 byte-stable default).
|
|
113
|
+
* DEFAULT ON (operator ruling 2026-06-12, after the #518 measurement closed both v0-win quadrants with no class
|
|
114
|
+
* down): omitting this builds the codex lexicon lazily with the frozen measured scales (biasScale 5.0 /
|
|
115
|
+
* annotationBiasScale 12.0). Pass `false` for the proposer-free baseline (the pre-2026-06-12 byte-stable default).
|
|
120
116
|
*/
|
|
121
117
|
spanProposer?: SpanProposerConfig | false;
|
|
122
118
|
/**
|
|
123
|
-
* Per-word BIO consistency repair (#727 + the admin-token fragmentation class). Default off →
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
* `ParseOptions.enforceWordConsistency` overrides this default.
|
|
119
|
+
* Per-word BIO consistency repair (#727 + the admin-token fragmentation class). Default off → byte-identical. When
|
|
120
|
+
* true, every `▁`-delimited word's pieces are forced to ONE tag by a confidence-weighted vote over the post-prior
|
|
121
|
+
* emissions (see word-consistency.ts). Per-parse `ParseOptions.enforceWordConsistency` overrides this default.
|
|
127
122
|
*/
|
|
128
123
|
enforceWordConsistency?: boolean;
|
|
129
124
|
}
|
|
130
125
|
/**
|
|
131
|
-
* Config for the Stage 2.7 span-proposer integration (see
|
|
132
|
-
* `NeuralAddressClassifierConfig.spanProposer`).
|
|
126
|
+
* Config for the Stage 2.7 span-proposer integration (see `NeuralAddressClassifierConfig.spanProposer`).
|
|
133
127
|
*/
|
|
134
128
|
export interface SpanProposerConfig extends SpanProposalPriorOpts {
|
|
135
129
|
/** Codex-backed designator vocabulary (`buildCodexSpanLexicon`). */
|
|
@@ -145,51 +139,54 @@ export declare class NeuralAddressClassifier {
|
|
|
145
139
|
private readonly endTransitions;
|
|
146
140
|
constructor(cfg: NeuralAddressClassifierConfig);
|
|
147
141
|
/**
|
|
148
|
-
* The default-ON Stage 2.7 config: codex lexicon (us/au/nz), frozen measured scales (the prior
|
|
149
|
-
*
|
|
142
|
+
* The default-ON Stage 2.7 config: codex lexicon (us/au/nz), frozen measured scales (the prior builder's own
|
|
143
|
+
* defaults). Built once per instance, only when a parse actually needs it.
|
|
150
144
|
*/
|
|
151
145
|
private defaultProposer;
|
|
152
146
|
/**
|
|
153
|
-
* One-call factory that resolves the weights package (or explicit paths), loads the tokenizer and
|
|
154
|
-
*
|
|
147
|
+
* One-call factory that resolves the weights package (or explicit paths), loads the tokenizer and ONNX runner, and
|
|
148
|
+
* returns a ready-to-use classifier.
|
|
155
149
|
*
|
|
156
|
-
* Resolution order: explicit paths in `opts` → `@mailwoman/neural-weights-<locale>` package →
|
|
157
|
-
*
|
|
150
|
+
* Resolution order: explicit paths in `opts` → `@mailwoman/neural-weights-<locale>` package → throws a single
|
|
151
|
+
* actionable error.
|
|
158
152
|
*
|
|
159
|
-
* **Node-only.** The dynamic imports keep `
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
153
|
+
* **Node-only.** The dynamic imports keep `ONNXRunner` (onnxruntime-node) + `resolveWeights` (uses Node fs) out of
|
|
154
|
+
* the static dependency graph, so this file can be bundled for the browser by `@mailwoman/neural-web`. Calling this
|
|
155
|
+
* method in a browser will throw at runtime — use `loadNeuralClassifierFromUrls` from `@mailwoman/neural-web`
|
|
156
|
+
* instead.
|
|
163
157
|
*/
|
|
164
158
|
static loadFromWeights(opts?: ResolveWeightsOpts & {
|
|
165
159
|
postcodeAnchorLookup?: AnchorLookup;
|
|
160
|
+
executionProviders?: string[];
|
|
166
161
|
}): Promise<NeuralAddressClassifier>;
|
|
167
162
|
/** Tokenize → infer → Viterbi (or argmax) → decoder tree. */
|
|
168
163
|
parse(text: string, opts?: ParseOpts): Promise<AddressTree>;
|
|
169
164
|
/**
|
|
170
|
-
* Like `parse`, but also returns the raw per-token logits and piece offsets needed for per-span
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
* the model's emissions, not the decode's opinions.
|
|
165
|
+
* Like `parse`, but also returns the raw per-token logits and piece offsets needed for per-span logit aggregation
|
|
166
|
+
* (Option C joint-reconcile integration). Shares the ENTIRE decode path with `parse` (one `#decode`, #481) —
|
|
167
|
+
* including the repair passes, which previously ran only in `parse`: reconcile must consume the same tokens the
|
|
168
|
+
* argmax path serves users, and the repair opts were silently ignored here before. `logits` stay RAW (pre-prior,
|
|
169
|
+
* pre-repair) — they are the model's emissions, not the decode's opinions.
|
|
176
170
|
*/
|
|
177
171
|
parseWithLogits(text: string, opts?: ParseOpts): Promise<ParseWithLogitsResult>;
|
|
178
|
-
|
|
172
|
+
parseJSON(text: string, opts?: ParseOpts): Promise<Partial<Record<ComponentTag, string>>>;
|
|
173
|
+
parseJSON(text: string, opts: ParseOpts & SerializeJSONOpts): Promise<Partial<Record<ComponentTag, string>> & {
|
|
174
|
+
unknown?: UnknownSpan[];
|
|
175
|
+
}>;
|
|
179
176
|
parseTuples(text: string, opts?: ParseOpts): Promise<Array<[ComponentTag, string]>>;
|
|
180
|
-
|
|
181
|
-
|
|
177
|
+
parseTuples(text: string, opts: ParseOpts & SerializeTuplesOpts): Promise<Array<[ComponentTag | "unknown", string]>>;
|
|
178
|
+
parseXML(text: string, opts?: ParseOpts & {
|
|
179
|
+
xml?: Parameters<typeof decodeAsXML>[1];
|
|
182
180
|
}): Promise<string>;
|
|
183
181
|
/**
|
|
184
|
-
* Guard against a silent label/emission shape overrun. When the model emits MORE logits per token
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
* the caller violated.
|
|
182
|
+
* Guard against a silent label/emission shape overrun. When the model emits MORE logits per token than the configured
|
|
183
|
+
* label vocabulary (e.g. a Stage 3 bundle loaded with the default Stage 2 labels), viterbi indexes past the
|
|
184
|
+
* transition matrix and dies with an opaque `Cannot read properties of undefined (reading '0')`. Fail fast here with
|
|
185
|
+
* a message that names the contract the caller violated.
|
|
189
186
|
*
|
|
190
|
-
* The opposite shape (model narrower than labels) is intentionally permitted — STAGE2_BIO_LABELS
|
|
191
|
-
*
|
|
192
|
-
*
|
|
187
|
+
* The opposite shape (model narrower than labels) is intentionally permitted — STAGE2_BIO_LABELS prefix-extends
|
|
188
|
+
* STAGE1_BIO_LABELS so a Stage 1 model loaded with Stage 2 labels decodes correctly via the first 15 logits. See
|
|
189
|
+
* labels.ts for the contract.
|
|
193
190
|
*/
|
|
194
191
|
private assertEmissionWidth;
|
|
195
192
|
}
|
|
@@ -203,98 +200,92 @@ export interface ParseWithLogitsResult {
|
|
|
203
200
|
}>;
|
|
204
201
|
}
|
|
205
202
|
/**
|
|
206
|
-
* Per-call opts for `parse()`. Threading a precomputed `QueryShape` here turns on the soft-prior
|
|
207
|
-
*
|
|
203
|
+
* Per-call opts for `parse()`. Threading a precomputed `QueryShape` here turns on the soft-prior bias path in the
|
|
204
|
+
* Viterbi decoder (Stage 2.4 boundary → Stage 3 encoder integration).
|
|
208
205
|
*/
|
|
209
206
|
export interface ParseOpts {
|
|
210
207
|
/**
|
|
211
|
-
* Precomputed `QueryShape` for this input (from `@mailwoman/query-shape`'s `computeQueryShape`).
|
|
212
|
-
*
|
|
213
|
-
*
|
|
208
|
+
* Precomputed `QueryShape` for this input (from `@mailwoman/query-shape`'s `computeQueryShape`). Known-format hits in
|
|
209
|
+
* the shape produce additive emission biases toward the matching BIO label. Typed structurally — no runtime
|
|
210
|
+
* dependency on `@mailwoman/query-shape`.
|
|
214
211
|
*/
|
|
215
212
|
queryShape?: QueryShapeLike;
|
|
216
213
|
/**
|
|
217
|
-
* Maximum bias magnitude in log-odds units. Default 1.0 — adds up to ~e^1 ≈ 2.7× odds to the
|
|
218
|
-
*
|
|
214
|
+
* Maximum bias magnitude in log-odds units. Default 1.0 — adds up to ~e^1 ≈ 2.7× odds to the favored label.
|
|
215
|
+
* Confidence-scaled, so a 0.6-confidence format hit gets +0.6 max bias.
|
|
219
216
|
*/
|
|
220
217
|
queryShapeBiasScale?: number;
|
|
221
218
|
/**
|
|
222
|
-
* Pre-built FST gazetteer matcher. When provided, gazetteer matches produce additive emission
|
|
223
|
-
* biases.
|
|
219
|
+
* Pre-built FST gazetteer matcher. When provided, gazetteer matches produce additive emission biases.
|
|
224
220
|
*/
|
|
225
|
-
fst?:
|
|
221
|
+
fst?: FSTMatcherLike;
|
|
226
222
|
/** Bias magnitude for FST gazetteer matches. Default 1.0. */
|
|
227
223
|
fstBiasScale?: number;
|
|
228
224
|
/**
|
|
229
|
-
* Pre-built street-morphology FST matcher. When provided, street-type affixes (Avenue, rue,
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
* tokens. Closes the v0.6.1 dependent_locality vacuum; see
|
|
225
|
+
* Pre-built street-morphology FST matcher. When provided, street-type affixes (Avenue, rue, Calle, Straße, …) produce
|
|
226
|
+
* additive emission biases toward `street_prefix`/`street_suffix` on the matched tokens AND toward `street` / away
|
|
227
|
+
* from `dependent_locality` on the adjacent name tokens. Closes the v0.6.1 dependent_locality vacuum; see
|
|
233
228
|
* `docs/articles/concepts/street-supplement-architecture.md` for the layered design.
|
|
234
229
|
*/
|
|
235
|
-
fstStreetMorphology?:
|
|
230
|
+
fstStreetMorphology?: FSTMatcherLike;
|
|
236
231
|
/** Override bias magnitudes for the morphology prior. */
|
|
237
232
|
fstStreetMorphologyOpts?: StreetMorphologyPriorOpts;
|
|
238
233
|
/**
|
|
239
|
-
* When true, run the deterministic postcode regex repair pass (v0.7 #35) on the decoded label
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
* v0.7 gate confirms it. See `./postcode-repair.ts`.
|
|
234
|
+
* When true, run the deterministic postcode regex repair pass (v0.7 #35) on the decoded label sequence before
|
|
235
|
+
* tree-building. Detects postcode-shaped substrings (GB/CA/NL/US/FR/… patterns) and snaps/adds the postcode span to
|
|
236
|
+
* the matched shape, fixing the SentencePiece-fragmentation failures catalogued in the 2026-05-29 postcode
|
|
237
|
+
* diagnostic. Off by default — opt-in until the v0.7 gate confirms it. See `./postcode-repair.ts`.
|
|
244
238
|
*/
|
|
245
239
|
postcodeRepair?: boolean;
|
|
246
240
|
/**
|
|
247
|
-
* Per-word BIO consistency repair (#727 + the admin-token fragmentation class). Overrides the
|
|
248
|
-
*
|
|
241
|
+
* Per-word BIO consistency repair (#727 + the admin-token fragmentation class). Overrides the classifier's
|
|
242
|
+
* `enforceWordConsistency` config default for this parse. See word-consistency.ts.
|
|
249
243
|
*/
|
|
250
244
|
enforceWordConsistency?: boolean;
|
|
251
245
|
/**
|
|
252
|
-
* When true, run the deterministic secondary-unit regex repair pass on the decoded label sequence
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
246
|
+
* When true, run the deterministic secondary-unit regex repair pass on the decoded label sequence before
|
|
247
|
+
* tree-building. Detects designator-shaped substrings ("Apt 4B", "Ste 12", "Unit 9400", bare "#104", …) and
|
|
248
|
+
* snaps/adds the unit span, fixing the unit-drop weakness the three-arena capability eval surfaced (postal
|
|
249
|
+
* secondary-unit 0% neural). Off by default — opt-in until the v0.7.2 arena re-run quantifies its delta. See
|
|
250
|
+
* `./unit-repair.ts`.
|
|
257
251
|
*/
|
|
258
252
|
unitRepair?: boolean;
|
|
259
253
|
/**
|
|
260
|
-
* When true AND the input is detected ALL-CAPS (registry/compliance data like `214 JONES RD,
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
* case-insensitively regardless).
|
|
254
|
+
* When true AND the input is detected ALL-CAPS (registry/compliance data like `214 JONES RD, ELKHART, TX 75839`),
|
|
255
|
+
* title-case the input before the model sees it. The model trains on mixed-case text, so all-caps is partly OOD — it
|
|
256
|
+
* drops/mis-bounds tokens (#690: `PALESTINE` → locality `ALESTINE`; all-caps locality 3/5 vs title-case 5/5).
|
|
257
|
+
* Detection-gated, so MIXED-case input is untouched (byte-stable). Off by default. On all-caps input the output
|
|
258
|
+
* values are title-cased (the SHOUTING is normalized away — better, and the resolver name-matches case-insensitively
|
|
259
|
+
* regardless).
|
|
267
260
|
*/
|
|
268
261
|
normalizeCase?: boolean;
|
|
269
262
|
/**
|
|
270
|
-
* Optional span-confidence calibrator (task #59). When provided, each decoded span's `conf=` is
|
|
271
|
-
*
|
|
272
|
-
*
|
|
273
|
-
*
|
|
263
|
+
* Optional span-confidence calibrator (task #59). When provided, each decoded span's `conf=` is mapped through it
|
|
264
|
+
* (isotonic lookup table → calibrated probability of correctness). OPT-IN — omit for the byte-stable default softmax
|
|
265
|
+
* confidence. Build one via `createCalibrator` (`@mailwoman/core/decoder`) from
|
|
266
|
+
* `data/eval/calibration/isotonic-<locale>-<version>.json`.
|
|
274
267
|
*/
|
|
275
268
|
calibrate?: Calibrator;
|
|
276
269
|
/** Per-parse override of the config-level `bridgePunctuationGaps` (see that doc). */
|
|
277
270
|
bridgePunctuationGaps?: boolean;
|
|
278
271
|
/**
|
|
279
|
-
* Per-parse switch for the config-level `spanProposer` (see that doc). `false` disables the
|
|
280
|
-
*
|
|
281
|
-
* stage without a configured lexicon.
|
|
272
|
+
* Per-parse switch for the config-level `spanProposer` (see that doc). `false` disables the configured proposer for
|
|
273
|
+
* this parse; `true`/omitted runs it when configured. Cannot enable the stage without a configured lexicon.
|
|
282
274
|
*/
|
|
283
275
|
spanProposer?: boolean;
|
|
284
276
|
/**
|
|
285
277
|
* Address-system conventions enforcement (#511 Tier A / #478's rules-as-constraints slice).
|
|
286
278
|
*
|
|
287
|
-
* - `"auto"` — detect the system from the model's locale head (`locale_logits` output, v1.1.0+
|
|
288
|
-
*
|
|
289
|
-
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
* that already know the locale, e.g. the pipeline's BCP-47 region).
|
|
279
|
+
* - `"auto"` — detect the system from the model's locale head (`locale_logits` output, v1.1.0+ exports; silently no-ops
|
|
280
|
+
* on models without it) and apply that system's codex conventions: forbidden tags become a hard emission mask
|
|
281
|
+
* before Viterbi, and a conventions postcode shape enables the snap-only postcode repair pass.
|
|
282
|
+
* - A `SystemCode` (`"fr"`, `"us"`, …) — apply that system's conventions unconditionally (callers that already know the
|
|
283
|
+
* locale, e.g. the pipeline's BCP-47 region).
|
|
293
284
|
* - Omit — byte-stable default: no detection, no mask (pre-#511 behavior).
|
|
294
285
|
*
|
|
295
|
-
* The detection threshold is deliberately high (0.8): the mask must never fire on a guess.
|
|
296
|
-
*
|
|
297
|
-
*
|
|
286
|
+
* The detection threshold is deliberately high (0.8): the mask must never fire on a guess. Measured motivation: the
|
|
287
|
+
* 2026-06-10 v1.1.0 gate, where US suffix logic fired inside French parses (`street_suffix: "Rue"`) and digit-splits
|
|
288
|
+
* corrupted leading FR postcodes.
|
|
298
289
|
*/
|
|
299
290
|
addressSystemConventions?: "auto" | SystemCode;
|
|
300
291
|
}
|
package/out/classifier.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAwB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,EAIN,WAAW,EACX,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAmC,KAAK,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAGpG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEzD,OAAO,EAA0B,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC5E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAEhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAA0C,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAGpG,OAAO,EAA2B,KAAK,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAE9F,OAAO,EAAuC,KAAK,yBAAyB,EAAE,MAAM,8BAA8B,CAAA;AAClH,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAGnD,OAAO,KAAK,EAAE,kBAAkB,EAAmB,MAAM,cAAc,CAAA;AAGvE;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC5B,KAAK,CACJ,QAAQ,EAAE,MAAM,EAAE,EAClB,MAAM,CAAC,EAAE;QAAE,QAAQ,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAAC,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;KAAE,EAC9F,SAAS,CAAC,EAAE;QAAE,QAAQ,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAAC,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;KAAE,GAC/F,OAAO,CAAC,WAAW,CAAC,CAAA;CACvB;AAED,MAAM,WAAW,6BAA6B;IAC7C,SAAS,EAAE,kBAAkB,CAAA;IAC7B,MAAM,EAAE,YAAY,CAAA;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAA;IAC7B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,CAAA;IACxB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC3B,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;IACzB;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAA;IACnC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC
|
|
1
|
+
{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAwB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,EAIN,WAAW,EACX,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAmC,KAAK,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAGpG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEzD,OAAO,EAA0B,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC5E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAEhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAA0C,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAGpG,OAAO,EAA2B,KAAK,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAE9F,OAAO,EAAuC,KAAK,yBAAyB,EAAE,MAAM,8BAA8B,CAAA;AAClH,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAGnD,OAAO,KAAK,EAAE,kBAAkB,EAAmB,MAAM,cAAc,CAAA;AAGvE;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC5B,KAAK,CACJ,QAAQ,EAAE,MAAM,EAAE,EAClB,MAAM,CAAC,EAAE;QAAE,QAAQ,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAAC,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;KAAE,EAC9F,SAAS,CAAC,EAAE;QAAE,QAAQ,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAAC,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;KAAE,GAC/F,OAAO,CAAC,WAAW,CAAC,CAAA;CACvB;AAED,MAAM,WAAW,6BAA6B;IAC7C,SAAS,EAAE,kBAAkB,CAAA;IAC7B,MAAM,EAAE,YAAY,CAAA;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAA;IAC7B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,CAAA;IACxB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC3B,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;IACzB;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAA;IACnC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC;;;;;;;;;OASG;IACH,6BAA6B,CAAC,EAAE,OAAO,CAAA;IACvC;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,GAAG,UAAU,CAAA;IAC9C;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B;;;;;;;;;;;OAWG;IACH,YAAY,CAAC,EAAE,kBAAkB,GAAG,KAAK,CAAA;IAEzC;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAA;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,qBAAqB;IAChE,oEAAoE;IACpE,OAAO,EAAE,mBAAmB,CAAA;CAC5B;AAED,qBAAa,uBAAuB;;IASvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IARhC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAY;IAGxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAC3C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;gBAEZ,GAAG,EAAE,6BAA6B;IAc/D;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB;;;;;;;;;;;OAWG;WACU,eAAe,CAC3B,IAAI,GAAE,kBAAkB,GAAG;QAAE,oBAAoB,CAAC,EAAE,YAAY,CAAC;QAAC,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;KAAO,GACpG,OAAO,CAAC,uBAAuB,CAAC;IAsGnC,6DAA6D;IACvD,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAYjE;;;;;;OAMG;IACG,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAqL/E,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzF,SAAS,CACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,GAAG,iBAAiB,GACjC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,GAAG;QAAE,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC;IAQzE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACnF,WAAW,CAChB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,GAAG,mBAAmB,GACnC,OAAO,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;IAQ/C,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG;QAAE,GAAG,CAAC,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAI7G;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB;CAY3B;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,WAAW,CAAA;IACjB,MAAM,EAAE,MAAM,EAAE,EAAE,CAAA;IAClB,MAAM,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAC7C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,CAAA;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;OAEG;IACH,GAAG,CAAC,EAAE,cAAc,CAAA;IACpB,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,cAAc,CAAA;IACpC,yDAAyD;IACzD,uBAAuB,CAAC,EAAE,yBAAyB,CAAA;IACnD;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAA;IAExB;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAEhC;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,UAAU,CAAA;IACtB,qFAAqF;IACrF,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB;;;;;;;;;;;;;OAaG;IACH,wBAAwB,CAAC,EAAE,MAAM,GAAG,UAAU,CAAA;CAC9C"}
|
package/out/classifier.js
CHANGED
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
* `@mailwoman/core` decoder. Single user-facing entrypoint: `parse(text)` returns an
|
|
8
8
|
* `AddressTree` ready for projection into JSON / tuple / XML.
|
|
9
9
|
*
|
|
10
|
-
* Convenience wrappers `
|
|
10
|
+
* Convenience wrappers `parseJSON` / `parseTuples` / `parseXML` project the tree on the way out.
|
|
11
11
|
*/
|
|
12
12
|
import { conventionsForSystem } from "@mailwoman/codex";
|
|
13
|
-
import { buildAddressTree,
|
|
13
|
+
import { buildAddressTree, decodeAsJSON, decodeAsTuples, decodeAsXML, } from "@mailwoman/core/decoder";
|
|
14
14
|
import { proposeSpans } from "@mailwoman/core/pipeline";
|
|
15
15
|
import { detectAddressSystem } from "./address-system.js";
|
|
16
16
|
import { normalizeInputCase } from "./case-normalize.js";
|
|
17
|
-
import {
|
|
17
|
+
import { buildFSTEmissionPriors } from "./fst-prior.js";
|
|
18
18
|
import { STAGE2_BIO_LABELS } from "./labels.js";
|
|
19
19
|
import { repairPostcodeLabels } from "./postcode-repair.js";
|
|
20
20
|
import { addEmissionMatrix, buildEmissionPriors } from "./query-shape-prior.js";
|
|
@@ -25,7 +25,7 @@ import { buildCodexSpanLexicon } from "./span-proposer-lexicon.js";
|
|
|
25
25
|
import { buildStreetMorphologyEmissionPriors } from "./street-morphology-prior.js";
|
|
26
26
|
import { MailwomanTokenizer } from "./tokenizer.js";
|
|
27
27
|
import { repairUnitLabels } from "./unit-repair.js";
|
|
28
|
-
import {
|
|
28
|
+
import { buildBIOEndMask, buildBIOStartMask, buildBIOTransitionMask, softmax, viterbi } from "./viterbi.js";
|
|
29
29
|
import { enforceWordConsistency } from "./word-consistency.js";
|
|
30
30
|
export class NeuralAddressClassifier {
|
|
31
31
|
cfg;
|
|
@@ -40,35 +40,35 @@ export class NeuralAddressClassifier {
|
|
|
40
40
|
this.cfg = cfg;
|
|
41
41
|
this.labels = cfg.labels ?? STAGE2_BIO_LABELS;
|
|
42
42
|
this.decodeMode = cfg.decode ?? "viterbi";
|
|
43
|
-
const structural =
|
|
43
|
+
const structural = buildBIOTransitionMask(this.labels);
|
|
44
44
|
if (cfg.transitions) {
|
|
45
45
|
this.transitions = addMatrices(structural, cfg.transitions);
|
|
46
46
|
}
|
|
47
47
|
else {
|
|
48
48
|
this.transitions = structural;
|
|
49
49
|
}
|
|
50
|
-
this.startTransitions = cfg.startTransitions ??
|
|
51
|
-
this.endTransitions = cfg.endTransitions ??
|
|
50
|
+
this.startTransitions = cfg.startTransitions ?? buildBIOStartMask(this.labels);
|
|
51
|
+
this.endTransitions = cfg.endTransitions ?? buildBIOEndMask(this.labels);
|
|
52
52
|
}
|
|
53
53
|
/**
|
|
54
|
-
* The default-ON Stage 2.7 config: codex lexicon (us/au/nz), frozen measured scales (the prior
|
|
55
|
-
*
|
|
54
|
+
* The default-ON Stage 2.7 config: codex lexicon (us/au/nz), frozen measured scales (the prior builder's own
|
|
55
|
+
* defaults). Built once per instance, only when a parse actually needs it.
|
|
56
56
|
*/
|
|
57
57
|
defaultProposer() {
|
|
58
58
|
this.#defaultProposerCfg ??= { lexicon: buildCodexSpanLexicon() };
|
|
59
59
|
return this.#defaultProposerCfg;
|
|
60
60
|
}
|
|
61
61
|
/**
|
|
62
|
-
* One-call factory that resolves the weights package (or explicit paths), loads the tokenizer and
|
|
63
|
-
*
|
|
62
|
+
* One-call factory that resolves the weights package (or explicit paths), loads the tokenizer and ONNX runner, and
|
|
63
|
+
* returns a ready-to-use classifier.
|
|
64
64
|
*
|
|
65
|
-
* Resolution order: explicit paths in `opts` → `@mailwoman/neural-weights-<locale>` package →
|
|
66
|
-
*
|
|
65
|
+
* Resolution order: explicit paths in `opts` → `@mailwoman/neural-weights-<locale>` package → throws a single
|
|
66
|
+
* actionable error.
|
|
67
67
|
*
|
|
68
|
-
* **Node-only.** The dynamic imports keep `
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
68
|
+
* **Node-only.** The dynamic imports keep `ONNXRunner` (onnxruntime-node) + `resolveWeights` (uses Node fs) out of
|
|
69
|
+
* the static dependency graph, so this file can be bundled for the browser by `@mailwoman/neural-web`. Calling this
|
|
70
|
+
* method in a browser will throw at runtime — use `loadNeuralClassifierFromUrls` from `@mailwoman/neural-web`
|
|
71
|
+
* instead.
|
|
72
72
|
*/
|
|
73
73
|
static async loadFromWeights(opts = {}) {
|
|
74
74
|
// /* webpackIgnore: true */ tells webpack to leave the dynamic import statement intact —
|
|
@@ -76,7 +76,7 @@ export class NeuralAddressClassifier {
|
|
|
76
76
|
// + node:fs) and throws cleanly in a browser if called. Without the directive, webpack
|
|
77
77
|
// pulls onnx-runner / weights into the browser chunk graph + then chokes on the Node-only
|
|
78
78
|
// builtins they reference.
|
|
79
|
-
const [{
|
|
79
|
+
const [{ ONNXRunner }, { resolveWeights, readLabelsFromModelCard, readCrfTransitions, readRequiredChannels }, { parseAnchorLookup }, { parseGazetteerLexicon }, { PostcodeBinaryResolver }, fs,] = await Promise.all([
|
|
80
80
|
import(/* webpackIgnore: true */ "./onnx-runner.js"),
|
|
81
81
|
import(/* webpackIgnore: true */ "./weights.js"),
|
|
82
82
|
import(/* webpackIgnore: true */ "./anchor-inference.js"),
|
|
@@ -89,7 +89,7 @@ export class NeuralAddressClassifier {
|
|
|
89
89
|
const crf = readCrfTransitions(resolved.crfTransitionsPath);
|
|
90
90
|
const [tokenizer, runner] = await Promise.all([
|
|
91
91
|
MailwomanTokenizer.loadFromFile(resolved.tokenizerPath),
|
|
92
|
-
|
|
92
|
+
ONNXRunner.create(resolved.modelPath, { executionProviders: opts.executionProviders }),
|
|
93
93
|
]);
|
|
94
94
|
// --- Soft-feed (#718 D1): feed the channels the SHIPPED model was trained against ----------
|
|
95
95
|
// The anchor-trained en-us model goes OOD when scored anchor-OFF (the #566/#685 crater: country
|
|
@@ -164,12 +164,11 @@ export class NeuralAddressClassifier {
|
|
|
164
164
|
return buildAddressTree(modelText, tokens, opts?.calibrate ? { calibrate: opts.calibrate } : undefined);
|
|
165
165
|
}
|
|
166
166
|
/**
|
|
167
|
-
* Like `parse`, but also returns the raw per-token logits and piece offsets needed for per-span
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
* the model's emissions, not the decode's opinions.
|
|
167
|
+
* Like `parse`, but also returns the raw per-token logits and piece offsets needed for per-span logit aggregation
|
|
168
|
+
* (Option C joint-reconcile integration). Shares the ENTIRE decode path with `parse` (one `#decode`, #481) —
|
|
169
|
+
* including the repair passes, which previously ran only in `parse`: reconcile must consume the same tokens the
|
|
170
|
+
* argmax path serves users, and the repair opts were silently ignored here before. `logits` stay RAW (pre-prior,
|
|
171
|
+
* pre-repair) — they are the model's emissions, not the decode's opinions.
|
|
173
172
|
*/
|
|
174
173
|
async parseWithLogits(text, opts) {
|
|
175
174
|
if (text.length === 0) {
|
|
@@ -183,9 +182,9 @@ export class NeuralAddressClassifier {
|
|
|
183
182
|
};
|
|
184
183
|
}
|
|
185
184
|
/**
|
|
186
|
-
* THE decode path (#481): tokenize → anchor/gazetteer features → infer → priors → CRF/argmax →
|
|
187
|
-
*
|
|
188
|
-
*
|
|
185
|
+
* THE decode path (#481): tokenize → anchor/gazetteer features → infer → priors → CRF/argmax → tokens → repairs. Both
|
|
186
|
+
* `parse` and `parseWithLogits` consume this — never fork it; the 2026-06 audit found three drift surfaces in the
|
|
187
|
+
* previous duplicated copies.
|
|
189
188
|
*/
|
|
190
189
|
async #decode(text, opts) {
|
|
191
190
|
const { pieces, ids } = this.cfg.tokenizer.encode(text);
|
|
@@ -221,7 +220,7 @@ export class NeuralAddressClassifier {
|
|
|
221
220
|
}))
|
|
222
221
|
: logits;
|
|
223
222
|
if (opts?.fst) {
|
|
224
|
-
emissions = addEmissionMatrix(emissions,
|
|
223
|
+
emissions = addEmissionMatrix(emissions, buildFSTEmissionPriors(opts.fst, pieces, this.labels, {
|
|
225
224
|
biasScale: opts.fstBiasScale ?? 1.0,
|
|
226
225
|
}));
|
|
227
226
|
}
|
|
@@ -306,25 +305,24 @@ export class NeuralAddressClassifier {
|
|
|
306
305
|
}
|
|
307
306
|
return { tokens, logits, pieces };
|
|
308
307
|
}
|
|
309
|
-
async
|
|
310
|
-
return
|
|
308
|
+
async parseJSON(text, opts = {}) {
|
|
309
|
+
return decodeAsJSON(await this.parse(text, opts), opts);
|
|
311
310
|
}
|
|
312
|
-
async parseTuples(text, opts) {
|
|
313
|
-
return decodeAsTuples(await this.parse(text, opts));
|
|
311
|
+
async parseTuples(text, opts = {}) {
|
|
312
|
+
return decodeAsTuples(await this.parse(text, opts), opts);
|
|
314
313
|
}
|
|
315
|
-
async
|
|
316
|
-
return
|
|
314
|
+
async parseXML(text, opts) {
|
|
315
|
+
return decodeAsXML(await this.parse(text, opts), opts?.xml);
|
|
317
316
|
}
|
|
318
317
|
/**
|
|
319
|
-
* Guard against a silent label/emission shape overrun. When the model emits MORE logits per token
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
* the caller violated.
|
|
318
|
+
* Guard against a silent label/emission shape overrun. When the model emits MORE logits per token than the configured
|
|
319
|
+
* label vocabulary (e.g. a Stage 3 bundle loaded with the default Stage 2 labels), viterbi indexes past the
|
|
320
|
+
* transition matrix and dies with an opaque `Cannot read properties of undefined (reading '0')`. Fail fast here with
|
|
321
|
+
* a message that names the contract the caller violated.
|
|
324
322
|
*
|
|
325
|
-
* The opposite shape (model narrower than labels) is intentionally permitted — STAGE2_BIO_LABELS
|
|
326
|
-
*
|
|
327
|
-
*
|
|
323
|
+
* The opposite shape (model narrower than labels) is intentionally permitted — STAGE2_BIO_LABELS prefix-extends
|
|
324
|
+
* STAGE1_BIO_LABELS so a Stage 1 model loaded with Stage 2 labels decodes correctly via the first 15 logits. See
|
|
325
|
+
* labels.ts for the contract.
|
|
328
326
|
*/
|
|
329
327
|
assertEmissionWidth(logits) {
|
|
330
328
|
if (logits.length === 0)
|
|
@@ -338,12 +336,11 @@ export class NeuralAddressClassifier {
|
|
|
338
336
|
}
|
|
339
337
|
}
|
|
340
338
|
/**
|
|
341
|
-
* Loud-degrade warning for the `loadFromWeights` soft-feed (#718 D1) — the Node mirror of
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
345
|
-
*
|
|
346
|
-
* surface).
|
|
339
|
+
* Loud-degrade warning for the `loadFromWeights` soft-feed (#718 D1) — the Node mirror of neural-web's
|
|
340
|
+
* `warnOnUnfedTrainedChannels`. Fired ONCE per channel per process: a model-card that declares a channel REQUIRED,
|
|
341
|
+
* paired with a package that didn't ship (or could not parse) its data, runs that channel OFF. Structural fallback (the
|
|
342
|
+
* parse still works), loud console (a silently anchor-OFF anchor-trained model is the #566/#685 OOD crater this fix
|
|
343
|
+
* exists to surface).
|
|
347
344
|
*/
|
|
348
345
|
const warnedUnfedChannels = new Set();
|
|
349
346
|
function warnUnfedChannel(channel, detail) {
|