mailwoman 7.5.0 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api-engine.ts +4 -1
- package/commands/corpus/shard/index.tsx +14 -0
- package/commands/eval/invariance.tsx +64 -0
- package/commands/gazetteer/pair-index.tsx +237 -0
- package/commands/release/hf.tsx +6 -0
- package/eval-harness/gauntlet/cases/regression.ts +1 -1
- package/eval-harness/invariance/command.ts +59 -0
- package/eval-harness/invariance/compare.ts +106 -0
- package/eval-harness/invariance/runner.ts +375 -0
- package/eval-harness/invariance/transforms.ts +322 -0
- package/gazetteer-pipeline/admin/index.ts +3 -15
- package/gazetteer-pipeline/pair-index.ts +196 -0
- package/out/api-engine.d.ts.map +1 -1
- package/out/api-engine.js +4 -1
- package/out/api-engine.js.map +1 -1
- package/out/commands/corpus/shard/index.d.ts +3 -0
- package/out/commands/corpus/shard/index.d.ts.map +1 -1
- package/out/commands/corpus/shard/index.js +12 -0
- package/out/commands/corpus/shard/index.js.map +1 -1
- package/out/commands/eval/invariance.d.ts +34 -0
- package/out/commands/eval/invariance.d.ts.map +1 -0
- package/out/commands/eval/invariance.js +52 -0
- package/out/commands/eval/invariance.js.map +1 -0
- package/out/commands/gazetteer/pair-index.d.ts +39 -0
- package/out/commands/gazetteer/pair-index.d.ts.map +1 -0
- package/out/commands/gazetteer/pair-index.js +189 -0
- package/out/commands/gazetteer/pair-index.js.map +1 -0
- package/out/commands/release/hf.d.ts +1 -0
- package/out/commands/release/hf.d.ts.map +1 -1
- package/out/commands/release/hf.js +4 -0
- package/out/commands/release/hf.js.map +1 -1
- package/out/eval-harness/gauntlet/cases/regression.js +1 -1
- package/out/eval-harness/gauntlet/cases/regression.js.map +1 -1
- package/out/eval-harness/invariance/command.d.ts +29 -0
- package/out/eval-harness/invariance/command.d.ts.map +1 -0
- package/out/eval-harness/invariance/command.js +36 -0
- package/out/eval-harness/invariance/command.js.map +1 -0
- package/out/eval-harness/invariance/compare.d.ts +42 -0
- package/out/eval-harness/invariance/compare.d.ts.map +1 -0
- package/out/eval-harness/invariance/compare.js +82 -0
- package/out/eval-harness/invariance/compare.js.map +1 -0
- package/out/eval-harness/invariance/runner.d.ts +96 -0
- package/out/eval-harness/invariance/runner.d.ts.map +1 -0
- package/out/eval-harness/invariance/runner.js +247 -0
- package/out/eval-harness/invariance/runner.js.map +1 -0
- package/out/eval-harness/invariance/transforms.d.ts +35 -0
- package/out/eval-harness/invariance/transforms.d.ts.map +1 -0
- package/out/eval-harness/invariance/transforms.js +273 -0
- package/out/eval-harness/invariance/transforms.js.map +1 -0
- package/out/gazetteer-pipeline/admin/index.d.ts.map +1 -1
- package/out/gazetteer-pipeline/admin/index.js +3 -12
- package/out/gazetteer-pipeline/admin/index.js.map +1 -1
- package/out/gazetteer-pipeline/pair-index.d.ts +94 -0
- package/out/gazetteer-pipeline/pair-index.d.ts.map +1 -0
- package/out/gazetteer-pipeline/pair-index.js +129 -0
- package/out/gazetteer-pipeline/pair-index.js.map +1 -0
- package/out/release-tools/publish-hf.d.ts +3 -0
- package/out/release-tools/publish-hf.d.ts.map +1 -1
- package/out/release-tools/publish-hf.js +23 -0
- package/out/release-tools/publish-hf.js.map +1 -1
- package/package.json +23 -23
- package/release-tools/publish-hf.ts +27 -0
package/api-engine.ts
CHANGED
|
@@ -88,7 +88,10 @@ function buildPreflightMessage(): string {
|
|
|
88
88
|
` curl -fSL https://public.sister.software/mailwoman/gazetteer/2026-07-07a/candidate.db \\`,
|
|
89
89
|
` -o ${conventionCandidate}`,
|
|
90
90
|
"",
|
|
91
|
-
" Then
|
|
91
|
+
" Then point `$MAILWOMAN_CANDIDATE_DB` at that file and re-run `serve`:",
|
|
92
|
+
` export MAILWOMAN_CANDIDATE_DB=${conventionCandidate}`,
|
|
93
|
+
"",
|
|
94
|
+
" Or point at your own gazetteer:",
|
|
92
95
|
" $MAILWOMAN_WOF_DB=<path1,path2,...> (admin WOF SQLite shard(s) — the FTS backend)",
|
|
93
96
|
" $MAILWOMAN_CANDIDATE_DB=<path> (candidate-table gazetteer — population-first, demo-parity)",
|
|
94
97
|
"",
|
|
@@ -43,9 +43,20 @@ const OptionsSchema = zod.object({
|
|
|
43
43
|
edgesDir: zod.string().optional().describe("intersection: TIGER EDGES dir"),
|
|
44
44
|
country: zod.string().optional().describe("locale: target country"),
|
|
45
45
|
intlFraction: zod.string().optional().describe("german/locale: international-order fraction"),
|
|
46
|
+
countryFraction: zod
|
|
47
|
+
.string()
|
|
48
|
+
.optional()
|
|
49
|
+
.describe("locale: fraction of rows that append an explicit country surface form. Default 0"),
|
|
50
|
+
districtAsLocality: zod
|
|
51
|
+
.boolean()
|
|
52
|
+
.optional()
|
|
53
|
+
.describe(
|
|
54
|
+
"locale: override the per-part districtAsLocality mapping (--no-district-as-locality to force off). Unset leaves each part's own value untouched"
|
|
55
|
+
),
|
|
46
56
|
bareProb: zod.string().optional().describe("street-bare: P(bare street)"),
|
|
47
57
|
hnProb: zod.string().optional().describe("street-bare: P(house number)"),
|
|
48
58
|
communes: zod.string().optional().describe("fr-admin-split: communes source"),
|
|
59
|
+
banDir: zod.string().optional().describe("fr-lieudit: BAN adresses-<dept>.csv directory"),
|
|
49
60
|
excludeSurfaces: zod
|
|
50
61
|
.string()
|
|
51
62
|
.optional()
|
|
@@ -97,9 +108,12 @@ const CorpusShard: CommandComponent<typeof OptionsSchema, typeof ArgumentsSchema
|
|
|
97
108
|
edgesDir: options.edgesDir,
|
|
98
109
|
country: options.country,
|
|
99
110
|
intlFraction: num(options.intlFraction),
|
|
111
|
+
countryFraction: num(options.countryFraction),
|
|
112
|
+
districtAsLocality: options.districtAsLocality,
|
|
100
113
|
bareProb: num(options.bareProb),
|
|
101
114
|
hnProb: num(options.hnProb),
|
|
102
115
|
communes: options.communes,
|
|
116
|
+
banDir: options.banDir,
|
|
103
117
|
excludeSurfaces: options.excludeSurfaces,
|
|
104
118
|
multilocaleCount: num(options.multilocaleCount),
|
|
105
119
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `mailwoman eval invariance` — the INVARIANCE MINI-SUITE: a standing, seconds-cheap
|
|
7
|
+
* metamorphic-invariance check (comma-drop / abbreviation-swap / case-fold / lowercase /
|
|
8
|
+
* whitespace-jitter / trailing-punct / idempotence) over `eval-harness/invariance/suite.jsonl`, meant
|
|
9
|
+
* to run in EVERY probe grade so distribution-shift collateral surfaces at 2k-probe cost instead of
|
|
10
|
+
* ship-prep cost. Compares decoded PARSE COMPONENTS only (no resolver) — cheap by construction. Exit
|
|
11
|
+
* nonzero on any LOST pair, or when the DEGRADED count exceeds `--max-degraded`. `--baseline` switches
|
|
12
|
+
* to regression mode: a violation the baseline ALSO exhibits on the same pair is reported but doesn't
|
|
13
|
+
* fail the gate — the shape probe grading uses to diff a candidate against v385.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { Text } from "ink"
|
|
17
|
+
import zod from "zod"
|
|
18
|
+
|
|
19
|
+
import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
|
|
20
|
+
import { runInvarianceCommand } from "../../eval-harness/invariance/command.ts"
|
|
21
|
+
|
|
22
|
+
export const description =
|
|
23
|
+
"Metamorphic invariance mini-suite (comma-drop/abbrev/case/idempotence) — standing probe guard"
|
|
24
|
+
|
|
25
|
+
const OptionsSchema = zod.object({
|
|
26
|
+
suite: zod.string().optional().describe("Alternate suite.jsonl path (default: the shipped fixture)"),
|
|
27
|
+
model: zod.string().optional().describe("Candidate ONNX (requires --tokenizer + --model-card)"),
|
|
28
|
+
tokenizer: zod.string().optional().describe("Candidate tokenizer path"),
|
|
29
|
+
modelCard: zod.string().optional().describe("Candidate model-card path"),
|
|
30
|
+
weightsCache: zod
|
|
31
|
+
.string()
|
|
32
|
+
.optional()
|
|
33
|
+
.describe(
|
|
34
|
+
"Package-shaped candidate weights dir (<root>/node_modules/@mailwoman/neural-weights-<locale>) — #718-safe, alternative to --model"
|
|
35
|
+
),
|
|
36
|
+
locale: zod.string().optional().describe("Locale tag for weights-package resolution (default en-US)"),
|
|
37
|
+
maxDegraded: zod.number().optional().describe("Fail if the NEW-violation DEGRADED count exceeds this (default 0)"),
|
|
38
|
+
baseline: zod
|
|
39
|
+
.string()
|
|
40
|
+
.optional()
|
|
41
|
+
.describe("Baseline ONNX for regression mode (requires --baseline-tokenizer + --baseline-model-card)"),
|
|
42
|
+
baselineTokenizer: zod.string().optional().describe("Baseline tokenizer path"),
|
|
43
|
+
baselineModelCard: zod.string().optional().describe("Baseline model-card path"),
|
|
44
|
+
baselineWeightsCache: zod
|
|
45
|
+
.string()
|
|
46
|
+
.optional()
|
|
47
|
+
.describe("Package-shaped baseline weights dir — alternative to --baseline + the two flags above"),
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
export { OptionsSchema as options }
|
|
51
|
+
|
|
52
|
+
const EvalInvariance: CommandComponent<typeof OptionsSchema> = ({ options }) => {
|
|
53
|
+
const state = useCommandTask(
|
|
54
|
+
() => runInvarianceCommand(options),
|
|
55
|
+
(exitCode) => exitCode
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
|
|
59
|
+
|
|
60
|
+
// The runner narrates its own report + verdict lines — rendering anything here would duplicate it.
|
|
61
|
+
return null
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export default EvalInvariance
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `mailwoman gazetteer pair-index` — build the PIX1 placetype-pair index (placetype-pair-prior
|
|
7
|
+
* arc, Task 3) from the HM Land Registry PPD tuples CSV (`corpus/src/tools/fetch/ppd.ts`'s
|
|
8
|
+
* `gb-tuples.csv`; columns `NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE`). Streams the CSV
|
|
9
|
+
* (CSVSpliterator, the `corpus/src/shard-recipes/locale.ts` `readTuples` idiom), folds
|
|
10
|
+
* child=CITY/parent=DISTRICT through `normalizeFSTToken` and tags every pair `dependent_locality`
|
|
11
|
+
* (`PairIndexBuilder`, `gazetteer-pipeline/pair-index.ts` — the extracted, unit-tested fold/dedupe/
|
|
12
|
+
* skip logic), then writes `pair-index-<country>.bin` via `serializePairIndex`.
|
|
13
|
+
*
|
|
14
|
+
* `--delta` is REQUIRED with no default: it's the soft-prior bias magnitude a probe hit will
|
|
15
|
+
* contribute at decode time, and the calibration task (not this one) owns the real value — a
|
|
16
|
+
* silent default here would let an uncalibrated number ship unnoticed.
|
|
17
|
+
*
|
|
18
|
+
* Self-verifying (the sealed-artifact spirit — see AGENTS.md's database section, which this
|
|
19
|
+
* mirrors for a flat binary): after writing, the command re-reads the bytes through a fresh
|
|
20
|
+
* `PairIndexResolver` and probes a few known (child, parent) pairs, printing `PROBE OK`/`PROBE
|
|
21
|
+
* MISS` lines rather than trusting the write silently succeeded.
|
|
22
|
+
*
|
|
23
|
+
* Also prints the raw (pre-fold) CITY word-length distribution (p50/p90/p99/max + a per-length
|
|
24
|
+
* count table) — this sizes the word-span window the decode-side prior (Task 4) walks.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { createReadStream, existsSync, writeFileSync } from "node:fs"
|
|
28
|
+
import { join } from "node:path"
|
|
29
|
+
|
|
30
|
+
import { dataRootPath, md5File } from "@mailwoman/core/utils"
|
|
31
|
+
// @mailwoman/neural's fst-prior and pair-index-resolver subpaths are self-contained (fst-prior only
|
|
32
|
+
// type-imports from a sibling module; pair-index-resolver only imports core/types) — safe value
|
|
33
|
+
// imports at module level, no heavy ONNX runtime pulled in (mirrors postcode-binary.tsx's comment).
|
|
34
|
+
import { normalizeFSTToken } from "@mailwoman/neural/fst-prior"
|
|
35
|
+
import { PairIndexResolver, serializePairIndex, type PairIndexHeader } from "@mailwoman/neural/pair-index-resolver"
|
|
36
|
+
import { Box, Text } from "ink"
|
|
37
|
+
import { CSVSpliterator } from "spliterator"
|
|
38
|
+
import zod from "zod"
|
|
39
|
+
|
|
40
|
+
import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
|
|
41
|
+
import { PairIndexBuilder, applyPairIndexHoldout } from "../../gazetteer-pipeline/pair-index.ts"
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The GB source's adjudicated production distinct-pair count — the cross-check this build must reproduce. Re-anchored
|
|
45
|
+
* (final-review fix) from the rung-3 census's raw 19,431 lines (`scratchpad/gb-probe-grade/census-gb-pairs.jsonl`) down
|
|
46
|
+
* to 19,209. This is not a bug fix — a real Task-3 cross-check rebuild fired the gate at exactly this delta and it was
|
|
47
|
+
* adjudicated as the production fold correctly MERGING punctuation-variant duplicates the raw census counted separately
|
|
48
|
+
* (e.g. "St Helens" vs "St. Helens" fold to the SAME `(child, parent)` key). The collision receipt: 221 merge groups —
|
|
49
|
+
* 220 groups where 2 raw census lines collapse to 1 production entry (220 × 1 collapsed line = 220) plus 1 group where
|
|
50
|
+
* 3 raw lines collapse to 1 entry (1 × 2 collapsed lines = 2) — 220 + 2 = 222 raw lines absorbed; 19,431 − 222 =
|
|
51
|
+
* 19,209. See `.superpowers/sdd/progress.md`'s Task 3 entry for the adjudication record. A mismatch AGAINST 19,209 on a
|
|
52
|
+
* real rebuild means this build's fold diverged from the adjudicated baseline, not that 19,209 is wrong; investigate
|
|
53
|
+
* before trusting the artifact.
|
|
54
|
+
*/
|
|
55
|
+
const EXPECTED_GB_PAIR_COUNT = 19_209
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The raw rung-3 census's pre-fold line count (`scratchpad/gb-probe-grade/census-gb-pairs.jsonl`) — retained as a named
|
|
59
|
+
* constant for provenance/debugging (e.g. diffing a future source refresh against this cycle's raw count), NOT the
|
|
60
|
+
* cross-check target. See {@link EXPECTED_GB_PAIR_COUNT}'s doc comment for why the production target is lower.
|
|
61
|
+
*/
|
|
62
|
+
const RUNG3_PRE_FOLD_CENSUS_LINE_COUNT = 19_431
|
|
63
|
+
|
|
64
|
+
/** Known (child, parent) pairs from the rung-3 GB census, probed after write as a self-check. */
|
|
65
|
+
const GB_PROBE_PAIRS: ReadonlyArray<readonly [city: string, district: string]> = [
|
|
66
|
+
["Fishburn", "Stockton-on-Tees"],
|
|
67
|
+
["Shoreditch", "London"],
|
|
68
|
+
["Sedgefield", "Stockton-on-Tees"],
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
const OptionsSchema = zod.object({
|
|
72
|
+
out: zod.string().default("docs/static/mailwoman").describe("Output dir for pair-index-<country>.bin"),
|
|
73
|
+
country: zod.string().default("gb").describe("ISO country code this shard is built for"),
|
|
74
|
+
source: zod.string().optional().describe("Source PPD tuples CSV. Default <data-root>/ppd/2026-07-22/gb-tuples.csv"),
|
|
75
|
+
delta: zod
|
|
76
|
+
.number()
|
|
77
|
+
.describe(
|
|
78
|
+
"REQUIRED, no default — the soft-prior bias magnitude a probe hit contributes at decode time. " +
|
|
79
|
+
"The calibration task supplies the real value; this command refuses to default it silently."
|
|
80
|
+
),
|
|
81
|
+
holdoutFraction: zod
|
|
82
|
+
.number()
|
|
83
|
+
.min(0)
|
|
84
|
+
.max(1)
|
|
85
|
+
.default(0)
|
|
86
|
+
.describe(
|
|
87
|
+
"DEV-ONLY falsifier flag (placetype-pair-prior arc, Task 6) — withhold this fraction of deduplicated pairs " +
|
|
88
|
+
"from the build (seeded, deterministic; see --holdout-seed), for measuring decode-layer degradation " +
|
|
89
|
+
"against pairs the index was never trained/built on. Default 0 = a normal, complete build. NEVER pass a " +
|
|
90
|
+
"nonzero value for a shipped artifact build."
|
|
91
|
+
),
|
|
92
|
+
holdoutSeed: zod
|
|
93
|
+
.number()
|
|
94
|
+
.default(42)
|
|
95
|
+
.describe("Seed for --holdout-fraction's deterministic withholding. Ignored when --holdout-fraction is 0."),
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
export { OptionsSchema as options }
|
|
99
|
+
|
|
100
|
+
const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options }) => {
|
|
101
|
+
const state = useCommandTask(async () => {
|
|
102
|
+
const country = options.country.toLowerCase()
|
|
103
|
+
const sourcePath = options.source ?? dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")
|
|
104
|
+
|
|
105
|
+
if (!existsSync(sourcePath)) {
|
|
106
|
+
throw new Error(`pair-index: source CSV not found: ${sourcePath}`)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const builder = new PairIndexBuilder()
|
|
110
|
+
let header: string[] | null = null
|
|
111
|
+
let cityIx = -1
|
|
112
|
+
let districtIx = -1
|
|
113
|
+
|
|
114
|
+
// Same CSVSpliterator idiom as `corpus/src/shard-recipes/locale.ts`'s `readTuples`: array mode, no header
|
|
115
|
+
// row consumed by the parser (`header: false`), so we build the column index off the first yielded row
|
|
116
|
+
// ourselves and skip forward from there.
|
|
117
|
+
for await (const cells of CSVSpliterator.fromAsync<string[]>(createReadStream(sourcePath), {
|
|
118
|
+
mode: "array",
|
|
119
|
+
header: false,
|
|
120
|
+
enableQuoteHandling: true,
|
|
121
|
+
})) {
|
|
122
|
+
if (header === null) {
|
|
123
|
+
header = cells.map((h) => h.trim().toUpperCase())
|
|
124
|
+
cityIx = header.indexOf("CITY")
|
|
125
|
+
districtIx = header.indexOf("DISTRICT")
|
|
126
|
+
|
|
127
|
+
if (cityIx < 0 || districtIx < 0) {
|
|
128
|
+
throw new Error(`pair-index: source header is missing CITY/DISTRICT: ${header.join(",")}`)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
continue
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
builder.addRow(cells[cityIx] ?? "", cells[districtIx] ?? "")
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const built = builder.finish()
|
|
138
|
+
const { rowsKept, rowsSkipped, distribution } = built
|
|
139
|
+
|
|
140
|
+
// Task 6 falsifier-board dev flag: withhold a deterministic fraction of pairs from the build so a
|
|
141
|
+
// downstream eval can measure decode-layer degradation on pairs the index never saw. A no-op
|
|
142
|
+
// (`entries === built.entries` by value) when --holdout-fraction is the 0 default.
|
|
143
|
+
const { kept: entries, heldOut } = applyPairIndexHoldout(
|
|
144
|
+
built.entries,
|
|
145
|
+
options.holdoutFraction,
|
|
146
|
+
options.holdoutSeed
|
|
147
|
+
)
|
|
148
|
+
const sourceMD5 = await md5File(sourcePath)
|
|
149
|
+
|
|
150
|
+
const pairIndexHeader: PairIndexHeader = {
|
|
151
|
+
country,
|
|
152
|
+
delta: options.delta,
|
|
153
|
+
schemaVersion: 1,
|
|
154
|
+
foldVersion: 1,
|
|
155
|
+
sourceMD5s: [sourceMD5],
|
|
156
|
+
buildDate: new Date().toISOString(),
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const bytes = serializePairIndex(pairIndexHeader, entries)
|
|
160
|
+
const outPath = join(options.out, `pair-index-${country}.bin`)
|
|
161
|
+
|
|
162
|
+
writeFileSync(outPath, bytes)
|
|
163
|
+
|
|
164
|
+
// Self-verifying readback: construct a fresh resolver over the bytes we just wrote (not the in-memory
|
|
165
|
+
// `entries`) and probe known pairs, rather than trusting the write silently succeeded.
|
|
166
|
+
const resolver = new PairIndexResolver(bytes)
|
|
167
|
+
const probeLines = GB_PROBE_PAIRS.map(([city, district]) => {
|
|
168
|
+
const child = normalizeFSTToken(city)
|
|
169
|
+
const parent = normalizeFSTToken(district)
|
|
170
|
+
const tag = resolver.probe(child, parent)
|
|
171
|
+
|
|
172
|
+
return tag
|
|
173
|
+
? `PROBE OK: fold("${city}")/fold("${district}") → "${child}"/"${parent}" → ${tag}`
|
|
174
|
+
: `PROBE MISS: fold("${city}")/fold("${district}") → "${child}"/"${parent}" → (no entry)`
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
const distLines = [
|
|
178
|
+
`CITY word-length distribution (raw, pre-fold; n=${distribution.totalRows.toLocaleString()}): ` +
|
|
179
|
+
`p50=${distribution.p50} p90=${distribution.p90} p99=${distribution.p99} max=${distribution.max}`,
|
|
180
|
+
...distribution.counts.map(
|
|
181
|
+
(b) => ` ${b.words} word${b.words === 1 ? "" : "s"}: ${b.rows.toLocaleString()} rows`
|
|
182
|
+
),
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
// The rung-3 cross-check assumes a COMPLETE build — a nonzero --holdout-fraction deliberately produces a
|
|
186
|
+
// smaller `entries.length` by design, so the strict count-match gate is meaningless (and would misreport
|
|
187
|
+
// "BLOCKED") under holdout. Gate against `built.entries.length` (pre-holdout) instead in that case.
|
|
188
|
+
const preHoldoutCount = built.entries.length
|
|
189
|
+
// Pre-fold context suffix — the raw rung-3 census line count, for provenance/debugging (e.g. diffing a
|
|
190
|
+
// future source refresh against this cycle's raw count). Not the cross-check target; see
|
|
191
|
+
// `EXPECTED_GB_PAIR_COUNT`'s doc comment for the 221-group collision receipt that separates the two numbers.
|
|
192
|
+
const preFoldSuffix = ` (pre-fold rung-3 census: ${RUNG3_PRE_FOLD_CENSUS_LINE_COUNT.toLocaleString()} lines)`
|
|
193
|
+
const gateLine =
|
|
194
|
+
country === "gb"
|
|
195
|
+
? options.holdoutFraction > 0
|
|
196
|
+
? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${EXPECTED_GB_PAIR_COUNT.toLocaleString()})${preFoldSuffix}`
|
|
197
|
+
: preHoldoutCount === EXPECTED_GB_PAIR_COUNT
|
|
198
|
+
? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (production baseline expects ${EXPECTED_GB_PAIR_COUNT.toLocaleString()})${preFoldSuffix}`
|
|
199
|
+
: `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != the production baseline's ${EXPECTED_GB_PAIR_COUNT.toLocaleString()} — investigate fold divergence before trusting this artifact${preFoldSuffix}`
|
|
200
|
+
: `(cross-check only registered for gb; ${preHoldoutCount.toLocaleString()} distinct pairs)`
|
|
201
|
+
|
|
202
|
+
const holdoutLine =
|
|
203
|
+
options.holdoutFraction > 0
|
|
204
|
+
? `HOLDOUT: withheld ${heldOut.length.toLocaleString()}/${preHoldoutCount.toLocaleString()} pairs ` +
|
|
205
|
+
`(fraction=${options.holdoutFraction}, seed=${options.holdoutSeed}) — ${entries.length.toLocaleString()} pairs written`
|
|
206
|
+
: undefined
|
|
207
|
+
|
|
208
|
+
return [
|
|
209
|
+
`pair-index-${country}.bin → ${outPath} (${bytes.length.toLocaleString()} bytes)`,
|
|
210
|
+
`rows kept ${rowsKept.toLocaleString()} / skipped ${rowsSkipped.toLocaleString()} (empty CITY)`,
|
|
211
|
+
`distinct pairs: ${entries.length.toLocaleString()}`,
|
|
212
|
+
...(holdoutLine ? [holdoutLine] : []),
|
|
213
|
+
gateLine,
|
|
214
|
+
...distLines,
|
|
215
|
+
...probeLines,
|
|
216
|
+
]
|
|
217
|
+
})
|
|
218
|
+
|
|
219
|
+
if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
|
|
220
|
+
|
|
221
|
+
if (state.status === "done") {
|
|
222
|
+
return (
|
|
223
|
+
<Box flexDirection="column">
|
|
224
|
+
{state.result.map((line, i) => (
|
|
225
|
+
<Text key={i} color={i === 0 ? "green" : undefined}>
|
|
226
|
+
{i === 0 ? "✓ " : " "}
|
|
227
|
+
{line}
|
|
228
|
+
</Text>
|
|
229
|
+
))}
|
|
230
|
+
</Box>
|
|
231
|
+
)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
return null
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export default GazetteerPairIndex
|
package/commands/release/hf.tsx
CHANGED
|
@@ -36,6 +36,12 @@ const OptionsSchema = zod.object({
|
|
|
36
36
|
modelSize: zod.string().optional().describe("Override the displayed model size (default: derived from --model)"),
|
|
37
37
|
steps: zod.number().optional().describe("Training steps recorded in releases.json (default 100000)"),
|
|
38
38
|
postcodes: zod.string().optional().describe("Comma-separated postcode soft-feed binaries (postcode-us.bin,…)"),
|
|
39
|
+
pairIndexes: zod
|
|
40
|
+
.string()
|
|
41
|
+
.optional()
|
|
42
|
+
.describe(
|
|
43
|
+
"Comma-separated placetype-pair-index soft-feed binaries (pair-index-gb.bin,…; placetype-pair-prior arc, Task 8)"
|
|
44
|
+
),
|
|
39
45
|
gazetteerLexicon: zod
|
|
40
46
|
.string()
|
|
41
47
|
.optional()
|
|
@@ -662,6 +662,6 @@ export const REGRESSION_CASES: SeedCase[] = [
|
|
|
662
662
|
expectLon: 2.3467,
|
|
663
663
|
expectToleranceM: 50_000,
|
|
664
664
|
addedAt: "2026-07-15",
|
|
665
|
-
note: "
|
|
665
|
+
note: "Passes on the staged v3.11.x candidates (task-8 record) — flip to pass WHEN a lineage successor promotes; shipped v385 still fails (verified 2026-07-23). The street/locality boundary that swallowed 'Paris' on 2026-07-15 has since flipped in experimental lineages (#727 span-head). Was the only miss in the operator's 10-row tricky list (9/10 passed at the time); tracked as span-head's class. See final-review fix-wave report / #1189 adjudication for context.",
|
|
666
666
|
},
|
|
667
667
|
]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* CLI-facing orchestration for `mailwoman eval invariance` — wires suite + transforms + runner
|
|
7
|
+
* together from argv-shaped options. Thin on purpose: the module the `invariance.tsx` command wraps
|
|
8
|
+
* narrates everything and owns only the exit code, matching `eval gate` / `eval error-analysis`.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { type ModelSelectOptions, loadSuite, buildParseFn, runInvarianceSuite } from "./runner.ts"
|
|
12
|
+
|
|
13
|
+
export interface InvarianceCommandOptions extends ModelSelectOptions {
|
|
14
|
+
/** Alternate suite fixture path. Default the shipped `suite.jsonl`. */
|
|
15
|
+
suite?: string
|
|
16
|
+
/** Fail the gate if the NEW-violation DEGRADED count exceeds this. Default 0. */
|
|
17
|
+
maxDegraded?: number
|
|
18
|
+
/**
|
|
19
|
+
* `--baseline` regression mode (probe-grading shape, e.g. v385): a baseline candidate ONNX graded on the SAME suite.
|
|
20
|
+
* Requires `baselineTokenizer` + `baselineModelCard`, or pass `baselineWeightsCache` instead for a package-shaped
|
|
21
|
+
* dir.
|
|
22
|
+
*/
|
|
23
|
+
baseline?: string
|
|
24
|
+
baselineTokenizer?: string
|
|
25
|
+
baselineModelCard?: string
|
|
26
|
+
/** Package-shaped baseline weights dir — alternative to `baseline` + the two flags above. */
|
|
27
|
+
baselineWeightsCache?: string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Run the invariance mini-suite from CLI-shaped options. Returns the process exit code (0 = PASS). */
|
|
31
|
+
export async function runInvarianceCommand(options: InvarianceCommandOptions): Promise<number> {
|
|
32
|
+
const rows = loadSuite(options.suite)
|
|
33
|
+
console.error(`[invariance] loaded ${rows.length} rows from ${options.suite ?? "the shipped suite.jsonl"}`)
|
|
34
|
+
|
|
35
|
+
console.error(`[invariance] loading candidate model…`)
|
|
36
|
+
const parse = await buildParseFn(options)
|
|
37
|
+
|
|
38
|
+
let baselineParse: Awaited<ReturnType<typeof buildParseFn>> | undefined
|
|
39
|
+
|
|
40
|
+
if (options.baselineWeightsCache || options.baseline) {
|
|
41
|
+
console.error(`[invariance] loading baseline model (regression mode)…`)
|
|
42
|
+
baselineParse = await buildParseFn({
|
|
43
|
+
weightsCache: options.baselineWeightsCache,
|
|
44
|
+
model: options.baselineWeightsCache ? undefined : options.baseline,
|
|
45
|
+
tokenizer: options.baselineTokenizer,
|
|
46
|
+
modelCard: options.baselineModelCard,
|
|
47
|
+
locale: options.locale,
|
|
48
|
+
})
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const result = await runInvarianceSuite({
|
|
52
|
+
rows,
|
|
53
|
+
parse,
|
|
54
|
+
...(baselineParse ? { baselineParse } : {}),
|
|
55
|
+
maxDegraded: options.maxDegraded,
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
return result.exitCode
|
|
59
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Component-level comparison for the invariance mini-suite — pure, no model, no I/O. Compares two
|
|
7
|
+
* decoded component maps (order-insensitive: a plain key→value record has no order) and classifies the
|
|
8
|
+
* pair INVARIANT / DEGRADED / LOST.
|
|
9
|
+
*
|
|
10
|
+
* `house_number` / `street` / `postcode` are treated as CRITICAL: they're the load-bearing tags a
|
|
11
|
+
* downstream geocoder needs to resolve a rooftop (the same three the #251/#1101 DIR-test failures broke
|
|
12
|
+
* on). A value change on a critical tag that's present in the original — including one token bleeding
|
|
13
|
+
* from a neighboring tag, e.g. a stripped comma pulling a directional suffix into the locality — is LOST
|
|
14
|
+
* even when the transformed parse is non-empty, because the address is no longer resolvable to the same
|
|
15
|
+
* place. A drift confined to non-critical tags (locality, region, dependent_locality, unit, …) is
|
|
16
|
+
* DEGRADED: recoverable, worth flagging, not ship-blocking on its own.
|
|
17
|
+
*
|
|
18
|
+
* A critical tag that's ABSENT in the original parse but PRESENT in the transformed one — a hallucination
|
|
19
|
+
* — is ALSO LOST, not DEGRADED and not ignored. (Adjudicated: review fix wave, 2026-07-23.) A missing
|
|
20
|
+
* critical tag degrades gracefully to a coarser admin-tier fallback; a hallucinated one can resolve to a
|
|
21
|
+
* SPECIFIC WRONG rooftop with high apparent confidence — worse than falling back, because nothing
|
|
22
|
+
* downstream knows to distrust it.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export const CRITICAL_TAGS = ["house_number", "street", "postcode"] as const
|
|
26
|
+
|
|
27
|
+
export type Verdict = "INVARIANT" | "DEGRADED" | "LOST"
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Ordinal severity — INVARIANT < DEGRADED < LOST. Used by the runner's `--baseline` regression gate to decide whether a
|
|
31
|
+
* candidate's verdict on a (row, transform) pair is "at least as bad as" the baseline's, not merely "also
|
|
32
|
+
* non-INVARIANT" (see runner.ts's `preExisting` computation).
|
|
33
|
+
*/
|
|
34
|
+
export const VERDICT_SEVERITY: Record<Verdict, number> = { INVARIANT: 0, DEGRADED: 1, LOST: 2 }
|
|
35
|
+
|
|
36
|
+
export interface CompareResult {
|
|
37
|
+
verdict: Verdict
|
|
38
|
+
/** Human-readable per-tag diff lines, empty for INVARIANT. */
|
|
39
|
+
diff: string[]
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Normalize a component value for comparison: trim, lowercase, collapse internal whitespace. Non-string/empty → "". */
|
|
43
|
+
function normVal(v: unknown): string {
|
|
44
|
+
if (typeof v !== "string") return ""
|
|
45
|
+
|
|
46
|
+
return v.trim().toLowerCase().replace(/\s+/g, " ")
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Compare a baseline (`original`) component map against a perturbed (`transformed`) one. Order-insensitive by
|
|
51
|
+
* construction — both are plain key→value records.
|
|
52
|
+
*/
|
|
53
|
+
export function compareComponents(
|
|
54
|
+
original: Record<string, string>,
|
|
55
|
+
transformed: Record<string, string>
|
|
56
|
+
): CompareResult {
|
|
57
|
+
const originalKeys = Object.keys(original).filter((k) => normVal(original[k]))
|
|
58
|
+
const transformedKeys = Object.keys(transformed).filter((k) => normVal(transformed[k]))
|
|
59
|
+
|
|
60
|
+
// LOST — the transformed parse is empty (or all-blank) while the original had components at all.
|
|
61
|
+
// "unresolvable-shaped": a fully collapsed decode, the parse-level analog of a resolver falling back
|
|
62
|
+
// to an admin-only tier with no coordinate.
|
|
63
|
+
if (transformedKeys.length === 0 && originalKeys.length > 0) {
|
|
64
|
+
return { verdict: "LOST", diff: ["transformed parse is empty"] }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const diff: string[] = []
|
|
68
|
+
let criticalBroken = false
|
|
69
|
+
|
|
70
|
+
for (const tag of CRITICAL_TAGS) {
|
|
71
|
+
const o = normVal(original[tag])
|
|
72
|
+
const t = normVal(transformed[tag])
|
|
73
|
+
|
|
74
|
+
if (!o) {
|
|
75
|
+
// Not present in the original. A hallucinated value on the TRANSFORMED side is still LOST — see
|
|
76
|
+
// the header doc comment (a wrong-but-confident rooftop is worse than a graceful fallback).
|
|
77
|
+
if (t) {
|
|
78
|
+
criticalBroken = true
|
|
79
|
+
diff.push(`${tag}: ∅ → "${transformed[tag]}" (hallucinated)`)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (o !== t) {
|
|
86
|
+
criticalBroken = true
|
|
87
|
+
diff.push(`${tag}: "${original[tag]}" → "${transformed[tag] ?? "∅"}"`)
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (criticalBroken) return { verdict: "LOST", diff }
|
|
92
|
+
|
|
93
|
+
// Non-critical drift: any key added/dropped, or any shared key's value changed.
|
|
94
|
+
const allKeys = new Set([...originalKeys, ...transformedKeys, ...Object.keys(transformed), ...Object.keys(original)])
|
|
95
|
+
|
|
96
|
+
for (const tag of allKeys) {
|
|
97
|
+
const o = normVal(original[tag])
|
|
98
|
+
const t = normVal(transformed[tag])
|
|
99
|
+
|
|
100
|
+
if (o !== t) {
|
|
101
|
+
diff.push(`${tag}: "${original[tag] ?? "∅"}" → "${transformed[tag] ?? "∅"}"`)
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return diff.length > 0 ? { verdict: "DEGRADED", diff } : { verdict: "INVARIANT", diff: [] }
|
|
106
|
+
}
|