mailwoman 7.4.0 → 7.6.0

package/README.md

@@ -76,6 +76,10 @@ mailwoman registry --sources config.json --out entities.geojson
 
 # Interactive TUI
 mailwoman parse --tui
+
+# Diagnose the install — what works, what's missing, and the one command to fix each gap
+mailwoman doctor          # human-readable checklist (exit 1 if a core check fails)
+mailwoman doctor --json   # machine-readable { checks: [{ id, label, status, detail, fix?, core }], exitCode }
 ```
 
 ## Library API

package/api-engine.ts

@@ -88,7 +88,10 @@ function buildPreflightMessage(): string {
 		`    curl -fSL https://public.sister.software/mailwoman/gazetteer/2026-07-07a/candidate.db \\`,
 		`      -o ${conventionCandidate}`,
 		"",
-		"  Then re-run `serve` (the file is auto-detected at that path), or point at your own:",
+		"  Then point `$MAILWOMAN_CANDIDATE_DB` at that file and re-run `serve`:",
+		`    export MAILWOMAN_CANDIDATE_DB=${conventionCandidate}`,
+		"",
+		"  Or point at your own gazetteer:",
 		"    $MAILWOMAN_WOF_DB=<path1,path2,...>    (admin WOF SQLite shard(s) — the FTS backend)",
 		"    $MAILWOMAN_CANDIDATE_DB=<path>         (candidate-table gazetteer — population-first, demo-parity)",
 		"",

package/commands/corpus/shard/index.tsx

@@ -43,9 +43,20 @@ const OptionsSchema = zod.object({
 	edgesDir: zod.string().optional().describe("intersection: TIGER EDGES dir"),
 	country: zod.string().optional().describe("locale: target country"),
 	intlFraction: zod.string().optional().describe("german/locale: international-order fraction"),
+	countryFraction: zod
+		.string()
+		.optional()
+		.describe("locale: fraction of rows that append an explicit country surface form. Default 0"),
+	districtAsLocality: zod
+		.boolean()
+		.optional()
+		.describe(
+			"locale: override the per-part districtAsLocality mapping (--no-district-as-locality to force off). Unset leaves each part's own value untouched"
+		),
 	bareProb: zod.string().optional().describe("street-bare: P(bare street)"),
 	hnProb: zod.string().optional().describe("street-bare: P(house number)"),
 	communes: zod.string().optional().describe("fr-admin-split: communes source"),
+	banDir: zod.string().optional().describe("fr-lieudit: BAN adresses-<dept>.csv directory"),
 	excludeSurfaces: zod
 		.string()
 		.optional()
@@ -97,9 +108,12 @@ const CorpusShard: CommandComponent<typeof OptionsSchema, typeof ArgumentsSchema
 			edgesDir: options.edgesDir,
 			country: options.country,
 			intlFraction: num(options.intlFraction),
+			countryFraction: num(options.countryFraction),
+			districtAsLocality: options.districtAsLocality,
 			bareProb: num(options.bareProb),
 			hnProb: num(options.hnProb),
 			communes: options.communes,
+			banDir: options.banDir,
 			excludeSurfaces: options.excludeSurfaces,
 			multilocaleCount: num(options.multilocaleCount),
 		}

package/commands/doctor.tsx

@@ -0,0 +1,120 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman doctor [--json]` — the out-of-box diagnostic. A consumer who just ran `npm i mailwoman`
+ *   runs this and learns exactly what works, what's missing, and the ONE command that fixes each gap.
+ *   Six checks: model weights, the fr-fr locale overlay (informational), the data root, the admin
+ *   gazetteer, the POI layer, and the Node + ONNX runtime.
+ *
+ *   Exit-code contract (meaning-of-zero, per `checks.ts`):
+ *
+ *   - 0 when every CORE check (weights + runtime) is `ok` — parse is ready, even with no data layers.
+ *   - 1 when any core check is not `ok`.
+ *
+ *   Data layers (data root, gazetteer, POI) are OPTIONAL: a gap is reported as missing/degraded with a
+ *   fix hint, never a hard failure — parse runs without them. All verdict logic is pure and unit-tested
+ *   in `doctor/checks.ts`; the IO seams live in `doctor/runner.ts`.
+ */
+
+import { Box, Text } from "ink"
+import type * as React from "react"
+import zod from "zod"
+
+import { type CommandComponent, useCommandTask } from "../cli-kit/index.ts"
+import { CheckStatus, type DoctorCheck, type DoctorReport } from "../doctor/checks.ts"
+import { runDoctor } from "../doctor/runner.ts"
+
+const OptionsSchema = zod.object({
+	json: zod
+		.boolean()
+		.optional()
+		.default(false)
+		.describe(
+			"Emit the report as JSON instead of a checklist: { checks: [{ id, label, status, detail, fix?, core }], exitCode } " +
+				"— a superset of { id, status, detail, fix? } (label + core aid machine consumers)."
+		),
+})
+
+export { OptionsSchema as options }
+
+/** The status glyph + ink color for a check outcome. */
+function statusGlyph(status: CheckStatus): { glyph: string; color: string } {
+	switch (status) {
+		case CheckStatus.OK:
+			return { glyph: "✓", color: "green" }
+		case CheckStatus.Degraded:
+			return { glyph: "⚠", color: "yellow" }
+		case CheckStatus.Missing:
+			return { glyph: "✗", color: "red" }
+	}
+}
+
+/** One check row: the glyph + label + detail, plus an indented `fix:` line and an `(optional)` tag for non-core gaps. */
+function CheckRow({ check }: { check: DoctorCheck }): React.ReactElement {
+	const { glyph, color } = statusGlyph(check.status)
+	const optional = !check.core && check.status !== CheckStatus.OK ? " (optional)" : ""
+
+	return (
+		<Box flexDirection="column">
+			<Text color={color}>
+				{glyph} {check.label}: {check.detail}
+				{optional}
+			</Text>
+			{check.fix ? <Text color="gray">{`    fix: ${check.fix}`}</Text> : null}
+		</Box>
+	)
+}
+
+/** The rendered checklist + PASS/FAIL summary. */
+function Report({ report }: { report: DoctorReport }): React.ReactElement {
+	const pass = report.exitCode === 0
+
+	return (
+		<Box flexDirection="column">
+			<Text bold>mailwoman doctor</Text>
+			<Text> </Text>
+			{report.checks.map((check) => (
+				<CheckRow key={check.id} check={check} />
+			))}
+			<Text> </Text>
+			<Text color={pass ? "green" : "red"} bold>
+				{pass ? "PASS — core checks ok (weights + runtime); parse is ready" : "FAIL — a core check is not ok"}
+			</Text>
+		</Box>
+	)
+}
+
+const DoctorCommand: CommandComponent<typeof OptionsSchema> = ({ options }) => {
+	// `--json` writes the report straight to stdout via `console.log` (mirroring `mailwoman openapi`) and
+	// renders NOTHING through Ink — Ink's `<Text>` hard-wraps to the terminal width, which would inject
+	// newlines mid-string and corrupt the JSON when piped. The checklist form renders through Ink normally.
+	const state = useCommandTask<DoctorReport>(
+		async () => {
+			const report = await runDoctor()
+
+			if (options.json) {
+				console.log(JSON.stringify(report, null, 2))
+			}
+
+			return report
+		},
+		(report) => report.exitCode
+	)
+
+	if (state.status === "error") {
+		return <Text color="red">{state.message}</Text>
+	}
+
+	// JSON payload already emitted in the task; give Ink nothing to draw (transient frames would pollute stdout).
+	if (options.json) return null
+
+	if (state.status !== "done") {
+		return <Text color="gray">running diagnostics…</Text>
+	}
+
+	return <Report report={state.result} />
+}
+
+export default DoctorCommand

package/commands/eval/invariance.tsx

@@ -0,0 +1,64 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman eval invariance` — the INVARIANCE MINI-SUITE: a standing, seconds-cheap
+ *   metamorphic-invariance check (comma-drop / abbreviation-swap / case-fold / lowercase /
+ *   whitespace-jitter / trailing-punct / idempotence) over `eval-harness/invariance/suite.jsonl`, meant
+ *   to run in EVERY probe grade so distribution-shift collateral surfaces at 2k-probe cost instead of
+ *   ship-prep cost. Compares decoded PARSE COMPONENTS only (no resolver) — cheap by construction. Exit
+ *   nonzero on any LOST pair, or when the DEGRADED count exceeds `--max-degraded`. `--baseline` switches
+ *   to regression mode: a violation the baseline ALSO exhibits on the same pair is reported but doesn't
+ *   fail the gate — the shape probe grading uses to diff a candidate against v385.
+ */
+
+import { Text } from "ink"
+import zod from "zod"
+
+import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
+import { runInvarianceCommand } from "../../eval-harness/invariance/command.ts"
+
+export const description =
+	"Metamorphic invariance mini-suite (comma-drop/abbrev/case/idempotence) — standing probe guard"
+
+const OptionsSchema = zod.object({
+	suite: zod.string().optional().describe("Alternate suite.jsonl path (default: the shipped fixture)"),
+	model: zod.string().optional().describe("Candidate ONNX (requires --tokenizer + --model-card)"),
+	tokenizer: zod.string().optional().describe("Candidate tokenizer path"),
+	modelCard: zod.string().optional().describe("Candidate model-card path"),
+	weightsCache: zod
+		.string()
+		.optional()
+		.describe(
+			"Package-shaped candidate weights dir (<root>/node_modules/@mailwoman/neural-weights-<locale>) — #718-safe, alternative to --model"
+		),
+	locale: zod.string().optional().describe("Locale tag for weights-package resolution (default en-US)"),
+	maxDegraded: zod.number().optional().describe("Fail if the NEW-violation DEGRADED count exceeds this (default 0)"),
+	baseline: zod
+		.string()
+		.optional()
+		.describe("Baseline ONNX for regression mode (requires --baseline-tokenizer + --baseline-model-card)"),
+	baselineTokenizer: zod.string().optional().describe("Baseline tokenizer path"),
+	baselineModelCard: zod.string().optional().describe("Baseline model-card path"),
+	baselineWeightsCache: zod
+		.string()
+		.optional()
+		.describe("Package-shaped baseline weights dir — alternative to --baseline + the two flags above"),
+})
+
+export { OptionsSchema as options }
+
+const EvalInvariance: CommandComponent<typeof OptionsSchema> = ({ options }) => {
+	const state = useCommandTask(
+		() => runInvarianceCommand(options),
+		(exitCode) => exitCode
+	)
+
+	if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
+
+	// The runner narrates its own report + verdict lines — rendering anything here would duplicate it.
+	return null
+}
+
+export default EvalInvariance

package/commands/gazetteer/pair-index.tsx

@@ -0,0 +1,237 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman gazetteer pair-index` — build the PIX1 placetype-pair index (placetype-pair-prior
+ *   arc, Task 3) from the HM Land Registry PPD tuples CSV (`corpus/src/tools/fetch/ppd.ts`'s
+ *   `gb-tuples.csv`; columns `NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE`). Streams the CSV
+ *   (CSVSpliterator, the `corpus/src/shard-recipes/locale.ts` `readTuples` idiom), folds
+ *   child=CITY/parent=DISTRICT through `normalizeFSTToken` and tags every pair `dependent_locality`
+ *   (`PairIndexBuilder`, `gazetteer-pipeline/pair-index.ts` — the extracted, unit-tested fold/dedupe/
+ *   skip logic), then writes `pair-index-<country>.bin` via `serializePairIndex`.
+ *
+ *   `--delta` is REQUIRED with no default: it's the soft-prior bias magnitude a probe hit will
+ *   contribute at decode time, and the calibration task (not this one) owns the real value — a
+ *   silent default here would let an uncalibrated number ship unnoticed.
+ *
+ *   Self-verifying (the sealed-artifact spirit — see AGENTS.md's database section, which this
+ *   mirrors for a flat binary): after writing, the command re-reads the bytes through a fresh
+ *   `PairIndexResolver` and probes a few known (child, parent) pairs, printing `PROBE OK`/`PROBE
+ *   MISS` lines rather than trusting the write silently succeeded.
+ *
+ *   Also prints the raw (pre-fold) CITY word-length distribution (p50/p90/p99/max + a per-length
+ *   count table) — this sizes the word-span window the decode-side prior (Task 4) walks.
+ */
+
+import { createReadStream, existsSync, writeFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { dataRootPath, md5File } from "@mailwoman/core/utils"
+// @mailwoman/neural's fst-prior and pair-index-resolver subpaths are self-contained (fst-prior only
+// type-imports from a sibling module; pair-index-resolver only imports core/types) — safe value
+// imports at module level, no heavy ONNX runtime pulled in (mirrors postcode-binary.tsx's comment).
+import { normalizeFSTToken } from "@mailwoman/neural/fst-prior"
+import { PairIndexResolver, serializePairIndex, type PairIndexHeader } from "@mailwoman/neural/pair-index-resolver"
+import { Box, Text } from "ink"
+import { CSVSpliterator } from "spliterator"
+import zod from "zod"
+
+import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
+import { PairIndexBuilder, applyPairIndexHoldout } from "../../gazetteer-pipeline/pair-index.ts"
+
+/**
+ * The GB source's adjudicated production distinct-pair count — the cross-check this build must reproduce. Re-anchored
+ * (final-review fix) from the rung-3 census's raw 19,431 lines (`scratchpad/gb-probe-grade/census-gb-pairs.jsonl`) down
+ * to 19,209. This is not a bug fix — a real Task-3 cross-check rebuild fired the gate at exactly this delta and it was
+ * adjudicated as the production fold correctly MERGING punctuation-variant duplicates the raw census counted separately
+ * (e.g. "St Helens" vs "St. Helens" fold to the SAME `(child, parent)` key). The collision receipt: 221 merge groups —
+ * 220 groups where 2 raw census lines collapse to 1 production entry (220 × 1 collapsed line = 220) plus 1 group where
+ * 3 raw lines collapse to 1 entry (1 × 2 collapsed lines = 2) — 220 + 2 = 222 raw lines absorbed; 19,431 − 222 =
+ * 19,209. See `.superpowers/sdd/progress.md`'s Task 3 entry for the adjudication record. A mismatch AGAINST 19,209 on a
+ * real rebuild means this build's fold diverged from the adjudicated baseline, not that 19,209 is wrong; investigate
+ * before trusting the artifact.
+ */
+const EXPECTED_GB_PAIR_COUNT = 19_209
+
+/**
+ * The raw rung-3 census's pre-fold line count (`scratchpad/gb-probe-grade/census-gb-pairs.jsonl`) — retained as a named
+ * constant for provenance/debugging (e.g. diffing a future source refresh against this cycle's raw count), NOT the
+ * cross-check target. See {@link EXPECTED_GB_PAIR_COUNT}'s doc comment for why the production target is lower.
+ */
+const RUNG3_PRE_FOLD_CENSUS_LINE_COUNT = 19_431
+
+/** Known (child, parent) pairs from the rung-3 GB census, probed after write as a self-check. */
+const GB_PROBE_PAIRS: ReadonlyArray<readonly [city: string, district: string]> = [
+	["Fishburn", "Stockton-on-Tees"],
+	["Shoreditch", "London"],
+	["Sedgefield", "Stockton-on-Tees"],
+]
+
+const OptionsSchema = zod.object({
+	out: zod.string().default("docs/static/mailwoman").describe("Output dir for pair-index-<country>.bin"),
+	country: zod.string().default("gb").describe("ISO country code this shard is built for"),
+	source: zod.string().optional().describe("Source PPD tuples CSV. Default <data-root>/ppd/2026-07-22/gb-tuples.csv"),
+	delta: zod
+		.number()
+		.describe(
+			"REQUIRED, no default — the soft-prior bias magnitude a probe hit contributes at decode time. " +
+				"The calibration task supplies the real value; this command refuses to default it silently."
+		),
+	holdoutFraction: zod
+		.number()
+		.min(0)
+		.max(1)
+		.default(0)
+		.describe(
+			"DEV-ONLY falsifier flag (placetype-pair-prior arc, Task 6) — withhold this fraction of deduplicated pairs " +
+				"from the build (seeded, deterministic; see --holdout-seed), for measuring decode-layer degradation " +
+				"against pairs the index was never trained/built on. Default 0 = a normal, complete build. NEVER pass a " +
+				"nonzero value for a shipped artifact build."
+		),
+	holdoutSeed: zod
+		.number()
+		.default(42)
+		.describe("Seed for --holdout-fraction's deterministic withholding. Ignored when --holdout-fraction is 0."),
+})
+
+export { OptionsSchema as options }
+
+const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options }) => {
+	const state = useCommandTask(async () => {
+		const country = options.country.toLowerCase()
+		const sourcePath = options.source ?? dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")
+
+		if (!existsSync(sourcePath)) {
+			throw new Error(`pair-index: source CSV not found: ${sourcePath}`)
+		}
+
+		const builder = new PairIndexBuilder()
+		let header: string[] | null = null
+		let cityIx = -1
+		let districtIx = -1
+
+		// Same CSVSpliterator idiom as `corpus/src/shard-recipes/locale.ts`'s `readTuples`: array mode, no header
+		// row consumed by the parser (`header: false`), so we build the column index off the first yielded row
+		// ourselves and skip forward from there.
+		for await (const cells of CSVSpliterator.fromAsync<string[]>(createReadStream(sourcePath), {
+			mode: "array",
+			header: false,
+			enableQuoteHandling: true,
+		})) {
+			if (header === null) {
+				header = cells.map((h) => h.trim().toUpperCase())
+				cityIx = header.indexOf("CITY")
+				districtIx = header.indexOf("DISTRICT")
+
+				if (cityIx < 0 || districtIx < 0) {
+					throw new Error(`pair-index: source header is missing CITY/DISTRICT: ${header.join(",")}`)
+				}
+
+				continue
+			}
+
+			builder.addRow(cells[cityIx] ?? "", cells[districtIx] ?? "")
+		}
+
+		const built = builder.finish()
+		const { rowsKept, rowsSkipped, distribution } = built
+
+		// Task 6 falsifier-board dev flag: withhold a deterministic fraction of pairs from the build so a
+		// downstream eval can measure decode-layer degradation on pairs the index never saw. A no-op
+		// (`entries === built.entries` by value) when --holdout-fraction is the 0 default.
+		const { kept: entries, heldOut } = applyPairIndexHoldout(
+			built.entries,
+			options.holdoutFraction,
+			options.holdoutSeed
+		)
+		const sourceMD5 = await md5File(sourcePath)
+
+		const pairIndexHeader: PairIndexHeader = {
+			country,
+			delta: options.delta,
+			schemaVersion: 1,
+			foldVersion: 1,
+			sourceMD5s: [sourceMD5],
+			buildDate: new Date().toISOString(),
+		}
+
+		const bytes = serializePairIndex(pairIndexHeader, entries)
+		const outPath = join(options.out, `pair-index-${country}.bin`)
+
+		writeFileSync(outPath, bytes)
+
+		// Self-verifying readback: construct a fresh resolver over the bytes we just wrote (not the in-memory
+		// `entries`) and probe known pairs, rather than trusting the write silently succeeded.
+		const resolver = new PairIndexResolver(bytes)
+		const probeLines = GB_PROBE_PAIRS.map(([city, district]) => {
+			const child = normalizeFSTToken(city)
+			const parent = normalizeFSTToken(district)
+			const tag = resolver.probe(child, parent)
+
+			return tag
+				? `PROBE OK: fold("${city}")/fold("${district}") → "${child}"/"${parent}" → ${tag}`
+				: `PROBE MISS: fold("${city}")/fold("${district}") → "${child}"/"${parent}" → (no entry)`
+		})
+
+		const distLines = [
+			`CITY word-length distribution (raw, pre-fold; n=${distribution.totalRows.toLocaleString()}): ` +
+				`p50=${distribution.p50} p90=${distribution.p90} p99=${distribution.p99} max=${distribution.max}`,
+			...distribution.counts.map(
+				(b) => `  ${b.words} word${b.words === 1 ? "" : "s"}: ${b.rows.toLocaleString()} rows`
+			),
+		]
+
+		// The rung-3 cross-check assumes a COMPLETE build — a nonzero --holdout-fraction deliberately produces a
+		// smaller `entries.length` by design, so the strict count-match gate is meaningless (and would misreport
+		// "BLOCKED") under holdout. Gate against `built.entries.length` (pre-holdout) instead in that case.
+		const preHoldoutCount = built.entries.length
+		// Pre-fold context suffix — the raw rung-3 census line count, for provenance/debugging (e.g. diffing a
+		// future source refresh against this cycle's raw count). Not the cross-check target; see
+		// `EXPECTED_GB_PAIR_COUNT`'s doc comment for the 221-group collision receipt that separates the two numbers.
+		const preFoldSuffix = ` (pre-fold rung-3 census: ${RUNG3_PRE_FOLD_CENSUS_LINE_COUNT.toLocaleString()} lines)`
+		const gateLine =
+			country === "gb"
+				? options.holdoutFraction > 0
+					? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${EXPECTED_GB_PAIR_COUNT.toLocaleString()})${preFoldSuffix}`
+					: preHoldoutCount === EXPECTED_GB_PAIR_COUNT
+						? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (production baseline expects ${EXPECTED_GB_PAIR_COUNT.toLocaleString()})${preFoldSuffix}`
+						: `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != the production baseline's ${EXPECTED_GB_PAIR_COUNT.toLocaleString()} — investigate fold divergence before trusting this artifact${preFoldSuffix}`
+				: `(cross-check only registered for gb; ${preHoldoutCount.toLocaleString()} distinct pairs)`
+
+		const holdoutLine =
+			options.holdoutFraction > 0
+				? `HOLDOUT: withheld ${heldOut.length.toLocaleString()}/${preHoldoutCount.toLocaleString()} pairs ` +
+					`(fraction=${options.holdoutFraction}, seed=${options.holdoutSeed}) — ${entries.length.toLocaleString()} pairs written`
+				: undefined
+
+		return [
+			`pair-index-${country}.bin → ${outPath} (${bytes.length.toLocaleString()} bytes)`,
+			`rows kept ${rowsKept.toLocaleString()} / skipped ${rowsSkipped.toLocaleString()} (empty CITY)`,
+			`distinct pairs: ${entries.length.toLocaleString()}`,
+			...(holdoutLine ? [holdoutLine] : []),
+			gateLine,
+			...distLines,
+			...probeLines,
+		]
+	})
+
+	if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
+
+	if (state.status === "done") {
+		return (
+			<Box flexDirection="column">
+				{state.result.map((line, i) => (
+					<Text key={i} color={i === 0 ? "green" : undefined}>
+						{i === 0 ? "✓ " : "  "}
+						{line}
+					</Text>
+				))}
+			</Box>
+		)
+	}
+
+	return null
+}
+
+export default GazetteerPairIndex

package/commands/release/hf.tsx

@@ -36,6 +36,12 @@ const OptionsSchema = zod.object({
 	modelSize: zod.string().optional().describe("Override the displayed model size (default: derived from --model)"),
 	steps: zod.number().optional().describe("Training steps recorded in releases.json (default 100000)"),
 	postcodes: zod.string().optional().describe("Comma-separated postcode soft-feed binaries (postcode-us.bin,…)"),
+	pairIndexes: zod
+		.string()
+		.optional()
+		.describe(
+			"Comma-separated placetype-pair-index soft-feed binaries (pair-index-gb.bin,…; placetype-pair-prior arc, Task 8)"
+		),
 	gazetteerLexicon: zod
 		.string()
 		.optional()