@mailwoman/neural-weights-fr-fr 8.3.0 → 8.5.0

package/model-card.json

@@ -1,8 +1,8 @@
 {
 	"name": "neural-weights-fr-fr",
 	"version": "6.5.0",
-	"model_lineage": "Ships the en-us v310 (v3.1.0-fr-fragment, #1143 T2) model — init_from v264 plus the synth-fr-fragment BAN shard @ 12.0; tokenizer v0.9.0-multisplice (73,143) UNCHANGED. One model serves both locales; see the en-us card for the full lineage + grade. NOTE for this locale specifically: the shard is BAN-derived (Base Adresse Nationale, Licence Ouverte 2.0) and the gains are measured on French — bare-street 0.215→0.715, street-particle 0.273→0.855, admin-street-homonym 0.087→0.517, with the bare-locality counter-guard holding exactly. This is the largest FR-facing model change since 5.9.0.",
-	"phase": "Stage 3 — multi-locale (FR served by the shared en-us v310 model, shipped as 6.4.0)",
+	"model_lineage": "Ships the en-us v310 (v3.1.0-fr-fragment, #1143 T2) model \u2014 init_from v264 plus the synth-fr-fragment BAN shard @ 12.0; tokenizer v0.9.0-multisplice (73,143) UNCHANGED. One model serves both locales; see the en-us card for the full lineage + grade. NOTE for this locale specifically: the shard is BAN-derived (Base Adresse Nationale, Licence Ouverte 2.0) and the gains are measured on French \u2014 bare-street 0.215\u21920.715, street-particle 0.273\u21920.855, admin-street-homonym 0.087\u21920.517, with the bare-locality counter-guard holding exactly. This is the largest FR-facing model change since 5.9.0.",
+	"phase": "Stage 3 \u2014 multi-locale (FR served by the shared en-us v310 model, shipped as 6.4.0)",
 	"license": "AGPL-3.0-only OR LicenseRef-Commercial",
 	"locale": "fr-fr",
 	"training": {
@@ -13,7 +13,7 @@
 		"note": "Identical artifact to neural-weights-en-us 6.4.0 / v310 fr-fragment (one multi-locale model serves both, copied at publish time). See that card's training/recipe for full provenance."
 	},
 	"requires": {
-		"$comment": "SHIP-CONFIG mirrored from neural-weights-en-us (#718/#721): this package ships en-us's model, so its required channels are identical — including `country` (the #1104 dedicated country soft-feed channel added in v263). createScorer / loadFromWeights read this and feed anchor + gazetteer + country + the conventions mask; running channel-OFF is the #566/#685 OOD crater.",
+		"$comment": "SHIP-CONFIG mirrored from neural-weights-en-us (#718/#721): this package ships en-us's model, so its required channels are identical \u2014 including `country` (the #1104 dedicated country soft-feed channel added in v263). createScorer / loadFromWeights read this and feed anchor + gazetteer + country + the conventions mask; running channel-OFF is the #566/#685 OOD crater.",
 		"anchor": {
 			"required": true
 		},
@@ -86,7 +86,7 @@
 		"I-intersection_b"
 	],
 	"eval": {
-		"$comment": "SUPERSEDED / STALE (#721). The per_component + calibration numbers below are the OLD v0.9.7-step-20000 checkpoint measured ANCHOR-OFF (country 0.21, postcode 0.69 — the #566/#685 OOD crater, NOT this package's real quality). Since the unified release CI ships en-us's v1.5.0 model for fr-fr too, the authoritative FR numbers are en-us v1.5.0-fr-order's, anchor-ON: assembled FR coord p50 1.5km / p90 5.3km, locality-match 76.3%; golden per-tag fr.house_number 99.3, fr.region 41.8 (open gap #330), fr.country 62.7, fr.postcode 99.5. See docs/articles/evals/2026-06-18-v150-vs-v170-head-to-head.md and neural-weights-en-us/model-card.json. Refresh or remove these stale fields per #721.",
+		"$comment": "SUPERSEDED / STALE (#721). The per_component + calibration numbers below are the OLD v0.9.7-step-20000 checkpoint measured ANCHOR-OFF (country 0.21, postcode 0.69 \u2014 the #566/#685 OOD crater, NOT this package's real quality). Since the unified release CI ships en-us's v1.5.0 model for fr-fr too, the authoritative FR numbers are en-us v1.5.0-fr-order's, anchor-ON: assembled FR coord p50 1.5km / p90 5.3km, locality-match 76.3%; golden per-tag fr.house_number 99.3, fr.region 41.8 (open gap #330), fr.country 62.7, fr.postcode 99.5. See docs/articles/evals/2026-06-18-v150-vs-v170-head-to-head.md and neural-weights-en-us/model-card.json. Refresh or remove these stale fields per #721.",
 		"n_entries": 4535,
 		"full_parse_exact_match": 0.08180815876515987,
 		"mean_token_confidence": 0.8062812768727202,
@@ -213,14 +213,35 @@
 				"n": 30517,
 				"acc": 0.6440344725890488
 			}
-		]
+		],
+		"r6_fr_lieudit": {
+			"$comment": "Pre-registered bars from docs/superpowers/plans/2026-08-01-r6-fr-lieudit-preregistration.md.",
+			"golden_board": {
+				"n": 80,
+				"emit_before": 0,
+				"emit_after": 80,
+				"tag_correct_before": 0,
+				"tag_correct_after": 76,
+				"tag_correct_pct": 95.0,
+				"bar_pct": 70.0,
+				"note": "mailwoman/eval-harness/fixtures/fr-lieudit-golden.jsonl. Prior-OFF leg on identical bytes stays 0/80. The 4 residual misses are span-boundary (dropped particles), not retrieval \u2014 the prior fired on all 80. DISCLOSURE: 80/80 of the board's pairs are in the index (both are BAN-derived), so this measures mechanism efficacy where the gazetteer has data, not generalization."
+			},
+			"venue_confound_board": {
+				"n": 60,
+				"false_positives": 0,
+				"fp_pct": 0.0,
+				"bar_pct": 2.0,
+				"note": "40 article-led lieu-dit surfaces (La/Le/Les/Saint\u2026, from 84,582 available) + 20 others, each opening a venue name"
+			},
+			"gauntlet": "PASS after both decode-path fixes; neural 498/498, mailwoman 521/521"
+		}
 	},
 	"known_failure_modes": [
 		"underperforms on Hawaiian addresses (sparse in training corpus)",
 		"particle-honorific kryptonite (e.g. FR 'Saint-Just-Saint-Rambert') if not in synth set",
 		"non-Latin scripts (CJK, Cyrillic) fall through to byte-fallback tokens; F1 unknown"
 	],
-	"notes": "v0.4.0 — issue #116. Same encoder geometry as v0.3.0 (8.87M params, 6L/256H/4-heads, 21 BIO labels, linear-chain CRF). Issue proposed (1) per-token CRF NLL normalization + (3) class-weighted CE biased toward coarse labels + (4) source-weight rebalance. Empirical iteration found that §1 and §3 destabilize sustained training at every LR tested (5e-4, 3e-4, 1.5e-4 — even the v0.3.0-safe LR), and on the golden v0.1.2 eval (4535 entries) they slightly REGRESS country/postcode F1 vs v0.3.0. SHIPPED recipe is the §4-only ablation (v0.3.0 dual-loss + v0.4.0 source-weight rebalance) at lr=1.5e-4, step 2200. Modest fine-label gains (street +0.03, house_number +0.01), modest coarse-F1 regression (country -0.07, postcode -0.07). §1/§3 deferred to v0.4.1 corpus-side investigation per issue's '2K divergence' clause. Full ablation matrix retrospective in LOG.md.",
+	"notes": "v0.4.0 \u2014 issue #116. Same encoder geometry as v0.3.0 (8.87M params, 6L/256H/4-heads, 21 BIO labels, linear-chain CRF). Issue proposed (1) per-token CRF NLL normalization + (3) class-weighted CE biased toward coarse labels + (4) source-weight rebalance. Empirical iteration found that \u00a71 and \u00a73 destabilize sustained training at every LR tested (5e-4, 3e-4, 1.5e-4 \u2014 even the v0.3.0-safe LR), and on the golden v0.1.2 eval (4535 entries) they slightly REGRESS country/postcode F1 vs v0.3.0. SHIPPED recipe is the \u00a74-only ablation (v0.3.0 dual-loss + v0.4.0 source-weight rebalance) at lr=1.5e-4, step 2200. Modest fine-label gains (street +0.03, house_number +0.01), modest coarse-F1 regression (country -0.07, postcode -0.07). \u00a71/\u00a73 deferred to v0.4.1 corpus-side investigation per issue's '2K divergence' clause. Full ablation matrix retrospective in LOG.md.",
 	"format": {
 		"model": "ONNX int8 dynamic",
 		"tokenizer": "SentencePiece unigram, byte_fallback=true, vocab_size=16000",
@@ -230,7 +251,24 @@
 	"files": {
 		"model": "model.onnx",
 		"tokenizer": "tokenizer.model",
-		"model_card": "model-card.json"
+		"model_card": "model-card.json",
+		"pair_index": "pair-index-fr.bin"
 	},
-	"base_relpath": "/data/models/checkpoints/v0_4_0-stableLR-source-only/step-002200"
+	"base_relpath": "/data/models/checkpoints/v0_4_0-stableLR-source-only/step-002200",
+	"fr_artifacts": {
+		"$comment": "Hierarchy campaign R6 (2026-08-01). The FR instance of the placetype-pair prior.",
+		"pair_index_fr_bin": {
+			"format": "PIX1 placetype-pair index \u2014 a retrieval-augmented (lieu-dit, commune) dependent_locality prior, e.g. (\"Pinsonnac\", \"Montpeyroux\")",
+			"source": "BAN \u2014 Base Adresse Nationale (DINUM/IGN, Licence Ouverte 2.0), raw adresses-<dept>.csv `nom_ld` field read through ban/sdk's cleanLieuDit filter. 101 d\u00e9partements, 1,684,844 rows carrying a clean lieu-dit. NOT WOF: WOF's French neighbourhood records are Paris quartiers, cartographic subdivisions that never appear in a postal address (the arrondissement is encoded in the postcode). The lieu-dit is the line French addresses actually carry \u2014 La Poste's line 5.",
+			"pairs": 199282,
+			"bytes": 5839518,
+			"md5": "f0999f1d5cd52a642ca7129157659578",
+			"delta_calibration": "\u03b4=10, transitionBeta=5 \u2014 inherited from the GB calibration, per R5's finding that the classifier's dependent_locality deficit is uniform across locales rather than GB-specific. The R6 bars were measured at exactly this pair.",
+			"country_gate": "hard-gated on the header's country (fr).",
+			"not_committed": "derived artifact, .gitignore'd \u2014 rebuilt via `mailwoman gazetteer pair-index --out neural-weights-fr-fr --country fr --delta 10 --transition-beta 5 --ban-dir <corpus/sources/ban>`",
+			"required_decode_fixes": "This artifact is INERT without two locale-general probe-chain fixes that shipped alongside it: a LEADING postcode strip on parent-candidate keys (France writes '12210 Montpeyroux'; the probe previously stripped only trailing postcodes, the Anglo/NZ convention) and NEWLINE as a segment boundary (computeGroupSegments scanned for commas only, so multi-line addresses collapsed to one segment and the segment path went structurally inert). With the artifact but without them the golden board stayed at 0/80.",
+			"$comment_md5": "REPRODUCIBILITY: a PIX1 header embeds `buildDate`, so a rebuild from identical sources produces different bytes and a different md5. This md5 therefore identifies the artifact STAGED TO HUGGING FACE for this release \u2014 it is a provenance record of the shipped bytes, NOT a check a local rebuild can reproduce. A mismatch after rebuilding is expected and is not corruption; verify pair COUNT and header delta/beta instead.",
+			"provenance_gap": "The PIX1 header's sourceMD5s is EMPTY for this artifact: the BAN source is a directory of 101 d\u00e9partement CSVs, and the build hashes individual files only. Hashing the full dump costs more than the provenance is worth at build time, so a BAN refresh is a deliberate act \u2014 rebuild with the artifact deleted rather than trusting a guard to notice."
+		}
+	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/neural-weights-fr-fr",
-  "version": "8.3.0",
+  "version": "8.5.0",
   "description": "Mailwoman neural-classifier weights for locale 'fr-fr'. Data-only overlay — shares the base model.onnx + tokenizer.model from @mailwoman/neural-weights-en-us (they are byte-identical); ships only the fr-specific data siblings. Loaded by @mailwoman/neural at runtime.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
@@ -11,6 +11,7 @@
   "files": [
     "model-card.json",
     "postcode-fr.bin",
+    "pair-index-fr.bin",
     "fst-fr-fr.bin",
     "fst-street-morphology.bin",
     "anchor-lexicon-v1.json",
@@ -31,7 +32,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@mailwoman/neural-weights-en-us": "8.3.0"
+    "@mailwoman/neural-weights-en-us": "8.5.0"
   },
   "mailwoman": {
     "baseWeights": "@mailwoman/neural-weights-en-us"

package/scripts/link-dev-weights.ts

@@ -180,3 +180,107 @@ if (existsSync(MORPHOLOGY_SRC)) {
 		`WARNING: missing ${MORPHOLOGY_SRC} — the street-context gate falls back to the per-process dictionary build.`
 	)
 }
+
+/**
+ * Placetype-pair index (hierarchy campaign R6, 2026-08-01): build `pair-index-fr.bin` from the raw BAN dump so
+ * `resolveWeights` surfaces `pairIndexPath` in dev and the placetype-pair prior is live for fr-fr.
+ *
+ * The FR source is BAN's `nom_ld` (lieu-dit), read through `ban/sdk`'s `cleanLieuDit` — NOT WOF. WOF's French
+ * neighbourhood records are Paris quartiers, which never appear in a postal address; the lieu-dit is the line French
+ * addresses actually carry. See `mailwoman/gazetteer-pipeline/lieudit-pairs.ts`.
+ *
+ * Freshness guard: the build streams ~26M BAN rows across 101 département files, so an unconditional rebuild would make
+ * every test run unusable. Peek the header and rebuild only when the calibrated delta or transition beta has moved.
+ * Unlike en-gb this does NOT md5 its source — BAN is a directory of 101 files, and hashing all of them costs more than
+ * the guard saves; a BAN refresh is a deliberate act, so re-run with the artifact deleted after one.
+ */
+const PAIR_INDEX_BIN_DEST = resolve(PKG_DIR, "pair-index-fr.bin")
+/**
+ * Calibrated soft-prior magnitudes — the pair the R6 bars were measured at (board 0/80 → 76/80, 0/60 confound FPs).
+ */
+const PAIR_INDEX_DELTA = 10
+const PAIR_INDEX_TRANSITION_BETA = 5
+const BAN_DIR = dataRootPath("corpus", "sources", "ban")
+
+/**
+ * Minimal PIX1 header reader — magic + header block only, reimplemented rather than imported so this data-only package
+ * gains no dependency on `@mailwoman/neural`. `neural/pair-index-resolver.ts` owns the format.
+ */
+function peekPairIndexHeaderFields(path: string): { delta: number; transitionBeta: number | undefined } {
+	const bytes = readFileSync(path)
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+	// "PIX1" little-endian.
+	const MAGIC = 0x31_58_49_50
+
+	if (view.getUint32(0, true) !== MAGIC) {
+		throw new Error(`pair index: bad magic reading ${path}`)
+	}
+
+	const headerLen = view.getUint32(4, true)
+	const header = JSON.parse(Buffer.from(bytes.subarray(8, 8 + headerLen)).toString("utf8")) as {
+		delta: number
+		transitionBeta?: number
+	}
+
+	return { delta: header.delta, transitionBeta: header.transitionBeta }
+}
+
+if (!existsSync(CLI)) {
+	console.error(`WARNING: ${CLI} not built — run \`yarn compile\` first, then re-run for pair-index-fr.bin.`)
+} else if (!existsSync(String(BAN_DIR))) {
+	console.error(
+		`WARNING: missing ${BAN_DIR} — pair-index-fr.bin not built; the placetype-pair prior stays inert for FR.`
+	)
+} else {
+	let needsRebuild = true
+
+	if (existsSync(PAIR_INDEX_BIN_DEST)) {
+		try {
+			const header = peekPairIndexHeaderFields(PAIR_INDEX_BIN_DEST)
+
+			if (header.delta !== PAIR_INDEX_DELTA) {
+				console.log(`rebuilding pair-index-fr.bin — delta ${header.delta} → ${PAIR_INDEX_DELTA}`)
+			} else if (header.transitionBeta !== PAIR_INDEX_TRANSITION_BETA) {
+				console.log(
+					`rebuilding pair-index-fr.bin — transitionBeta ${header.transitionBeta} → ${PAIR_INDEX_TRANSITION_BETA}`
+				)
+			} else {
+				needsRebuild = false
+			}
+		} catch (error) {
+			console.log(`rebuilding pair-index-fr.bin — header unreadable (${(error as Error).message})`)
+		}
+	}
+
+	if (!needsRebuild) {
+		console.log(`skipped pair-index-fr.bin build — ${PAIR_INDEX_BIN_DEST} is current`)
+	} else {
+		const result = spawnSync(
+			process.execPath,
+			[
+				CLI,
+				"gazetteer",
+				"pair-index",
+				"--out",
+				PKG_DIR,
+				"--country",
+				"fr",
+				"--delta",
+				String(PAIR_INDEX_DELTA),
+				"--transition-beta",
+				String(PAIR_INDEX_TRANSITION_BETA),
+				"--ban-dir",
+				String(BAN_DIR),
+			],
+			{ stdio: "inherit" }
+		)
+
+		if (result.status !== 0 || !existsSync(PAIR_INDEX_BIN_DEST)) {
+			console.error(`FAILED: gazetteer pair-index --country fr (exit ${result.status})`)
+
+			process.exit(1)
+		}
+
+		console.log(`built pair-index-fr.bin ← ${BAN_DIR}`)
+	}
+}