mailwoman 8.4.0 → 8.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/commands/filer/linkage-eval.tsx +57 -0
  2. package/commands/gazetteer/build/bdc.tsx +7 -0
  3. package/commands/gazetteer/census.tsx +190 -0
  4. package/commands/gazetteer/pair-index.tsx +158 -24
  5. package/eval-harness/gauntlet/cases/regression.ts +205 -5
  6. package/eval-harness/gauntlet/harness.ts +4 -0
  7. package/eval-harness/gauntlet/regression.ts +7 -1
  8. package/gazetteer-pipeline/borough-pairs.ts +0 -0
  9. package/gazetteer-pipeline/lieudit-pairs.ts +0 -0
  10. package/gazetteer-pipeline/pair-index.ts +8 -0
  11. package/gazetteer-pipeline/placetype-census.ts +191 -0
  12. package/geocode-core.ts +15 -0
  13. package/out/commands/filer/linkage-eval.d.ts +20 -0
  14. package/out/commands/filer/linkage-eval.d.ts.map +1 -0
  15. package/out/commands/filer/linkage-eval.js +40 -0
  16. package/out/commands/filer/linkage-eval.js.map +1 -0
  17. package/out/commands/gazetteer/build/bdc.d.ts.map +1 -1
  18. package/out/commands/gazetteer/build/bdc.js +6 -1
  19. package/out/commands/gazetteer/build/bdc.js.map +1 -1
  20. package/out/commands/gazetteer/census.d.ts +35 -0
  21. package/out/commands/gazetteer/census.d.ts.map +1 -0
  22. package/out/commands/gazetteer/census.js +137 -0
  23. package/out/commands/gazetteer/census.js.map +1 -0
  24. package/out/commands/gazetteer/pair-index.d.ts +3 -0
  25. package/out/commands/gazetteer/pair-index.d.ts.map +1 -1
  26. package/out/commands/gazetteer/pair-index.js +129 -23
  27. package/out/commands/gazetteer/pair-index.js.map +1 -1
  28. package/out/eval-harness/gauntlet/cases/regression.d.ts.map +1 -1
  29. package/out/eval-harness/gauntlet/cases/regression.js +205 -5
  30. package/out/eval-harness/gauntlet/cases/regression.js.map +1 -1
  31. package/out/eval-harness/gauntlet/harness.d.ts +2 -0
  32. package/out/eval-harness/gauntlet/harness.d.ts.map +1 -1
  33. package/out/eval-harness/gauntlet/harness.js +2 -0
  34. package/out/eval-harness/gauntlet/harness.js.map +1 -1
  35. package/out/eval-harness/gauntlet/regression.d.ts.map +1 -1
  36. package/out/eval-harness/gauntlet/regression.js +7 -1
  37. package/out/eval-harness/gauntlet/regression.js.map +1 -1
  38. package/out/gazetteer-pipeline/borough-pairs.d.ts +33 -0
  39. package/out/gazetteer-pipeline/borough-pairs.d.ts.map +1 -0
  40. package/out/gazetteer-pipeline/borough-pairs.js +0 -0
  41. package/out/gazetteer-pipeline/borough-pairs.js.map +1 -0
  42. package/out/gazetteer-pipeline/lieudit-pairs.d.ts +58 -0
  43. package/out/gazetteer-pipeline/lieudit-pairs.d.ts.map +1 -0
  44. package/out/gazetteer-pipeline/lieudit-pairs.js +0 -0
  45. package/out/gazetteer-pipeline/lieudit-pairs.js.map +1 -0
  46. package/out/gazetteer-pipeline/pair-index.d.ts +5 -0
  47. package/out/gazetteer-pipeline/pair-index.d.ts.map +1 -1
  48. package/out/gazetteer-pipeline/pair-index.js +7 -0
  49. package/out/gazetteer-pipeline/pair-index.js.map +1 -1
  50. package/out/gazetteer-pipeline/placetype-census.d.ts +70 -0
  51. package/out/gazetteer-pipeline/placetype-census.d.ts.map +1 -0
  52. package/out/gazetteer-pipeline/placetype-census.js +142 -0
  53. package/out/gazetteer-pipeline/placetype-census.js.map +1 -0
  54. package/out/geocode-core.d.ts +13 -0
  55. package/out/geocode-core.d.ts.map +1 -1
  56. package/out/geocode-core.js +2 -0
  57. package/out/geocode-core.js.map +1 -1
  58. package/package.json +25 -25
@@ -0,0 +1,57 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `mailwoman filer linkage-eval` — the 3b Task 4 corporate-family recovery eval. Geocode-free,
7
+ * database-free (builds its own scratch `filer.db` artifacts from an embedded synthetic corpus); emits
8
+ * the markdown scorecard to stdout and, when `--out-md` is given, to that path too. `--date` overrides
9
+ * the report's dated H1 so the committed scorecard can be regenerated without editing code.
10
+ */
11
+
12
+ import { filerLinkageEval } from "@mailwoman/filer/tools"
13
+ import { Text } from "ink"
14
+ import zod from "zod"
15
+
16
+ import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
17
+
18
+ const OptionsSchema = zod.object({
19
+ outMd: zod.string().optional().describe("Also write the markdown report here"),
20
+ date: zod.string().optional().describe("Report date (YYYY-MM-DD) for the H1 — defaults to today"),
21
+ })
22
+
23
+ export { OptionsSchema as options }
24
+
25
+ /**
26
+ * `null` renders as `N/A`, never as `0.000` — the withheld run makes no positive call, so its precision and F1 are
27
+ * undefined rather than zero (see `linkage-metrics.ts`).
28
+ */
29
+ function formatScoreValue(value: number | null): string {
30
+ return value === null ? "N/A" : value.toFixed(3)
31
+ }
32
+
33
+ const FilerLinkageEval: CommandComponent<typeof OptionsSchema> = ({ options }) => {
34
+ const state = useCommandTask(() =>
35
+ filerLinkageEval({ outMd: options.outMd, date: options.date }, (line) => console.error(line))
36
+ )
37
+
38
+ if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
39
+
40
+ if (state.status === "done") {
41
+ const { withheld, control } = state.result
42
+
43
+ // Both rows, always: the withheld number is the measurement and the control is what makes it falsifiable.
44
+ return (
45
+ <Text color="green">
46
+ linkage-eval · withheld: recall {formatScoreValue(withheld.score.recall)} ({withheld.score.truePositivePairs}/
47
+ {withheld.score.truthPositivePairs} pairs, F1 {formatScoreValue(withheld.score.f1)}) · control: F1{" "}
48
+ {formatScoreValue(control.score.f1)} ({control.score.truePositivePairs}/{control.score.truthPositivePairs}{" "}
49
+ pairs)
50
+ </Text>
51
+ )
52
+ }
53
+
54
+ return null
55
+ }
56
+
57
+ export default FilerLinkageEval
@@ -40,6 +40,7 @@ import {
40
40
  createBDCClient,
41
41
  createTIGERBlockCentroidLookup,
42
42
  downloadBDCFile,
43
+ formatBDCThrottleStats,
43
44
  resolveLatestVintage,
44
45
  retrieveAvailabilityFiles,
45
46
  retrieveFilingDates,
@@ -140,6 +141,12 @@ const GazetteerBuildBDC: CommandComponent<typeof OptionsSchema> = ({ options })
140
141
  csvPaths.push(await downloadBDCFile(client, file, cacheDir))
141
142
  }
142
143
 
144
+ // The FCC caps this API at ten requests per minute — six seconds a call — so a national run is
145
+ // throttle-bound by construction and the interesting number is how much of the wall clock went to
146
+ // waiting rather than transferring. Printed once the network phase is over, on stderr with the rest
147
+ // of the progress stream, so a rate change can be assessed against a measurement.
148
+ console.error(`▸ ${formatBDCThrottleStats(client.throttleStats())}`)
149
+
143
150
  const out = options.out ?? dataRootPath("bdc", "bdc.db")
144
151
  const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
145
152
  const tigerDBPath = dataRootPath("tiger", "tiger.db")
@@ -0,0 +1,190 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `mailwoman gazetteer census` — build the PCN1 placetype census (hierarchy campaign R4c) from the
7
+ * shipped WOF admin DB. Counts each locality-class parent's children through the projection table
8
+ * (`gazetteer-pipeline/placetype-census.ts`), folds parent surfaces with the SAME `normalizeFSTToken`
9
+ * the PIX1 pair index uses — so a consumer folds once and probes both artifacts — and writes
10
+ * `placetype-census-<country>.bin`.
11
+ *
12
+ * Fold collisions SUM. Two distinct raw parents that fold together ("St Helens" / "St. Helens") are
13
+ * one census node whose counts are the union of both; the serializer refuses duplicate parents, so a
14
+ * merge bug surfaces as a throw rather than a silently halved count.
15
+ *
16
+ * `--delta` is deliberately OPTIONAL and unset by default, unlike the pair index's required one: R4c
17
+ * ships the census as data + loader + offline probe with NO decode wiring. A calibrated delta is a
18
+ * later rung's output; writing one now would put an unmeasured bias into a shipped artifact.
19
+ *
20
+ * Self-verifying (the sealed-artifact spirit): after writing, the command re-reads its own bytes
21
+ * through a fresh `PlacetypeCensusResolver` and probes known parents, printing PROBE OK/MISS with the
22
+ * node's dependent-locality share and lift rather than trusting the write.
23
+ */
24
+
25
+ import { existsSync, writeFileSync } from "node:fs"
26
+ import { join } from "node:path"
27
+
28
+ import type { ComponentTag } from "@mailwoman/core/types"
29
+ import { dataRootPath, md5File } from "@mailwoman/core/utils"
30
+ import { normalizeFSTToken } from "@mailwoman/neural/fst-prior"
31
+ import {
32
+ PlacetypeCensusResolver,
33
+ serializePlacetypeCensus,
34
+ type PlacetypeCensusHeader,
35
+ type PlacetypeCensusNode,
36
+ } from "@mailwoman/neural/placetype-census"
37
+ import { Box, Text } from "ink"
38
+ import zod from "zod"
39
+
40
+ import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
41
+ import { buildPlacetypeCensus, toBaseRates } from "../../gazetteer-pipeline/placetype-census.ts"
42
+
43
+ /**
44
+ * Known parents probed after write, PER COUNTRY. Probing another country's names against a freshly built census prints
45
+ * reassuring-looking misses that verify nothing (the lesson the pair-index command's en-nz first build taught).
46
+ */
47
+ const PROBE_PARENTS_BY_COUNTRY: Readonly<Record<string, readonly string[]>> = {
48
+ gb: ["London", "Manchester", "Birmingham"],
49
+ us: ["New York", "Chicago", "Springfield"],
50
+ }
51
+
52
+ const OptionsSchema = zod.object({
53
+ out: zod.string().default("docs/static/mailwoman").describe("Output dir for placetype-census-<country>.bin"),
54
+ country: zod.string().default("gb").describe("ISO country code this census is built for"),
55
+ source: zod.string().optional().describe("WOF admin DB. Default <data-root>/wof/admin-global-priority.db"),
56
+ delta: zod
57
+ .number()
58
+ .optional()
59
+ .describe(
60
+ "OPTIONAL soft-prior magnitude written into the header. Omit (the default) for an un-wired census — R4c " +
61
+ "ships data + loader + offline probe only; a calibration rung supplies the real value."
62
+ ),
63
+ })
64
+
65
+ export { OptionsSchema as options }
66
+
67
+ const GazetteerCensus: CommandComponent<typeof OptionsSchema> = ({ options }) => {
68
+ const state = useCommandTask(async () => {
69
+ const country = options.country.toLowerCase()
70
+ const sourcePath = options.source ?? String(dataRootPath("wof", "admin-global-priority.db"))
71
+
72
+ if (!existsSync(sourcePath)) {
73
+ throw new Error(`census: source WOF admin DB not found: ${sourcePath}`)
74
+ }
75
+
76
+ const built = buildPlacetypeCensus(sourcePath, country.toUpperCase())
77
+
78
+ if (built.unmappedPlacetypes.length) {
79
+ throw new Error(
80
+ `census: source carries placetypes absent from PLACETYPE_PROJECTION: ${built.unmappedPlacetypes.join(", ")} — ` +
81
+ `extend the projection table (and plan/reference/placetype-evidence.mdx) before building.`
82
+ )
83
+ }
84
+
85
+ // Fold, merging collisions by SUMMING counts — the serializer throws on duplicates, so a merge bug is loud.
86
+ const folded = new Map<string, PlacetypeCensusNode>()
87
+ let collisions = 0
88
+
89
+ for (const node of built.nodes) {
90
+ const parent = normalizeFSTToken(node.parent)
91
+
92
+ if (!parent) continue
93
+
94
+ const existing = folded.get(parent)
95
+
96
+ if (!existing) {
97
+ folded.set(parent, { parent, counts: { ...node.counts }, total: node.total })
98
+
99
+ continue
100
+ }
101
+
102
+ collisions++
103
+
104
+ for (const [tag, n] of Object.entries(node.counts) as Array<[ComponentTag, number]>) {
105
+ existing.counts[tag] = (existing.counts[tag] ?? 0) + n
106
+ }
107
+
108
+ existing.total += node.total
109
+ }
110
+
111
+ const nodes = [...folded.values()]
112
+ const baseRates = toBaseRates(built.countryTotals)
113
+ const sourceMD5 = await md5File(sourcePath)
114
+
115
+ const header: PlacetypeCensusHeader = {
116
+ country,
117
+ schemaVersion: 1,
118
+ foldVersion: 1,
119
+ sourceMD5s: [sourceMD5],
120
+ buildDate: new Date().toISOString(),
121
+ baseRates,
122
+ ...(options.delta !== undefined ? { delta: options.delta } : {}),
123
+ }
124
+
125
+ const bytes = serializePlacetypeCensus(header, nodes)
126
+ const outPath = join(options.out, `placetype-census-${country}.bin`)
127
+
128
+ writeFileSync(outPath, bytes)
129
+
130
+ // Self-verifying readback over the bytes just written, not the in-memory nodes.
131
+ const resolver = new PlacetypeCensusResolver(bytes)
132
+ const probeParents = PROBE_PARENTS_BY_COUNTRY[country]
133
+
134
+ if (!probeParents) {
135
+ throw new Error(
136
+ `census: no self-check probe parents registered for country "${country}" — add an entry to ` +
137
+ `PROBE_PARENTS_BY_COUNTRY (probing another country's names verifies nothing).`
138
+ )
139
+ }
140
+
141
+ const probeLines = probeParents.map((raw) => {
142
+ const parent = normalizeFSTToken(raw)
143
+ const node = resolver.probe(parent)
144
+
145
+ if (!node) return `PROBE MISS: fold("${raw}") → "${parent}" → (no node)`
146
+
147
+ const depLoc = node.counts.dependent_locality ?? 0
148
+ const share = resolver.share(parent, "dependent_locality")
149
+ const lift = resolver.lift(parent, "dependent_locality")
150
+
151
+ return (
152
+ `PROBE OK: "${parent}" → ${node.total.toLocaleString()} children, ` +
153
+ `${depLoc.toLocaleString()} dependent_locality (share ${(share * 100).toFixed(1)}%, lift ${lift.toFixed(1)}×)`
154
+ )
155
+ })
156
+
157
+ const baseRateLines = (Object.entries(baseRates) as Array<[ComponentTag, number]>)
158
+ .toSorted((a, b) => b[1] - a[1])
159
+ .map(([tag, rate]) => ` ${tag}: ${(rate * 100).toFixed(2)}%`)
160
+
161
+ return [
162
+ `placetype-census-${country}.bin → ${outPath} (${bytes.length.toLocaleString()} bytes)`,
163
+ `header: ${options.delta !== undefined ? `delta=${options.delta}` : "(no delta — un-wired census)"}`,
164
+ `nodes: ${nodes.length.toLocaleString()} (${built.nodes.length.toLocaleString()} pre-fold, ${collisions.toLocaleString()} fold collisions merged)`,
165
+ `child links counted: ${built.links.toLocaleString()}`,
166
+ "country base rates:",
167
+ ...baseRateLines,
168
+ ...probeLines,
169
+ ]
170
+ })
171
+
172
+ if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
173
+
174
+ if (state.status === "done") {
175
+ return (
176
+ <Box flexDirection="column">
177
+ {state.result.map((line, i) => (
178
+ <Text key={i} color={i === 0 ? "green" : undefined}>
179
+ {i === 0 ? "✓ " : " "}
180
+ {line}
181
+ </Text>
182
+ ))}
183
+ </Box>
184
+ )
185
+ }
186
+
187
+ return null
188
+ }
189
+
190
+ export default GazetteerCensus
@@ -24,7 +24,7 @@
24
24
  * count table) — this sizes the word-span window the decode-side prior (Task 4) walks.
25
25
  */
26
26
 
27
- import { createReadStream, existsSync, writeFileSync } from "node:fs"
27
+ import { createReadStream, existsSync, readFileSync, writeFileSync } from "node:fs"
28
28
  import { join } from "node:path"
29
29
 
30
30
  import { dataRootPath, md5File } from "@mailwoman/core/utils"
@@ -38,6 +38,8 @@ import { CSVSpliterator } from "spliterator"
38
38
  import zod from "zod"
39
39
 
40
40
  import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
41
+ import { extractBoroughPairs } from "../../gazetteer-pipeline/borough-pairs.ts"
42
+ import { extractLieuDitPairs } from "../../gazetteer-pipeline/lieudit-pairs.ts"
41
43
  import { PairIndexBuilder, applyPairIndexHoldout } from "../../gazetteer-pipeline/pair-index.ts"
42
44
 
43
45
  /**
@@ -61,6 +63,15 @@ const EXPECTED_GB_PAIR_COUNT = 19_209
61
63
  */
62
64
  const RUNG3_PRE_FOLD_CENSUS_LINE_COUNT = 19_431
63
65
 
66
+ /**
67
+ * The US instance's cross-check (hierarchy campaign R5), measured 2026-08-01 against the shipped
68
+ * `admin-global-priority.db`. Unlike GB there is no postal register in the mix — every pair is WOF-sourced (borough +
69
+ * neighbourhood children under locality/localadmin/borough parents, see `PAIR_PLACETYPES_BY_COUNTRY`), so this number
70
+ * tracks the WOF snapshot alone. A snapshot refresh legitimately moves it; re-anchor the constant deliberately, after
71
+ * inspecting the diff, rather than relaxing the gate.
72
+ */
73
+ const EXPECTED_US_PAIR_COUNT = 49_033
74
+
64
75
  /**
65
76
  * Known (child, parent) pairs probed after write as a self-check — PER COUNTRY, keyed by the `--country` code. Probing
66
77
  * another country's names against a freshly built index prints reassuring-looking `PROBE MISS` lines that verify
@@ -78,6 +89,32 @@ const PROBE_PAIRS_BY_COUNTRY: Readonly<Record<string, ReadonlyArray<readonly [ci
78
89
  // The repeated-name convention's identity pair — the (x,x) evidence the segment rule keys on.
79
90
  ["Mangawhai", "Mangawhai"],
80
91
  ],
92
+ // R5 (hierarchy campaign): the US instance's probes. Unlike GB/NZ these are NOT postal-format
93
+ // dependent localities — USPS routes city/state/ZIP — they are the borough/neighbourhood class
94
+ // the schema's umbrella term covers, sourced from WOF rather than a postal register.
95
+ us: [
96
+ ["Astoria", "Queens"],
97
+ ["Park Slope", "Brooklyn"],
98
+ ["Manhattan", "New York"],
99
+ ],
100
+ // R6: the FR instance. These are LIEUX-DITS under their communes (BAN `nom_ld`), not quartiers —
101
+ // see gazetteer-pipeline/lieudit-pairs.ts for why the French source differs from the US one.
102
+ fr: [
103
+ ["Pinsonnac", "Montpeyroux"],
104
+ ["Line", "Salignac-Eyvigues"],
105
+ ],
106
+ }
107
+
108
+ /**
109
+ * Split a comma-separated path list, tolerating whitespace and an absent value. Each secondary source stays its OWN
110
+ * file rather than being pre-merged into a blob, so every one keeps a distinct provenance md5 in the header — which is
111
+ * what lets a freshness guard notice that exactly one of them changed.
112
+ */
113
+ function splitPathList(value: string | undefined): string[] {
114
+ return (value ?? "")
115
+ .split(",")
116
+ .map((path) => path.trim())
117
+ .filter(Boolean)
81
118
  }
82
119
 
83
120
  const OptionsSchema = zod.object({
@@ -110,6 +147,22 @@ const OptionsSchema = zod.object({
110
147
  "against pairs the index was never trained/built on. Default 0 = a normal, complete build. NEVER pass a " +
111
148
  "nonzero value for a shipped artifact build."
112
149
  ),
150
+ pairsJsonl: zod
151
+ .string()
152
+ .optional()
153
+ .describe(
154
+ "Secondary pairs JSONL ({child, parent} per line) merged through the same fold — hierarchy campaign R3 (ONSPD London wards etc.)"
155
+ ),
156
+ boroughDb: zod
157
+ .string()
158
+ .optional()
159
+ .describe("WOF admin DB path — merge borough (child, parent) pairs for the country (hierarchy campaign R2)"),
160
+ banDir: zod
161
+ .string()
162
+ .optional()
163
+ .describe(
164
+ "BAN adresses-<dept>.csv directory — merge FR (lieu-dit, commune) pairs through ban/sdk's cleanLieuDit filter (hierarchy campaign R6)"
165
+ ),
113
166
  holdoutSeed: zod
114
167
  .number()
115
168
  .default(42)
@@ -121,12 +174,23 @@ export { OptionsSchema as options }
121
174
  const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options }) => {
122
175
  const state = useCommandTask(async () => {
123
176
  const country = options.country.toLowerCase()
124
- const sourcePath = options.source ?? dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")
177
+ // The PPD tuples CSV is a GB national register — there is no equivalent for the US instance (USPS routes
178
+ // city/state/ZIP, so no postal source carries dependent localities). A country whose pairs come entirely from
179
+ // the WOF/secondary sources below runs with NO CSV rather than being handed an empty one.
180
+ const sourcePath =
181
+ options.source ?? (country === "gb" ? String(dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")) : undefined)
125
182
 
126
- if (!existsSync(sourcePath)) {
183
+ if (sourcePath && !existsSync(sourcePath)) {
127
184
  throw new Error(`pair-index: source CSV not found: ${sourcePath}`)
128
185
  }
129
186
 
187
+ if (!sourcePath && !options.boroughDb && !options.pairsJsonl && !options.banDir) {
188
+ throw new Error(
189
+ `pair-index: country "${country}" has no PPD default — pass --source, --borough-db, --pairs-jsonl or --ban-dir, ` +
190
+ `or the build would write an empty index.`
191
+ )
192
+ }
193
+
130
194
  const builder = new PairIndexBuilder()
131
195
  let header: string[] | null = null
132
196
  let cityIx = -1
@@ -135,24 +199,82 @@ const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options })
135
199
  // Same CSVSpliterator idiom as `corpus/src/shard-recipes/locale.ts`'s `readTuples`: array mode, no header
136
200
  // row consumed by the parser (`header: false`), so we build the column index off the first yielded row
137
201
  // ourselves and skip forward from there.
138
- for await (const cells of CSVSpliterator.fromAsync<string[]>(createReadStream(sourcePath), {
139
- mode: "array",
140
- header: false,
141
- enableQuoteHandling: true,
142
- })) {
143
- if (header === null) {
144
- header = cells.map((h) => h.trim().toUpperCase())
145
- cityIx = header.indexOf("CITY")
146
- districtIx = header.indexOf("DISTRICT")
147
-
148
- if (cityIx < 0 || districtIx < 0) {
149
- throw new Error(`pair-index: source header is missing CITY/DISTRICT: ${header.join(",")}`)
202
+ if (sourcePath) {
203
+ for await (const cells of CSVSpliterator.fromAsync<string[]>(createReadStream(sourcePath), {
204
+ mode: "array",
205
+ header: false,
206
+ enableQuoteHandling: true,
207
+ })) {
208
+ if (header === null) {
209
+ header = cells.map((h) => h.trim().toUpperCase())
210
+ cityIx = header.indexOf("CITY")
211
+ districtIx = header.indexOf("DISTRICT")
212
+
213
+ if (cityIx < 0 || districtIx < 0) {
214
+ throw new Error(`pair-index: source header is missing CITY/DISTRICT: ${header.join(",")}`)
215
+ }
216
+
217
+ continue
150
218
  }
151
219
 
152
- continue
220
+ builder.addRow(cells[cityIx] ?? "", cells[districtIx] ?? "")
153
221
  }
222
+ }
223
+
224
+ // R2 (hierarchy campaign): borough pairs from the WOF admin DB, through the SAME fold/dedupe
225
+ // as the CSV rows (boroughs project onto dependent_locality — plan/reference/placetype-evidence).
226
+ let boroughsAdded = 0
154
227
 
155
- builder.addRow(cells[cityIx] ?? "", cells[districtIx] ?? "")
228
+ if (options.boroughDb) {
229
+ const before = builder.distinctCount
230
+
231
+ for (const pair of extractBoroughPairs(options.boroughDb, country.toUpperCase())) {
232
+ builder.addRow(pair.child, pair.parent)
233
+ }
234
+
235
+ boroughsAdded = builder.distinctCount - before
236
+
237
+ console.error(`pair-index: +${boroughsAdded} distinct borough pairs (WOF admin DB)`)
238
+ }
239
+
240
+ // R6: FR lieu-dit pairs from the raw BAN dump. Streams ~26M rows, so it is the slowest source by far —
241
+ // deliberately opt-in per build rather than a default.
242
+ if (options.banDir) {
243
+ const before = builder.distinctCount
244
+ const { pairs, rowsWithLieuDit, filesRead } = await extractLieuDitPairs(options.banDir)
245
+
246
+ for (const pair of pairs) {
247
+ builder.addRow(pair.child, pair.parent)
248
+ }
249
+
250
+ boroughsAdded += builder.distinctCount - before
251
+
252
+ console.error(
253
+ `pair-index: +${builder.distinctCount - before} distinct lieu-dit pairs ` +
254
+ `(${filesRead} départements, ${rowsWithLieuDit.toLocaleString()} rows with a clean lieu-dit)`
255
+ )
256
+ }
257
+
258
+ // R3: generic secondary pairs (ONSPD-derived London ward pairs; future NI/IE sources) — the
259
+ // same fold/dedupe path, counted into the cross-check delta alongside the boroughs.
260
+ if (options.pairsJsonl) {
261
+ for (const path of splitPathList(options.pairsJsonl)) {
262
+ const before = builder.distinctCount
263
+
264
+ for (const line of readFileSync(path, "utf8").split("\n")) {
265
+ if (!line.trim()) continue
266
+
267
+ const pair = JSON.parse(line) as { child: string; parent: string }
268
+
269
+ builder.addRow(pair.child, pair.parent)
270
+ }
271
+
272
+ boroughsAdded += builder.distinctCount - before
273
+
274
+ console.error(
275
+ `pair-index: +${builder.distinctCount - before} distinct secondary pairs (${path.split("/").pop()})`
276
+ )
277
+ }
156
278
  }
157
279
 
158
280
  const built = builder.finish()
@@ -167,7 +289,13 @@ const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options })
167
289
  options.holdoutSeed
168
290
  )
169
291
 
170
- const sourceMD5 = await md5File(sourcePath)
292
+ // Provenance covers every source that contributed rows, in the order they were folded in — a US build has no
293
+ // CSV at all, so an unconditional single-element array would have claimed a source the artifact never read.
294
+ const sourceMD5s = [
295
+ ...(sourcePath ? [await md5File(sourcePath)] : []),
296
+ ...(options.boroughDb ? [await md5File(options.boroughDb)] : []),
297
+ ...(await Promise.all(splitPathList(options.pairsJsonl).map((path) => md5File(path)))),
298
+ ]
171
299
 
172
300
  // `transitionBeta` is spread conditionally so an omitted flag writes NO header key at all (a
173
301
  // beta-less binary, byte-shape-identical to every pre-TRANSITION-BETA artifact) — not a null/0.
@@ -176,7 +304,7 @@ const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options })
176
304
  delta: options.delta,
177
305
  schemaVersion: 1,
178
306
  foldVersion: 1,
179
- sourceMD5s: [sourceMD5],
307
+ sourceMD5s,
180
308
  buildDate: new Date().toISOString(),
181
309
  ...(options.transitionBeta !== undefined ? { transitionBeta: options.transitionBeta } : {}),
182
310
  }
@@ -228,11 +356,17 @@ const GazetteerPairIndex: CommandComponent<typeof OptionsSchema> = ({ options })
228
356
  const gateLine =
229
357
  country === "gb"
230
358
  ? options.holdoutFraction > 0
231
- ? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${EXPECTED_GB_PAIR_COUNT.toLocaleString()})${preFoldSuffix}`
232
- : preHoldoutCount === EXPECTED_GB_PAIR_COUNT
233
- ? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (production baseline expects ${EXPECTED_GB_PAIR_COUNT.toLocaleString()})${preFoldSuffix}`
234
- : `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != the production baseline's ${EXPECTED_GB_PAIR_COUNT.toLocaleString()} — investigate fold divergence before trusting this artifact${preFoldSuffix}`
235
- : `(cross-check only registered for gb; ${preHoldoutCount.toLocaleString()} distinct pairs)`
359
+ ? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${(EXPECTED_GB_PAIR_COUNT + boroughsAdded).toLocaleString()})${preFoldSuffix}`
360
+ : preHoldoutCount === EXPECTED_GB_PAIR_COUNT + boroughsAdded
361
+ ? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (baseline ${EXPECTED_GB_PAIR_COUNT.toLocaleString()} + ${boroughsAdded} borough)${preFoldSuffix}`
362
+ : `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != baseline ${EXPECTED_GB_PAIR_COUNT.toLocaleString()} + ${boroughsAdded} borough — investigate fold divergence before trusting this artifact${preFoldSuffix}`
363
+ : country === "us"
364
+ ? options.holdoutFraction > 0
365
+ ? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${EXPECTED_US_PAIR_COUNT.toLocaleString()})`
366
+ : preHoldoutCount === EXPECTED_US_PAIR_COUNT
367
+ ? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (all WOF-sourced; no postal register)`
368
+ : `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != ${EXPECTED_US_PAIR_COUNT.toLocaleString()} — a WOF snapshot refresh moves this number, so re-anchor the constant DELIBERATELY (with the diff inspected) rather than trusting the artifact`
369
+ : `(cross-check only registered for gb/us; ${preHoldoutCount.toLocaleString()} distinct pairs)`
236
370
 
237
371
  const holdoutLine =
238
372
  options.holdoutFraction > 0