mailwoman 8.3.0 → 8.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/commands/filer/linkage-eval.tsx +57 -0
  2. package/commands/gazetteer/build/bdc.tsx +210 -0
  3. package/commands/gazetteer/build/poi.tsx +131 -7
  4. package/commands/gazetteer/census.tsx +190 -0
  5. package/commands/gazetteer/pair-index.tsx +158 -24
  6. package/eval-harness/gates/v7.0.0-base.json +1 -0
  7. package/eval-harness/gauntlet/cases/regression.ts +1112 -3
  8. package/eval-harness/gauntlet/harness.ts +61 -5
  9. package/eval-harness/gauntlet/regression.ts +12 -2
  10. package/eval-harness/promotion-gate.ts +3 -1
  11. package/gazetteer-pipeline/borough-pairs.ts +0 -0
  12. package/gazetteer-pipeline/lieudit-pairs.ts +0 -0
  13. package/gazetteer-pipeline/pair-index.ts +8 -0
  14. package/gazetteer-pipeline/placetype-census.ts +191 -0
  15. package/gazetteer-pipeline/poi/build-poi.ts +114 -20
  16. package/geocode-core.ts +15 -0
  17. package/out/commands/filer/linkage-eval.d.ts +20 -0
  18. package/out/commands/filer/linkage-eval.d.ts.map +1 -0
  19. package/out/commands/filer/linkage-eval.js +40 -0
  20. package/out/commands/filer/linkage-eval.js.map +1 -0
  21. package/out/commands/gazetteer/build/bdc.d.ts +43 -0
  22. package/out/commands/gazetteer/build/bdc.d.ts.map +1 -0
  23. package/out/commands/gazetteer/build/bdc.js +153 -0
  24. package/out/commands/gazetteer/build/bdc.js.map +1 -0
  25. package/out/commands/gazetteer/build/poi.d.ts +18 -0
  26. package/out/commands/gazetteer/build/poi.d.ts.map +1 -1
  27. package/out/commands/gazetteer/build/poi.js +103 -7
  28. package/out/commands/gazetteer/build/poi.js.map +1 -1
  29. package/out/commands/gazetteer/census.d.ts +35 -0
  30. package/out/commands/gazetteer/census.d.ts.map +1 -0
  31. package/out/commands/gazetteer/census.js +137 -0
  32. package/out/commands/gazetteer/census.js.map +1 -0
  33. package/out/commands/gazetteer/pair-index.d.ts +3 -0
  34. package/out/commands/gazetteer/pair-index.d.ts.map +1 -1
  35. package/out/commands/gazetteer/pair-index.js +129 -23
  36. package/out/commands/gazetteer/pair-index.js.map +1 -1
  37. package/out/eval-harness/gauntlet/cases/regression.d.ts.map +1 -1
  38. package/out/eval-harness/gauntlet/cases/regression.js +1106 -3
  39. package/out/eval-harness/gauntlet/cases/regression.js.map +1 -1
  40. package/out/eval-harness/gauntlet/harness.d.ts +9 -0
  41. package/out/eval-harness/gauntlet/harness.d.ts.map +1 -1
  42. package/out/eval-harness/gauntlet/harness.js +48 -8
  43. package/out/eval-harness/gauntlet/harness.js.map +1 -1
  44. package/out/eval-harness/gauntlet/regression.d.ts.map +1 -1
  45. package/out/eval-harness/gauntlet/regression.js +12 -2
  46. package/out/eval-harness/gauntlet/regression.js.map +1 -1
  47. package/out/eval-harness/promotion-gate.d.ts.map +1 -1
  48. package/out/eval-harness/promotion-gate.js +3 -1
  49. package/out/eval-harness/promotion-gate.js.map +1 -1
  50. package/out/gazetteer-pipeline/borough-pairs.d.ts +33 -0
  51. package/out/gazetteer-pipeline/borough-pairs.d.ts.map +1 -0
  52. package/out/gazetteer-pipeline/borough-pairs.js +0 -0
  53. package/out/gazetteer-pipeline/borough-pairs.js.map +1 -0
  54. package/out/gazetteer-pipeline/lieudit-pairs.d.ts +58 -0
  55. package/out/gazetteer-pipeline/lieudit-pairs.d.ts.map +1 -0
  56. package/out/gazetteer-pipeline/lieudit-pairs.js +0 -0
  57. package/out/gazetteer-pipeline/lieudit-pairs.js.map +1 -0
  58. package/out/gazetteer-pipeline/pair-index.d.ts +5 -0
  59. package/out/gazetteer-pipeline/pair-index.d.ts.map +1 -1
  60. package/out/gazetteer-pipeline/pair-index.js +7 -0
  61. package/out/gazetteer-pipeline/pair-index.js.map +1 -1
  62. package/out/gazetteer-pipeline/placetype-census.d.ts +70 -0
  63. package/out/gazetteer-pipeline/placetype-census.d.ts.map +1 -0
  64. package/out/gazetteer-pipeline/placetype-census.js +142 -0
  65. package/out/gazetteer-pipeline/placetype-census.js.map +1 -0
  66. package/out/gazetteer-pipeline/poi/build-poi.d.ts +51 -1
  67. package/out/gazetteer-pipeline/poi/build-poi.d.ts.map +1 -1
  68. package/out/gazetteer-pipeline/poi/build-poi.js +70 -17
  69. package/out/gazetteer-pipeline/poi/build-poi.js.map +1 -1
  70. package/out/geocode-core.d.ts +13 -0
  71. package/out/geocode-core.d.ts.map +1 -1
  72. package/out/geocode-core.js +2 -0
  73. package/out/geocode-core.js.map +1 -1
  74. package/package.json +25 -23
@@ -0,0 +1,57 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `mailwoman filer linkage-eval` — the 3b Task 4 corporate-family recovery eval. Geocode-free,
7
+ * database-free (builds its own scratch `filer.db` artifacts from an embedded synthetic corpus); emits
8
+ * the markdown scorecard to stdout and, when `--out-md` is given, to that path too. `--date` overrides
9
+ * the report's dated H1 so the committed scorecard can be regenerated without editing code.
10
+ */
11
+
12
+ import { filerLinkageEval } from "@mailwoman/filer/tools"
13
+ import { Text } from "ink"
14
+ import zod from "zod"
15
+
16
+ import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
17
+
18
+ const OptionsSchema = zod.object({
19
+ outMd: zod.string().optional().describe("Also write the markdown report here"),
20
+ date: zod.string().optional().describe("Report date (YYYY-MM-DD) for the H1 — defaults to today"),
21
+ })
22
+
23
+ export { OptionsSchema as options }
24
+
25
+ /**
26
+ * `null` renders as `N/A`, never as `0.000` — the withheld run makes no positive call, so its precision and F1 are
27
+ * undefined rather than zero (see `linkage-metrics.ts`).
28
+ */
29
+ function formatScoreValue(value: number | null): string {
30
+ return value === null ? "N/A" : value.toFixed(3)
31
+ }
32
+
33
+ const FilerLinkageEval: CommandComponent<typeof OptionsSchema> = ({ options }) => {
34
+ const state = useCommandTask(() =>
35
+ filerLinkageEval({ outMd: options.outMd, date: options.date }, (line) => console.error(line))
36
+ )
37
+
38
+ if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
39
+
40
+ if (state.status === "done") {
41
+ const { withheld, control } = state.result
42
+
43
+ // Both rows, always: the withheld number is the measurement and the control is what makes it falsifiable.
44
+ return (
45
+ <Text color="green">
46
+ linkage-eval · withheld: recall {formatScoreValue(withheld.score.recall)} ({withheld.score.truePositivePairs}/
47
+ {withheld.score.truthPositivePairs} pairs, F1 {formatScoreValue(withheld.score.f1)}) · control: F1{" "}
48
+ {formatScoreValue(control.score.f1)} ({control.score.truePositivePairs}/{control.score.truthPositivePairs}{" "}
49
+ pairs)
50
+ </Text>
51
+ )
52
+ }
53
+
54
+ return null
55
+ }
56
+
57
+ export default FilerLinkageEval
@@ -0,0 +1,210 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `mailwoman gazetteer build bdc` — the FCC BDC availability ingest + sealed `bdc.db` layer build
7
+ * (2a Task 8). Thin wiring only: list → download → build lives in `@mailwoman/bdc/sdk`
8
+ * (`buildBDCDatabase`, list-files.ts, download.ts), so it stays unit-testable without Ink/Pastel or
9
+ * network in the loop. Mirrors `poi.tsx`'s progress (stderr) / summary (stdout) split.
10
+ *
11
+ * First cut wires Fixed Broadband provider availability only (the primary wireline dataset) — mobile
12
+ * broadband/voice subcategories are a future flag, not a scope gap in this command's shape.
13
+ *
14
+ * `--provider-list-path` (3a Task 8, decision 6) opts INTO populating `bdc_provider` —
15
+ * `bdc_availability` itself is unaffected either way. When given, `parseProviderList`
16
+ * (`@mailwoman/filer/sdk`) streams the FCC provider-list CSV as `BuildBDCOptions.providers`, and
17
+ * filer.db (`--filer-db-path`, default `<data-root>/filer/filer.db`) is opened read-only to resolve
18
+ * each multi-FRN provider's primary FRN (`readFRNFilingCandidates` + `pickPrimaryFRN`, imported
19
+ * rather than reimplemented — see `build-bdc.ts`'s `BuildBDCOptions.filerDB` docstring). Omitting
20
+ * `--provider-list-path` leaves `bdc_provider` empty, exactly as before Task 8.
21
+ *
22
+ * Both paths are `existsSync`-guarded BEFORE any download/build work starts (review fix round 1,
23
+ * IMPORTANT-2): `populateBDCProviderTable` only runs after `writeLayerManifest`, i.e. at the very END
24
+ * of a full build — an unguarded typo'd `--provider-list-path` would otherwise surface as a raw ENOENT
25
+ * only after a nationwide availability ingest had already finished, discarding hours of work.
26
+ * `--filer-db-path` given without `--provider-list-path` is a loud error, not a silent no-op (MINOR):
27
+ * filer.db is only ever read to resolve a multi-FRN primary FRN, so it does nothing without a provider
28
+ * list to resolve FRNs FOR.
29
+ */
30
+
31
+ import { execFileSync } from "node:child_process"
32
+ import { existsSync } from "node:fs"
33
+ import { DatabaseSync } from "node:sqlite"
34
+
35
+ import {
36
+ BDCFileCategory,
37
+ BDCFilingDataType,
38
+ BDCProviderSubCategory,
39
+ buildBDCDatabase,
40
+ createBDCClient,
41
+ createTIGERBlockCentroidLookup,
42
+ downloadBDCFile,
43
+ formatBDCThrottleStats,
44
+ resolveLatestVintage,
45
+ retrieveAvailabilityFiles,
46
+ retrieveFilingDates,
47
+ } from "@mailwoman/bdc/sdk"
48
+ import { DatabaseClient } from "@mailwoman/core/kysley/client"
49
+ import { dataRootPath } from "@mailwoman/core/utils"
50
+ import type { FilerDatabase } from "@mailwoman/filer"
51
+ import { parseProviderList } from "@mailwoman/filer/sdk"
52
+ import { Box, Text } from "ink"
53
+ import zod from "zod"
54
+
55
+ import { type CommandComponent, useCommandTask } from "../../../cli-kit/index.ts"
56
+ import { artifactSizeMB } from "../../../gazetteer-pipeline/admin/index.ts"
57
+
58
+ const OptionsSchema = zod.object({
59
+ state: zod.string().describe('2-digit FCC state/territory FIPS code (e.g. "06" for California)'),
60
+ asOfDate: zod.string().optional().describe("FCC filing as_of_date (YYYY-MM-DD). Default: latest available"),
61
+ out: zod.string().optional().describe("bdc.db output path. Default <data-root>/bdc/bdc.db"),
62
+ includeLocationIds: zod
63
+ .boolean()
64
+ .default(false)
65
+ .describe("Populate bdc_availability.location_id (opaque BSL join key; NULL by default)"),
66
+ providerListPath: zod
67
+ .string()
68
+ .optional()
69
+ .describe(
70
+ "Path to the FCC BDC provider list CSV (frn/provider_id/holding_company). When given, populates " +
71
+ "bdc_provider (decision 6, 3a task 8); omit to leave bdc_provider empty, as before task 8."
72
+ ),
73
+ filerDbPath: zod
74
+ .string()
75
+ .optional()
76
+ .describe(
77
+ "Path to filer.db, used to resolve a multi-FRN provider's primary FRN when --provider-list-path is given. " +
78
+ "Default <data-root>/filer/filer.db"
79
+ ),
80
+ })
81
+
82
+ export { OptionsSchema as options }
83
+
84
+ const GazetteerBuildBDC: CommandComponent<typeof OptionsSchema> = ({ options }) => {
85
+ const state = useCommandTask(async () => {
86
+ // Fail-fast guards (review fix round 1, IMPORTANT-2 / MINOR) — checked BEFORE any network/download work
87
+ // starts. `populateBDCProviderTable` only runs after the availability ingest AND writeLayerManifest, so
88
+ // without this, a typo'd --provider-list-path would surface only at the very end of a full national
89
+ // build, discarding hours of work for a check that costs microseconds up front.
90
+ if (options.filerDbPath && !options.providerListPath) {
91
+ throw new Error(
92
+ "gazetteer build bdc: --filer-db-path was given without --provider-list-path — filer.db is only read " +
93
+ "to resolve a multi-FRN provider's primary FRN, so it has no effect without a provider list. Pass " +
94
+ "--provider-list-path too, or drop --filer-db-path."
95
+ )
96
+ }
97
+
98
+ let filerDbPath: string | undefined
99
+
100
+ if (options.providerListPath) {
101
+ if (!existsSync(options.providerListPath)) {
102
+ throw new Error(`gazetteer build bdc: --provider-list-path not found: "${options.providerListPath}"`)
103
+ }
104
+
105
+ filerDbPath = options.filerDbPath ?? dataRootPath("filer", "filer.db")
106
+
107
+ if (!existsSync(filerDbPath)) {
108
+ throw new Error(`gazetteer build bdc: filer.db not found at "${filerDbPath}" (--filer-db-path)`)
109
+ }
110
+ }
111
+
112
+ const client = createBDCClient()
113
+
114
+ let asOfDate = options.asOfDate
115
+
116
+ if (!asOfDate) {
117
+ console.error("▸ resolving latest as_of_date...")
118
+
119
+ const dates = await retrieveFilingDates(client, { filingType: BDCFilingDataType.Availability })
120
+ asOfDate = resolveLatestVintage(dates, BDCFilingDataType.Availability)
121
+ }
122
+
123
+ console.error(`▸ as-of-date: ${asOfDate}`)
124
+
125
+ const files = await retrieveAvailabilityFiles(client, {
126
+ asOfDate,
127
+ category: BDCFileCategory.Provider,
128
+ subcategory: BDCProviderSubCategory.FixedBroadband,
129
+ })
130
+
131
+ const stateFiles = files.filter((file) => file.stateCode === options.state)
132
+
133
+ console.error(`▸ ${stateFiles.length} provider file(s) for state ${options.state} @ ${asOfDate}`)
134
+
135
+ const cacheDir = dataRootPath("bdc", "cache", "availability", asOfDate, options.state)
136
+ const csvPaths: string[] = []
137
+
138
+ for (const file of stateFiles) {
139
+ console.error(` downloading ${file.fileName}...`)
140
+
141
+ csvPaths.push(await downloadBDCFile(client, file, cacheDir))
142
+ }
143
+
144
+ // The FCC caps this API at ten requests per minute — six seconds a call — so a national run is
145
+ // throttle-bound by construction and the interesting number is how much of the wall clock went to
146
+ // waiting rather than transferring. Printed once the network phase is over, on stderr with the rest
147
+ // of the progress stream, so a rate change can be assessed against a measurement.
148
+ console.error(`▸ ${formatBDCThrottleStats(client.throttleStats())}`)
149
+
150
+ const out = options.out ?? dataRootPath("bdc", "bdc.db")
151
+ const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
152
+ const tigerDBPath = dataRootPath("tiger", "tiger.db")
153
+
154
+ console.error(`▸ build: ${out}`)
155
+
156
+ // --provider-list-path (3a Task 8, decision 6) opts into populating bdc_provider — omitted, `providers`/
157
+ // `filerDB` stay undefined and buildBDCDatabase's default (byte-identical) path runs unchanged. Both
158
+ // paths were already existsSync-validated above, so this can't ENOENT.
159
+ let filerDB: DatabaseClient<FilerDatabase> | undefined
160
+
161
+ if (filerDbPath) {
162
+ console.error(`▸ provider list: ${options.providerListPath} (filer.db: ${filerDbPath})`)
163
+
164
+ filerDB = new DatabaseClient<FilerDatabase>({ database: new DatabaseSync(filerDbPath, { readOnly: true }) })
165
+ }
166
+
167
+ try {
168
+ const result = await buildBDCDatabase({
169
+ csvPaths,
170
+ out,
171
+ asOfDate,
172
+ buildSHA,
173
+ includeLocationIDs: options.includeLocationIds,
174
+ blockCentroids: createTIGERBlockCentroidLookup(tigerDBPath),
175
+ providers: options.providerListPath ? parseProviderList(options.providerListPath) : undefined,
176
+ filerDB,
177
+ onProgress: (message) => console.error(` [bdc] ${message}`),
178
+ })
179
+
180
+ return [
181
+ `bdc.db: ${out} (${artifactSizeMB(out)} MB)`,
182
+ `${result.rows.toLocaleString()} rows · ${result.providers} provider(s) · ${result.deduped.toLocaleString()} deduped` +
183
+ ` · ${result.unknownGeoids.toLocaleString()} unknown geoid(s) · ${result.coverageCells.toLocaleString()} coverage cells` +
184
+ (options.providerListPath ? ` · ${result.providersPopulated.toLocaleString()} bdc_provider row(s)` : ""),
185
+ `manifest: name=bdc tier=shipped license=public-domain source=fcc-bdc sourceVintage=${asOfDate} buildSHA=${buildSHA}`,
186
+ ]
187
+ } finally {
188
+ await filerDB?.destroy()
189
+ }
190
+ })
191
+
192
+ if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
193
+
194
+ if (state.status === "done") {
195
+ return (
196
+ <Box flexDirection="column">
197
+ {state.result.map((line, i) => (
198
+ <Text key={i} color={i === 0 ? "green" : undefined}>
199
+ {i === 0 ? "✓ " : " "}
200
+ {line}
201
+ </Text>
202
+ ))}
203
+ </Box>
204
+ )
205
+ }
206
+
207
+ return null // progress streams to stderr until the summary lands
208
+ }
209
+
210
+ export default GazetteerBuildBDC
@@ -8,40 +8,164 @@
8
8
  * materialize/seal (`buildPOIDatabase`) logic lives in `gazetteer-pipeline/poi/build-poi.ts`, so it
9
9
  * stays unit-testable without Ink/Pastel in the loop. Mirrors `overture-ingest.tsx`'s progress
10
10
  * (stderr) / summary (stdout) split.
11
+ *
12
+ * `--source osm` (bdc 2b task 3, decisions 3/5): a second, build-local branch alongside the default
13
+ * Overture path — same command, same `buildPOIDatabase` seam, different inputs. It streams
14
+ * `@mailwoman/osm/sdk`'s `extractOSMPOIs` over a Geofabrik `.osm.pbf` extract (telecom-infrastructure
15
+ * categories only, task 2), stamps the invocation's `--country` onto every row (a bare OSM feature
16
+ * carries no country property — see `extract-poi.ts`'s module docstring), and derives res-6 coverage
17
+ * by polyfilling the extract's declared `--bbox` (`bboxCoverageCells`, decision 5) rather than only
18
+ * the cells rows happened to land in — an infra-only extract is sparse by category, so "no plant in a
19
+ * well-surveyed cell" needs its own zero-observed-rows coverage row, not silent absence. The manifest
20
+ * swaps to `tier: build-local`, `license: ODbL-1.0`, OSM attribution — the default Overture branch
21
+ * below is UNTOUCHED and stays byte-identical when `--source` is omitted.
11
22
  */
12
23
 
13
24
  import { execFileSync } from "node:child_process"
14
25
 
26
+ import { LayerTier } from "@mailwoman/core/layers"
15
27
  import { dataRootPath } from "@mailwoman/core/utils"
16
28
  import { Box, Text } from "ink"
17
29
  import zod from "zod"
18
30
 
19
31
  import { type CommandComponent, useCommandTask } from "../../../cli-kit/index.ts"
20
32
  import { artifactSizeMB } from "../../../gazetteer-pipeline/admin/index.ts"
21
- import { buildPOIDatabase, DEFAULT_RELEASE, ingestPlaces } from "../../../gazetteer-pipeline/poi/build-poi.ts"
33
+ import {
34
+ bboxCoverageCells,
35
+ buildPOIDatabase,
36
+ DEFAULT_RELEASE,
37
+ ingestPlaces,
38
+ type BBox,
39
+ type POISourceRow,
40
+ } from "../../../gazetteer-pipeline/poi/build-poi.ts"
22
41
 
23
42
  const DEFAULT_COUNTRIES = "US,CA,MX,FR"
24
43
 
25
44
  const OptionsSchema = zod.object({
26
- release: zod.string().optional().describe(`Pinned Overture release. Default ${DEFAULT_RELEASE}`),
27
- countries: zod.string().optional().describe(`ISO 3166-1 alpha-2, comma-separated. Default ${DEFAULT_COUNTRIES}`),
28
- out: zod.string().optional().describe("poi.db output path. Default <data-root>/poi/poi.db"),
29
- limit: zod.string().optional().describe("Cap rows per country (debug)"),
45
+ source: zod
46
+ .enum(["overture", "osm"])
47
+ .default("overture")
48
+ .describe("Data source: overture (default, shipped) or osm (build-local, ODbL — requires --pbf/--country/--bbox)"),
49
+ release: zod
50
+ .string()
51
+ .optional()
52
+ .describe(`overture: pinned release, default ${DEFAULT_RELEASE}. osm: the extract's vintage/date tag (required)`),
53
+ countries: zod
54
+ .string()
55
+ .optional()
56
+ .describe(`overture only: ISO 3166-1 alpha-2, comma-separated. Default ${DEFAULT_COUNTRIES}`),
57
+ out: zod
58
+ .string()
59
+ .optional()
60
+ .describe("poi.db output path. Default <data-root>/poi/poi.db (overture) or poi/poi-osm-<cc>.db (osm)"),
61
+ limit: zod.string().optional().describe("overture only: cap rows per country (debug)"),
30
62
  skipIngest: zod
31
63
  .boolean()
32
64
  .default(false)
33
- .describe("Skip the DuckDB/S3 ingest; build from already-materialized per-country Parquet"),
65
+ .describe("overture only: skip the DuckDB/S3 ingest; build from already-materialized per-country Parquet"),
66
+ pbf: zod.string().optional().describe("osm: path to a Geofabrik .osm.pbf extract (required)"),
67
+ country: zod
68
+ .string()
69
+ .optional()
70
+ .describe("osm: ISO 3166-1 alpha-2 country stamped onto every extracted row (required)"),
71
+ bbox: zod
72
+ .string()
73
+ .optional()
74
+ .describe("osm: extract bounding box as 'minLon,minLat,maxLon,maxLat' — polyfills res-6 coverage cells (required)"),
34
75
  })
35
76
 
36
77
  export { OptionsSchema as options }
37
78
 
79
+ /**
80
+ * `--bbox` field count: `minLon,minLat,maxLon,maxLat`.
81
+ */
82
+ const BBOX_FIELD_COUNT = 4
83
+
84
+ /**
85
+ * Parse `--bbox "minLon,minLat,maxLon,maxLat"` into a {@link BBox}. Throws with the raw input echoed back on any
86
+ * shape/finiteness mismatch — a silently-mis-parsed bbox would corrupt coverage silently, so fail loud instead.
87
+ */
88
+ function parseBBoxFlag(raw: string): BBox {
89
+ const parts = raw.split(",").map((s) => Number(s.trim()))
90
+
91
+ if (parts.length !== BBOX_FIELD_COUNT || parts.some((n) => !Number.isFinite(n))) {
92
+ throw new Error(
93
+ `--bbox must be 4 comma-separated numbers "minLon,minLat,maxLon,maxLat", got ${JSON.stringify(raw)}`
94
+ )
95
+ }
96
+
97
+ const [minLon, minLat, maxLon, maxLat] = parts as [number, number, number, number]
98
+
99
+ return { minLon, minLat, maxLon, maxLat }
100
+ }
101
+
38
102
  const GazetteerBuildPOI: CommandComponent<typeof OptionsSchema> = ({ options }) => {
39
103
  const state = useCommandTask(async () => {
104
+ const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
105
+
106
+ if (options.source === "osm") {
107
+ const { pbf, country: rawCountry, release, bbox: bboxFlag } = options
108
+
109
+ if (!pbf || !rawCountry || !release || !bboxFlag) {
110
+ throw new Error(
111
+ "gazetteer build poi --source osm requires --pbf <extract.osm.pbf> --country <cc> --release <vintage> " +
112
+ "--bbox <minLon,minLat,maxLon,maxLat>"
113
+ )
114
+ }
115
+
116
+ const country = rawCountry.trim().toUpperCase()
117
+ const bbox = parseBBoxFlag(bboxFlag)
118
+ const out = options.out ?? dataRootPath("poi", `poi-osm-${country.toLowerCase()}.db`)
119
+
120
+ console.error(`▸ extract: OSM telecom-infrastructure POIs from ${pbf} (country=${country})`)
121
+
122
+ // Buffered (not streamed straight into buildPOIDatabase): telecom-infra extracts are
123
+ // category-sparse — a few thousand rows even for a whole country — and the array is walked
124
+ // twice, once here to derive coverage, once as buildPOIDatabase's `rows` seam.
125
+ const rows: POISourceRow[] = []
126
+
127
+ // DYNAMIC import, load-bearing: @mailwoman/osm is UNPUBLISHED (ODbL counsel sign-off
128
+ // pending — see osm/README.md), and Pastel loads every command module eagerly on --help,
129
+ // so a top-level import breaks the whole published CLI on a clean install (caught by the
130
+ // smoke test's clean-install leg, 2026-07-31). The osm source branch is build-local by
131
+ // design; it may only resolve its SDK when actually invoked.
132
+ const { extractOSMPOIs } = await import("@mailwoman/osm/sdk")
133
+
134
+ for await (const row of extractOSMPOIs(pbf)) {
135
+ // Task 3 hand-off from task 2: extractOSMPOIs yields `country: ""` (a bare OSM feature
136
+ // carries no country property) — stamp the invocation's --country before buildPOIDatabase
137
+ // ever sees the row.
138
+ rows.push({ ...row, country })
139
+ }
140
+
141
+ console.error(`▸ build: ${out}`)
142
+
143
+ const coverageCellsOverride = bboxCoverageCells(bbox, rows)
144
+
145
+ const result = await buildPOIDatabase({
146
+ rows,
147
+ out,
148
+ release,
149
+ buildSHA,
150
+ source: "osm",
151
+ tier: LayerTier.BuildLocal,
152
+ coverageCellsOverride,
153
+ createdAt: new Date().toISOString(),
154
+ onProgress: (phase, message) => console.error(` [${phase}] ${message}`),
155
+ })
156
+
157
+ return [
158
+ `poi.db (osm/${country}): ${out} (${artifactSizeMB(out)} MB)`,
159
+ `${result.rows.toLocaleString()} rows · ${result.categories} categories` +
160
+ ` · ${result.skipped.toLocaleString()} skipped (non-finite coords) · ${result.coverageCells.toLocaleString()} coverage cells`,
161
+ `manifest: name=poi tier=build-local source=osm sourceVintage=${release} buildSHA=${buildSHA}`,
162
+ ]
163
+ }
164
+
40
165
  const release = options.release ?? DEFAULT_RELEASE
41
166
  const countries = (options.countries ?? DEFAULT_COUNTRIES).split(",").map((c) => c.trim().toUpperCase())
42
167
  const limit = options.limit ? Number.parseInt(options.limit, 10) : undefined
43
168
  const out = options.out ?? dataRootPath("poi", "poi.db")
44
- const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
45
169
 
46
170
  let parquetPaths: string[]
47
171
 
@@ -0,0 +1,190 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `mailwoman gazetteer census` — build the PCN1 placetype census (hierarchy campaign R4c) from the
7
+ * shipped WOF admin DB. Counts each locality-class parent's children through the projection table
8
+ * (`gazetteer-pipeline/placetype-census.ts`), folds parent surfaces with the SAME `normalizeFSTToken`
9
+ * the PIX1 pair index uses — so a consumer folds once and probes both artifacts — and writes
10
+ * `placetype-census-<country>.bin`.
11
+ *
12
+ * Fold collisions SUM. Two distinct raw parents that fold together ("St Helens" / "St. Helens") are
13
+ * one census node whose counts are the union of both; the serializer refuses duplicate parents, so a
14
+ * merge bug surfaces as a throw rather than a silently halved count.
15
+ *
16
+ * `--delta` is deliberately OPTIONAL and unset by default, unlike the pair index's required one: R4c
17
+ * ships the census as data + loader + offline probe with NO decode wiring. A calibrated delta is a
18
+ * later rung's output; writing one now would put an unmeasured bias into a shipped artifact.
19
+ *
20
+ * Self-verifying (the sealed-artifact spirit): after writing, the command re-reads its own bytes
21
+ * through a fresh `PlacetypeCensusResolver` and probes known parents, printing PROBE OK/MISS with the
22
+ * node's dependent-locality share and lift rather than trusting the write.
23
+ */
24
+
25
+ import { existsSync, writeFileSync } from "node:fs"
26
+ import { join } from "node:path"
27
+
28
+ import type { ComponentTag } from "@mailwoman/core/types"
29
+ import { dataRootPath, md5File } from "@mailwoman/core/utils"
30
+ import { normalizeFSTToken } from "@mailwoman/neural/fst-prior"
31
+ import {
32
+ PlacetypeCensusResolver,
33
+ serializePlacetypeCensus,
34
+ type PlacetypeCensusHeader,
35
+ type PlacetypeCensusNode,
36
+ } from "@mailwoman/neural/placetype-census"
37
+ import { Box, Text } from "ink"
38
+ import zod from "zod"
39
+
40
+ import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
41
+ import { buildPlacetypeCensus, toBaseRates } from "../../gazetteer-pipeline/placetype-census.ts"
42
+
43
+ /**
44
+ * Known parents probed after write, PER COUNTRY. Probing another country's names against a freshly built census prints
45
+ * reassuring-looking misses that verify nothing (the lesson the pair-index command's en-nz first build taught).
46
+ */
47
+ const PROBE_PARENTS_BY_COUNTRY: Readonly<Record<string, readonly string[]>> = {
48
+ gb: ["London", "Manchester", "Birmingham"],
49
+ us: ["New York", "Chicago", "Springfield"],
50
+ }
51
+
52
+ const OptionsSchema = zod.object({
53
+ out: zod.string().default("docs/static/mailwoman").describe("Output dir for placetype-census-<country>.bin"),
54
+ country: zod.string().default("gb").describe("ISO country code this census is built for"),
55
+ source: zod.string().optional().describe("WOF admin DB. Default <data-root>/wof/admin-global-priority.db"),
56
+ delta: zod
57
+ .number()
58
+ .optional()
59
+ .describe(
60
+ "OPTIONAL soft-prior magnitude written into the header. Omit (the default) for an un-wired census — R4c " +
61
+ "ships data + loader + offline probe only; a calibration rung supplies the real value."
62
+ ),
63
+ })
64
+
65
+ export { OptionsSchema as options }
66
+
67
+ const GazetteerCensus: CommandComponent<typeof OptionsSchema> = ({ options }) => {
68
+ const state = useCommandTask(async () => {
69
+ const country = options.country.toLowerCase()
70
+ const sourcePath = options.source ?? String(dataRootPath("wof", "admin-global-priority.db"))
71
+
72
+ if (!existsSync(sourcePath)) {
73
+ throw new Error(`census: source WOF admin DB not found: ${sourcePath}`)
74
+ }
75
+
76
+ const built = buildPlacetypeCensus(sourcePath, country.toUpperCase())
77
+
78
+ if (built.unmappedPlacetypes.length) {
79
+ throw new Error(
80
+ `census: source carries placetypes absent from PLACETYPE_PROJECTION: ${built.unmappedPlacetypes.join(", ")} — ` +
81
+ `extend the projection table (and plan/reference/placetype-evidence.mdx) before building.`
82
+ )
83
+ }
84
+
85
+ // Fold, merging collisions by SUMMING counts — the serializer throws on duplicates, so a merge bug is loud.
86
+ const folded = new Map<string, PlacetypeCensusNode>()
87
+ let collisions = 0
88
+
89
+ for (const node of built.nodes) {
90
+ const parent = normalizeFSTToken(node.parent)
91
+
92
+ if (!parent) continue
93
+
94
+ const existing = folded.get(parent)
95
+
96
+ if (!existing) {
97
+ folded.set(parent, { parent, counts: { ...node.counts }, total: node.total })
98
+
99
+ continue
100
+ }
101
+
102
+ collisions++
103
+
104
+ for (const [tag, n] of Object.entries(node.counts) as Array<[ComponentTag, number]>) {
105
+ existing.counts[tag] = (existing.counts[tag] ?? 0) + n
106
+ }
107
+
108
+ existing.total += node.total
109
+ }
110
+
111
+ const nodes = [...folded.values()]
112
+ const baseRates = toBaseRates(built.countryTotals)
113
+ const sourceMD5 = await md5File(sourcePath)
114
+
115
+ const header: PlacetypeCensusHeader = {
116
+ country,
117
+ schemaVersion: 1,
118
+ foldVersion: 1,
119
+ sourceMD5s: [sourceMD5],
120
+ buildDate: new Date().toISOString(),
121
+ baseRates,
122
+ ...(options.delta !== undefined ? { delta: options.delta } : {}),
123
+ }
124
+
125
+ const bytes = serializePlacetypeCensus(header, nodes)
126
+ const outPath = join(options.out, `placetype-census-${country}.bin`)
127
+
128
+ writeFileSync(outPath, bytes)
129
+
130
+ // Self-verifying readback over the bytes just written, not the in-memory nodes.
131
+ const resolver = new PlacetypeCensusResolver(bytes)
132
+ const probeParents = PROBE_PARENTS_BY_COUNTRY[country]
133
+
134
+ if (!probeParents) {
135
+ throw new Error(
136
+ `census: no self-check probe parents registered for country "${country}" — add an entry to ` +
137
+ `PROBE_PARENTS_BY_COUNTRY (probing another country's names verifies nothing).`
138
+ )
139
+ }
140
+
141
+ const probeLines = probeParents.map((raw) => {
142
+ const parent = normalizeFSTToken(raw)
143
+ const node = resolver.probe(parent)
144
+
145
+ if (!node) return `PROBE MISS: fold("${raw}") → "${parent}" → (no node)`
146
+
147
+ const depLoc = node.counts.dependent_locality ?? 0
148
+ const share = resolver.share(parent, "dependent_locality")
149
+ const lift = resolver.lift(parent, "dependent_locality")
150
+
151
+ return (
152
+ `PROBE OK: "${parent}" → ${node.total.toLocaleString()} children, ` +
153
+ `${depLoc.toLocaleString()} dependent_locality (share ${(share * 100).toFixed(1)}%, lift ${lift.toFixed(1)}×)`
154
+ )
155
+ })
156
+
157
+ const baseRateLines = (Object.entries(baseRates) as Array<[ComponentTag, number]>)
158
+ .toSorted((a, b) => b[1] - a[1])
159
+ .map(([tag, rate]) => ` ${tag}: ${(rate * 100).toFixed(2)}%`)
160
+
161
+ return [
162
+ `placetype-census-${country}.bin → ${outPath} (${bytes.length.toLocaleString()} bytes)`,
163
+ `header: ${options.delta !== undefined ? `delta=${options.delta}` : "(no delta — un-wired census)"}`,
164
+ `nodes: ${nodes.length.toLocaleString()} (${built.nodes.length.toLocaleString()} pre-fold, ${collisions.toLocaleString()} fold collisions merged)`,
165
+ `child links counted: ${built.links.toLocaleString()}`,
166
+ "country base rates:",
167
+ ...baseRateLines,
168
+ ...probeLines,
169
+ ]
170
+ })
171
+
172
+ if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
173
+
174
+ if (state.status === "done") {
175
+ return (
176
+ <Box flexDirection="column">
177
+ {state.result.map((line, i) => (
178
+ <Text key={i} color={i === 0 ? "green" : undefined}>
179
+ {i === 0 ? "✓ " : " "}
180
+ {line}
181
+ </Text>
182
+ ))}
183
+ </Box>
184
+ )
185
+ }
186
+
187
+ return null
188
+ }
189
+
190
+ export default GazetteerCensus