mailwoman 8.3.0 → 8.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/filer/linkage-eval.tsx +57 -0
- package/commands/gazetteer/build/bdc.tsx +210 -0
- package/commands/gazetteer/build/poi.tsx +131 -7
- package/commands/gazetteer/census.tsx +190 -0
- package/commands/gazetteer/pair-index.tsx +158 -24
- package/eval-harness/gates/v7.0.0-base.json +1 -0
- package/eval-harness/gauntlet/cases/regression.ts +1112 -3
- package/eval-harness/gauntlet/harness.ts +61 -5
- package/eval-harness/gauntlet/regression.ts +12 -2
- package/eval-harness/promotion-gate.ts +3 -1
- package/gazetteer-pipeline/borough-pairs.ts +0 -0
- package/gazetteer-pipeline/lieudit-pairs.ts +0 -0
- package/gazetteer-pipeline/pair-index.ts +8 -0
- package/gazetteer-pipeline/placetype-census.ts +191 -0
- package/gazetteer-pipeline/poi/build-poi.ts +114 -20
- package/geocode-core.ts +15 -0
- package/out/commands/filer/linkage-eval.d.ts +20 -0
- package/out/commands/filer/linkage-eval.d.ts.map +1 -0
- package/out/commands/filer/linkage-eval.js +40 -0
- package/out/commands/filer/linkage-eval.js.map +1 -0
- package/out/commands/gazetteer/build/bdc.d.ts +43 -0
- package/out/commands/gazetteer/build/bdc.d.ts.map +1 -0
- package/out/commands/gazetteer/build/bdc.js +153 -0
- package/out/commands/gazetteer/build/bdc.js.map +1 -0
- package/out/commands/gazetteer/build/poi.d.ts +18 -0
- package/out/commands/gazetteer/build/poi.d.ts.map +1 -1
- package/out/commands/gazetteer/build/poi.js +103 -7
- package/out/commands/gazetteer/build/poi.js.map +1 -1
- package/out/commands/gazetteer/census.d.ts +35 -0
- package/out/commands/gazetteer/census.d.ts.map +1 -0
- package/out/commands/gazetteer/census.js +137 -0
- package/out/commands/gazetteer/census.js.map +1 -0
- package/out/commands/gazetteer/pair-index.d.ts +3 -0
- package/out/commands/gazetteer/pair-index.d.ts.map +1 -1
- package/out/commands/gazetteer/pair-index.js +129 -23
- package/out/commands/gazetteer/pair-index.js.map +1 -1
- package/out/eval-harness/gauntlet/cases/regression.d.ts.map +1 -1
- package/out/eval-harness/gauntlet/cases/regression.js +1106 -3
- package/out/eval-harness/gauntlet/cases/regression.js.map +1 -1
- package/out/eval-harness/gauntlet/harness.d.ts +9 -0
- package/out/eval-harness/gauntlet/harness.d.ts.map +1 -1
- package/out/eval-harness/gauntlet/harness.js +48 -8
- package/out/eval-harness/gauntlet/harness.js.map +1 -1
- package/out/eval-harness/gauntlet/regression.d.ts.map +1 -1
- package/out/eval-harness/gauntlet/regression.js +12 -2
- package/out/eval-harness/gauntlet/regression.js.map +1 -1
- package/out/eval-harness/promotion-gate.d.ts.map +1 -1
- package/out/eval-harness/promotion-gate.js +3 -1
- package/out/eval-harness/promotion-gate.js.map +1 -1
- package/out/gazetteer-pipeline/borough-pairs.d.ts +33 -0
- package/out/gazetteer-pipeline/borough-pairs.d.ts.map +1 -0
- package/out/gazetteer-pipeline/borough-pairs.js +0 -0
- package/out/gazetteer-pipeline/borough-pairs.js.map +1 -0
- package/out/gazetteer-pipeline/lieudit-pairs.d.ts +58 -0
- package/out/gazetteer-pipeline/lieudit-pairs.d.ts.map +1 -0
- package/out/gazetteer-pipeline/lieudit-pairs.js +0 -0
- package/out/gazetteer-pipeline/lieudit-pairs.js.map +1 -0
- package/out/gazetteer-pipeline/pair-index.d.ts +5 -0
- package/out/gazetteer-pipeline/pair-index.d.ts.map +1 -1
- package/out/gazetteer-pipeline/pair-index.js +7 -0
- package/out/gazetteer-pipeline/pair-index.js.map +1 -1
- package/out/gazetteer-pipeline/placetype-census.d.ts +70 -0
- package/out/gazetteer-pipeline/placetype-census.d.ts.map +1 -0
- package/out/gazetteer-pipeline/placetype-census.js +142 -0
- package/out/gazetteer-pipeline/placetype-census.js.map +1 -0
- package/out/gazetteer-pipeline/poi/build-poi.d.ts +51 -1
- package/out/gazetteer-pipeline/poi/build-poi.d.ts.map +1 -1
- package/out/gazetteer-pipeline/poi/build-poi.js +70 -17
- package/out/gazetteer-pipeline/poi/build-poi.js.map +1 -1
- package/out/geocode-core.d.ts +13 -0
- package/out/geocode-core.d.ts.map +1 -1
- package/out/geocode-core.js +2 -0
- package/out/geocode-core.js.map +1 -1
- package/package.json +25 -23
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `mailwoman filer linkage-eval` — the 3b Task 4 corporate-family recovery eval. Geocode-free,
|
|
7
|
+
* database-free (builds its own scratch `filer.db` artifacts from an embedded synthetic corpus); emits
|
|
8
|
+
* the markdown scorecard to stdout and, when `--out-md` is given, to that path too. `--date` overrides
|
|
9
|
+
* the report's dated H1 so the committed scorecard can be regenerated without editing code.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { filerLinkageEval } from "@mailwoman/filer/tools"
|
|
13
|
+
import { Text } from "ink"
|
|
14
|
+
import zod from "zod"
|
|
15
|
+
|
|
16
|
+
import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
|
|
17
|
+
|
|
18
|
+
const OptionsSchema = zod.object({
|
|
19
|
+
outMd: zod.string().optional().describe("Also write the markdown report here"),
|
|
20
|
+
date: zod.string().optional().describe("Report date (YYYY-MM-DD) for the H1 — defaults to today"),
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
export { OptionsSchema as options }
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* `null` renders as `N/A`, never as `0.000` — the withheld run makes no positive call, so its precision and F1 are
|
|
27
|
+
* undefined rather than zero (see `linkage-metrics.ts`).
|
|
28
|
+
*/
|
|
29
|
+
function formatScoreValue(value: number | null): string {
|
|
30
|
+
return value === null ? "N/A" : value.toFixed(3)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const FilerLinkageEval: CommandComponent<typeof OptionsSchema> = ({ options }) => {
|
|
34
|
+
const state = useCommandTask(() =>
|
|
35
|
+
filerLinkageEval({ outMd: options.outMd, date: options.date }, (line) => console.error(line))
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
|
|
39
|
+
|
|
40
|
+
if (state.status === "done") {
|
|
41
|
+
const { withheld, control } = state.result
|
|
42
|
+
|
|
43
|
+
// Both rows, always: the withheld number is the measurement and the control is what makes it falsifiable.
|
|
44
|
+
return (
|
|
45
|
+
<Text color="green">
|
|
46
|
+
linkage-eval · withheld: recall {formatScoreValue(withheld.score.recall)} ({withheld.score.truePositivePairs}/
|
|
47
|
+
{withheld.score.truthPositivePairs} pairs, F1 {formatScoreValue(withheld.score.f1)}) · control: F1{" "}
|
|
48
|
+
{formatScoreValue(control.score.f1)} ({control.score.truePositivePairs}/{control.score.truthPositivePairs}{" "}
|
|
49
|
+
pairs)
|
|
50
|
+
</Text>
|
|
51
|
+
)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return null
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export default FilerLinkageEval
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `mailwoman gazetteer build bdc` — the FCC BDC availability ingest + sealed `bdc.db` layer build
|
|
7
|
+
* (2a Task 8). Thin wiring only: list → download → build lives in `@mailwoman/bdc/sdk`
|
|
8
|
+
* (`buildBDCDatabase`, list-files.ts, download.ts), so it stays unit-testable without Ink/Pastel or
|
|
9
|
+
* network in the loop. Mirrors `poi.tsx`'s progress (stderr) / summary (stdout) split.
|
|
10
|
+
*
|
|
11
|
+
* First cut wires Fixed Broadband provider availability only (the primary wireline dataset) — mobile
|
|
12
|
+
* broadband/voice subcategories are a future flag, not a scope gap in this command's shape.
|
|
13
|
+
*
|
|
14
|
+
* `--provider-list-path` (3a Task 8, decision 6) opts INTO populating `bdc_provider` —
|
|
15
|
+
* `bdc_availability` itself is unaffected either way. When given, `parseProviderList`
|
|
16
|
+
* (`@mailwoman/filer/sdk`) streams the FCC provider-list CSV as `BuildBDCOptions.providers`, and
|
|
17
|
+
* filer.db (`--filer-db-path`, default `<data-root>/filer/filer.db`) is opened read-only to resolve
|
|
18
|
+
* each multi-FRN provider's primary FRN (`readFRNFilingCandidates` + `pickPrimaryFRN`, imported
|
|
19
|
+
* rather than reimplemented — see `build-bdc.ts`'s `BuildBDCOptions.filerDB` docstring). Omitting
|
|
20
|
+
* `--provider-list-path` leaves `bdc_provider` empty, exactly as before Task 8.
|
|
21
|
+
*
|
|
22
|
+
* Both paths are `existsSync`-guarded BEFORE any download/build work starts (review fix round 1,
|
|
23
|
+
* IMPORTANT-2): `populateBDCProviderTable` only runs after `writeLayerManifest`, i.e. at the very END
|
|
24
|
+
* of a full build — an unguarded typo'd `--provider-list-path` would otherwise surface as a raw ENOENT
|
|
25
|
+
* only after a nationwide availability ingest had already finished, discarding hours of work.
|
|
26
|
+
* `--filer-db-path` given without `--provider-list-path` is a loud error, not a silent no-op (MINOR):
|
|
27
|
+
* filer.db is only ever read to resolve a multi-FRN primary FRN, so it does nothing without a provider
|
|
28
|
+
* list to resolve FRNs FOR.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { execFileSync } from "node:child_process"
|
|
32
|
+
import { existsSync } from "node:fs"
|
|
33
|
+
import { DatabaseSync } from "node:sqlite"
|
|
34
|
+
|
|
35
|
+
import {
|
|
36
|
+
BDCFileCategory,
|
|
37
|
+
BDCFilingDataType,
|
|
38
|
+
BDCProviderSubCategory,
|
|
39
|
+
buildBDCDatabase,
|
|
40
|
+
createBDCClient,
|
|
41
|
+
createTIGERBlockCentroidLookup,
|
|
42
|
+
downloadBDCFile,
|
|
43
|
+
formatBDCThrottleStats,
|
|
44
|
+
resolveLatestVintage,
|
|
45
|
+
retrieveAvailabilityFiles,
|
|
46
|
+
retrieveFilingDates,
|
|
47
|
+
} from "@mailwoman/bdc/sdk"
|
|
48
|
+
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
49
|
+
import { dataRootPath } from "@mailwoman/core/utils"
|
|
50
|
+
import type { FilerDatabase } from "@mailwoman/filer"
|
|
51
|
+
import { parseProviderList } from "@mailwoman/filer/sdk"
|
|
52
|
+
import { Box, Text } from "ink"
|
|
53
|
+
import zod from "zod"
|
|
54
|
+
|
|
55
|
+
import { type CommandComponent, useCommandTask } from "../../../cli-kit/index.ts"
|
|
56
|
+
import { artifactSizeMB } from "../../../gazetteer-pipeline/admin/index.ts"
|
|
57
|
+
|
|
58
|
+
const OptionsSchema = zod.object({
|
|
59
|
+
state: zod.string().describe('2-digit FCC state/territory FIPS code (e.g. "06" for California)'),
|
|
60
|
+
asOfDate: zod.string().optional().describe("FCC filing as_of_date (YYYY-MM-DD). Default: latest available"),
|
|
61
|
+
out: zod.string().optional().describe("bdc.db output path. Default <data-root>/bdc/bdc.db"),
|
|
62
|
+
includeLocationIds: zod
|
|
63
|
+
.boolean()
|
|
64
|
+
.default(false)
|
|
65
|
+
.describe("Populate bdc_availability.location_id (opaque BSL join key; NULL by default)"),
|
|
66
|
+
providerListPath: zod
|
|
67
|
+
.string()
|
|
68
|
+
.optional()
|
|
69
|
+
.describe(
|
|
70
|
+
"Path to the FCC BDC provider list CSV (frn/provider_id/holding_company). When given, populates " +
|
|
71
|
+
"bdc_provider (decision 6, 3a task 8); omit to leave bdc_provider empty, as before task 8."
|
|
72
|
+
),
|
|
73
|
+
filerDbPath: zod
|
|
74
|
+
.string()
|
|
75
|
+
.optional()
|
|
76
|
+
.describe(
|
|
77
|
+
"Path to filer.db, used to resolve a multi-FRN provider's primary FRN when --provider-list-path is given. " +
|
|
78
|
+
"Default <data-root>/filer/filer.db"
|
|
79
|
+
),
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
export { OptionsSchema as options }
|
|
83
|
+
|
|
84
|
+
const GazetteerBuildBDC: CommandComponent<typeof OptionsSchema> = ({ options }) => {
|
|
85
|
+
const state = useCommandTask(async () => {
|
|
86
|
+
// Fail-fast guards (review fix round 1, IMPORTANT-2 / MINOR) — checked BEFORE any network/download work
|
|
87
|
+
// starts. `populateBDCProviderTable` only runs after the availability ingest AND writeLayerManifest, so
|
|
88
|
+
// without this, a typo'd --provider-list-path would surface only at the very end of a full national
|
|
89
|
+
// build, discarding hours of work for a check that costs microseconds up front.
|
|
90
|
+
if (options.filerDbPath && !options.providerListPath) {
|
|
91
|
+
throw new Error(
|
|
92
|
+
"gazetteer build bdc: --filer-db-path was given without --provider-list-path — filer.db is only read " +
|
|
93
|
+
"to resolve a multi-FRN provider's primary FRN, so it has no effect without a provider list. Pass " +
|
|
94
|
+
"--provider-list-path too, or drop --filer-db-path."
|
|
95
|
+
)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let filerDbPath: string | undefined
|
|
99
|
+
|
|
100
|
+
if (options.providerListPath) {
|
|
101
|
+
if (!existsSync(options.providerListPath)) {
|
|
102
|
+
throw new Error(`gazetteer build bdc: --provider-list-path not found: "${options.providerListPath}"`)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
filerDbPath = options.filerDbPath ?? dataRootPath("filer", "filer.db")
|
|
106
|
+
|
|
107
|
+
if (!existsSync(filerDbPath)) {
|
|
108
|
+
throw new Error(`gazetteer build bdc: filer.db not found at "${filerDbPath}" (--filer-db-path)`)
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const client = createBDCClient()
|
|
113
|
+
|
|
114
|
+
let asOfDate = options.asOfDate
|
|
115
|
+
|
|
116
|
+
if (!asOfDate) {
|
|
117
|
+
console.error("▸ resolving latest as_of_date...")
|
|
118
|
+
|
|
119
|
+
const dates = await retrieveFilingDates(client, { filingType: BDCFilingDataType.Availability })
|
|
120
|
+
asOfDate = resolveLatestVintage(dates, BDCFilingDataType.Availability)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
console.error(`▸ as-of-date: ${asOfDate}`)
|
|
124
|
+
|
|
125
|
+
const files = await retrieveAvailabilityFiles(client, {
|
|
126
|
+
asOfDate,
|
|
127
|
+
category: BDCFileCategory.Provider,
|
|
128
|
+
subcategory: BDCProviderSubCategory.FixedBroadband,
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
const stateFiles = files.filter((file) => file.stateCode === options.state)
|
|
132
|
+
|
|
133
|
+
console.error(`▸ ${stateFiles.length} provider file(s) for state ${options.state} @ ${asOfDate}`)
|
|
134
|
+
|
|
135
|
+
const cacheDir = dataRootPath("bdc", "cache", "availability", asOfDate, options.state)
|
|
136
|
+
const csvPaths: string[] = []
|
|
137
|
+
|
|
138
|
+
for (const file of stateFiles) {
|
|
139
|
+
console.error(` downloading ${file.fileName}...`)
|
|
140
|
+
|
|
141
|
+
csvPaths.push(await downloadBDCFile(client, file, cacheDir))
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// The FCC caps this API at ten requests per minute — six seconds a call — so a national run is
|
|
145
|
+
// throttle-bound by construction and the interesting number is how much of the wall clock went to
|
|
146
|
+
// waiting rather than transferring. Printed once the network phase is over, on stderr with the rest
|
|
147
|
+
// of the progress stream, so a rate change can be assessed against a measurement.
|
|
148
|
+
console.error(`▸ ${formatBDCThrottleStats(client.throttleStats())}`)
|
|
149
|
+
|
|
150
|
+
const out = options.out ?? dataRootPath("bdc", "bdc.db")
|
|
151
|
+
const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
|
|
152
|
+
const tigerDBPath = dataRootPath("tiger", "tiger.db")
|
|
153
|
+
|
|
154
|
+
console.error(`▸ build: ${out}`)
|
|
155
|
+
|
|
156
|
+
// --provider-list-path (3a Task 8, decision 6) opts into populating bdc_provider — omitted, `providers`/
|
|
157
|
+
// `filerDB` stay undefined and buildBDCDatabase's default (byte-identical) path runs unchanged. Both
|
|
158
|
+
// paths were already existsSync-validated above, so this can't ENOENT.
|
|
159
|
+
let filerDB: DatabaseClient<FilerDatabase> | undefined
|
|
160
|
+
|
|
161
|
+
if (filerDbPath) {
|
|
162
|
+
console.error(`▸ provider list: ${options.providerListPath} (filer.db: ${filerDbPath})`)
|
|
163
|
+
|
|
164
|
+
filerDB = new DatabaseClient<FilerDatabase>({ database: new DatabaseSync(filerDbPath, { readOnly: true }) })
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
try {
|
|
168
|
+
const result = await buildBDCDatabase({
|
|
169
|
+
csvPaths,
|
|
170
|
+
out,
|
|
171
|
+
asOfDate,
|
|
172
|
+
buildSHA,
|
|
173
|
+
includeLocationIDs: options.includeLocationIds,
|
|
174
|
+
blockCentroids: createTIGERBlockCentroidLookup(tigerDBPath),
|
|
175
|
+
providers: options.providerListPath ? parseProviderList(options.providerListPath) : undefined,
|
|
176
|
+
filerDB,
|
|
177
|
+
onProgress: (message) => console.error(` [bdc] ${message}`),
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
return [
|
|
181
|
+
`bdc.db: ${out} (${artifactSizeMB(out)} MB)`,
|
|
182
|
+
`${result.rows.toLocaleString()} rows · ${result.providers} provider(s) · ${result.deduped.toLocaleString()} deduped` +
|
|
183
|
+
` · ${result.unknownGeoids.toLocaleString()} unknown geoid(s) · ${result.coverageCells.toLocaleString()} coverage cells` +
|
|
184
|
+
(options.providerListPath ? ` · ${result.providersPopulated.toLocaleString()} bdc_provider row(s)` : ""),
|
|
185
|
+
`manifest: name=bdc tier=shipped license=public-domain source=fcc-bdc sourceVintage=${asOfDate} buildSHA=${buildSHA}`,
|
|
186
|
+
]
|
|
187
|
+
} finally {
|
|
188
|
+
await filerDB?.destroy()
|
|
189
|
+
}
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
|
|
193
|
+
|
|
194
|
+
if (state.status === "done") {
|
|
195
|
+
return (
|
|
196
|
+
<Box flexDirection="column">
|
|
197
|
+
{state.result.map((line, i) => (
|
|
198
|
+
<Text key={i} color={i === 0 ? "green" : undefined}>
|
|
199
|
+
{i === 0 ? "✓ " : " "}
|
|
200
|
+
{line}
|
|
201
|
+
</Text>
|
|
202
|
+
))}
|
|
203
|
+
</Box>
|
|
204
|
+
)
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return null // progress streams to stderr until the summary lands
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
export default GazetteerBuildBDC
|
|
@@ -8,40 +8,164 @@
|
|
|
8
8
|
* materialize/seal (`buildPOIDatabase`) logic lives in `gazetteer-pipeline/poi/build-poi.ts`, so it
|
|
9
9
|
* stays unit-testable without Ink/Pastel in the loop. Mirrors `overture-ingest.tsx`'s progress
|
|
10
10
|
* (stderr) / summary (stdout) split.
|
|
11
|
+
*
|
|
12
|
+
* `--source osm` (bdc 2b task 3, decisions 3/5): a second, build-local branch alongside the default
|
|
13
|
+
* Overture path — same command, same `buildPOIDatabase` seam, different inputs. It streams
|
|
14
|
+
* `@mailwoman/osm/sdk`'s `extractOSMPOIs` over a Geofabrik `.osm.pbf` extract (telecom-infrastructure
|
|
15
|
+
* categories only, task 2), stamps the invocation's `--country` onto every row (a bare OSM feature
|
|
16
|
+
* carries no country property — see `extract-poi.ts`'s module docstring), and derives res-6 coverage
|
|
17
|
+
* by polyfilling the extract's declared `--bbox` (`bboxCoverageCells`, decision 5) rather than only
|
|
18
|
+
* the cells rows happened to land in — an infra-only extract is sparse by category, so "no plant in a
|
|
19
|
+
* well-surveyed cell" needs its own zero-observed-rows coverage row, not silent absence. The manifest
|
|
20
|
+
* swaps to `tier: build-local`, `license: ODbL-1.0`, OSM attribution — the default Overture branch
|
|
21
|
+
* below is UNTOUCHED and stays byte-identical when `--source` is omitted.
|
|
11
22
|
*/
|
|
12
23
|
|
|
13
24
|
import { execFileSync } from "node:child_process"
|
|
14
25
|
|
|
26
|
+
import { LayerTier } from "@mailwoman/core/layers"
|
|
15
27
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
16
28
|
import { Box, Text } from "ink"
|
|
17
29
|
import zod from "zod"
|
|
18
30
|
|
|
19
31
|
import { type CommandComponent, useCommandTask } from "../../../cli-kit/index.ts"
|
|
20
32
|
import { artifactSizeMB } from "../../../gazetteer-pipeline/admin/index.ts"
|
|
21
|
-
import {
|
|
33
|
+
import {
|
|
34
|
+
bboxCoverageCells,
|
|
35
|
+
buildPOIDatabase,
|
|
36
|
+
DEFAULT_RELEASE,
|
|
37
|
+
ingestPlaces,
|
|
38
|
+
type BBox,
|
|
39
|
+
type POISourceRow,
|
|
40
|
+
} from "../../../gazetteer-pipeline/poi/build-poi.ts"
|
|
22
41
|
|
|
23
42
|
const DEFAULT_COUNTRIES = "US,CA,MX,FR"
|
|
24
43
|
|
|
25
44
|
const OptionsSchema = zod.object({
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
45
|
+
source: zod
|
|
46
|
+
.enum(["overture", "osm"])
|
|
47
|
+
.default("overture")
|
|
48
|
+
.describe("Data source: overture (default, shipped) or osm (build-local, ODbL — requires --pbf/--country/--bbox)"),
|
|
49
|
+
release: zod
|
|
50
|
+
.string()
|
|
51
|
+
.optional()
|
|
52
|
+
.describe(`overture: pinned release, default ${DEFAULT_RELEASE}. osm: the extract's vintage/date tag (required)`),
|
|
53
|
+
countries: zod
|
|
54
|
+
.string()
|
|
55
|
+
.optional()
|
|
56
|
+
.describe(`overture only: ISO 3166-1 alpha-2, comma-separated. Default ${DEFAULT_COUNTRIES}`),
|
|
57
|
+
out: zod
|
|
58
|
+
.string()
|
|
59
|
+
.optional()
|
|
60
|
+
.describe("poi.db output path. Default <data-root>/poi/poi.db (overture) or poi/poi-osm-<cc>.db (osm)"),
|
|
61
|
+
limit: zod.string().optional().describe("overture only: cap rows per country (debug)"),
|
|
30
62
|
skipIngest: zod
|
|
31
63
|
.boolean()
|
|
32
64
|
.default(false)
|
|
33
|
-
.describe("
|
|
65
|
+
.describe("overture only: skip the DuckDB/S3 ingest; build from already-materialized per-country Parquet"),
|
|
66
|
+
pbf: zod.string().optional().describe("osm: path to a Geofabrik .osm.pbf extract (required)"),
|
|
67
|
+
country: zod
|
|
68
|
+
.string()
|
|
69
|
+
.optional()
|
|
70
|
+
.describe("osm: ISO 3166-1 alpha-2 country stamped onto every extracted row (required)"),
|
|
71
|
+
bbox: zod
|
|
72
|
+
.string()
|
|
73
|
+
.optional()
|
|
74
|
+
.describe("osm: extract bounding box as 'minLon,minLat,maxLon,maxLat' — polyfills res-6 coverage cells (required)"),
|
|
34
75
|
})
|
|
35
76
|
|
|
36
77
|
export { OptionsSchema as options }
|
|
37
78
|
|
|
79
|
+
/**
|
|
80
|
+
* `--bbox` field count: `minLon,minLat,maxLon,maxLat`.
|
|
81
|
+
*/
|
|
82
|
+
const BBOX_FIELD_COUNT = 4
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Parse `--bbox "minLon,minLat,maxLon,maxLat"` into a {@link BBox}. Throws with the raw input echoed back on any
|
|
86
|
+
* shape/finiteness mismatch — a silently-mis-parsed bbox would corrupt coverage silently, so fail loud instead.
|
|
87
|
+
*/
|
|
88
|
+
function parseBBoxFlag(raw: string): BBox {
|
|
89
|
+
const parts = raw.split(",").map((s) => Number(s.trim()))
|
|
90
|
+
|
|
91
|
+
if (parts.length !== BBOX_FIELD_COUNT || parts.some((n) => !Number.isFinite(n))) {
|
|
92
|
+
throw new Error(
|
|
93
|
+
`--bbox must be 4 comma-separated numbers "minLon,minLat,maxLon,maxLat", got ${JSON.stringify(raw)}`
|
|
94
|
+
)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const [minLon, minLat, maxLon, maxLat] = parts as [number, number, number, number]
|
|
98
|
+
|
|
99
|
+
return { minLon, minLat, maxLon, maxLat }
|
|
100
|
+
}
|
|
101
|
+
|
|
38
102
|
const GazetteerBuildPOI: CommandComponent<typeof OptionsSchema> = ({ options }) => {
|
|
39
103
|
const state = useCommandTask(async () => {
|
|
104
|
+
const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
|
|
105
|
+
|
|
106
|
+
if (options.source === "osm") {
|
|
107
|
+
const { pbf, country: rawCountry, release, bbox: bboxFlag } = options
|
|
108
|
+
|
|
109
|
+
if (!pbf || !rawCountry || !release || !bboxFlag) {
|
|
110
|
+
throw new Error(
|
|
111
|
+
"gazetteer build poi --source osm requires --pbf <extract.osm.pbf> --country <cc> --release <vintage> " +
|
|
112
|
+
"--bbox <minLon,minLat,maxLon,maxLat>"
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const country = rawCountry.trim().toUpperCase()
|
|
117
|
+
const bbox = parseBBoxFlag(bboxFlag)
|
|
118
|
+
const out = options.out ?? dataRootPath("poi", `poi-osm-${country.toLowerCase()}.db`)
|
|
119
|
+
|
|
120
|
+
console.error(`▸ extract: OSM telecom-infrastructure POIs from ${pbf} (country=${country})`)
|
|
121
|
+
|
|
122
|
+
// Buffered (not streamed straight into buildPOIDatabase): telecom-infra extracts are
|
|
123
|
+
// category-sparse — a few thousand rows even for a whole country — and the array is walked
|
|
124
|
+
// twice, once here to derive coverage, once as buildPOIDatabase's `rows` seam.
|
|
125
|
+
const rows: POISourceRow[] = []
|
|
126
|
+
|
|
127
|
+
// DYNAMIC import, load-bearing: @mailwoman/osm is UNPUBLISHED (ODbL counsel sign-off
|
|
128
|
+
// pending — see osm/README.md), and Pastel loads every command module eagerly on --help,
|
|
129
|
+
// so a top-level import breaks the whole published CLI on a clean install (caught by the
|
|
130
|
+
// smoke test's clean-install leg, 2026-07-31). The osm source branch is build-local by
|
|
131
|
+
// design; it may only resolve its SDK when actually invoked.
|
|
132
|
+
const { extractOSMPOIs } = await import("@mailwoman/osm/sdk")
|
|
133
|
+
|
|
134
|
+
for await (const row of extractOSMPOIs(pbf)) {
|
|
135
|
+
// Task 3 hand-off from task 2: extractOSMPOIs yields `country: ""` (a bare OSM feature
|
|
136
|
+
// carries no country property) — stamp the invocation's --country before buildPOIDatabase
|
|
137
|
+
// ever sees the row.
|
|
138
|
+
rows.push({ ...row, country })
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
console.error(`▸ build: ${out}`)
|
|
142
|
+
|
|
143
|
+
const coverageCellsOverride = bboxCoverageCells(bbox, rows)
|
|
144
|
+
|
|
145
|
+
const result = await buildPOIDatabase({
|
|
146
|
+
rows,
|
|
147
|
+
out,
|
|
148
|
+
release,
|
|
149
|
+
buildSHA,
|
|
150
|
+
source: "osm",
|
|
151
|
+
tier: LayerTier.BuildLocal,
|
|
152
|
+
coverageCellsOverride,
|
|
153
|
+
createdAt: new Date().toISOString(),
|
|
154
|
+
onProgress: (phase, message) => console.error(` [${phase}] ${message}`),
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
return [
|
|
158
|
+
`poi.db (osm/${country}): ${out} (${artifactSizeMB(out)} MB)`,
|
|
159
|
+
`${result.rows.toLocaleString()} rows · ${result.categories} categories` +
|
|
160
|
+
` · ${result.skipped.toLocaleString()} skipped (non-finite coords) · ${result.coverageCells.toLocaleString()} coverage cells`,
|
|
161
|
+
`manifest: name=poi tier=build-local source=osm sourceVintage=${release} buildSHA=${buildSHA}`,
|
|
162
|
+
]
|
|
163
|
+
}
|
|
164
|
+
|
|
40
165
|
const release = options.release ?? DEFAULT_RELEASE
|
|
41
166
|
const countries = (options.countries ?? DEFAULT_COUNTRIES).split(",").map((c) => c.trim().toUpperCase())
|
|
42
167
|
const limit = options.limit ? Number.parseInt(options.limit, 10) : undefined
|
|
43
168
|
const out = options.out ?? dataRootPath("poi", "poi.db")
|
|
44
|
-
const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
|
|
45
169
|
|
|
46
170
|
let parquetPaths: string[]
|
|
47
171
|
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `mailwoman gazetteer census` — build the PCN1 placetype census (hierarchy campaign R4c) from the
|
|
7
|
+
* shipped WOF admin DB. Counts each locality-class parent's children through the projection table
|
|
8
|
+
* (`gazetteer-pipeline/placetype-census.ts`), folds parent surfaces with the SAME `normalizeFSTToken`
|
|
9
|
+
* the PIX1 pair index uses — so a consumer folds once and probes both artifacts — and writes
|
|
10
|
+
* `placetype-census-<country>.bin`.
|
|
11
|
+
*
|
|
12
|
+
* Fold collisions SUM. Two distinct raw parents that fold together ("St Helens" / "St. Helens") are
|
|
13
|
+
* one census node whose counts are the union of both; the serializer refuses duplicate parents, so a
|
|
14
|
+
* merge bug surfaces as a throw rather than a silently halved count.
|
|
15
|
+
*
|
|
16
|
+
* `--delta` is deliberately OPTIONAL and unset by default, unlike the pair index's required one: R4c
|
|
17
|
+
* ships the census as data + loader + offline probe with NO decode wiring. A calibrated delta is a
|
|
18
|
+
* later rung's output; writing one now would put an unmeasured bias into a shipped artifact.
|
|
19
|
+
*
|
|
20
|
+
* Self-verifying (the sealed-artifact spirit): after writing, the command re-reads its own bytes
|
|
21
|
+
* through a fresh `PlacetypeCensusResolver` and probes known parents, printing PROBE OK/MISS with the
|
|
22
|
+
* node's dependent-locality share and lift rather than trusting the write.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { existsSync, writeFileSync } from "node:fs"
|
|
26
|
+
import { join } from "node:path"
|
|
27
|
+
|
|
28
|
+
import type { ComponentTag } from "@mailwoman/core/types"
|
|
29
|
+
import { dataRootPath, md5File } from "@mailwoman/core/utils"
|
|
30
|
+
import { normalizeFSTToken } from "@mailwoman/neural/fst-prior"
|
|
31
|
+
import {
|
|
32
|
+
PlacetypeCensusResolver,
|
|
33
|
+
serializePlacetypeCensus,
|
|
34
|
+
type PlacetypeCensusHeader,
|
|
35
|
+
type PlacetypeCensusNode,
|
|
36
|
+
} from "@mailwoman/neural/placetype-census"
|
|
37
|
+
import { Box, Text } from "ink"
|
|
38
|
+
import zod from "zod"
|
|
39
|
+
|
|
40
|
+
import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
|
|
41
|
+
import { buildPlacetypeCensus, toBaseRates } from "../../gazetteer-pipeline/placetype-census.ts"
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Known parents probed after write, PER COUNTRY. Probing another country's names against a freshly built census prints
|
|
45
|
+
* reassuring-looking misses that verify nothing (the lesson the pair-index command's en-nz first build taught).
|
|
46
|
+
*/
|
|
47
|
+
const PROBE_PARENTS_BY_COUNTRY: Readonly<Record<string, readonly string[]>> = {
|
|
48
|
+
gb: ["London", "Manchester", "Birmingham"],
|
|
49
|
+
us: ["New York", "Chicago", "Springfield"],
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const OptionsSchema = zod.object({
|
|
53
|
+
out: zod.string().default("docs/static/mailwoman").describe("Output dir for placetype-census-<country>.bin"),
|
|
54
|
+
country: zod.string().default("gb").describe("ISO country code this census is built for"),
|
|
55
|
+
source: zod.string().optional().describe("WOF admin DB. Default <data-root>/wof/admin-global-priority.db"),
|
|
56
|
+
delta: zod
|
|
57
|
+
.number()
|
|
58
|
+
.optional()
|
|
59
|
+
.describe(
|
|
60
|
+
"OPTIONAL soft-prior magnitude written into the header. Omit (the default) for an un-wired census — R4c " +
|
|
61
|
+
"ships data + loader + offline probe only; a calibration rung supplies the real value."
|
|
62
|
+
),
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
export { OptionsSchema as options }
|
|
66
|
+
|
|
67
|
+
const GazetteerCensus: CommandComponent<typeof OptionsSchema> = ({ options }) => {
|
|
68
|
+
const state = useCommandTask(async () => {
|
|
69
|
+
const country = options.country.toLowerCase()
|
|
70
|
+
const sourcePath = options.source ?? String(dataRootPath("wof", "admin-global-priority.db"))
|
|
71
|
+
|
|
72
|
+
if (!existsSync(sourcePath)) {
|
|
73
|
+
throw new Error(`census: source WOF admin DB not found: ${sourcePath}`)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const built = buildPlacetypeCensus(sourcePath, country.toUpperCase())
|
|
77
|
+
|
|
78
|
+
if (built.unmappedPlacetypes.length) {
|
|
79
|
+
throw new Error(
|
|
80
|
+
`census: source carries placetypes absent from PLACETYPE_PROJECTION: ${built.unmappedPlacetypes.join(", ")} — ` +
|
|
81
|
+
`extend the projection table (and plan/reference/placetype-evidence.mdx) before building.`
|
|
82
|
+
)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Fold, merging collisions by SUMMING counts — the serializer throws on duplicates, so a merge bug is loud.
|
|
86
|
+
const folded = new Map<string, PlacetypeCensusNode>()
|
|
87
|
+
let collisions = 0
|
|
88
|
+
|
|
89
|
+
for (const node of built.nodes) {
|
|
90
|
+
const parent = normalizeFSTToken(node.parent)
|
|
91
|
+
|
|
92
|
+
if (!parent) continue
|
|
93
|
+
|
|
94
|
+
const existing = folded.get(parent)
|
|
95
|
+
|
|
96
|
+
if (!existing) {
|
|
97
|
+
folded.set(parent, { parent, counts: { ...node.counts }, total: node.total })
|
|
98
|
+
|
|
99
|
+
continue
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
collisions++
|
|
103
|
+
|
|
104
|
+
for (const [tag, n] of Object.entries(node.counts) as Array<[ComponentTag, number]>) {
|
|
105
|
+
existing.counts[tag] = (existing.counts[tag] ?? 0) + n
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
existing.total += node.total
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const nodes = [...folded.values()]
|
|
112
|
+
const baseRates = toBaseRates(built.countryTotals)
|
|
113
|
+
const sourceMD5 = await md5File(sourcePath)
|
|
114
|
+
|
|
115
|
+
const header: PlacetypeCensusHeader = {
|
|
116
|
+
country,
|
|
117
|
+
schemaVersion: 1,
|
|
118
|
+
foldVersion: 1,
|
|
119
|
+
sourceMD5s: [sourceMD5],
|
|
120
|
+
buildDate: new Date().toISOString(),
|
|
121
|
+
baseRates,
|
|
122
|
+
...(options.delta !== undefined ? { delta: options.delta } : {}),
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const bytes = serializePlacetypeCensus(header, nodes)
|
|
126
|
+
const outPath = join(options.out, `placetype-census-${country}.bin`)
|
|
127
|
+
|
|
128
|
+
writeFileSync(outPath, bytes)
|
|
129
|
+
|
|
130
|
+
// Self-verifying readback over the bytes just written, not the in-memory nodes.
|
|
131
|
+
const resolver = new PlacetypeCensusResolver(bytes)
|
|
132
|
+
const probeParents = PROBE_PARENTS_BY_COUNTRY[country]
|
|
133
|
+
|
|
134
|
+
if (!probeParents) {
|
|
135
|
+
throw new Error(
|
|
136
|
+
`census: no self-check probe parents registered for country "${country}" — add an entry to ` +
|
|
137
|
+
`PROBE_PARENTS_BY_COUNTRY (probing another country's names verifies nothing).`
|
|
138
|
+
)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const probeLines = probeParents.map((raw) => {
|
|
142
|
+
const parent = normalizeFSTToken(raw)
|
|
143
|
+
const node = resolver.probe(parent)
|
|
144
|
+
|
|
145
|
+
if (!node) return `PROBE MISS: fold("${raw}") → "${parent}" → (no node)`
|
|
146
|
+
|
|
147
|
+
const depLoc = node.counts.dependent_locality ?? 0
|
|
148
|
+
const share = resolver.share(parent, "dependent_locality")
|
|
149
|
+
const lift = resolver.lift(parent, "dependent_locality")
|
|
150
|
+
|
|
151
|
+
return (
|
|
152
|
+
`PROBE OK: "${parent}" → ${node.total.toLocaleString()} children, ` +
|
|
153
|
+
`${depLoc.toLocaleString()} dependent_locality (share ${(share * 100).toFixed(1)}%, lift ${lift.toFixed(1)}×)`
|
|
154
|
+
)
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
const baseRateLines = (Object.entries(baseRates) as Array<[ComponentTag, number]>)
|
|
158
|
+
.toSorted((a, b) => b[1] - a[1])
|
|
159
|
+
.map(([tag, rate]) => ` ${tag}: ${(rate * 100).toFixed(2)}%`)
|
|
160
|
+
|
|
161
|
+
return [
|
|
162
|
+
`placetype-census-${country}.bin → ${outPath} (${bytes.length.toLocaleString()} bytes)`,
|
|
163
|
+
`header: ${options.delta !== undefined ? `delta=${options.delta}` : "(no delta — un-wired census)"}`,
|
|
164
|
+
`nodes: ${nodes.length.toLocaleString()} (${built.nodes.length.toLocaleString()} pre-fold, ${collisions.toLocaleString()} fold collisions merged)`,
|
|
165
|
+
`child links counted: ${built.links.toLocaleString()}`,
|
|
166
|
+
"country base rates:",
|
|
167
|
+
...baseRateLines,
|
|
168
|
+
...probeLines,
|
|
169
|
+
]
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
|
|
173
|
+
|
|
174
|
+
if (state.status === "done") {
|
|
175
|
+
return (
|
|
176
|
+
<Box flexDirection="column">
|
|
177
|
+
{state.result.map((line, i) => (
|
|
178
|
+
<Text key={i} color={i === 0 ? "green" : undefined}>
|
|
179
|
+
{i === 0 ? "✓ " : " "}
|
|
180
|
+
{line}
|
|
181
|
+
</Text>
|
|
182
|
+
))}
|
|
183
|
+
</Box>
|
|
184
|
+
)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return null
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export default GazetteerCensus
|