mailwoman 8.3.0 → 8.4.0

package/commands/gazetteer/build/bdc.tsx

@@ -0,0 +1,203 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman gazetteer build bdc` — the FCC BDC availability ingest + sealed `bdc.db` layer build
+ *   (2a Task 8). Thin wiring only: list → download → build lives in `@mailwoman/bdc/sdk`
+ *   (`buildBDCDatabase`, list-files.ts, download.ts), so it stays unit-testable without Ink/Pastel or
+ *   network in the loop. Mirrors `poi.tsx`'s progress (stderr) / summary (stdout) split.
+ *
+ *   First cut wires Fixed Broadband provider availability only (the primary wireline dataset) — mobile
+ *   broadband/voice subcategories are a future flag, not a scope gap in this command's shape.
+ *
+ *   `--provider-list-path` (3a Task 8, decision 6) opts INTO populating `bdc_provider` —
+ *   `bdc_availability` itself is unaffected either way. When given, `parseProviderList`
+ *   (`@mailwoman/filer/sdk`) streams the FCC provider-list CSV as `BuildBDCOptions.providers`, and
+ *   filer.db (`--filer-db-path`, default `<data-root>/filer/filer.db`) is opened read-only to resolve
+ *   each multi-FRN provider's primary FRN (`readFRNFilingCandidates` + `pickPrimaryFRN`, imported
+ *   rather than reimplemented — see `build-bdc.ts`'s `BuildBDCOptions.filerDB` docstring). Omitting
+ *   `--provider-list-path` leaves `bdc_provider` empty, exactly as before Task 8.
+ *
+ *   Both paths are `existsSync`-guarded BEFORE any download/build work starts (review fix round 1,
+ *   IMPORTANT-2): `populateBDCProviderTable` only runs after `writeLayerManifest`, i.e. at the very END
+ *   of a full build — an unguarded typo'd `--provider-list-path` would otherwise surface as a raw ENOENT
+ *   only after a nationwide availability ingest had already finished, discarding hours of work.
+ *   `--filer-db-path` given without `--provider-list-path` is a loud error, not a silent no-op (MINOR):
+ *   filer.db is only ever read to resolve a multi-FRN primary FRN, so it does nothing without a provider
+ *   list to resolve FRNs FOR.
+ */
+
+import { execFileSync } from "node:child_process"
+import { existsSync } from "node:fs"
+import { DatabaseSync } from "node:sqlite"
+
+import {
+	BDCFileCategory,
+	BDCFilingDataType,
+	BDCProviderSubCategory,
+	buildBDCDatabase,
+	createBDCClient,
+	createTIGERBlockCentroidLookup,
+	downloadBDCFile,
+	resolveLatestVintage,
+	retrieveAvailabilityFiles,
+	retrieveFilingDates,
+} from "@mailwoman/bdc/sdk"
+import { DatabaseClient } from "@mailwoman/core/kysley/client"
+import { dataRootPath } from "@mailwoman/core/utils"
+import type { FilerDatabase } from "@mailwoman/filer"
+import { parseProviderList } from "@mailwoman/filer/sdk"
+import { Box, Text } from "ink"
+import zod from "zod"
+
+import { type CommandComponent, useCommandTask } from "../../../cli-kit/index.ts"
+import { artifactSizeMB } from "../../../gazetteer-pipeline/admin/index.ts"
+
+const OptionsSchema = zod.object({
+	state: zod.string().describe('2-digit FCC state/territory FIPS code (e.g. "06" for California)'),
+	asOfDate: zod.string().optional().describe("FCC filing as_of_date (YYYY-MM-DD). Default: latest available"),
+	out: zod.string().optional().describe("bdc.db output path. Default <data-root>/bdc/bdc.db"),
+	includeLocationIds: zod
+		.boolean()
+		.default(false)
+		.describe("Populate bdc_availability.location_id (opaque BSL join key; NULL by default)"),
+	providerListPath: zod
+		.string()
+		.optional()
+		.describe(
+			"Path to the FCC BDC provider list CSV (frn/provider_id/holding_company). When given, populates " +
+				"bdc_provider (decision 6, 3a task 8); omit to leave bdc_provider empty, as before task 8."
+		),
+	filerDbPath: zod
+		.string()
+		.optional()
+		.describe(
+			"Path to filer.db, used to resolve a multi-FRN provider's primary FRN when --provider-list-path is given. " +
+				"Default <data-root>/filer/filer.db"
+		),
+})
+
+export { OptionsSchema as options }
+
+const GazetteerBuildBDC: CommandComponent<typeof OptionsSchema> = ({ options }) => {
+	const state = useCommandTask(async () => {
+		// Fail-fast guards (review fix round 1, IMPORTANT-2 / MINOR) — checked BEFORE any network/download work
+		// starts. `populateBDCProviderTable` only runs after the availability ingest AND writeLayerManifest, so
+		// without this, a typo'd --provider-list-path would surface only at the very end of a full national
+		// build, discarding hours of work for a check that costs microseconds up front.
+		if (options.filerDbPath && !options.providerListPath) {
+			throw new Error(
+				"gazetteer build bdc: --filer-db-path was given without --provider-list-path — filer.db is only read " +
+					"to resolve a multi-FRN provider's primary FRN, so it has no effect without a provider list. Pass " +
+					"--provider-list-path too, or drop --filer-db-path."
+			)
+		}
+
+		let filerDbPath: string | undefined
+
+		if (options.providerListPath) {
+			if (!existsSync(options.providerListPath)) {
+				throw new Error(`gazetteer build bdc: --provider-list-path not found: "${options.providerListPath}"`)
+			}
+
+			filerDbPath = options.filerDbPath ?? dataRootPath("filer", "filer.db")
+
+			if (!existsSync(filerDbPath)) {
+				throw new Error(`gazetteer build bdc: filer.db not found at "${filerDbPath}" (--filer-db-path)`)
+			}
+		}
+
+		const client = createBDCClient()
+
+		let asOfDate = options.asOfDate
+
+		if (!asOfDate) {
+			console.error("▸ resolving latest as_of_date...")
+
+			const dates = await retrieveFilingDates(client, { filingType: BDCFilingDataType.Availability })
+			asOfDate = resolveLatestVintage(dates, BDCFilingDataType.Availability)
+		}
+
+		console.error(`▸ as-of-date: ${asOfDate}`)
+
+		const files = await retrieveAvailabilityFiles(client, {
+			asOfDate,
+			category: BDCFileCategory.Provider,
+			subcategory: BDCProviderSubCategory.FixedBroadband,
+		})
+
+		const stateFiles = files.filter((file) => file.stateCode === options.state)
+
+		console.error(`▸ ${stateFiles.length} provider file(s) for state ${options.state} @ ${asOfDate}`)
+
+		const cacheDir = dataRootPath("bdc", "cache", "availability", asOfDate, options.state)
+		const csvPaths: string[] = []
+
+		for (const file of stateFiles) {
+			console.error(`  downloading ${file.fileName}...`)
+
+			csvPaths.push(await downloadBDCFile(client, file, cacheDir))
+		}
+
+		const out = options.out ?? dataRootPath("bdc", "bdc.db")
+		const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
+		const tigerDBPath = dataRootPath("tiger", "tiger.db")
+
+		console.error(`▸ build: ${out}`)
+
+		// --provider-list-path (3a Task 8, decision 6) opts into populating bdc_provider — omitted, `providers`/
+		// `filerDB` stay undefined and buildBDCDatabase's default (byte-identical) path runs unchanged. Both
+		// paths were already existsSync-validated above, so this can't ENOENT.
+		let filerDB: DatabaseClient<FilerDatabase> | undefined
+
+		if (filerDbPath) {
+			console.error(`▸ provider list: ${options.providerListPath} (filer.db: ${filerDbPath})`)
+
+			filerDB = new DatabaseClient<FilerDatabase>({ database: new DatabaseSync(filerDbPath, { readOnly: true }) })
+		}
+
+		try {
+			const result = await buildBDCDatabase({
+				csvPaths,
+				out,
+				asOfDate,
+				buildSHA,
+				includeLocationIDs: options.includeLocationIds,
+				blockCentroids: createTIGERBlockCentroidLookup(tigerDBPath),
+				providers: options.providerListPath ? parseProviderList(options.providerListPath) : undefined,
+				filerDB,
+				onProgress: (message) => console.error(`  [bdc] ${message}`),
+			})
+
+			return [
+				`bdc.db: ${out} (${artifactSizeMB(out)} MB)`,
+				`${result.rows.toLocaleString()} rows · ${result.providers} provider(s) · ${result.deduped.toLocaleString()} deduped` +
+					` · ${result.unknownGeoids.toLocaleString()} unknown geoid(s) · ${result.coverageCells.toLocaleString()} coverage cells` +
+					(options.providerListPath ? ` · ${result.providersPopulated.toLocaleString()} bdc_provider row(s)` : ""),
+				`manifest: name=bdc tier=shipped license=public-domain source=fcc-bdc sourceVintage=${asOfDate} buildSHA=${buildSHA}`,
+			]
+		} finally {
+			await filerDB?.destroy()
+		}
+	})
+
+	if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
+
+	if (state.status === "done") {
+		return (
+			<Box flexDirection="column">
+				{state.result.map((line, i) => (
+					<Text key={i} color={i === 0 ? "green" : undefined}>
+						{i === 0 ? "✓ " : "  "}
+						{line}
+					</Text>
+				))}
+			</Box>
+		)
+	}
+
+	return null // progress streams to stderr until the summary lands
+}
+
+export default GazetteerBuildBDC

package/commands/gazetteer/build/poi.tsx

@@ -8,40 +8,164 @@
  *   materialize/seal (`buildPOIDatabase`) logic lives in `gazetteer-pipeline/poi/build-poi.ts`, so it
  *   stays unit-testable without Ink/Pastel in the loop. Mirrors `overture-ingest.tsx`'s progress
  *   (stderr) / summary (stdout) split.
+ *
+ *   `--source osm` (bdc 2b task 3, decisions 3/5): a second, build-local branch alongside the default
+ *   Overture path — same command, same `buildPOIDatabase` seam, different inputs. It streams
+ *   `@mailwoman/osm/sdk`'s `extractOSMPOIs` over a Geofabrik `.osm.pbf` extract (telecom-infrastructure
+ *   categories only, task 2), stamps the invocation's `--country` onto every row (a bare OSM feature
+ *   carries no country property — see `extract-poi.ts`'s module docstring), and derives res-6 coverage
+ *   by polyfilling the extract's declared `--bbox` (`bboxCoverageCells`, decision 5) rather than only
+ *   the cells rows happened to land in — an infra-only extract is sparse by category, so "no plant in a
+ *   well-surveyed cell" needs its own zero-observed-rows coverage row, not silent absence. The manifest
+ *   swaps to `tier: build-local`, `license: ODbL-1.0`, OSM attribution — the default Overture branch
+ *   below is UNTOUCHED and stays byte-identical when `--source` is omitted.
  */
 
 import { execFileSync } from "node:child_process"
 
+import { LayerTier } from "@mailwoman/core/layers"
 import { dataRootPath } from "@mailwoman/core/utils"
 import { Box, Text } from "ink"
 import zod from "zod"
 
 import { type CommandComponent, useCommandTask } from "../../../cli-kit/index.ts"
 import { artifactSizeMB } from "../../../gazetteer-pipeline/admin/index.ts"
-import { buildPOIDatabase, DEFAULT_RELEASE, ingestPlaces } from "../../../gazetteer-pipeline/poi/build-poi.ts"
+import {
+	bboxCoverageCells,
+	buildPOIDatabase,
+	DEFAULT_RELEASE,
+	ingestPlaces,
+	type BBox,
+	type POISourceRow,
+} from "../../../gazetteer-pipeline/poi/build-poi.ts"
 
 const DEFAULT_COUNTRIES = "US,CA,MX,FR"
 
 const OptionsSchema = zod.object({
-	release: zod.string().optional().describe(`Pinned Overture release. Default ${DEFAULT_RELEASE}`),
-	countries: zod.string().optional().describe(`ISO 3166-1 alpha-2, comma-separated. Default ${DEFAULT_COUNTRIES}`),
-	out: zod.string().optional().describe("poi.db output path. Default <data-root>/poi/poi.db"),
-	limit: zod.string().optional().describe("Cap rows per country (debug)"),
+	source: zod
+		.enum(["overture", "osm"])
+		.default("overture")
+		.describe("Data source: overture (default, shipped) or osm (build-local, ODbL — requires --pbf/--country/--bbox)"),
+	release: zod
+		.string()
+		.optional()
+		.describe(`overture: pinned release, default ${DEFAULT_RELEASE}. osm: the extract's vintage/date tag (required)`),
+	countries: zod
+		.string()
+		.optional()
+		.describe(`overture only: ISO 3166-1 alpha-2, comma-separated. Default ${DEFAULT_COUNTRIES}`),
+	out: zod
+		.string()
+		.optional()
+		.describe("poi.db output path. Default <data-root>/poi/poi.db (overture) or poi/poi-osm-<cc>.db (osm)"),
+	limit: zod.string().optional().describe("overture only: cap rows per country (debug)"),
 	skipIngest: zod
 		.boolean()
 		.default(false)
-		.describe("Skip the DuckDB/S3 ingest; build from already-materialized per-country Parquet"),
+		.describe("overture only: skip the DuckDB/S3 ingest; build from already-materialized per-country Parquet"),
+	pbf: zod.string().optional().describe("osm: path to a Geofabrik .osm.pbf extract (required)"),
+	country: zod
+		.string()
+		.optional()
+		.describe("osm: ISO 3166-1 alpha-2 country stamped onto every extracted row (required)"),
+	bbox: zod
+		.string()
+		.optional()
+		.describe("osm: extract bounding box as 'minLon,minLat,maxLon,maxLat' — polyfills res-6 coverage cells (required)"),
 })
 
 export { OptionsSchema as options }
 
+/**
+ * `--bbox` field count: `minLon,minLat,maxLon,maxLat`.
+ */
+const BBOX_FIELD_COUNT = 4
+
+/**
+ * Parse `--bbox "minLon,minLat,maxLon,maxLat"` into a {@link BBox}. Throws with the raw input echoed back on any
+ * shape/finiteness mismatch — a silently-mis-parsed bbox would corrupt coverage silently, so fail loud instead.
+ */
+function parseBBoxFlag(raw: string): BBox {
+	const parts = raw.split(",").map((s) => Number(s.trim()))
+
+	if (parts.length !== BBOX_FIELD_COUNT || parts.some((n) => !Number.isFinite(n))) {
+		throw new Error(
+			`--bbox must be 4 comma-separated numbers "minLon,minLat,maxLon,maxLat", got ${JSON.stringify(raw)}`
+		)
+	}
+
+	const [minLon, minLat, maxLon, maxLat] = parts as [number, number, number, number]
+
+	return { minLon, minLat, maxLon, maxLat }
+}
+
 const GazetteerBuildPOI: CommandComponent<typeof OptionsSchema> = ({ options }) => {
 	const state = useCommandTask(async () => {
+		const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
+
+		if (options.source === "osm") {
+			const { pbf, country: rawCountry, release, bbox: bboxFlag } = options
+
+			if (!pbf || !rawCountry || !release || !bboxFlag) {
+				throw new Error(
+					"gazetteer build poi --source osm requires --pbf <extract.osm.pbf> --country <cc> --release <vintage> " +
+						"--bbox <minLon,minLat,maxLon,maxLat>"
+				)
+			}
+
+			const country = rawCountry.trim().toUpperCase()
+			const bbox = parseBBoxFlag(bboxFlag)
+			const out = options.out ?? dataRootPath("poi", `poi-osm-${country.toLowerCase()}.db`)
+
+			console.error(`▸ extract: OSM telecom-infrastructure POIs from ${pbf} (country=${country})`)
+
+			// Buffered (not streamed straight into buildPOIDatabase): telecom-infra extracts are
+			// category-sparse — a few thousand rows even for a whole country — and the array is walked
+			// twice, once here to derive coverage, once as buildPOIDatabase's `rows` seam.
+			const rows: POISourceRow[] = []
+
+			// DYNAMIC import, load-bearing: @mailwoman/osm is UNPUBLISHED (ODbL counsel sign-off
+			// pending — see osm/README.md), and Pastel loads every command module eagerly on --help,
+			// so a top-level import breaks the whole published CLI on a clean install (caught by the
+			// smoke test's clean-install leg, 2026-07-31). The osm source branch is build-local by
+			// design; it may only resolve its SDK when actually invoked.
+			const { extractOSMPOIs } = await import("@mailwoman/osm/sdk")
+
+			for await (const row of extractOSMPOIs(pbf)) {
+				// Task 3 hand-off from task 2: extractOSMPOIs yields `country: ""` (a bare OSM feature
+				// carries no country property) — stamp the invocation's --country before buildPOIDatabase
+				// ever sees the row.
+				rows.push({ ...row, country })
+			}
+
+			console.error(`▸ build: ${out}`)
+
+			const coverageCellsOverride = bboxCoverageCells(bbox, rows)
+
+			const result = await buildPOIDatabase({
+				rows,
+				out,
+				release,
+				buildSHA,
+				source: "osm",
+				tier: LayerTier.BuildLocal,
+				coverageCellsOverride,
+				createdAt: new Date().toISOString(),
+				onProgress: (phase, message) => console.error(`  [${phase}] ${message}`),
+			})
+
+			return [
+				`poi.db (osm/${country}): ${out} (${artifactSizeMB(out)} MB)`,
+				`${result.rows.toLocaleString()} rows · ${result.categories} categories` +
+					` · ${result.skipped.toLocaleString()} skipped (non-finite coords) · ${result.coverageCells.toLocaleString()} coverage cells`,
+				`manifest: name=poi tier=build-local source=osm sourceVintage=${release} buildSHA=${buildSHA}`,
+			]
+		}
+
 		const release = options.release ?? DEFAULT_RELEASE
 		const countries = (options.countries ?? DEFAULT_COUNTRIES).split(",").map((c) => c.trim().toUpperCase())
 		const limit = options.limit ? Number.parseInt(options.limit, 10) : undefined
 		const out = options.out ?? dataRootPath("poi", "poi.db")
-		const buildSHA = execFileSync("git", ["rev-parse", "--short", "HEAD"]).toString().trim()
 
 		let parquetPaths: string[]
 

package/eval-harness/gates/v7.0.0-base.json

@@ -1,4 +1,5 @@
 {
+	"label": "v7.0.0-base",
 	"$comment": "Gate spec for the v7.0.0 base lineage — cut from the v4.0.1-base-60k candidate's SHIP-CONFIG readings (int8, locality lexicon v6 per the G8 DE-fold revert; v7 trained the channel, v6 ships — DE surfaces run the trained absence identity). Philosophy inherited from v6.0.0-shipped-baseline; margins now use each metric's REAL support with WILSON intervals (the normal approximation under-margins boundary readings — 100% at n=36 priced at 1pp where Wilson prices ~10pp; the v6-era 2.0pp fallback class is retired).",
 	"$margin_rationale": "floor = ship-config reading - max(1.0pp, 2 x the downward Wilson 95% half-width at the metric's own support). Supports = each instrument's tp+fn or fixture n, recorded in $supports.",
 	"$supports": {