@mailwoman/bdc 8.3.0

package/sdk/common.ts

@@ -0,0 +1,363 @@
+/**
+ * @copyright Sister Software.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file BDC file metadata model — dictionaries for the FCC's `bdc_file` listing rows, the
+ *   raw-to-parsed record shape, and file-ordering comparators.
+ */
+
+import type { Tagged } from "type-fest"
+
+import type { BroadbandTechnologyCode } from "./technologies.ts"
+
+/**
+ * Unique identifier for an FCC BDC broadband provider.
+ *
+ * Re-homed locally as `Tagged<number, ...>` — the Nexus original imported this from `@isp.nexus/fcc`
+ * (`BroadbandProvider.ts`), where it's defined identically.
+ *
+ * @category BDC
+ * @category FCC
+ */
+export type ProviderID = Tagged<number, "ProviderID">
+
+/**
+ * The data type of the file, e.g. what kind of data is in the file.
+ *
+ * @category BDC
+ * @category FCC
+ */
+export const BDCFilingDataType = {
+	/**
+	 * The file contains data about the availability of broadband with a specific provider.
+	 */
+	Availability: "availability",
+	/**
+	 * The file contains data about the challenge process.
+	 */
+	Challenge: "challenge",
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCFilingDataType = (typeof BDCFilingDataType)[keyof typeof BDCFilingDataType]
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export const BDCGISFileType = {
+	ShapeFile: 1,
+	GeoPackage: 2,
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCGISFileType = (typeof BDCGISFileType)[keyof typeof BDCGISFileType]
+
+/**
+ * The type of file, e.g. what format the file is in.
+ *
+ * @category BDC
+ * @category FCC
+ */
+export const BDCFileFormat = {
+	CSV: "csv",
+	GIS: "gis",
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCFileFormat = (typeof BDCFileFormat)[keyof typeof BDCFileFormat]
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export const BDCFileCategory = {
+	Provider: "Provider",
+	Summary: "Summary",
+	State: "State",
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCFileCategory = (typeof BDCFileCategory)[keyof typeof BDCFileCategory]
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export const BDCProviderSubCategory = {
+	FixedBroadband: "Fixed Broadband",
+	MobileBroadband: "Mobile Broadband",
+	MobileVoice: "Mobile Voice",
+	SupportingData: "Supporting Data",
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCProviderSubCategory = (typeof BDCProviderSubCategory)[keyof typeof BDCProviderSubCategory]
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export const BDCSummarySubCategory = {
+	BroadbandSummaryByGeography: "Broadband Summary by Geography Type",
+	ProviderSummaryByGeography: "Provider Summary by Geography Type",
+	ProviderSummaryFixedBroadband: "Provider Summary - Fixed Broadband",
+	ProviderSummaryMobileBroadband: "Provider Summary - Mobile Broadband",
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCSummarySubCategory = (typeof BDCSummarySubCategory)[keyof typeof BDCSummarySubCategory]
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export const BDCStateSubCategory = {
+	FixedBroadband: "Fixed Broadband",
+	MobileBroadband: "Mobile Broadband",
+	MobileVoice: "Mobile Voice",
+} as const
+
+/**
+ * @category BDC
+ * @category FCC
+ */
+export type BDCStateSubCategory = (typeof BDCStateSubCategory)[keyof typeof BDCStateSubCategory]
+
+export type BDCSubCategory = BDCProviderSubCategory | BDCStateSubCategory | BDCSummarySubCategory
+
+/**
+ * A single row from the FCC's BDC file listing, as returned by the API before parsing.
+ *
+ * @category BDC
+ * @category FCC
+ */
+export interface RawBDCFile {
+	file_id: number
+	category: BDCFileCategory
+	subcategory: BDCSubCategory
+	/**
+	 * Comma-separated list of technology codes.
+	 *
+	 * Nullable in live data: the FCC's `/map/downloads/listAvailabilityData` response carries `technology_code: null` for
+	 * at least some State-category rows (first observed in the live FCC smoke test, see
+	 * `.superpowers/sdd/2026-07-30-bdc-2b-plan/live-smoke-findings.md`). Guarded in {@linkcode parseRawBDCFile} — a null
+	 * value parses to an empty `technologyCodes` set rather than throwing.
+	 *
+	 * @see {@link BroadbandTechnologyCode}
+	 */
+	technology_code: string | null
+	technology_code_desc: string
+	/**
+	 * 2-digit state or territory FIPS code.
+	 *
+	 * Loosely typed as `string` for now. The Nexus original was `AdminLevel1Code` (via `@isp.nexus/tiger`); this port
+	 * drops that dependency, same as `data-collection.ts`'s `FCCStateID`. Task 2c tightens this against
+	 * `@mailwoman/tiger` if a downstream dictionary needs the literal union.
+	 *
+	 * Nullable in live data for rows not scoped to a specific state (e.g. Provider-category rows). Guarded in
+	 * {@linkcode parseRawBDCFile} — a null value parses to an empty `stateCode` string.
+	 */
+	state_fips: string | null
+	/**
+	 * State or territory name.
+	 *
+	 * Loosely typed as `string` — the Nexus original was `StateName` (via `@isp.nexus/tiger`). Same task-2c deferral as
+	 * `state_fips` above.
+	 */
+	state_name: string
+	/**
+	 * Nullable in live data for rows not scoped to a specific provider (e.g. State/Summary-category rows). Guarded in
+	 * {@linkcode parseRawBDCFile} — a null value parses to a `providerID` of `0`.
+	 */
+	provider_id: string | null
+	/**
+	 * Nullable in live data — travels with `provider_id` (see above). Guarded in {@linkcode parseRawBDCFile} — a null
+	 * value parses to an empty `providerName` string.
+	 */
+	provider_name: string | null
+	file_type: string
+	file_name: string
+	record_count: string
+}
+
+/**
+ * A parsed FCC BDC file-listing entry.
+ *
+ * @category BDC
+ * @category FCC
+ */
+export interface BDCFile {
+	/**
+	 * Unique identifier for the file, defined by the FCC.
+	 */
+	fileID: number
+
+	revision: Date
+	vintage: Date
+
+	/**
+	 * The date the file was was downloaded, parsed, and stored in the database.
+	 */
+	synchronizedAt?: Date
+
+	/**
+	 * The category of the file.
+	 */
+	category: BDCFileCategory
+	/**
+	 * The subcategory of the file.
+	 */
+	subcategory: BDCSubCategory
+	/**
+	 * The technology codes in the file. Empty when the raw `technology_code` was `null`.
+	 */
+	technologyCodes: Set<BroadbandTechnologyCode>
+	/**
+	 * The state or territory FIPS code.
+	 *
+	 * Loosely typed as `string` — see {@linkcode RawBDCFile} for the task-2c deferral. Empty string when the raw
+	 * `state_fips` was `null`.
+	 */
+	stateCode: string
+	/**
+	 * The provider ID associated with the file. `0` when the raw `provider_id` was `null` (no specific provider — see
+	 * {@linkcode RawBDCFile}).
+	 */
+	providerID: ProviderID
+	/**
+	 * The provider name associated with the file. Empty string when the raw `provider_name` was `null`.
+	 */
+	providerName: string
+	/**
+	 * The number of records in the file.
+	 */
+	recordCount: number
+	/**
+	 * The type of file, e.g. what format the file is in.
+	 */
+	fileType: string
+	/**
+	 * The name of the file, as provided by the FCC.
+	 */
+	fileName: string
+}
+
+const BDCFileNamePattern = /([A-Z])(\d+)_(\d{2})([a-z]{3})(\d{4})$/
+
+const MonthAbbreviation = {
+	jan: 0,
+	feb: 1,
+	mar: 2,
+	apr: 3,
+	may: 4,
+	jun: 5,
+	jul: 6,
+	aug: 7,
+	sep: 8,
+	oct: 9,
+	nov: 10,
+	dec: 11,
+} as const
+
+export type MonthAbbreviation = keyof typeof MonthAbbreviation
+
+const VintageMonthLetter = {
+	/**
+	 * December
+	 */
+	D: 11,
+	/**
+	 * June
+	 */
+	J: 5,
+}
+
+type VintageMonthLetter = keyof typeof VintageMonthLetter
+
+/**
+ * Given a BDC file, parse the components of the file name.
+ */
+export function parseBDCFileTimestamps(fileName: string) {
+	const match = fileName.match(BDCFileNamePattern)
+
+	if (!match) throw new Error(`Invalid BDC file name: ${fileName}`)
+	const [, vintageMonthLetter, vintageYearAbbreviation, revisionDay, revisionMonthAbbreviation, revisionYear] = match
+
+	const revisionMonth = MonthAbbreviation[revisionMonthAbbreviation as MonthAbbreviation]
+	const revision = new Date(Number.parseInt(revisionYear!, 10), revisionMonth, Number.parseInt(revisionDay!, 10))
+
+	const vintageMonth = VintageMonthLetter[vintageMonthLetter as VintageMonthLetter]
+	const vintageYear = Number.parseInt(`20${vintageYearAbbreviation}`, 10)
+
+	const vintage = new Date(vintageYear, vintageMonth)
+
+	return {
+		revision,
+		vintage,
+	}
+}
+
+/**
+ * Parses a raw BDC file-listing entry into a {@linkcode BDCFile}.
+ */
+export function parseRawBDCFile(raw: RawBDCFile): BDCFile {
+	const parsedBDC: BDCFile = {
+		...parseBDCFileTimestamps(raw.file_name),
+		fileName: raw.file_name,
+		fileType: raw.file_type,
+		fileID: raw.file_id,
+		recordCount: Number.parseInt(raw.record_count, 10),
+		category: raw.category,
+		subcategory: raw.subcategory,
+		technologyCodes: new Set(
+			raw.technology_code === null
+				? []
+				: raw.technology_code.split(",").map((code) => Number.parseInt(code, 10) as BroadbandTechnologyCode)
+		),
+		stateCode: raw.state_fips ?? "",
+		providerID: (raw.provider_id === null ? 0 : Number.parseInt(raw.provider_id, 10)) as ProviderID,
+		providerName: raw.provider_name ?? "",
+	}
+
+	return parsedBDC
+}
+
+/**
+ * Comparator for sorting {@linkcode BDCFile} records ascending by revision date.
+ */
+export function compareRevisionAsc(a: BDCFile, b: BDCFile): number {
+	return a.revision.getTime() - b.revision.getTime()
+}
+
+/**
+ * Comparator for sorting {@linkcode BDCFile} records ascending by provider ID.
+ */
+export function compareProviderIDAsc(a: BDCFile, b: BDCFile): number {
+	return a.providerID - b.providerID
+}
+
+/**
+ * Comparator for sorting {@linkcode BDCFile} records ascending by state FIPS code.
+ */
+export function compareStateCodeAsc(a: BDCFile, b: BDCFile): number {
+	return Number.parseInt(a.stateCode, 10) - Number.parseInt(b.stateCode, 10)
+}

package/sdk/data-collection.ts

@@ -0,0 +1,268 @@
+/**
+ * @copyright Sister Software.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file FCC Broadband Data Collection System
+ */
+
+import type { BuildingTypeCode } from "./buildings.ts"
+import type { BroadbandServicableLocationID } from "./location.ts"
+
+/**
+ * Snake_case identifier for a US State or Territory.
+ *
+ * Loosely typed as `string` for now. The Nexus original derived this from `StateName` (via `@isp.nexus/tiger`); this
+ * port drops that dependency since 2a doesn't consume the literal union anywhere. Task 2c tightens this against
+ * `@mailwoman/tiger`'s `StateName` if a downstream dictionary needs it.
+ *
+ * @category FCC
+ * @title FCC State Identifier
+ */
+export type FCCStateID = string
+
+/**
+ * Code indicating confidence in the association between the textual address and the location.
+ *
+ * @category NTIA
+ * @category FCC
+ */
+export const AddressConfidenceCode = {
+	High: "1",
+	Medium: "2",
+	Low: "3",
+	SuccessfulChallenge: "4",
+} as const
+
+/**
+ * @category NTIA
+ * @category FCC
+ */
+export type AddressConfidenceCode = (typeof AddressConfidenceCode)[keyof typeof AddressConfidenceCode]
+
+/**
+ * A modeled land use designation estimated from assembled county assessor information.
+ *
+ * @category NTIA
+ * @category FCC
+ */
+export const LandUseCode = {
+	Residential: 1,
+	Land: 2,
+	Business: 3,
+	Unknown: 4,
+	Agriculture: 5,
+	Community: 6,
+	Industrial: 7,
+	Recreation: 8,
+	Utility: 9,
+	MixedUse: 10,
+	Transportation: 11,
+	Water: 12,
+	Communications: 13,
+	Row: 14,
+	Wireless: 15,
+	Other: 0,
+} as const
+
+/**
+ * @category NTIA
+ * @category FCC
+ */
+export type LandUseCode = (typeof LandUseCode)[keyof typeof LandUseCode]
+
+/**
+ * Flag indicating if the location is a broadband serviceable location.
+ */
+export const BSLFlag = {
+	Serviceable: 1,
+	NotServiceable: 0,
+} as const
+
+export type BSLFlag = (typeof BSLFlag)[keyof typeof BSLFlag]
+
+/**
+ * Interface representing a record from the NTIA data dictionary.
+ *
+ * @category NTIA
+ * @title NTIA Record
+ */
+export interface NTIARecord {
+	/**
+	 * Unique ID for the Fabric location. Remains persistent across versions, even if the location's position or building
+	 * changes.
+	 *
+	 * @title Location ID
+	 * @pattern ^\d{10}$
+	 */
+	location_id: BroadbandServicableLocationID
+
+	/**
+	 * Primary postal address excluding city, state, and ZIP code.
+	 *
+	 * @example
+	 * 	123 Main St
+	 *
+	 * @title Primary Address
+	 */
+	address_primary: string
+
+	/**
+	 * City name from the postal address. Also known as the locality.
+	 *
+	 * @example
+	 * 	San Francisco
+	 *
+	 * @title City
+	 */
+	city: string
+
+	/**
+	 * 2-character state abbreviation from the postal address.
+	 *
+	 * Loosely typed as `string` — see {@linkcode FCCStateID} for why the tiger-backed literal union is deferred to task
+	 * 2c.
+	 *
+	 * @example
+	 * 	CA
+	 *
+	 * @title State Abbreviation
+	 */
+	state: string
+
+	/**
+	 * 5-digit ZIP code associated with the address.
+	 *
+	 * @example
+	 * 	94103
+	 *
+	 * @title ZIP Code
+	 */
+	zip: string
+
+	/**
+	 * USPS ZIP+4 extension.
+	 *
+	 * @example
+	 * 	e.g. 94103-1234
+	 *
+	 * @title ZIP+4
+	 */
+	zip_suffix?: string
+
+	/**
+	 * Estimate of the number of units at the location. Includes both residential and non-residential units.
+	 *
+	 * @example
+	 * 	10
+	 *
+	 * @title Unit Count
+	 */
+	unit_count: number
+
+	/**
+	 * Flag indicating if the location is a broadband serviceable location. 1 for serviceable, 0 for not.
+	 *
+	 * @example
+	 * 	1
+	 *
+	 * @title BSL Flag
+	 */
+	bsl_flag: BSLFlag
+
+	/**
+	 * Code indicating the type of building at the location. Can be residential (R), non-residential (B), mixed (X), group
+	 * quarters (G), CAI (C), enterprise (E), or other (O).
+	 *
+	 * @example
+	 * 	R
+	 *
+	 * @title Building Type Code
+	 */
+	building_type_code: BuildingTypeCode
+
+	/**
+	 * Modeled land use designation from county assessor information. Possible values range from 0 (Other) to 15
+	 * (Wireless).
+	 *
+	 * @example
+	 * 	1
+	 *
+	 * @title Land Use Code
+	 */
+	land_use_code: LandUseCode | null
+
+	/**
+	 * Code indicating confidence in the association between the textual address and the location. 1 = High, 2 = Medium, 3
+	 * = Low, 4 = successful address challenge.
+	 *
+	 * @example
+	 * 	1
+	 *
+	 * @title Address Confidence Code
+	 */
+	address_confidence_code: AddressConfidenceCode
+
+	/**
+	 * 5-digit TIGER 2020 identifier for the county based on the latitude and longitude.
+	 *
+	 * @example
+	 * 	06075
+	 *
+	 * @title County GEOID
+	 */
+	county_geoid: string
+
+	/**
+	 * 15-digit 2020 U.S. Census Bureau FIPS code for the census block, based on the latitude and longitude.
+	 *
+	 * @example
+	 * 	060750001000123
+	 *
+	 * @title Block GEOID
+	 */
+	block_geoid: string
+
+	/**
+	 * H3 hex cell ID, level 9, for geospatial referencing.
+	 *
+	 * @example
+	 * 	8928308280fffff
+	 *
+	 * @title H3 Cell ID (9)
+	 */
+	h3_9: string
+
+	/**
+	 * Latitude coordinate of the location in decimal degrees. Uses WGS84 format with 5-digit precision.
+	 *
+	 * @example
+	 * 	37.774929
+	 *
+	 * @minimum -90
+	 * @maximum 90
+	 * @title Latitude
+	 */
+	latitude: number
+
+	/**
+	 * Longitude coordinate of the location in decimal degrees. Uses WGS84 format with 5-digit precision.
+	 *
+	 * @example
+	 * 	e.g. -122.419416
+	 *
+	 * @minimum -180
+	 * @maximum 180
+	 * @title Longitude
+	 */
+	longitude: number
+
+	/**
+	 * FCC Fabric Release Date, present only for tier 2, 3, and 4 licensees.
+	 *
+	 * @example
+	 * 	07212023
+	 *
+	 * @title FCC Fabric Release Date
+	 */
+	fcc_rel?: string
+}

package/sdk/download.ts

@@ -0,0 +1,91 @@
+/**
+ * @copyright Sister Software.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file FCC BDC availability-file download + zip extraction.
+ *
+ *   Re-homed from Nexus's `sync/fcc/bdc/download-file.ts` (relicense-by-copy, no provenance headers),
+ *   trimmed hard: the Nexus original downloaded AND cached the `.zip`, extracted it, THEN wrote a Parquet
+ *   file with a row-count integrity check. All Parquet machinery is dropped here — 2a's `bdc.db` is
+ *   SQLite, not Parquet-backed (see Task 7+) — and the `.zip` itself isn't cached either; only the
+ *   extracted CSV is written to `destinationDir`, and its presence alone is the cache check.
+ *
+ *   The zip-extraction library also changes: the Nexus original's `extractSingleFileZip` used `adm-zip`
+ *   (a repo-wide Nexus dependency). No unzip dependency exists anywhere in this repo — every workspace
+ *   `package.json` was checked, `tiger/` and `osm/` included — so `yauzl-promise` is added to `bdc` alone
+ *   (noted in this task's commit body per the brief).
+ */
+
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+
+import { fromBuffer } from "yauzl-promise"
+
+import type { BDCClient } from "./client.ts"
+import { BDCFilingDataType, type BDCFile } from "./common.ts"
+
+/**
+ * Extract the first file entry of a zip archive buffer into a single in-memory `Buffer`.
+ *
+ * BDC availability downloads are always a single zip-wrapped CSV, so — like the Nexus original — this doesn't walk
+ * every entry, just the first non-directory one.
+ */
+async function extractSingleFileZip(zippedBuffer: Buffer): Promise<Buffer> {
+	const zip = await fromBuffer(zippedBuffer)
+
+	try {
+		for await (const entry of zip) {
+			if (entry.filename.endsWith("/")) continue
+
+			const readStream = await entry.openReadStream()
+			const chunks: Buffer[] = []
+
+			for await (const chunk of readStream) {
+				chunks.push(chunk)
+			}
+
+			return Buffer.concat(chunks)
+		}
+
+		throw new Error("extractSingleFileZip: no file entries found in zip archive.")
+	} finally {
+		await zip.close()
+	}
+}
+
+/**
+ * Download and cache an FCC BDC availability file, extracting its zip-wrapped CSV to `destinationDir`.
+ *
+ * Cache-if-exists: if the extracted CSV already exists at the destination path, this returns immediately without
+ * issuing any network request. Otherwise it downloads the zip via `client`, extracts the CSV, writes it to
+ * `destinationDir`, and returns the written path. Only the extracted CSV is ever cached — the intermediate `.zip` is
+ * never written to disk.
+ *
+ * THIS FILE OWNS THE CACHE FOR THE DOWNLOAD PATH, which is why `BDCClient.getArrayBuffer` switches the client's own
+ * response cache off: the `existsSync`-equivalent check above is the real cache hit, and running a
+ * multi-hundred-megabyte archive through a JSON-validating disk cache would write a second, unreadable copy of a file
+ * already on disk here.
+ *
+ * @returns The path of the extracted (and now cached) CSV file.
+ */
+export async function downloadBDCFile(client: BDCClient, file: BDCFile, destinationDir: string): Promise<string> {
+	const csvPath = path.join(destinationDir, `${file.fileName}.csv`)
+
+	const alreadyCached = await fs
+		.access(csvPath)
+		.then(() => true)
+		.catch(() => false)
+
+	if (alreadyCached) return csvPath
+
+	const zippedArrayBuffer = await client.getArrayBuffer(
+		`/map/downloads/downloadFile/${BDCFilingDataType.Availability}/${file.fileID}`
+	)
+
+	const csvBuffer = await extractSingleFileZip(Buffer.from(zippedArrayBuffer))
+
+	await fs.mkdir(destinationDir, { recursive: true })
+	await fs.writeFile(csvPath, csvBuffer)
+
+	return csvPath
+}