@mailwoman/bdc 9.0.0 → 9.2.0

package/sdk/download.ts

@@ -3,56 +3,16 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  * @file FCC BDC availability-file download + zip extraction.
- *
- *   Re-homed from Nexus's `sync/fcc/bdc/download-file.ts` (relicense-by-copy, no provenance headers),
- *   trimmed hard: the Nexus original downloaded AND cached the `.zip`, extracted it, THEN wrote a Parquet
- *   file with a row-count integrity check. All Parquet machinery is dropped here — `bdc.db` is
- *   SQLite, not Parquet-backed (`bdc/schema.ts`) — and the `.zip` itself isn't cached either; only the
- *   extracted CSV is written to `destinationDir`, and its presence alone is the cache check.
- *
- *   The zip-extraction library also changes: the Nexus original's `extractSingleFileZip` used `adm-zip`
- *   (a repo-wide Nexus dependency). No unzip dependency exists anywhere in this repo — every workspace
- *   `package.json` was checked, `tiger/` and `osm/` included — so `yauzl-promise` is a `bdc`-only
- *   dependency.
  */
 
 import * as fs from "node:fs/promises"
 import * as path from "node:path"
 
-import { fromBuffer } from "yauzl-promise"
+import { extractSingleFileZip } from "@mailwoman/core/fs/zip"
 
 import type { BDCClient } from "./client.ts"
 import { BDCFilingDataType, type BDCFile } from "./common.ts"
 
-/**
- * Extract the first file entry of a zip archive buffer into a single in-memory `Buffer`.
- *
- * BDC availability downloads are always a single zip-wrapped CSV, so — like the Nexus original — this doesn't walk
- * every entry, just the first non-directory one.
- */
-async function extractSingleFileZip(zippedBuffer: Buffer): Promise<Buffer> {
-	const zip = await fromBuffer(zippedBuffer)
-
-	try {
-		for await (const entry of zip) {
-			if (entry.filename.endsWith("/")) continue
-
-			const readStream = await entry.openReadStream()
-			const chunks: Buffer[] = []
-
-			for await (const chunk of readStream) {
-				chunks.push(chunk)
-			}
-
-			return Buffer.concat(chunks)
-		}
-
-		throw new Error("extractSingleFileZip: no file entries found in zip archive.")
-	} finally {
-		await zip.close()
-	}
-}
-
 /**
  * Download and cache an FCC BDC availability file, extracting its zip-wrapped CSV to `destinationDir`.
  *

package/sdk/filing-landscape.ts

@@ -34,10 +34,10 @@
  */
 
 import type { DatabaseClient } from "@mailwoman/core/kysley/client"
-import { readLayerCoverage, readLayerManifest, type LayerContractDatabase } from "@mailwoman/core/layers"
+import { readLayerCoverage, readLayerManifest } from "@mailwoman/core/layers"
 import { expandH3Cell, shortCellToInt, type H3Cell, type H3CellShort } from "@mailwoman/spatial"
 import { cellToParent } from "h3-js"
-import { sql, type Kysely } from "kysely"
+import { sql } from "kysely"
 
 import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, type BDCDatabase } from "../schema.ts"
 
@@ -139,14 +139,6 @@ const speedBucketCaseSQL = sql<string>`CASE
 	ELSE ${BDC_SPEED_BUCKET_GIGABIT}
 END`
 
-/**
- * `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
- * in `DB` — same narrowing cast as `build-bdc.ts`'s `asContractDB`.
- */
-function asContractDB(kdb: DatabaseClient<BDCDatabase>): Kysely<LayerContractDatabase> {
-	return kdb as unknown as Kysely<LayerContractDatabase>
-}
-
 /**
  * Reconstruct the res-6 ancestor of a res-9 short-cell int WITHOUT a centroid — see the module docstring for why the
  * centroid is the wrong input. Exported so tests can assert this agrees, cell-for-cell, with `build-bdc.ts`'s own
@@ -184,7 +176,7 @@ export async function filingLandscape(
 
 	// Read (and validate) the manifest FIRST — a broken/missing manifest must throw before any block is
 	// classified, never fall through to an "unstamped" answer (gate 4).
-	const manifest = await readLayerManifest(asContractDB(db))
+	const manifest = await readLayerManifest(db)
 
 	const requestedUnits: ReadonlyArray<string | number> = query.geoids ?? query.h3Cells!
 	const unitColumn = query.geoids ? ("geoid" as const) : ("h3_cell" as const)
@@ -230,7 +222,7 @@ export async function filingLandscape(
 		}
 
 		const res6Parent = res9ShortCellToRes6Parent(candidateCell)
-		const coverage = await readLayerCoverage(asContractDB(db), res6Parent)
+		const coverage = await readLayerCoverage(db, res6Parent)
 
 		if (coverage === undefined) {
 			unknownBlockCount++

package/sdk/parsing.ts

@@ -2,45 +2,84 @@
  * @copyright Sister Software.
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- * @file FCC BDC availability CSV byte-level parser.
- *
- *   Re-homed from Nexus's `sync/fcc/bdc/parsing.ts` (relicense-by-copy, no provenance headers): the same
- *   byte-scanning generator over the FCC's 12-column availability CSV — see the Nexus `RawBSLAvailabilityRow`
- *   tuple (`sync/fcc/bdc/block-aggregator.ts`) for the column order this scan assumes: frn, provider_id,
- *   brand_name, location_id, technology, max_advertised_download_speed, max_advertised_upload_speed,
- *   low_latency, business_residential_code, state_usps, block_geoid, h3_res8_id. Changed only where the
- *   pre-registered 2a decisions require: `geoid` (column 10, `block_geoid`) decodes to an ASCII string
- *   instead of staying a raw `Uint8Array` slice (decision 3), and `location_id` (column 3) stays a string
- *   instead of `parseInt`ing it, preserving leading zeros (decision 1). `business_residential_code` is
- *   likewise decoded to its ASCII string here — the Nexus original kept only the field's first raw byte as
- *   a bare `number`, which this port's `BDCAvailabilityRow` shape doesn't call for.
- *
- *   Columns 0-2 (frn, provider_id, brand_name) and 9, 11 (state_usps, h3_res8_id) are scanned over but
- *   never sliced into the output record: `provider_id` comes from the `providerID` parameter instead (the
- *   FCC partitions availability files per provider, so the caller already knows it), and FRN/brand/state/H3
- *   join concerns are out of scope for 2a (decision 8 — provider identity is a 2c registry-join seam).
+ * @file FCC BDC availability CSV row reader.
+ *
+ *   Streams the FCC's 12-column availability CSV through `CSVSpliterator` in `array` mode and projects the
+ *   seven columns 2a keeps. Columns 0-2 (`frn`, `provider_id`, `brand_name`) and 9, 11 (`state_usps`,
+ *   `h3_res8_id`) are read past and never emitted: `provider_id` comes from the {@linkcode ProviderID}
+ *   parameter instead (the FCC partitions availability files per provider, so the caller already knows it),
+ *   and FRN/brand/state/H3 join concerns are a 2c registry-join seam.
+ *
+ *   Two projection decisions are load-bearing and pre-registered. `location_id` (column 3) stays a STRING —
+ *   the FCC's values are zero-padded 10-digit strings and `parseInt` would lose the leading zeros (decision
+ *   1). `geoid` (column 10) is a string joining `TIGERBlockTable.GEOID` (decision 3).
+ *
+ *   ## Why the source is a resource, not a Buffer
+ *
+ *   This read is STREAMING because the files do not fit the alternative. One state × one technology —
+ *   `bdc_48_FibertothePremises_fixed_broadband_D25` — is 920 MB and 10,369,043 rows, and a national run
+ *   spans every state × every technology. The previous byte scanner took a `Buffer`, so its caller opened
+ *   with `readFile(csvPath)` and held the whole file resident per file.
+ *
+ *   ## Why quoting is not optional here
+ *
+ *   Measured on that same file: 421,555 rows carry one embedded comma inside a quoted `brand_name` and 327
+ *   carry two ("FiberFirst, LLC", "Valor Telecommunications of Texas, LP"). A delimiter scan blind to quotes
+ *   shifts every column right of `brand_name` on 4% of rows — measured exactly, a quote-blind
+ *   `String.split(",")` mismatches 81,095 of 2,000,000 real rows. `enableQuoteHandling` also keeps an
+ *   embedded NEWLINE inside its row — no row in that file needs it (the 12/13/14-field line counts sum
+ *   exactly to `wc -l`, so no record is split across lines), but the guarantee is what makes the reader safe
+ *   on a file nobody has measured yet.
+ *
+ *   ## What this costs
+ *
+ *   Measured on the full 920 MB / 10,369,042-row file, projecting these 7 of 12 columns, identical checksums
+ *   across every arm. One arm per process — arms sharing a process mis-rank, because the JIT warms across
+ *   them:
+ *
+ *   ```
+ *   hand-rolled byte scan, whole buffer   778 ns/row   peak RSS 978 MB
+ *   spliterator 6.2.0, streaming        2,435 ns/row   peak RSS 112 MB
+ *   spliterator patched, streaming        860 ns/row   peak RSS 100 MB
+ *   ```
+ *
+ *   The 6.2.0 row is why this looked like a regression when it landed: CSVSpliterator decoded once per
+ *   COLUMN, and `TextDecoder`'s per-call overhead dominates at column sizes. Fixed upstream in
+ *   sister-software/spliterator#6 by decoding the row once — so the streaming path now runs within 10% of a
+ *   whole-buffer byte scan while holding 9.8x less memory. **Requires a spliterator release carrying that
+ *   fix**; on 6.2.0 this reader is correct and roughly 3x slower.
+ *
+ *   Do not route around it by hand-rolling a splitter here again. The staging insert this feeds measures 615
+ *   ns/row, so the parse does sit on the critical path — but a local fast-and-quote-blind parser is exactly
+ *   the trade that produced the scanner this replaced, and the quote-blind version is 4% wrong.
  */
 
+import type { AsyncDataResource } from "spliterator"
+import { CSVSpliterator } from "spliterator"
+
 import type { ProviderID } from "./common.ts"
 
 /**
- * Byte values this scanner switches on.
+ * Column positions in the FCC's 12-column availability CSV. Named rather than sliced by offset so a reader can check
+ * them against the header row without counting commas:
  *
- * Local to this file — no generic newline-delimited-file utility exists elsewhere in the repo to import. Nexus's
- * equivalent (`LineDelimitedCharacter`) lived in `@isp.nexus/sdk/files`, a workspace this port doesn't carry over, and
- * repo convention forbids TS `enum` anyway (`erasableSyntaxOnly`).
+ * `frn,provider_id,brand_name,location_id,technology,max_advertised_download_speed,`
+ * `max_advertised_upload_speed,low_latency,business_residential_code,state_usps,block_geoid,h3_res8_id`
  */
-const CSVByte = {
-	Newline: 10,
-	Comma: 44,
-	DoubleQuote: 34,
-	One: 49,
+const Column = {
+	LocationID: 3,
+	Technology: 4,
+	MaxAdvertisedDownloadSpeed: 5,
+	MaxAdvertisedUploadSpeed: 6,
+	LowLatency: 7,
+	BusinessResidentialCode: 8,
+	BlockGeoID: 10,
 } as const
 
 /**
  * A single parsed row of FCC BDC availability data.
  *
- * @see {@linkcode takeAvailabilityLine}
+ * @see {@linkcode readAvailabilityRows}
  */
 export interface BDCAvailabilityRow {
 	provider_id: number
@@ -55,70 +94,64 @@ export interface BDCAvailabilityRow {
 	low_latency: 0 | 1
 	business_residential_code: string
 	/**
-	 * Decoded to an ASCII string at the parse boundary (2a decision 3) — joins `TIGERBlockTable.GEOID` (note: uppercase
-	 * column on that side).
+	 * Joins `TIGERBlockTable.GEOID` (note: uppercase column on that side) — 2a decision 3.
 	 */
 	geoid: string
 }
 
 /**
- * Given a buffer containing FCC BDC availability CSV data, yield each data row (the header row is skipped) as a
- * {@linkcode BDCAvailabilityRow}.
+ * Project one already-split CSV row onto {@linkcode BDCAvailabilityRow}.
  *
- * A byte-level scan, not a general CSV parser: it tracks comma/newline byte positions directly and only toggles
- * quote-awareness via a running double-quote count (an odd count means the scanner is currently inside a quoted field,
- * so a comma there isn't a column delimiter) — matching the Nexus original's approach for this specific, known-shaped
- * 12-column file rather than reaching for a general CSV library.
+ * Separate from the iteration so the sync and async readers cannot drift in what they emit — the one-function
+ * discipline. Not exported: a caller with a split row wants {@linkcode readAvailabilityRowsSync}.
  */
-export function* takeAvailabilityLine(csvBuffer: Buffer, providerID: ProviderID): Iterable<BDCAvailabilityRow> {
-	// Skip the header row.
-	let byteIndex = csvBuffer.indexOf(CSVByte.Newline) + 1
-	const contentDelimiters = new Uint32Array(12) // 12 columns
-	contentDelimiters[0] = byteIndex
-	let delimiterIndex = 1
-	let doubleQuoteCount = 0
-
-	while (byteIndex < csvBuffer.length) {
-		const byte = csvBuffer[byteIndex]
-
-		if (byte === CSVByte.DoubleQuote) {
-			doubleQuoteCount++
-		}
-
-		if (byte === CSVByte.Comma && doubleQuoteCount % 2 === 0) {
-			contentDelimiters[delimiterIndex] = byteIndex
-
-			delimiterIndex++
-		}
-
-		if (byte === CSVByte.Newline) {
-			contentDelimiters[delimiterIndex] = CSVByte.Newline
-			const slices: Buffer[] = []
-
-			// Skip columns 0-2 (frn, provider_id, brand_name) — see the file header for why.
-			for (let i = 3; i < contentDelimiters.length; i++) {
-				const start = contentDelimiters[i]! + 1
-				const end = contentDelimiters[i + 1]
-
-				slices.push(csvBuffer.subarray(start, end))
-			}
+function projectRow(columns: readonly string[], providerID: ProviderID): BDCAvailabilityRow {
+	return {
+		provider_id: providerID,
+		location_id: columns[Column.LocationID] ?? "",
+		technology_code: Number.parseInt(columns[Column.Technology] ?? "", 10),
+		max_advertised_download_speed: Number.parseInt(columns[Column.MaxAdvertisedDownloadSpeed] ?? "", 10),
+		max_advertised_upload_speed: Number.parseInt(columns[Column.MaxAdvertisedUploadSpeed] ?? "", 10),
+		low_latency: columns[Column.LowLatency] === "1" ? 1 : 0,
+		business_residential_code: columns[Column.BusinessResidentialCode] ?? "",
+		geoid: columns[Column.BlockGeoID] ?? "",
+	}
+}
 
-			const record: BDCAvailabilityRow = {
-				provider_id: providerID,
-				location_id: slices[0]!.toString("ascii"),
-				technology_code: Number.parseInt(slices[1]!.toString(), 10),
-				max_advertised_download_speed: Number.parseInt(slices[2]!.toString(), 10),
-				max_advertised_upload_speed: Number.parseInt(slices[3]!.toString(), 10),
-				low_latency: slices[4]![0] === CSVByte.One ? 1 : 0,
-				business_residential_code: slices[5]!.toString("ascii"),
-				geoid: slices[7]!.toString("ascii"),
-			}
+/**
+ * Shared reader options. `header: true` consumes the first row as the header even in `array` mode;
+ * `enableQuoteHandling` is what makes the 421,882 quoted-brand rows keep their column alignment. `crlf` already
+ * defaults to `true` for CSV (RFC 4180), so a CRLF file does not leak `\r` into the last column.
+ */
+const READER_OPTIONS = { mode: "array", enableQuoteHandling: true } as const
 
-			yield record
-			delimiterIndex = 1
-			doubleQuoteCount = 0
-		}
+/**
+ * Stream an FCC BDC availability CSV, yielding every data row. The header row is consumed, never emitted.
+ *
+ * `source` is anything `spliterator` can open asynchronously — a path string, a `path-ts` builder, a URL, a file
+ * handle, or an async chunk iterator. Prefer handing it the path and letting it own the read: that is what keeps a 920
+ * MB file off the heap.
+ */
+export async function* readAvailabilityRows(
+	source: AsyncDataResource,
+	providerID: ProviderID
+): AsyncIterable<BDCAvailabilityRow> {
+	for await (const columns of CSVSpliterator.fromAsync<string[]>(source, READER_OPTIONS)) {
+		yield projectRow(columns, providerID)
+	}
+}
 
-		byteIndex++
+/**
+ * Synchronous sibling for an in-memory buffer — fixtures and tests, never the build path.
+ *
+ * Kept because a test that must assert on a literal CSV should not have to stand up a stream to do it. If you are
+ * reaching for this against a file on disk, reach for {@linkcode readAvailabilityRows} instead.
+ */
+export function* readAvailabilityRowsSync(
+	csvBuffer: Buffer | string,
+	providerID: ProviderID
+): Iterable<BDCAvailabilityRow> {
+	for (const columns of CSVSpliterator.from<string[]>(csvBuffer, READER_OPTIONS)) {
+		yield projectRow(columns, providerID)
 	}
 }

package/sdk/plausibility.ts

@@ -113,11 +113,15 @@
  */
 
 import type { DatabaseClient } from "@mailwoman/core/kysley/client"
-import { readLayerCoverage, readLayerManifest, type LayerContractDatabase } from "@mailwoman/core/layers"
+import {
+	readLayerCoverage,
+	readLayerManifest,
+	type LayerContractDatabase,
+	type LayerContractHandle,
+} from "@mailwoman/core/layers"
 import type { POILookup } from "@mailwoman/resolver-wof-sqlite/poi-lookup"
 import { shortCellToInt, type H3Cell, type PointLiteral } from "@mailwoman/spatial"
 import { latLngToCell } from "h3-js"
-import type { Kysely } from "kysely"
 
 import { BDC_H3_RESOLUTION, type BDCDatabase } from "../schema.ts"
 import {
@@ -295,15 +299,6 @@ const SPEED_BUCKET_RANK: Readonly<Record<string, number>> = {
 	[BDC_SPEED_BUCKET_GIGABIT]: 3,
 }
 
-/**
- * `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
- * in `DB` — same cast idiom as `filing-landscape.ts`'s own private `asContractDB` (decision 8: reuse, never re-derive;
- * copied rather than imported since the original is module-private).
- */
-function asContractDB(kdb: DatabaseClient<BDCDatabase>): Kysely<LayerContractDatabase> {
-	return kdb as unknown as Kysely<LayerContractDatabase>
-}
-
 /**
  * `true` when `filing` corroborates the claim: same `technology_code`, AND `filing.speed_bucket` ranks at or above the
  * claimed download speed's own bucket. A different tech, or a same-tech but LESSER filing, is `false` — never disproof,
@@ -373,7 +368,7 @@ function combineCoverage(
  */
 async function assertLayerSpineResolution(
 	layer: "bdc" | "poi",
-	contractDB: Kysely<LayerContractDatabase>,
+	contractDB: LayerContractHandle,
 	expectedResolution: number
 ): Promise<void> {
 	const manifest = await readLayerManifest(contractDB)
@@ -430,7 +425,7 @@ export async function plausibilityCheck(claim: PlausibilityClaim, deps: Plausibi
 	// independently per WIRED layer, not only when both are present: a poi-only call still joins poi's coverage
 	// table against a BDC_H3_RESOLUTION-derived cell (below) and must not do so unchecked.
 	if (deps.bdcDB) {
-		await assertLayerSpineResolution("bdc", asContractDB(deps.bdcDB), BDC_H3_RESOLUTION)
+		await assertLayerSpineResolution("bdc", deps.bdcDB, BDC_H3_RESOLUTION)
 	}
 
 	if (deps.poi) {