@mailwoman/bdc 9.0.0 → 9.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/sdk/build-bdc.ts CHANGED
@@ -8,8 +8,7 @@
8
8
  *
9
9
  * Mirrors `mailwoman/gazetteer-pipeline/poi/build-poi.ts`'s shape closely: the same build-tuning
10
10
  * pragmas, the same single-pass `Map<number, number>` coverage aggregation taken during the load
11
- * (no second scan), the same {@link asContractDB} Kysely-invariance cast for the shared
12
- * `@mailwoman/core/layers` calls, and the same `writeLayerManifest` → `sealDatabase` tail.
11
+ * (no second scan), and the same `writeLayerManifest` `sealDatabase` tail.
13
12
  *
14
13
  * Two differences from that precedent, both deliberate:
15
14
  *
@@ -51,23 +50,23 @@
51
50
  * the house rule exists for.
52
51
  */
53
52
 
54
- import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"
55
- import { readFile } from "node:fs/promises"
56
- import { dirname } from "node:path"
53
+ import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs"
54
+ import { open } from "node:fs/promises"
55
+ import { basename, dirname, join } from "node:path"
57
56
  import { DatabaseSync } from "node:sqlite"
58
57
 
59
58
  import { DatabaseClient } from "@mailwoman/core/kysley/client"
60
59
  import {
60
+ CoverageBasis,
61
61
  createLayerCoverageTable,
62
62
  createLayerManifestTable,
63
63
  LayerFreshnessPolicy,
64
64
  LayerTier,
65
65
  writeLayerCoverage,
66
66
  writeLayerManifest,
67
- type LayerContractDatabase,
68
67
  } from "@mailwoman/core/layers"
69
68
  import { tryParsingJSON } from "@mailwoman/core/objects"
70
- import { openBuiltDatabase, sealDatabase } from "@mailwoman/core/utils"
69
+ import { openBuiltDatabase, sealDatabase, swapDatabaseIntoPlace } from "@mailwoman/core/utils"
71
70
  import type { FilerDatabase } from "@mailwoman/filer"
72
71
  // `pickPrimaryFRN`/`readFRNFilingCandidates` are loaded via a LAZY `await import("@mailwoman/filer/sdk")`
73
72
  // inside `populateBDCProviderTable`, not a top-level runtime import — see that function's docstring
@@ -88,7 +87,7 @@ import {
88
87
  type BDCProviderTable,
89
88
  } from "../schema.ts"
90
89
  import type { ProviderID } from "./common.ts"
91
- import { takeAvailabilityLine, type BDCAvailabilityRow } from "./parsing.ts"
90
+ import { readAvailabilityRows, type BDCAvailabilityRow } from "./parsing.ts"
92
91
 
93
92
  /**
94
93
  * Rows committed per `BEGIN`/`COMMIT` batch during both the staging load and the materialize pass — matches
@@ -206,16 +205,6 @@ export interface BuildBDCResult {
206
205
  providersPopulated: number
207
206
  }
208
207
 
209
- /**
210
- * `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
211
- * in `DB` — narrows a `DatabaseClient<BDCDatabase>` handle back down for the `@mailwoman/core/layers` calls. Exact
212
- * precedent: `build-poi.ts`'s own `asContractDB`; see that file for the full rationale (tried widening the shared
213
- * package's signatures first — breaks THEIR internal `insertInto`/`selectFrom` calls instead).
214
- */
215
- function asContractDB(kdb: DatabaseClient<BDCDatabase>): DatabaseClient<LayerContractDatabase> {
216
- return kdb as unknown as DatabaseClient<LayerContractDatabase>
217
- }
218
-
219
208
  /**
220
209
  * Create the build-only `bdc_stage` table — deliberately NOT part of the public {@link BDCDatabase} interface (it's
221
210
  * dropped before the artifact seals, so it never appears in the shipped schema). Built via Kysely's schema builder per
@@ -308,17 +297,43 @@ export function peekProviderID(csvBuffer: Buffer, csvPath?: string): ProviderID
308
297
  }
309
298
 
310
299
  /**
311
- * Reads each of `csvPaths` fully into memory, peeks its `provider_id` ({@linkcode peekProviderID}, passing the path
312
- * through so a malformed file's error names it), then yields every row via `takeAvailabilityLine`. This is the
300
+ * Bytes read to peek the `provider_id`. Only the header row plus the first data row are needed and an FCC availability
301
+ * row is ~110 bytes, so this is three orders of magnitude of slack. A file shorter than this simply reads short —
302
+ * {@linkcode peekProviderID} already reports a header-only or empty file by message.
303
+ */
304
+ const PROVIDER_ID_PEEK_BYTES = 64 * 1024
305
+
306
+ /**
307
+ * Read the head of a CSV, for {@linkcode peekProviderID}.
308
+ *
309
+ * The point is what it does NOT do. `provider_id` is a constant per file, so establishing it needs the first data row
310
+ * and nothing else; `readFile(csvPath)` was resident-loading the entire file to read one column of one row. The
311
+ * measured file that motivated this is 920 MB for a single state × technology.
312
+ */
313
+ async function readCSVHead(csvPath: string): Promise<Buffer> {
314
+ const handle = await open(csvPath)
315
+
316
+ try {
317
+ const buffer = Buffer.allocUnsafe(PROVIDER_ID_PEEK_BYTES)
318
+ const { bytesRead } = await handle.read(buffer, 0, PROVIDER_ID_PEEK_BYTES, 0)
319
+
320
+ return buffer.subarray(0, bytesRead)
321
+ } finally {
322
+ await handle.close()
323
+ }
324
+ }
325
+
326
+ /**
327
+ * Peeks each file's `provider_id` off its head ({@linkcode peekProviderID}, passing the path through so a malformed
328
+ * file's error names it), then STREAMS every row via `readAvailabilityRows` — the file is never resident. This is the
313
329
  * production counterpart to the test seam's injected `rows` — exercised by `build-bdc.test.ts` only for the
314
330
  * malformed-provider-id rejection path, same as `build-poi.ts`'s `readParquetRows`.
315
331
  */
316
332
  async function* readAvailabilityRowsFromCSVPaths(csvPaths: readonly string[]): AsyncIterable<BDCAvailabilityRow> {
317
333
  for (const csvPath of csvPaths) {
318
- const buffer = await readFile(csvPath)
319
- const providerID = peekProviderID(buffer, csvPath)
334
+ const providerID = peekProviderID(await readCSVHead(csvPath), csvPath)
320
335
 
321
- yield* takeAvailabilityLine(buffer, providerID)
336
+ yield* readAvailabilityRows(csvPath, providerID)
322
337
  }
323
338
  }
324
339
 
@@ -336,11 +351,18 @@ interface GeoJSONMultiPolygon {
336
351
  }
337
352
 
338
353
  /**
339
- * Naive (vertex-average, NOT area-weighted) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s) only
340
- * (interior rings/holes are ignored). A known simplification, not an oversight: census blocks are small relative to a
341
- * res-9 H3 cell (~174m edge), so the vertex-average and a proper area-weighted centroid land in the same cell for all
342
- * but pathologically elongated or holed block shapes. A precise area-weighted centroid is a reasonable future upgrade
343
- * if that ever proves wrong in practice no polygon-centroid library is pulled in for this first cut.
354
+ * Area-weighted (shoelace) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s), area-weighted across
355
+ * rings for a MultiPolygon. Interior rings/holes are still ignored a hole moves a block's centroid far less than the
356
+ * vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
357
+ *
358
+ * This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
359
+ * falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
360
+ * vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
361
+ * edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
362
+ * squiggly natural edge and drag a vertex-average toward it.
363
+ *
364
+ * A degenerate geometry with zero total ring area (a sliver the shoelace annihilates) falls back to the vertex average
365
+ * — a weaker answer beats none, and the fallback is exactly the old behavior.
344
366
  *
345
367
  * Returns `undefined` for anything that doesn't parse as one of the two geometry types (including `null` geometry).
346
368
  */
@@ -358,21 +380,49 @@ export function geometryCentroid(geometryJSON: string | null): { lat: number; lo
358
380
  ? geometry.coordinates.map((polygon) => polygon[0] ?? [])
359
381
  : []
360
382
 
383
+ let totalArea = 0
384
+ let weightedLon = 0
385
+ let weightedLat = 0
361
386
  let sumLon = 0
362
387
  let sumLat = 0
363
388
  let count = 0
364
389
 
365
390
  for (const ring of exteriorRings) {
366
- for (const point of ring) {
367
- const [lon, lat] = point
391
+ let ringArea = 0
392
+ let ringLon = 0
393
+ let ringLat = 0
368
394
 
369
- if (typeof lon !== "number" || typeof lat !== "number") continue
395
+ for (let i = 0; i < ring.length - 1; i++) {
396
+ const [x1, y1] = ring[i]!
397
+ const [x2, y2] = ring[i + 1]!
370
398
 
371
- sumLon += lon
372
- sumLat += lat
399
+ if (typeof x1 !== "number" || typeof y1 !== "number" || typeof x2 !== "number" || typeof y2 !== "number") {
400
+ continue
401
+ }
402
+
403
+ const cross = x1 * y2 - x2 * y1
404
+ ringArea += cross
405
+ ringLon += (x1 + x2) * cross
406
+ ringLat += (y1 + y2) * cross
407
+ // The vertex-average fallback accumulates alongside — one pass, both answers.
408
+ sumLon += x1
409
+ sumLat += y1
373
410
 
374
411
  count++
375
412
  }
413
+
414
+ ringArea /= 2
415
+
416
+ if (ringArea === 0) continue
417
+
418
+ const weight = Math.abs(ringArea)
419
+ totalArea += weight
420
+ weightedLon += (ringLon / (6 * ringArea)) * weight
421
+ weightedLat += (ringLat / (6 * ringArea)) * weight
422
+ }
423
+
424
+ if (totalArea > 0) {
425
+ return { lat: weightedLat / totalArea, lon: weightedLon / totalArea }
376
426
  }
377
427
 
378
428
  if (count === 0) return undefined
@@ -536,6 +586,22 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
536
586
 
537
587
  mkdirSync(dirname(options.out), { recursive: true })
538
588
 
589
+ // A crash inside a PRIOR run's swap can leave the slot empty while the previous version sits
590
+ // parked aside — restore it before building, so a failure in THIS run still leaves an artifact
591
+ // serving. Both aside spellings: this builder's old `.prev` and swapDatabaseIntoPlace's `.old-<pid>`.
592
+ if (!existsSync(options.out)) {
593
+ const base = basename(options.out)
594
+
595
+ const parked = readdirSync(dirname(options.out)).find(
596
+ (name) => name === `${base}.prev` || name.startsWith(`${base}.old-`)
597
+ )
598
+
599
+ if (parked) {
600
+ renameSync(join(dirname(options.out), parked), options.out)
601
+ progress(`restored ${parked} into place (a prior run crashed mid-swap)`)
602
+ }
603
+ }
604
+
539
605
  const rowSource: AsyncIterable<BDCAvailabilityRow> | Iterable<BDCAvailabilityRow> =
540
606
  options.rows ?? readAvailabilityRowsFromCSVPaths(options.csvPaths!)
541
607
 
@@ -544,264 +610,274 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
544
610
  db.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;")
545
611
  const kdb = new DatabaseClient<BDCDatabase>({ database: db })
546
612
 
547
- progress("creating manifest/coverage/availability/provider/stage tables")
548
- await createLayerManifestTable(asContractDB(kdb))
549
- await createLayerCoverageTable(asContractDB(kdb))
550
- await createBDCAvailabilityTable(kdb)
551
- await createBDCProviderTable(kdb)
552
- await createBDCStageTable(kdb)
613
+ // Assigned at the end of the try — the tallies live inside its scope; the seal + swap do not.
614
+ let result: BuildBDCResult
553
615
 
554
- const insStage = db.prepare(
555
- `INSERT OR IGNORE INTO bdc_stage (
616
+ try {
617
+ progress("creating manifest/coverage/availability/provider/stage tables")
618
+ await createLayerManifestTable(kdb)
619
+ await createLayerCoverageTable(kdb)
620
+ await createBDCAvailabilityTable(kdb)
621
+ await createBDCProviderTable(kdb)
622
+ await createBDCStageTable(kdb)
623
+
624
+ const insStage = db.prepare(
625
+ `INSERT OR IGNORE INTO bdc_stage (
556
626
  geoid, provider_id, technology_code, location_id,
557
627
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
558
628
  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
559
- )
560
-
561
- let staged = 0
562
- let batch = 0
563
-
564
- progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)")
565
- db.exec("BEGIN")
566
-
567
- for await (const row of rowSource) {
568
- insStage.run(
569
- row.geoid,
570
- row.provider_id,
571
- row.technology_code,
572
- row.location_id,
573
- row.max_advertised_download_speed,
574
- row.max_advertised_upload_speed,
575
- row.low_latency,
576
- row.business_residential_code
577
629
  )
578
630
 
579
- staged++
631
+ let staged = 0
632
+ let batch = 0
633
+
634
+ progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)")
635
+ db.exec("BEGIN")
636
+
637
+ for await (const row of rowSource) {
638
+ insStage.run(
639
+ row.geoid,
640
+ row.provider_id,
641
+ row.technology_code,
642
+ row.location_id,
643
+ row.max_advertised_download_speed,
644
+ row.max_advertised_upload_speed,
645
+ row.low_latency,
646
+ row.business_residential_code
647
+ )
580
648
 
581
- batch++
649
+ staged++
582
650
 
583
- if (batch >= STAGE_BATCH_SIZE) {
584
- db.exec("COMMIT")
585
- db.exec("BEGIN")
586
- batch = 0
587
- }
588
- }
651
+ batch++
589
652
 
590
- db.exec("COMMIT")
653
+ if (batch >= STAGE_BATCH_SIZE) {
654
+ db.exec("COMMIT")
655
+ db.exec("BEGIN")
656
+ batch = 0
657
+ }
658
+ }
591
659
 
592
- const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get() as {
593
- staged_count: number
594
- }
660
+ db.exec("COMMIT")
595
661
 
596
- const deduped = staged - stagedCountRow.staged_count
662
+ const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get() as {
663
+ staged_count: number
664
+ }
597
665
 
598
- progress(
599
- `staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`
600
- )
666
+ const deduped = staged - stagedCountRow.staged_count
601
667
 
602
- const centroidCache = new Map<string, { h3Cell: number; coverageCell: number } | null>()
603
- /**
604
- * Res-6 short-cell int → observed row count, aggregated during materialize (one pass, no second scan) — matches
605
- * `build-poi.ts`'s `coverage` Map.
606
- */
607
- const coverage = new Map<number, number>()
608
- const providers = new Set<number>()
609
- let unknownGeoids = 0
610
- let inserted = 0
668
+ progress(
669
+ `staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`
670
+ )
611
671
 
612
- const insAvailability = db.prepare(
613
- `INSERT INTO bdc_availability (
672
+ const centroidCache = new Map<string, { h3Cell: number; coverageCell: number } | null>()
673
+ /**
674
+ * Res-6 short-cell int → observed row count, aggregated during materialize (one pass, no second scan) — matches
675
+ * `build-poi.ts`'s `coverage` Map.
676
+ */
677
+ const coverage = new Map<number, number>()
678
+ const providers = new Set<number>()
679
+ let unknownGeoids = 0
680
+ let inserted = 0
681
+
682
+ const insAvailability = db.prepare(
683
+ `INSERT INTO bdc_availability (
614
684
  h3_cell, geoid, wof_id, provider_id, technology_code,
615
685
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code, location_id
616
686
  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
617
- )
618
-
619
- // The FCC's per-provider CSVs are per-BSL: the SAME (geoid, provider_id, technology_code, speeds, low_latency,
620
- // business_residential_code) tuple can repeat once per Broadband Serviceable Location within that block (a
621
- // dense urban block can carry ~100 BSLs) — `bdc_stage`'s natural key includes `location_id`, so those BSL rows
622
- // all survive the staging dedup as distinct staged rows. In `includeLocationIDs` mode that's correct: every BSL
623
- // is a real, distinct row the caller asked to keep. In the default (NULL `location_id`) mode, when those BSLs
624
- // ALSO share identical speeds/flags, they'd otherwise materialize as byte-identical rows, inflating `result.rows`
625
- // and `layer_coverage.observed_rows` by the BSL count (~100x at real scale) — `SELECT DISTINCT`
626
- // over every column EXCEPT `location_id` collapses those byte-identical BSL duplicates down to one row.
627
- // IMPORTANT — this is NOT a guarantee of one row per (geoid, provider_id, technology_code) triple: BSLs at the
628
- // same triple with DIFFERING speeds/flags are NOT the same tuple, so `SELECT DISTINCT` does not merge them —
629
- // they survive as multiple NULL-`location_id` rows at that one triple. Accepted, not a bug; see the module
630
- // docstring and `filing-landscape.ts`'s docstring for the read-side consequence.
631
- const stageStmt = options.includeLocationIDs
632
- ? db.prepare(
633
- `SELECT geoid, provider_id, technology_code, location_id,
687
+ )
688
+
689
+ // The FCC's per-provider CSVs are per-BSL: the SAME (geoid, provider_id, technology_code, speeds, low_latency,
690
+ // business_residential_code) tuple can repeat once per Broadband Serviceable Location within that block (a
691
+ // dense urban block can carry ~100 BSLs) — `bdc_stage`'s natural key includes `location_id`, so those BSL rows
692
+ // all survive the staging dedup as distinct staged rows. In `includeLocationIDs` mode that's correct: every BSL
693
+ // is a real, distinct row the caller asked to keep. In the default (NULL `location_id`) mode, when those BSLs
694
+ // ALSO share identical speeds/flags, they'd otherwise materialize as byte-identical rows, inflating `result.rows`
695
+ // and `layer_coverage.observed_rows` by the BSL count (~100x at real scale) — `SELECT DISTINCT`
696
+ // over every column EXCEPT `location_id` collapses those byte-identical BSL duplicates down to one row.
697
+ // IMPORTANT — this is NOT a guarantee of one row per (geoid, provider_id, technology_code) triple: BSLs at the
698
+ // same triple with DIFFERING speeds/flags are NOT the same tuple, so `SELECT DISTINCT` does not merge them —
699
+ // they survive as multiple NULL-`location_id` rows at that one triple. Accepted, not a bug; see the module
700
+ // docstring and `filing-landscape.ts`'s docstring for the read-side consequence.
701
+ const stageStmt = options.includeLocationIDs
702
+ ? db.prepare(
703
+ `SELECT geoid, provider_id, technology_code, location_id,
634
704
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
635
705
  FROM bdc_stage`
636
- )
637
- : db.prepare(
638
- `SELECT DISTINCT geoid, provider_id, technology_code,
706
+ )
707
+ : db.prepare(
708
+ `SELECT DISTINCT geoid, provider_id, technology_code,
639
709
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
640
710
  FROM bdc_stage`
711
+ )
712
+
713
+ progress(
714
+ "materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)"
715
+ )
716
+
717
+ db.exec("BEGIN")
718
+ batch = 0
719
+
720
+ for (const row of stageStmt.iterate() as IterableIterator<BDCStageRow>) {
721
+ let resolved = centroidCache.get(row.geoid)
722
+
723
+ if (resolved === undefined) {
724
+ const centroid = options.blockCentroids(row.geoid)
725
+
726
+ resolved = centroid
727
+ ? (() => {
728
+ // Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
729
+ // independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
730
+ // exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
731
+ // disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
732
+ // boundary artifacts). Deriving both `h3_cell` and the coverage cell from
733
+ // the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
734
+ // exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
735
+ // applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
736
+ // the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
737
+ const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION) as H3Cell
738
+
739
+ return {
740
+ h3Cell: shortCellToInt(fullRes9Cell),
741
+ coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION) as H3Cell),
742
+ }
743
+ })()
744
+ : null
745
+
746
+ centroidCache.set(row.geoid, resolved)
747
+ }
748
+
749
+ if (!resolved) {
750
+ unknownGeoids++
751
+
752
+ continue
753
+ }
754
+
755
+ insAvailability.run(
756
+ resolved.h3Cell,
757
+ row.geoid,
758
+ // wof_id stays NULL here — WOF point-in-polygon resolution against the block centroid is a later
759
+ // registry-join task, the same decision-8 scoping schema.ts documents for `bdc_provider`.
760
+ null,
761
+ row.provider_id,
762
+ row.technology_code,
763
+ row.max_advertised_download_speed,
764
+ row.max_advertised_upload_speed,
765
+ row.low_latency,
766
+ row.business_residential_code,
767
+ options.includeLocationIDs ? (row.location_id ?? null) : null
641
768
  )
642
769
 
643
- progress(
644
- "materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)"
645
- )
646
-
647
- db.exec("BEGIN")
648
- batch = 0
649
-
650
- for (const row of stageStmt.iterate() as IterableIterator<BDCStageRow>) {
651
- let resolved = centroidCache.get(row.geoid)
652
-
653
- if (resolved === undefined) {
654
- const centroid = options.blockCentroids(row.geoid)
655
-
656
- resolved = centroid
657
- ? (() => {
658
- // Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
659
- // independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
660
- // exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
661
- // disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
662
- // boundary artifacts). Deriving both `h3_cell` and the coverage cell from
663
- // the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
664
- // exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
665
- // applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
666
- // the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
667
- const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION) as H3Cell
668
-
669
- return {
670
- h3Cell: shortCellToInt(fullRes9Cell),
671
- coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION) as H3Cell),
672
- }
673
- })()
674
- : null
675
-
676
- centroidCache.set(row.geoid, resolved)
677
- }
770
+ inserted++
771
+ providers.add(row.provider_id)
772
+ coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1)
678
773
 
679
- if (!resolved) {
680
- unknownGeoids++
774
+ batch++
681
775
 
682
- continue
776
+ if (batch >= STAGE_BATCH_SIZE) {
777
+ db.exec("COMMIT")
778
+ db.exec("BEGIN")
779
+ batch = 0
780
+ }
683
781
  }
684
782
 
685
- insAvailability.run(
686
- resolved.h3Cell,
687
- row.geoid,
688
- // wof_id stays NULL here WOF point-in-polygon resolution against the block centroid is a later
689
- // registry-join task, the same decision-8 scoping schema.ts documents for `bdc_provider`.
690
- null,
691
- row.provider_id,
692
- row.technology_code,
693
- row.max_advertised_download_speed,
694
- row.max_advertised_upload_speed,
695
- row.low_latency,
696
- row.business_residential_code,
697
- options.includeLocationIDs ? (row.location_id ?? null) : null
783
+ db.exec("COMMIT")
784
+
785
+ progress(
786
+ `materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
787
+ `(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`
698
788
  )
699
789
 
700
- inserted++
701
- providers.add(row.provider_id)
702
- coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1)
790
+ await kdb.schema.dropTable("bdc_stage").execute()
791
+
792
+ progress("geoid index (index-after-load see schema.ts)")
793
+ await createBDCGeoidIndex(kdb)
794
+
795
+ // Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
796
+ // have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
797
+ // were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
798
+ const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
799
+ h3Cell,
800
+ completeness: 1,
801
+ basis: CoverageBasis.SourcePresent,
802
+ observedRows,
803
+ }))
804
+
805
+ await writeLayerCoverage(kdb, coverageCells)
806
+
807
+ progress("writing layer manifest")
808
+
809
+ await writeLayerManifest(kdb, {
810
+ name: "bdc",
811
+ version: options.asOfDate,
812
+ schemaVersion: 1,
813
+ tier: LayerTier.Shipped,
814
+ license: "public-domain",
815
+ attribution: BDC_ATTRIBUTION,
816
+ source: "fcc-bdc",
817
+ sourceVintage: options.asOfDate,
818
+ buildCmd: "mailwoman gazetteer build bdc",
819
+ buildSHA: options.buildSHA,
820
+ freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
821
+ spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
822
+ createdAt: new Date().toISOString(),
823
+ })
824
+
825
+ // bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
826
+ // `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
827
+ // `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
828
+ let providersPopulated = 0
829
+
830
+ if (options.providers) {
831
+ progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)")
832
+
833
+ providersPopulated = await populateBDCProviderTable(
834
+ kdb,
835
+ options.providers,
836
+ options.filerDB,
837
+ options.primaryFRNAsOf ?? options.asOfDate
838
+ )
703
839
 
704
- batch++
840
+ progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`)
841
+ }
705
842
 
706
- if (batch >= STAGE_BATCH_SIZE) {
707
- db.exec("COMMIT")
708
- db.exec("BEGIN")
709
- batch = 0
843
+ progress("finalize: ANALYZE + VACUUM")
844
+ db.exec("ANALYZE")
845
+ // page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
846
+ // `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
847
+ // same discipline).
848
+ db.exec("PRAGMA page_size=8192")
849
+ db.exec("VACUUM")
850
+ await kdb.destroy()
851
+
852
+ result = {
853
+ out: options.out,
854
+ rows: inserted,
855
+ deduped,
856
+ providers: providers.size,
857
+ coverageCells: coverageCells.length,
858
+ unknownGeoids,
859
+ providersPopulated,
860
+ }
861
+ } catch (error) {
862
+ // A mid-build throw must not leak the handle or orphan the staging file. The original error
863
+ // always wins over anything the cleanup itself throws.
864
+ try {
865
+ await kdb.destroy()
866
+ } catch {
867
+ // The handle may already be closed or mid-statement — nothing more to release.
710
868
  }
711
- }
712
869
 
713
- db.exec("COMMIT")
714
-
715
- progress(
716
- `materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
717
- `(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`
718
- )
719
-
720
- await kdb.schema.dropTable("bdc_stage").execute()
721
-
722
- progress("geoid index (index-after-load — see schema.ts)")
723
- await createBDCGeoidIndex(kdb)
724
-
725
- // Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
726
- // have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
727
- // were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
728
- const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
729
- h3Cell,
730
- completeness: 1,
731
- observedRows,
732
- }))
733
-
734
- await writeLayerCoverage(asContractDB(kdb), coverageCells)
735
-
736
- progress("writing layer manifest")
737
-
738
- await writeLayerManifest(asContractDB(kdb), {
739
- name: "bdc",
740
- version: options.asOfDate,
741
- schemaVersion: 1,
742
- tier: LayerTier.Shipped,
743
- license: "public-domain",
744
- attribution: BDC_ATTRIBUTION,
745
- source: "fcc-bdc",
746
- sourceVintage: options.asOfDate,
747
- buildCmd: "mailwoman gazetteer build bdc",
748
- buildSHA: options.buildSHA,
749
- freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
750
- spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
751
- createdAt: new Date().toISOString(),
752
- })
753
-
754
- // bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
755
- // `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
756
- // `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
757
- let providersPopulated = 0
758
-
759
- if (options.providers) {
760
- progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)")
761
-
762
- providersPopulated = await populateBDCProviderTable(
763
- kdb,
764
- options.providers,
765
- options.filerDB,
766
- options.primaryFRNAsOf ?? options.asOfDate
767
- )
870
+ rmSync(buildingPath, { force: true })
768
871
 
769
- progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`)
872
+ throw error
770
873
  }
771
874
 
772
- progress("finalize: ANALYZE + VACUUM")
773
- db.exec("ANALYZE")
774
- // page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
775
- // `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
776
- // same discipline).
777
- db.exec("PRAGMA page_size=8192")
778
- db.exec("VACUUM")
779
- await kdb.destroy()
780
-
781
875
  progress("seal")
782
876
  sealDatabase(buildingPath)
783
877
 
784
- // Atomic move-into-place the previous version is moved ASIDE FIRST, per the AGENTS.md database house rule
785
- // ("build successfully, then move the previous version to a temp directory, and then move the new version into
786
- // place"). Mirrors `mailwoman/eval-harness/gauntlet/build-regression-db.ts`'s `${output}.prev` swap. Deliberate
787
- // deviation from `build-poi.ts`'s direct-write — see the module docstring.
788
- if (existsSync(options.out)) {
789
- renameSync(options.out, `${options.out}.prev`)
790
- }
791
-
792
- renameSync(buildingPath, options.out)
793
-
794
- if (existsSync(`${options.out}.prev`)) {
795
- rmSync(`${options.out}.prev`)
796
- }
878
+ // Atomic move-into-place via the shared helper (the AGENTS.md database house rule): prior
879
+ // version aside first, forward rename restored on failure so the slot is never left empty.
880
+ swapDatabaseIntoPlace(buildingPath, options.out)
797
881
 
798
- return {
799
- out: options.out,
800
- rows: inserted,
801
- deduped,
802
- providers: providers.size,
803
- coverageCells: coverageCells.length,
804
- unknownGeoids,
805
- providersPopulated,
806
- }
882
+ return result
807
883
  }