@mailwoman/bdc 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/sdk/build-bdc.d.ts +13 -7
- package/out/sdk/build-bdc.d.ts.map +1 -1
- package/out/sdk/build-bdc.js +254 -190
- package/out/sdk/build-bdc.js.map +1 -1
- package/out/sdk/download.d.ts +0 -11
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +1 -37
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/filing-landscape.d.ts.map +1 -1
- package/out/sdk/filing-landscape.js +2 -9
- package/out/sdk/filing-landscape.js.map +1 -1
- package/out/sdk/parsing.d.ts +65 -27
- package/out/sdk/parsing.d.ts.map +1 -1
- package/out/sdk/parsing.js +103 -70
- package/out/sdk/parsing.js.map +1 -1
- package/out/sdk/plausibility.d.ts.map +1 -1
- package/out/sdk/plausibility.js +2 -10
- package/out/sdk/plausibility.js.map +1 -1
- package/package.json +106 -13
- package/sdk/build-bdc.ts +329 -253
- package/sdk/download.ts +1 -41
- package/sdk/filing-landscape.ts +4 -12
- package/sdk/parsing.ts +115 -82
- package/sdk/plausibility.ts +8 -13
package/sdk/build-bdc.ts
CHANGED
|
@@ -8,8 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Mirrors `mailwoman/gazetteer-pipeline/poi/build-poi.ts`'s shape closely: the same build-tuning
|
|
10
10
|
* pragmas, the same single-pass `Map<number, number>` coverage aggregation taken during the load
|
|
11
|
-
* (no second scan), the same
|
|
12
|
-
* `@mailwoman/core/layers` calls, and the same `writeLayerManifest` → `sealDatabase` tail.
|
|
11
|
+
* (no second scan), and the same `writeLayerManifest` → `sealDatabase` tail.
|
|
13
12
|
*
|
|
14
13
|
* Two differences from that precedent, both deliberate:
|
|
15
14
|
*
|
|
@@ -51,23 +50,23 @@
|
|
|
51
50
|
* the house rule exists for.
|
|
52
51
|
*/
|
|
53
52
|
|
|
54
|
-
import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"
|
|
55
|
-
import {
|
|
56
|
-
import { dirname } from "node:path"
|
|
53
|
+
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs"
|
|
54
|
+
import { open } from "node:fs/promises"
|
|
55
|
+
import { basename, dirname, join } from "node:path"
|
|
57
56
|
import { DatabaseSync } from "node:sqlite"
|
|
58
57
|
|
|
59
58
|
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
60
59
|
import {
|
|
60
|
+
CoverageBasis,
|
|
61
61
|
createLayerCoverageTable,
|
|
62
62
|
createLayerManifestTable,
|
|
63
63
|
LayerFreshnessPolicy,
|
|
64
64
|
LayerTier,
|
|
65
65
|
writeLayerCoverage,
|
|
66
66
|
writeLayerManifest,
|
|
67
|
-
type LayerContractDatabase,
|
|
68
67
|
} from "@mailwoman/core/layers"
|
|
69
68
|
import { tryParsingJSON } from "@mailwoman/core/objects"
|
|
70
|
-
import { openBuiltDatabase, sealDatabase } from "@mailwoman/core/utils"
|
|
69
|
+
import { openBuiltDatabase, sealDatabase, swapDatabaseIntoPlace } from "@mailwoman/core/utils"
|
|
71
70
|
import type { FilerDatabase } from "@mailwoman/filer"
|
|
72
71
|
// `pickPrimaryFRN`/`readFRNFilingCandidates` are loaded via a LAZY `await import("@mailwoman/filer/sdk")`
|
|
73
72
|
// inside `populateBDCProviderTable`, not a top-level runtime import — see that function's docstring
|
|
@@ -88,7 +87,7 @@ import {
|
|
|
88
87
|
type BDCProviderTable,
|
|
89
88
|
} from "../schema.ts"
|
|
90
89
|
import type { ProviderID } from "./common.ts"
|
|
91
|
-
import {
|
|
90
|
+
import { readAvailabilityRows, type BDCAvailabilityRow } from "./parsing.ts"
|
|
92
91
|
|
|
93
92
|
/**
|
|
94
93
|
* Rows committed per `BEGIN`/`COMMIT` batch during both the staging load and the materialize pass — matches
|
|
@@ -206,16 +205,6 @@ export interface BuildBDCResult {
|
|
|
206
205
|
providersPopulated: number
|
|
207
206
|
}
|
|
208
207
|
|
|
209
|
-
/**
|
|
210
|
-
* `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
|
|
211
|
-
* in `DB` — narrows a `DatabaseClient<BDCDatabase>` handle back down for the `@mailwoman/core/layers` calls. Exact
|
|
212
|
-
* precedent: `build-poi.ts`'s own `asContractDB`; see that file for the full rationale (tried widening the shared
|
|
213
|
-
* package's signatures first — breaks THEIR internal `insertInto`/`selectFrom` calls instead).
|
|
214
|
-
*/
|
|
215
|
-
function asContractDB(kdb: DatabaseClient<BDCDatabase>): DatabaseClient<LayerContractDatabase> {
|
|
216
|
-
return kdb as unknown as DatabaseClient<LayerContractDatabase>
|
|
217
|
-
}
|
|
218
|
-
|
|
219
208
|
/**
|
|
220
209
|
* Create the build-only `bdc_stage` table — deliberately NOT part of the public {@link BDCDatabase} interface (it's
|
|
221
210
|
* dropped before the artifact seals, so it never appears in the shipped schema). Built via Kysely's schema builder per
|
|
@@ -308,17 +297,43 @@ export function peekProviderID(csvBuffer: Buffer, csvPath?: string): ProviderID
|
|
|
308
297
|
}
|
|
309
298
|
|
|
310
299
|
/**
|
|
311
|
-
*
|
|
312
|
-
*
|
|
300
|
+
* Bytes read to peek the `provider_id`. Only the header row plus the first data row are needed and an FCC availability
|
|
301
|
+
* row is ~110 bytes, so this is three orders of magnitude of slack. A file shorter than this simply reads short —
|
|
302
|
+
* {@linkcode peekProviderID} already reports a header-only or empty file by message.
|
|
303
|
+
*/
|
|
304
|
+
const PROVIDER_ID_PEEK_BYTES = 64 * 1024
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Read the head of a CSV, for {@linkcode peekProviderID}.
|
|
308
|
+
*
|
|
309
|
+
* The point is what it does NOT do. `provider_id` is a constant per file, so establishing it needs the first data row
|
|
310
|
+
* and nothing else; `readFile(csvPath)` was resident-loading the entire file to read one column of one row. The
|
|
311
|
+
* measured file that motivated this is 920 MB for a single state × technology.
|
|
312
|
+
*/
|
|
313
|
+
async function readCSVHead(csvPath: string): Promise<Buffer> {
|
|
314
|
+
const handle = await open(csvPath)
|
|
315
|
+
|
|
316
|
+
try {
|
|
317
|
+
const buffer = Buffer.allocUnsafe(PROVIDER_ID_PEEK_BYTES)
|
|
318
|
+
const { bytesRead } = await handle.read(buffer, 0, PROVIDER_ID_PEEK_BYTES, 0)
|
|
319
|
+
|
|
320
|
+
return buffer.subarray(0, bytesRead)
|
|
321
|
+
} finally {
|
|
322
|
+
await handle.close()
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Peeks each file's `provider_id` off its head ({@linkcode peekProviderID}, passing the path through so a malformed
|
|
328
|
+
* file's error names it), then STREAMS every row via `readAvailabilityRows` — the file is never resident. This is the
|
|
313
329
|
* production counterpart to the test seam's injected `rows` — exercised by `build-bdc.test.ts` only for the
|
|
314
330
|
* malformed-provider-id rejection path, same as `build-poi.ts`'s `readParquetRows`.
|
|
315
331
|
*/
|
|
316
332
|
async function* readAvailabilityRowsFromCSVPaths(csvPaths: readonly string[]): AsyncIterable<BDCAvailabilityRow> {
|
|
317
333
|
for (const csvPath of csvPaths) {
|
|
318
|
-
const
|
|
319
|
-
const providerID = peekProviderID(buffer, csvPath)
|
|
334
|
+
const providerID = peekProviderID(await readCSVHead(csvPath), csvPath)
|
|
320
335
|
|
|
321
|
-
yield*
|
|
336
|
+
yield* readAvailabilityRows(csvPath, providerID)
|
|
322
337
|
}
|
|
323
338
|
}
|
|
324
339
|
|
|
@@ -336,11 +351,18 @@ interface GeoJSONMultiPolygon {
|
|
|
336
351
|
}
|
|
337
352
|
|
|
338
353
|
/**
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
354
|
+
* Area-weighted (shoelace) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s), area-weighted across
|
|
355
|
+
* rings for a MultiPolygon. Interior rings/holes are still ignored — a hole moves a block's centroid far less than the
|
|
356
|
+
* vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
|
|
357
|
+
*
|
|
358
|
+
* This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
|
|
359
|
+
* falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
|
|
360
|
+
* vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
|
|
361
|
+
* edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
|
|
362
|
+
* squiggly natural edge and drag a vertex-average toward it.
|
|
363
|
+
*
|
|
364
|
+
* A degenerate geometry with zero total ring area (a sliver the shoelace annihilates) falls back to the vertex average
|
|
365
|
+
* — a weaker answer beats none, and the fallback is exactly the old behavior.
|
|
344
366
|
*
|
|
345
367
|
* Returns `undefined` for anything that doesn't parse as one of the two geometry types (including `null` geometry).
|
|
346
368
|
*/
|
|
@@ -358,21 +380,49 @@ export function geometryCentroid(geometryJSON: string | null): { lat: number; lo
|
|
|
358
380
|
? geometry.coordinates.map((polygon) => polygon[0] ?? [])
|
|
359
381
|
: []
|
|
360
382
|
|
|
383
|
+
let totalArea = 0
|
|
384
|
+
let weightedLon = 0
|
|
385
|
+
let weightedLat = 0
|
|
361
386
|
let sumLon = 0
|
|
362
387
|
let sumLat = 0
|
|
363
388
|
let count = 0
|
|
364
389
|
|
|
365
390
|
for (const ring of exteriorRings) {
|
|
366
|
-
|
|
367
|
-
|
|
391
|
+
let ringArea = 0
|
|
392
|
+
let ringLon = 0
|
|
393
|
+
let ringLat = 0
|
|
368
394
|
|
|
369
|
-
|
|
395
|
+
for (let i = 0; i < ring.length - 1; i++) {
|
|
396
|
+
const [x1, y1] = ring[i]!
|
|
397
|
+
const [x2, y2] = ring[i + 1]!
|
|
370
398
|
|
|
371
|
-
|
|
372
|
-
|
|
399
|
+
if (typeof x1 !== "number" || typeof y1 !== "number" || typeof x2 !== "number" || typeof y2 !== "number") {
|
|
400
|
+
continue
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
const cross = x1 * y2 - x2 * y1
|
|
404
|
+
ringArea += cross
|
|
405
|
+
ringLon += (x1 + x2) * cross
|
|
406
|
+
ringLat += (y1 + y2) * cross
|
|
407
|
+
// The vertex-average fallback accumulates alongside — one pass, both answers.
|
|
408
|
+
sumLon += x1
|
|
409
|
+
sumLat += y1
|
|
373
410
|
|
|
374
411
|
count++
|
|
375
412
|
}
|
|
413
|
+
|
|
414
|
+
ringArea /= 2
|
|
415
|
+
|
|
416
|
+
if (ringArea === 0) continue
|
|
417
|
+
|
|
418
|
+
const weight = Math.abs(ringArea)
|
|
419
|
+
totalArea += weight
|
|
420
|
+
weightedLon += (ringLon / (6 * ringArea)) * weight
|
|
421
|
+
weightedLat += (ringLat / (6 * ringArea)) * weight
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
if (totalArea > 0) {
|
|
425
|
+
return { lat: weightedLat / totalArea, lon: weightedLon / totalArea }
|
|
376
426
|
}
|
|
377
427
|
|
|
378
428
|
if (count === 0) return undefined
|
|
@@ -536,6 +586,22 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
|
|
|
536
586
|
|
|
537
587
|
mkdirSync(dirname(options.out), { recursive: true })
|
|
538
588
|
|
|
589
|
+
// A crash inside a PRIOR run's swap can leave the slot empty while the previous version sits
|
|
590
|
+
// parked aside — restore it before building, so a failure in THIS run still leaves an artifact
|
|
591
|
+
// serving. Both aside spellings: this builder's old `.prev` and swapDatabaseIntoPlace's `.old-<pid>`.
|
|
592
|
+
if (!existsSync(options.out)) {
|
|
593
|
+
const base = basename(options.out)
|
|
594
|
+
|
|
595
|
+
const parked = readdirSync(dirname(options.out)).find(
|
|
596
|
+
(name) => name === `${base}.prev` || name.startsWith(`${base}.old-`)
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
if (parked) {
|
|
600
|
+
renameSync(join(dirname(options.out), parked), options.out)
|
|
601
|
+
progress(`restored ${parked} into place (a prior run crashed mid-swap)`)
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
539
605
|
const rowSource: AsyncIterable<BDCAvailabilityRow> | Iterable<BDCAvailabilityRow> =
|
|
540
606
|
options.rows ?? readAvailabilityRowsFromCSVPaths(options.csvPaths!)
|
|
541
607
|
|
|
@@ -544,264 +610,274 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
|
|
|
544
610
|
db.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;")
|
|
545
611
|
const kdb = new DatabaseClient<BDCDatabase>({ database: db })
|
|
546
612
|
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
await createLayerCoverageTable(asContractDB(kdb))
|
|
550
|
-
await createBDCAvailabilityTable(kdb)
|
|
551
|
-
await createBDCProviderTable(kdb)
|
|
552
|
-
await createBDCStageTable(kdb)
|
|
613
|
+
// Assigned at the end of the try — the tallies live inside its scope; the seal + swap do not.
|
|
614
|
+
let result: BuildBDCResult
|
|
553
615
|
|
|
554
|
-
|
|
555
|
-
|
|
616
|
+
try {
|
|
617
|
+
progress("creating manifest/coverage/availability/provider/stage tables")
|
|
618
|
+
await createLayerManifestTable(kdb)
|
|
619
|
+
await createLayerCoverageTable(kdb)
|
|
620
|
+
await createBDCAvailabilityTable(kdb)
|
|
621
|
+
await createBDCProviderTable(kdb)
|
|
622
|
+
await createBDCStageTable(kdb)
|
|
623
|
+
|
|
624
|
+
const insStage = db.prepare(
|
|
625
|
+
`INSERT OR IGNORE INTO bdc_stage (
|
|
556
626
|
geoid, provider_id, technology_code, location_id,
|
|
557
627
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
|
|
558
628
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
559
|
-
)
|
|
560
|
-
|
|
561
|
-
let staged = 0
|
|
562
|
-
let batch = 0
|
|
563
|
-
|
|
564
|
-
progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)")
|
|
565
|
-
db.exec("BEGIN")
|
|
566
|
-
|
|
567
|
-
for await (const row of rowSource) {
|
|
568
|
-
insStage.run(
|
|
569
|
-
row.geoid,
|
|
570
|
-
row.provider_id,
|
|
571
|
-
row.technology_code,
|
|
572
|
-
row.location_id,
|
|
573
|
-
row.max_advertised_download_speed,
|
|
574
|
-
row.max_advertised_upload_speed,
|
|
575
|
-
row.low_latency,
|
|
576
|
-
row.business_residential_code
|
|
577
629
|
)
|
|
578
630
|
|
|
579
|
-
staged
|
|
631
|
+
let staged = 0
|
|
632
|
+
let batch = 0
|
|
633
|
+
|
|
634
|
+
progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)")
|
|
635
|
+
db.exec("BEGIN")
|
|
636
|
+
|
|
637
|
+
for await (const row of rowSource) {
|
|
638
|
+
insStage.run(
|
|
639
|
+
row.geoid,
|
|
640
|
+
row.provider_id,
|
|
641
|
+
row.technology_code,
|
|
642
|
+
row.location_id,
|
|
643
|
+
row.max_advertised_download_speed,
|
|
644
|
+
row.max_advertised_upload_speed,
|
|
645
|
+
row.low_latency,
|
|
646
|
+
row.business_residential_code
|
|
647
|
+
)
|
|
580
648
|
|
|
581
|
-
|
|
649
|
+
staged++
|
|
582
650
|
|
|
583
|
-
|
|
584
|
-
db.exec("COMMIT")
|
|
585
|
-
db.exec("BEGIN")
|
|
586
|
-
batch = 0
|
|
587
|
-
}
|
|
588
|
-
}
|
|
651
|
+
batch++
|
|
589
652
|
|
|
590
|
-
|
|
653
|
+
if (batch >= STAGE_BATCH_SIZE) {
|
|
654
|
+
db.exec("COMMIT")
|
|
655
|
+
db.exec("BEGIN")
|
|
656
|
+
batch = 0
|
|
657
|
+
}
|
|
658
|
+
}
|
|
591
659
|
|
|
592
|
-
|
|
593
|
-
staged_count: number
|
|
594
|
-
}
|
|
660
|
+
db.exec("COMMIT")
|
|
595
661
|
|
|
596
|
-
|
|
662
|
+
const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get() as {
|
|
663
|
+
staged_count: number
|
|
664
|
+
}
|
|
597
665
|
|
|
598
|
-
|
|
599
|
-
`staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`
|
|
600
|
-
)
|
|
666
|
+
const deduped = staged - stagedCountRow.staged_count
|
|
601
667
|
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
* `build-poi.ts`'s `coverage` Map.
|
|
606
|
-
*/
|
|
607
|
-
const coverage = new Map<number, number>()
|
|
608
|
-
const providers = new Set<number>()
|
|
609
|
-
let unknownGeoids = 0
|
|
610
|
-
let inserted = 0
|
|
668
|
+
progress(
|
|
669
|
+
`staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`
|
|
670
|
+
)
|
|
611
671
|
|
|
612
|
-
|
|
613
|
-
|
|
672
|
+
const centroidCache = new Map<string, { h3Cell: number; coverageCell: number } | null>()
|
|
673
|
+
/**
|
|
674
|
+
* Res-6 short-cell int → observed row count, aggregated during materialize (one pass, no second scan) — matches
|
|
675
|
+
* `build-poi.ts`'s `coverage` Map.
|
|
676
|
+
*/
|
|
677
|
+
const coverage = new Map<number, number>()
|
|
678
|
+
const providers = new Set<number>()
|
|
679
|
+
let unknownGeoids = 0
|
|
680
|
+
let inserted = 0
|
|
681
|
+
|
|
682
|
+
const insAvailability = db.prepare(
|
|
683
|
+
`INSERT INTO bdc_availability (
|
|
614
684
|
h3_cell, geoid, wof_id, provider_id, technology_code,
|
|
615
685
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code, location_id
|
|
616
686
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
// The FCC's per-provider CSVs are per-BSL: the SAME (geoid, provider_id, technology_code, speeds, low_latency,
|
|
690
|
+
// business_residential_code) tuple can repeat once per Broadband Serviceable Location within that block (a
|
|
691
|
+
// dense urban block can carry ~100 BSLs) — `bdc_stage`'s natural key includes `location_id`, so those BSL rows
|
|
692
|
+
// all survive the staging dedup as distinct staged rows. In `includeLocationIDs` mode that's correct: every BSL
|
|
693
|
+
// is a real, distinct row the caller asked to keep. In the default (NULL `location_id`) mode, when those BSLs
|
|
694
|
+
// ALSO share identical speeds/flags, they'd otherwise materialize as byte-identical rows, inflating `result.rows`
|
|
695
|
+
// and `layer_coverage.observed_rows` by the BSL count (~100x at real scale) — `SELECT DISTINCT`
|
|
696
|
+
// over every column EXCEPT `location_id` collapses those byte-identical BSL duplicates down to one row.
|
|
697
|
+
// IMPORTANT — this is NOT a guarantee of one row per (geoid, provider_id, technology_code) triple: BSLs at the
|
|
698
|
+
// same triple with DIFFERING speeds/flags are NOT the same tuple, so `SELECT DISTINCT` does not merge them —
|
|
699
|
+
// they survive as multiple NULL-`location_id` rows at that one triple. Accepted, not a bug; see the module
|
|
700
|
+
// docstring and `filing-landscape.ts`'s docstring for the read-side consequence.
|
|
701
|
+
const stageStmt = options.includeLocationIDs
|
|
702
|
+
? db.prepare(
|
|
703
|
+
`SELECT geoid, provider_id, technology_code, location_id,
|
|
634
704
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
|
|
635
705
|
FROM bdc_stage`
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
706
|
+
)
|
|
707
|
+
: db.prepare(
|
|
708
|
+
`SELECT DISTINCT geoid, provider_id, technology_code,
|
|
639
709
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
|
|
640
710
|
FROM bdc_stage`
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
progress(
|
|
714
|
+
"materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)"
|
|
715
|
+
)
|
|
716
|
+
|
|
717
|
+
db.exec("BEGIN")
|
|
718
|
+
batch = 0
|
|
719
|
+
|
|
720
|
+
for (const row of stageStmt.iterate() as IterableIterator<BDCStageRow>) {
|
|
721
|
+
let resolved = centroidCache.get(row.geoid)
|
|
722
|
+
|
|
723
|
+
if (resolved === undefined) {
|
|
724
|
+
const centroid = options.blockCentroids(row.geoid)
|
|
725
|
+
|
|
726
|
+
resolved = centroid
|
|
727
|
+
? (() => {
|
|
728
|
+
// Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
|
|
729
|
+
// independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
|
|
730
|
+
// exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
|
|
731
|
+
// disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
|
|
732
|
+
// boundary artifacts). Deriving both `h3_cell` and the coverage cell from
|
|
733
|
+
// the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
|
|
734
|
+
// exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
|
|
735
|
+
// applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
|
|
736
|
+
// the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
|
|
737
|
+
const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION) as H3Cell
|
|
738
|
+
|
|
739
|
+
return {
|
|
740
|
+
h3Cell: shortCellToInt(fullRes9Cell),
|
|
741
|
+
coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION) as H3Cell),
|
|
742
|
+
}
|
|
743
|
+
})()
|
|
744
|
+
: null
|
|
745
|
+
|
|
746
|
+
centroidCache.set(row.geoid, resolved)
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
if (!resolved) {
|
|
750
|
+
unknownGeoids++
|
|
751
|
+
|
|
752
|
+
continue
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
insAvailability.run(
|
|
756
|
+
resolved.h3Cell,
|
|
757
|
+
row.geoid,
|
|
758
|
+
// wof_id stays NULL here — WOF point-in-polygon resolution against the block centroid is a later
|
|
759
|
+
// registry-join task, the same decision-8 scoping schema.ts documents for `bdc_provider`.
|
|
760
|
+
null,
|
|
761
|
+
row.provider_id,
|
|
762
|
+
row.technology_code,
|
|
763
|
+
row.max_advertised_download_speed,
|
|
764
|
+
row.max_advertised_upload_speed,
|
|
765
|
+
row.low_latency,
|
|
766
|
+
row.business_residential_code,
|
|
767
|
+
options.includeLocationIDs ? (row.location_id ?? null) : null
|
|
641
768
|
)
|
|
642
769
|
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
db.exec("BEGIN")
|
|
648
|
-
batch = 0
|
|
649
|
-
|
|
650
|
-
for (const row of stageStmt.iterate() as IterableIterator<BDCStageRow>) {
|
|
651
|
-
let resolved = centroidCache.get(row.geoid)
|
|
652
|
-
|
|
653
|
-
if (resolved === undefined) {
|
|
654
|
-
const centroid = options.blockCentroids(row.geoid)
|
|
655
|
-
|
|
656
|
-
resolved = centroid
|
|
657
|
-
? (() => {
|
|
658
|
-
// Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
|
|
659
|
-
// independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
|
|
660
|
-
// exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
|
|
661
|
-
// disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
|
|
662
|
-
// boundary artifacts). Deriving both `h3_cell` and the coverage cell from
|
|
663
|
-
// the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
|
|
664
|
-
// exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
|
|
665
|
-
// applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
|
|
666
|
-
// the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
|
|
667
|
-
const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION) as H3Cell
|
|
668
|
-
|
|
669
|
-
return {
|
|
670
|
-
h3Cell: shortCellToInt(fullRes9Cell),
|
|
671
|
-
coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION) as H3Cell),
|
|
672
|
-
}
|
|
673
|
-
})()
|
|
674
|
-
: null
|
|
675
|
-
|
|
676
|
-
centroidCache.set(row.geoid, resolved)
|
|
677
|
-
}
|
|
770
|
+
inserted++
|
|
771
|
+
providers.add(row.provider_id)
|
|
772
|
+
coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1)
|
|
678
773
|
|
|
679
|
-
|
|
680
|
-
unknownGeoids++
|
|
774
|
+
batch++
|
|
681
775
|
|
|
682
|
-
|
|
776
|
+
if (batch >= STAGE_BATCH_SIZE) {
|
|
777
|
+
db.exec("COMMIT")
|
|
778
|
+
db.exec("BEGIN")
|
|
779
|
+
batch = 0
|
|
780
|
+
}
|
|
683
781
|
}
|
|
684
782
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
null,
|
|
691
|
-
row.provider_id,
|
|
692
|
-
row.technology_code,
|
|
693
|
-
row.max_advertised_download_speed,
|
|
694
|
-
row.max_advertised_upload_speed,
|
|
695
|
-
row.low_latency,
|
|
696
|
-
row.business_residential_code,
|
|
697
|
-
options.includeLocationIDs ? (row.location_id ?? null) : null
|
|
783
|
+
db.exec("COMMIT")
|
|
784
|
+
|
|
785
|
+
progress(
|
|
786
|
+
`materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
|
|
787
|
+
`(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`
|
|
698
788
|
)
|
|
699
789
|
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
790
|
+
await kdb.schema.dropTable("bdc_stage").execute()
|
|
791
|
+
|
|
792
|
+
progress("geoid index (index-after-load — see schema.ts)")
|
|
793
|
+
await createBDCGeoidIndex(kdb)
|
|
794
|
+
|
|
795
|
+
// Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
|
|
796
|
+
// have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
|
|
797
|
+
// were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
|
|
798
|
+
const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
|
|
799
|
+
h3Cell,
|
|
800
|
+
completeness: 1,
|
|
801
|
+
basis: CoverageBasis.SourcePresent,
|
|
802
|
+
observedRows,
|
|
803
|
+
}))
|
|
804
|
+
|
|
805
|
+
await writeLayerCoverage(kdb, coverageCells)
|
|
806
|
+
|
|
807
|
+
progress("writing layer manifest")
|
|
808
|
+
|
|
809
|
+
await writeLayerManifest(kdb, {
|
|
810
|
+
name: "bdc",
|
|
811
|
+
version: options.asOfDate,
|
|
812
|
+
schemaVersion: 1,
|
|
813
|
+
tier: LayerTier.Shipped,
|
|
814
|
+
license: "public-domain",
|
|
815
|
+
attribution: BDC_ATTRIBUTION,
|
|
816
|
+
source: "fcc-bdc",
|
|
817
|
+
sourceVintage: options.asOfDate,
|
|
818
|
+
buildCmd: "mailwoman gazetteer build bdc",
|
|
819
|
+
buildSHA: options.buildSHA,
|
|
820
|
+
freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
|
|
821
|
+
spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
|
|
822
|
+
createdAt: new Date().toISOString(),
|
|
823
|
+
})
|
|
824
|
+
|
|
825
|
+
// bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
|
|
826
|
+
// `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
|
|
827
|
+
// `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
|
|
828
|
+
let providersPopulated = 0
|
|
829
|
+
|
|
830
|
+
if (options.providers) {
|
|
831
|
+
progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)")
|
|
832
|
+
|
|
833
|
+
providersPopulated = await populateBDCProviderTable(
|
|
834
|
+
kdb,
|
|
835
|
+
options.providers,
|
|
836
|
+
options.filerDB,
|
|
837
|
+
options.primaryFRNAsOf ?? options.asOfDate
|
|
838
|
+
)
|
|
703
839
|
|
|
704
|
-
|
|
840
|
+
progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`)
|
|
841
|
+
}
|
|
705
842
|
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
843
|
+
progress("finalize: ANALYZE + VACUUM")
|
|
844
|
+
db.exec("ANALYZE")
|
|
845
|
+
// page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
|
|
846
|
+
// `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
|
|
847
|
+
// same discipline).
|
|
848
|
+
db.exec("PRAGMA page_size=8192")
|
|
849
|
+
db.exec("VACUUM")
|
|
850
|
+
await kdb.destroy()
|
|
851
|
+
|
|
852
|
+
result = {
|
|
853
|
+
out: options.out,
|
|
854
|
+
rows: inserted,
|
|
855
|
+
deduped,
|
|
856
|
+
providers: providers.size,
|
|
857
|
+
coverageCells: coverageCells.length,
|
|
858
|
+
unknownGeoids,
|
|
859
|
+
providersPopulated,
|
|
860
|
+
}
|
|
861
|
+
} catch (error) {
|
|
862
|
+
// A mid-build throw must not leak the handle or orphan the staging file. The original error
|
|
863
|
+
// always wins over anything the cleanup itself throws.
|
|
864
|
+
try {
|
|
865
|
+
await kdb.destroy()
|
|
866
|
+
} catch {
|
|
867
|
+
// The handle may already be closed or mid-statement — nothing more to release.
|
|
710
868
|
}
|
|
711
|
-
}
|
|
712
869
|
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
progress(
|
|
716
|
-
`materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
|
|
717
|
-
`(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`
|
|
718
|
-
)
|
|
719
|
-
|
|
720
|
-
await kdb.schema.dropTable("bdc_stage").execute()
|
|
721
|
-
|
|
722
|
-
progress("geoid index (index-after-load — see schema.ts)")
|
|
723
|
-
await createBDCGeoidIndex(kdb)
|
|
724
|
-
|
|
725
|
-
// Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
|
|
726
|
-
// have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
|
|
727
|
-
// were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
|
|
728
|
-
const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
|
|
729
|
-
h3Cell,
|
|
730
|
-
completeness: 1,
|
|
731
|
-
observedRows,
|
|
732
|
-
}))
|
|
733
|
-
|
|
734
|
-
await writeLayerCoverage(asContractDB(kdb), coverageCells)
|
|
735
|
-
|
|
736
|
-
progress("writing layer manifest")
|
|
737
|
-
|
|
738
|
-
await writeLayerManifest(asContractDB(kdb), {
|
|
739
|
-
name: "bdc",
|
|
740
|
-
version: options.asOfDate,
|
|
741
|
-
schemaVersion: 1,
|
|
742
|
-
tier: LayerTier.Shipped,
|
|
743
|
-
license: "public-domain",
|
|
744
|
-
attribution: BDC_ATTRIBUTION,
|
|
745
|
-
source: "fcc-bdc",
|
|
746
|
-
sourceVintage: options.asOfDate,
|
|
747
|
-
buildCmd: "mailwoman gazetteer build bdc",
|
|
748
|
-
buildSHA: options.buildSHA,
|
|
749
|
-
freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
|
|
750
|
-
spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
|
|
751
|
-
createdAt: new Date().toISOString(),
|
|
752
|
-
})
|
|
753
|
-
|
|
754
|
-
// bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
|
|
755
|
-
// `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
|
|
756
|
-
// `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
|
|
757
|
-
let providersPopulated = 0
|
|
758
|
-
|
|
759
|
-
if (options.providers) {
|
|
760
|
-
progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)")
|
|
761
|
-
|
|
762
|
-
providersPopulated = await populateBDCProviderTable(
|
|
763
|
-
kdb,
|
|
764
|
-
options.providers,
|
|
765
|
-
options.filerDB,
|
|
766
|
-
options.primaryFRNAsOf ?? options.asOfDate
|
|
767
|
-
)
|
|
870
|
+
rmSync(buildingPath, { force: true })
|
|
768
871
|
|
|
769
|
-
|
|
872
|
+
throw error
|
|
770
873
|
}
|
|
771
874
|
|
|
772
|
-
progress("finalize: ANALYZE + VACUUM")
|
|
773
|
-
db.exec("ANALYZE")
|
|
774
|
-
// page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
|
|
775
|
-
// `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
|
|
776
|
-
// same discipline).
|
|
777
|
-
db.exec("PRAGMA page_size=8192")
|
|
778
|
-
db.exec("VACUUM")
|
|
779
|
-
await kdb.destroy()
|
|
780
|
-
|
|
781
875
|
progress("seal")
|
|
782
876
|
sealDatabase(buildingPath)
|
|
783
877
|
|
|
784
|
-
// Atomic move-into-place
|
|
785
|
-
//
|
|
786
|
-
|
|
787
|
-
// deviation from `build-poi.ts`'s direct-write — see the module docstring.
|
|
788
|
-
if (existsSync(options.out)) {
|
|
789
|
-
renameSync(options.out, `${options.out}.prev`)
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
renameSync(buildingPath, options.out)
|
|
793
|
-
|
|
794
|
-
if (existsSync(`${options.out}.prev`)) {
|
|
795
|
-
rmSync(`${options.out}.prev`)
|
|
796
|
-
}
|
|
878
|
+
// Atomic move-into-place via the shared helper (the AGENTS.md database house rule): prior
|
|
879
|
+
// version aside first, forward rename restored on failure so the slot is never left empty.
|
|
880
|
+
swapDatabaseIntoPlace(buildingPath, options.out)
|
|
797
881
|
|
|
798
|
-
return
|
|
799
|
-
out: options.out,
|
|
800
|
-
rows: inserted,
|
|
801
|
-
deduped,
|
|
802
|
-
providers: providers.size,
|
|
803
|
-
coverageCells: coverageCells.length,
|
|
804
|
-
unknownGeoids,
|
|
805
|
-
providersPopulated,
|
|
806
|
-
}
|
|
882
|
+
return result
|
|
807
883
|
}
|