@mailwoman/bdc 9.1.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/sdk/build-bdc.d.ts +13 -7
- package/out/sdk/build-bdc.d.ts.map +1 -1
- package/out/sdk/build-bdc.js +254 -190
- package/out/sdk/build-bdc.js.map +1 -1
- package/out/sdk/filing-landscape.d.ts.map +1 -1
- package/out/sdk/filing-landscape.js +2 -9
- package/out/sdk/filing-landscape.js.map +1 -1
- package/out/sdk/parsing.d.ts +65 -27
- package/out/sdk/parsing.d.ts.map +1 -1
- package/out/sdk/parsing.js +103 -70
- package/out/sdk/parsing.js.map +1 -1
- package/out/sdk/plausibility.d.ts.map +1 -1
- package/out/sdk/plausibility.js +2 -10
- package/out/sdk/plausibility.js.map +1 -1
- package/package.json +106 -13
- package/sdk/build-bdc.ts +329 -253
- package/sdk/filing-landscape.ts +4 -12
- package/sdk/parsing.ts +115 -82
- package/sdk/plausibility.ts +8 -13
package/out/sdk/build-bdc.d.ts
CHANGED
|
@@ -8,8 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Mirrors `mailwoman/gazetteer-pipeline/poi/build-poi.ts`'s shape closely: the same build-tuning
|
|
10
10
|
* pragmas, the same single-pass `Map<number, number>` coverage aggregation taken during the load
|
|
11
|
-
* (no second scan), the same
|
|
12
|
-
* `@mailwoman/core/layers` calls, and the same `writeLayerManifest` → `sealDatabase` tail.
|
|
11
|
+
* (no second scan), and the same `writeLayerManifest` → `sealDatabase` tail.
|
|
13
12
|
*
|
|
14
13
|
* Two differences from that precedent, both deliberate:
|
|
15
14
|
*
|
|
@@ -181,11 +180,18 @@ export interface BuildBDCResult {
|
|
|
181
180
|
*/
|
|
182
181
|
export declare function peekProviderID(csvBuffer: Buffer, csvPath?: string): ProviderID;
|
|
183
182
|
/**
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
183
|
+
* Area-weighted (shoelace) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s), area-weighted across
|
|
184
|
+
* rings for a MultiPolygon. Interior rings/holes are still ignored — a hole moves a block's centroid far less than the
|
|
185
|
+
* vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
|
|
186
|
+
*
|
|
187
|
+
* This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
|
|
188
|
+
* falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
|
|
189
|
+
* vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
|
|
190
|
+
* edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
|
|
191
|
+
* squiggly natural edge and drag a vertex-average toward it.
|
|
192
|
+
*
|
|
193
|
+
* A degenerate geometry with zero total ring area (a sliver the shoelace annihilates) falls back to the vertex average
|
|
194
|
+
* — a weaker answer beats none, and the fallback is exactly the old behavior.
|
|
189
195
|
*
|
|
190
196
|
* Returns `undefined` for anything that doesn't parse as one of the two geometry types (including `null` geometry).
|
|
191
197
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build-bdc.d.ts","sourceRoot":"","sources":["../../sdk/build-bdc.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"build-bdc.d.ts","sourceRoot":"","sources":["../../sdk/build-bdc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAOH,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAY9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAKrD,OAAO,KAAK,EAAO,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAchE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAwB,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAQ5E;;;GAGG;AACH,eAAO,MAAM,eAAe,QAEqF,CAAA;AAEjH,MAAM,WAAW,eAAe;IAC/B;;;OAGG;IACH,IAAI,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACvE;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IACX;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAA;IAC3E,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IACtC;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,CAAA;IACtE;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,aAAa,CAAC,CAAA;IACvC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;OAGG;IACH,kBAAkB,EAAE,MAAM,CAAA;CAC1B;AA4CD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,UAAU,CA+B9E;AAwDD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CA8DtG;AAED;;;;;;;GAOG;AACH,wBAAgB,8BAA8B,CAC7C,WAAW,EAAE,MAAM,GACjB,CAAC,KAAK,EAAE,MAAM,KAAK;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAS7D;AAmHD;;;;GAIG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAuTxF"}
|
package/out/sdk/build-bdc.js
CHANGED
|
@@ -8,8 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Mirrors `mailwoman/gazetteer-pipeline/poi/build-poi.ts`'s shape closely: the same build-tuning
|
|
10
10
|
* pragmas, the same single-pass `Map<number, number>` coverage aggregation taken during the load
|
|
11
|
-
* (no second scan), the same
|
|
12
|
-
* `@mailwoman/core/layers` calls, and the same `writeLayerManifest` → `sealDatabase` tail.
|
|
11
|
+
* (no second scan), and the same `writeLayerManifest` → `sealDatabase` tail.
|
|
13
12
|
*
|
|
14
13
|
* Two differences from that precedent, both deliberate:
|
|
15
14
|
*
|
|
@@ -50,18 +49,18 @@
|
|
|
50
49
|
* mid-build crash mustn't cost the previously-good artifact, which is exactly the failure mode
|
|
51
50
|
* the house rule exists for.
|
|
52
51
|
*/
|
|
53
|
-
import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
|
|
54
|
-
import {
|
|
55
|
-
import { dirname } from "node:path";
|
|
52
|
+
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs";
|
|
53
|
+
import { open } from "node:fs/promises";
|
|
54
|
+
import { basename, dirname, join } from "node:path";
|
|
56
55
|
import { DatabaseSync } from "node:sqlite";
|
|
57
56
|
import { DatabaseClient } from "@mailwoman/core/kysley/client";
|
|
58
|
-
import { createLayerCoverageTable, createLayerManifestTable, LayerFreshnessPolicy, LayerTier, writeLayerCoverage, writeLayerManifest, } from "@mailwoman/core/layers";
|
|
57
|
+
import { CoverageBasis, createLayerCoverageTable, createLayerManifestTable, LayerFreshnessPolicy, LayerTier, writeLayerCoverage, writeLayerManifest, } from "@mailwoman/core/layers";
|
|
59
58
|
import { tryParsingJSON } from "@mailwoman/core/objects";
|
|
60
|
-
import { openBuiltDatabase, sealDatabase } from "@mailwoman/core/utils";
|
|
59
|
+
import { openBuiltDatabase, sealDatabase, swapDatabaseIntoPlace } from "@mailwoman/core/utils";
|
|
61
60
|
import { shortCellToInt } from "@mailwoman/spatial";
|
|
62
61
|
import { cellToParent, latLngToCell } from "h3-js";
|
|
63
62
|
import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, createBDCAvailabilityTable, createBDCGeoidIndex, createBDCProviderTable, } from "../schema.js";
|
|
64
|
-
import {
|
|
63
|
+
import { readAvailabilityRows } from "./parsing.js";
|
|
65
64
|
/**
|
|
66
65
|
* Rows committed per `BEGIN`/`COMMIT` batch during both the staging load and the materialize pass — matches
|
|
67
66
|
* `build-poi.ts`'s `STAGE_BATCH_SIZE` discipline.
|
|
@@ -73,15 +72,6 @@ const STAGE_BATCH_SIZE = 10_000;
|
|
|
73
72
|
*/
|
|
74
73
|
export const BDC_ATTRIBUTION = "FCC Broadband Data Collection. This workspace never ingests, ships, or derives data from the Fabric: " +
|
|
75
74
|
"location_id is carried only as an opaque join key that a licensed user may join against their own Fabric copy.";
|
|
76
|
-
/**
|
|
77
|
-
* `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
|
|
78
|
-
* in `DB` — narrows a `DatabaseClient<BDCDatabase>` handle back down for the `@mailwoman/core/layers` calls. Exact
|
|
79
|
-
* precedent: `build-poi.ts`'s own `asContractDB`; see that file for the full rationale (tried widening the shared
|
|
80
|
-
* package's signatures first — breaks THEIR internal `insertInto`/`selectFrom` calls instead).
|
|
81
|
-
*/
|
|
82
|
-
function asContractDB(kdb) {
|
|
83
|
-
return kdb;
|
|
84
|
-
}
|
|
85
75
|
/**
|
|
86
76
|
* Create the build-only `bdc_stage` table — deliberately NOT part of the public {@link BDCDatabase} interface (it's
|
|
87
77
|
* dropped before the artifact seals, so it never appears in the shipped schema). Built via Kysely's schema builder per
|
|
@@ -145,24 +135,54 @@ export function peekProviderID(csvBuffer, csvPath) {
|
|
|
145
135
|
return providerID;
|
|
146
136
|
}
|
|
147
137
|
/**
|
|
148
|
-
*
|
|
149
|
-
*
|
|
138
|
+
* Bytes read to peek the `provider_id`. Only the header row plus the first data row are needed and an FCC availability
|
|
139
|
+
* row is ~110 bytes, so this is three orders of magnitude of slack. A file shorter than this simply reads short —
|
|
140
|
+
* {@linkcode peekProviderID} already reports a header-only or empty file by message.
|
|
141
|
+
*/
|
|
142
|
+
const PROVIDER_ID_PEEK_BYTES = 64 * 1024;
|
|
143
|
+
/**
|
|
144
|
+
* Read the head of a CSV, for {@linkcode peekProviderID}.
|
|
145
|
+
*
|
|
146
|
+
* The point is what it does NOT do. `provider_id` is a constant per file, so establishing it needs the first data row
|
|
147
|
+
* and nothing else; `readFile(csvPath)` was resident-loading the entire file to read one column of one row. The
|
|
148
|
+
* measured file that motivated this is 920 MB for a single state × technology.
|
|
149
|
+
*/
|
|
150
|
+
async function readCSVHead(csvPath) {
|
|
151
|
+
const handle = await open(csvPath);
|
|
152
|
+
try {
|
|
153
|
+
const buffer = Buffer.allocUnsafe(PROVIDER_ID_PEEK_BYTES);
|
|
154
|
+
const { bytesRead } = await handle.read(buffer, 0, PROVIDER_ID_PEEK_BYTES, 0);
|
|
155
|
+
return buffer.subarray(0, bytesRead);
|
|
156
|
+
}
|
|
157
|
+
finally {
|
|
158
|
+
await handle.close();
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Peeks each file's `provider_id` off its head ({@linkcode peekProviderID}, passing the path through so a malformed
|
|
163
|
+
* file's error names it), then STREAMS every row via `readAvailabilityRows` — the file is never resident. This is the
|
|
150
164
|
* production counterpart to the test seam's injected `rows` — exercised by `build-bdc.test.ts` only for the
|
|
151
165
|
* malformed-provider-id rejection path, same as `build-poi.ts`'s `readParquetRows`.
|
|
152
166
|
*/
|
|
153
167
|
async function* readAvailabilityRowsFromCSVPaths(csvPaths) {
|
|
154
168
|
for (const csvPath of csvPaths) {
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
yield* takeAvailabilityLine(buffer, providerID);
|
|
169
|
+
const providerID = peekProviderID(await readCSVHead(csvPath), csvPath);
|
|
170
|
+
yield* readAvailabilityRows(csvPath, providerID);
|
|
158
171
|
}
|
|
159
172
|
}
|
|
160
173
|
/**
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
174
|
+
* Area-weighted (shoelace) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s), area-weighted across
|
|
175
|
+
* rings for a MultiPolygon. Interior rings/holes are still ignored — a hole moves a block's centroid far less than the
|
|
176
|
+
* vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
|
|
177
|
+
*
|
|
178
|
+
* This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
|
|
179
|
+
* falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
|
|
180
|
+
* vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
|
|
181
|
+
* edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
|
|
182
|
+
* squiggly natural edge and drag a vertex-average toward it.
|
|
183
|
+
*
|
|
184
|
+
* A degenerate geometry with zero total ring area (a sliver the shoelace annihilates) falls back to the vertex average
|
|
185
|
+
* — a weaker answer beats none, and the fallback is exactly the old behavior.
|
|
166
186
|
*
|
|
167
187
|
* Returns `undefined` for anything that doesn't parse as one of the two geometry types (including `null` geometry).
|
|
168
188
|
*/
|
|
@@ -177,18 +197,41 @@ export function geometryCentroid(geometryJSON) {
|
|
|
177
197
|
: geometry.type === "MultiPolygon"
|
|
178
198
|
? geometry.coordinates.map((polygon) => polygon[0] ?? [])
|
|
179
199
|
: [];
|
|
200
|
+
let totalArea = 0;
|
|
201
|
+
let weightedLon = 0;
|
|
202
|
+
let weightedLat = 0;
|
|
180
203
|
let sumLon = 0;
|
|
181
204
|
let sumLat = 0;
|
|
182
205
|
let count = 0;
|
|
183
206
|
for (const ring of exteriorRings) {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
207
|
+
let ringArea = 0;
|
|
208
|
+
let ringLon = 0;
|
|
209
|
+
let ringLat = 0;
|
|
210
|
+
for (let i = 0; i < ring.length - 1; i++) {
|
|
211
|
+
const [x1, y1] = ring[i];
|
|
212
|
+
const [x2, y2] = ring[i + 1];
|
|
213
|
+
if (typeof x1 !== "number" || typeof y1 !== "number" || typeof x2 !== "number" || typeof y2 !== "number") {
|
|
187
214
|
continue;
|
|
188
|
-
|
|
189
|
-
|
|
215
|
+
}
|
|
216
|
+
const cross = x1 * y2 - x2 * y1;
|
|
217
|
+
ringArea += cross;
|
|
218
|
+
ringLon += (x1 + x2) * cross;
|
|
219
|
+
ringLat += (y1 + y2) * cross;
|
|
220
|
+
// The vertex-average fallback accumulates alongside — one pass, both answers.
|
|
221
|
+
sumLon += x1;
|
|
222
|
+
sumLat += y1;
|
|
190
223
|
count++;
|
|
191
224
|
}
|
|
225
|
+
ringArea /= 2;
|
|
226
|
+
if (ringArea === 0)
|
|
227
|
+
continue;
|
|
228
|
+
const weight = Math.abs(ringArea);
|
|
229
|
+
totalArea += weight;
|
|
230
|
+
weightedLon += (ringLon / (6 * ringArea)) * weight;
|
|
231
|
+
weightedLat += (ringLat / (6 * ringArea)) * weight;
|
|
232
|
+
}
|
|
233
|
+
if (totalArea > 0) {
|
|
234
|
+
return { lat: weightedLat / totalArea, lon: weightedLon / totalArea };
|
|
192
235
|
}
|
|
193
236
|
if (count === 0)
|
|
194
237
|
return undefined;
|
|
@@ -313,185 +356,206 @@ export async function buildBDCDatabase(options) {
|
|
|
313
356
|
rmSync(buildingPath);
|
|
314
357
|
}
|
|
315
358
|
mkdirSync(dirname(options.out), { recursive: true });
|
|
359
|
+
// A crash inside a PRIOR run's swap can leave the slot empty while the previous version sits
|
|
360
|
+
// parked aside — restore it before building, so a failure in THIS run still leaves an artifact
|
|
361
|
+
// serving. Both aside spellings: this builder's old `.prev` and swapDatabaseIntoPlace's `.old-<pid>`.
|
|
362
|
+
if (!existsSync(options.out)) {
|
|
363
|
+
const base = basename(options.out);
|
|
364
|
+
const parked = readdirSync(dirname(options.out)).find((name) => name === `${base}.prev` || name.startsWith(`${base}.old-`));
|
|
365
|
+
if (parked) {
|
|
366
|
+
renameSync(join(dirname(options.out), parked), options.out);
|
|
367
|
+
progress(`restored ${parked} into place (a prior run crashed mid-swap)`);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
316
370
|
const rowSource = options.rows ?? readAvailabilityRowsFromCSVPaths(options.csvPaths);
|
|
317
371
|
const db = new DatabaseSync(buildingPath);
|
|
318
372
|
// Build-tuning pragmas — identical to build-poi.ts's discipline.
|
|
319
373
|
db.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;");
|
|
320
374
|
const kdb = new DatabaseClient({ database: db });
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
375
|
+
// Assigned at the end of the try — the tallies live inside its scope; the seal + swap do not.
|
|
376
|
+
let result;
|
|
377
|
+
try {
|
|
378
|
+
progress("creating manifest/coverage/availability/provider/stage tables");
|
|
379
|
+
await createLayerManifestTable(kdb);
|
|
380
|
+
await createLayerCoverageTable(kdb);
|
|
381
|
+
await createBDCAvailabilityTable(kdb);
|
|
382
|
+
await createBDCProviderTable(kdb);
|
|
383
|
+
await createBDCStageTable(kdb);
|
|
384
|
+
const insStage = db.prepare(`INSERT OR IGNORE INTO bdc_stage (
|
|
328
385
|
geoid, provider_id, technology_code, location_id,
|
|
329
386
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
|
|
330
387
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
388
|
+
let staged = 0;
|
|
389
|
+
let batch = 0;
|
|
390
|
+
progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)");
|
|
391
|
+
db.exec("BEGIN");
|
|
392
|
+
for await (const row of rowSource) {
|
|
393
|
+
insStage.run(row.geoid, row.provider_id, row.technology_code, row.location_id, row.max_advertised_download_speed, row.max_advertised_upload_speed, row.low_latency, row.business_residential_code);
|
|
394
|
+
staged++;
|
|
395
|
+
batch++;
|
|
396
|
+
if (batch >= STAGE_BATCH_SIZE) {
|
|
397
|
+
db.exec("COMMIT");
|
|
398
|
+
db.exec("BEGIN");
|
|
399
|
+
batch = 0;
|
|
400
|
+
}
|
|
343
401
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
const insAvailability = db.prepare(`INSERT INTO bdc_availability (
|
|
402
|
+
db.exec("COMMIT");
|
|
403
|
+
const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get();
|
|
404
|
+
const deduped = staged - stagedCountRow.staged_count;
|
|
405
|
+
progress(`staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`);
|
|
406
|
+
const centroidCache = new Map();
|
|
407
|
+
/**
|
|
408
|
+
* Res-6 short-cell int → observed row count, aggregated during materialize (one pass, no second scan) — matches
|
|
409
|
+
* `build-poi.ts`'s `coverage` Map.
|
|
410
|
+
*/
|
|
411
|
+
const coverage = new Map();
|
|
412
|
+
const providers = new Set();
|
|
413
|
+
let unknownGeoids = 0;
|
|
414
|
+
let inserted = 0;
|
|
415
|
+
const insAvailability = db.prepare(`INSERT INTO bdc_availability (
|
|
359
416
|
h3_cell, geoid, wof_id, provider_id, technology_code,
|
|
360
417
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code, location_id
|
|
361
418
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
419
|
+
// The FCC's per-provider CSVs are per-BSL: the SAME (geoid, provider_id, technology_code, speeds, low_latency,
|
|
420
|
+
// business_residential_code) tuple can repeat once per Broadband Serviceable Location within that block (a
|
|
421
|
+
// dense urban block can carry ~100 BSLs) — `bdc_stage`'s natural key includes `location_id`, so those BSL rows
|
|
422
|
+
// all survive the staging dedup as distinct staged rows. In `includeLocationIDs` mode that's correct: every BSL
|
|
423
|
+
// is a real, distinct row the caller asked to keep. In the default (NULL `location_id`) mode, when those BSLs
|
|
424
|
+
// ALSO share identical speeds/flags, they'd otherwise materialize as byte-identical rows, inflating `result.rows`
|
|
425
|
+
// and `layer_coverage.observed_rows` by the BSL count (~100x at real scale) — `SELECT DISTINCT`
|
|
426
|
+
// over every column EXCEPT `location_id` collapses those byte-identical BSL duplicates down to one row.
|
|
427
|
+
// IMPORTANT — this is NOT a guarantee of one row per (geoid, provider_id, technology_code) triple: BSLs at the
|
|
428
|
+
// same triple with DIFFERING speeds/flags are NOT the same tuple, so `SELECT DISTINCT` does not merge them —
|
|
429
|
+
// they survive as multiple NULL-`location_id` rows at that one triple. Accepted, not a bug; see the module
|
|
430
|
+
// docstring and `filing-landscape.ts`'s docstring for the read-side consequence.
|
|
431
|
+
const stageStmt = options.includeLocationIDs
|
|
432
|
+
? db.prepare(`SELECT geoid, provider_id, technology_code, location_id,
|
|
376
433
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
|
|
377
434
|
FROM bdc_stage`)
|
|
378
|
-
|
|
435
|
+
: db.prepare(`SELECT DISTINCT geoid, provider_id, technology_code,
|
|
379
436
|
max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
|
|
380
437
|
FROM bdc_stage`);
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
438
|
+
progress("materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)");
|
|
439
|
+
db.exec("BEGIN");
|
|
440
|
+
batch = 0;
|
|
441
|
+
for (const row of stageStmt.iterate()) {
|
|
442
|
+
let resolved = centroidCache.get(row.geoid);
|
|
443
|
+
if (resolved === undefined) {
|
|
444
|
+
const centroid = options.blockCentroids(row.geoid);
|
|
445
|
+
resolved = centroid
|
|
446
|
+
? (() => {
|
|
447
|
+
// Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
|
|
448
|
+
// independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
|
|
449
|
+
// exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
|
|
450
|
+
// disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
|
|
451
|
+
// boundary artifacts). Deriving both `h3_cell` and the coverage cell from
|
|
452
|
+
// the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
|
|
453
|
+
// exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
|
|
454
|
+
// applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
|
|
455
|
+
// the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
|
|
456
|
+
const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION);
|
|
457
|
+
return {
|
|
458
|
+
h3Cell: shortCellToInt(fullRes9Cell),
|
|
459
|
+
coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION)),
|
|
460
|
+
};
|
|
461
|
+
})()
|
|
462
|
+
: null;
|
|
463
|
+
centroidCache.set(row.geoid, resolved);
|
|
464
|
+
}
|
|
465
|
+
if (!resolved) {
|
|
466
|
+
unknownGeoids++;
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
insAvailability.run(resolved.h3Cell, row.geoid,
|
|
470
|
+
// wof_id stays NULL here — WOF point-in-polygon resolution against the block centroid is a later
|
|
471
|
+
// registry-join task, the same decision-8 scoping schema.ts documents for `bdc_provider`.
|
|
472
|
+
null, row.provider_id, row.technology_code, row.max_advertised_download_speed, row.max_advertised_upload_speed, row.low_latency, row.business_residential_code, options.includeLocationIDs ? (row.location_id ?? null) : null);
|
|
473
|
+
inserted++;
|
|
474
|
+
providers.add(row.provider_id);
|
|
475
|
+
coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1);
|
|
476
|
+
batch++;
|
|
477
|
+
if (batch >= STAGE_BATCH_SIZE) {
|
|
478
|
+
db.exec("COMMIT");
|
|
479
|
+
db.exec("BEGIN");
|
|
480
|
+
batch = 0;
|
|
481
|
+
}
|
|
411
482
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
483
|
+
db.exec("COMMIT");
|
|
484
|
+
progress(`materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
|
|
485
|
+
`(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`);
|
|
486
|
+
await kdb.schema.dropTable("bdc_stage").execute();
|
|
487
|
+
progress("geoid index (index-after-load — see schema.ts)");
|
|
488
|
+
await createBDCGeoidIndex(kdb);
|
|
489
|
+
// Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
|
|
490
|
+
// have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
|
|
491
|
+
// were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
|
|
492
|
+
const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
|
|
493
|
+
h3Cell,
|
|
494
|
+
completeness: 1,
|
|
495
|
+
basis: CoverageBasis.SourcePresent,
|
|
496
|
+
observedRows,
|
|
497
|
+
}));
|
|
498
|
+
await writeLayerCoverage(kdb, coverageCells);
|
|
499
|
+
progress("writing layer manifest");
|
|
500
|
+
await writeLayerManifest(kdb, {
|
|
501
|
+
name: "bdc",
|
|
502
|
+
version: options.asOfDate,
|
|
503
|
+
schemaVersion: 1,
|
|
504
|
+
tier: LayerTier.Shipped,
|
|
505
|
+
license: "public-domain",
|
|
506
|
+
attribution: BDC_ATTRIBUTION,
|
|
507
|
+
source: "fcc-bdc",
|
|
508
|
+
sourceVintage: options.asOfDate,
|
|
509
|
+
buildCmd: "mailwoman gazetteer build bdc",
|
|
510
|
+
buildSHA: options.buildSHA,
|
|
511
|
+
freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
|
|
512
|
+
spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
|
|
513
|
+
createdAt: new Date().toISOString(),
|
|
514
|
+
});
|
|
515
|
+
// bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
|
|
516
|
+
// `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
|
|
517
|
+
// `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
|
|
518
|
+
let providersPopulated = 0;
|
|
519
|
+
if (options.providers) {
|
|
520
|
+
progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)");
|
|
521
|
+
providersPopulated = await populateBDCProviderTable(kdb, options.providers, options.filerDB, options.primaryFRNAsOf ?? options.asOfDate);
|
|
522
|
+
progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`);
|
|
424
523
|
}
|
|
524
|
+
progress("finalize: ANALYZE + VACUUM");
|
|
525
|
+
db.exec("ANALYZE");
|
|
526
|
+
// page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
|
|
527
|
+
// `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
|
|
528
|
+
// same discipline).
|
|
529
|
+
db.exec("PRAGMA page_size=8192");
|
|
530
|
+
db.exec("VACUUM");
|
|
531
|
+
await kdb.destroy();
|
|
532
|
+
result = {
|
|
533
|
+
out: options.out,
|
|
534
|
+
rows: inserted,
|
|
535
|
+
deduped,
|
|
536
|
+
providers: providers.size,
|
|
537
|
+
coverageCells: coverageCells.length,
|
|
538
|
+
unknownGeoids,
|
|
539
|
+
providersPopulated,
|
|
540
|
+
};
|
|
425
541
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
completeness: 1,
|
|
438
|
-
observedRows,
|
|
439
|
-
}));
|
|
440
|
-
await writeLayerCoverage(asContractDB(kdb), coverageCells);
|
|
441
|
-
progress("writing layer manifest");
|
|
442
|
-
await writeLayerManifest(asContractDB(kdb), {
|
|
443
|
-
name: "bdc",
|
|
444
|
-
version: options.asOfDate,
|
|
445
|
-
schemaVersion: 1,
|
|
446
|
-
tier: LayerTier.Shipped,
|
|
447
|
-
license: "public-domain",
|
|
448
|
-
attribution: BDC_ATTRIBUTION,
|
|
449
|
-
source: "fcc-bdc",
|
|
450
|
-
sourceVintage: options.asOfDate,
|
|
451
|
-
buildCmd: "mailwoman gazetteer build bdc",
|
|
452
|
-
buildSHA: options.buildSHA,
|
|
453
|
-
freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
|
|
454
|
-
spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
|
|
455
|
-
createdAt: new Date().toISOString(),
|
|
456
|
-
});
|
|
457
|
-
// bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
|
|
458
|
-
// `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
|
|
459
|
-
// `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
|
|
460
|
-
let providersPopulated = 0;
|
|
461
|
-
if (options.providers) {
|
|
462
|
-
progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)");
|
|
463
|
-
providersPopulated = await populateBDCProviderTable(kdb, options.providers, options.filerDB, options.primaryFRNAsOf ?? options.asOfDate);
|
|
464
|
-
progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`);
|
|
542
|
+
catch (error) {
|
|
543
|
+
// A mid-build throw must not leak the handle or orphan the staging file. The original error
|
|
544
|
+
// always wins over anything the cleanup itself throws.
|
|
545
|
+
try {
|
|
546
|
+
await kdb.destroy();
|
|
547
|
+
}
|
|
548
|
+
catch {
|
|
549
|
+
// The handle may already be closed or mid-statement — nothing more to release.
|
|
550
|
+
}
|
|
551
|
+
rmSync(buildingPath, { force: true });
|
|
552
|
+
throw error;
|
|
465
553
|
}
|
|
466
|
-
progress("finalize: ANALYZE + VACUUM");
|
|
467
|
-
db.exec("ANALYZE");
|
|
468
|
-
// page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
|
|
469
|
-
// `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
|
|
470
|
-
// same discipline).
|
|
471
|
-
db.exec("PRAGMA page_size=8192");
|
|
472
|
-
db.exec("VACUUM");
|
|
473
|
-
await kdb.destroy();
|
|
474
554
|
progress("seal");
|
|
475
555
|
sealDatabase(buildingPath);
|
|
476
|
-
// Atomic move-into-place
|
|
477
|
-
//
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
if (existsSync(options.out)) {
|
|
481
|
-
renameSync(options.out, `${options.out}.prev`);
|
|
482
|
-
}
|
|
483
|
-
renameSync(buildingPath, options.out);
|
|
484
|
-
if (existsSync(`${options.out}.prev`)) {
|
|
485
|
-
rmSync(`${options.out}.prev`);
|
|
486
|
-
}
|
|
487
|
-
return {
|
|
488
|
-
out: options.out,
|
|
489
|
-
rows: inserted,
|
|
490
|
-
deduped,
|
|
491
|
-
providers: providers.size,
|
|
492
|
-
coverageCells: coverageCells.length,
|
|
493
|
-
unknownGeoids,
|
|
494
|
-
providersPopulated,
|
|
495
|
-
};
|
|
556
|
+
// Atomic move-into-place via the shared helper (the AGENTS.md database house rule): prior
|
|
557
|
+
// version aside first, forward rename restored on failure so the slot is never left empty.
|
|
558
|
+
swapDatabaseIntoPlace(buildingPath, options.out);
|
|
559
|
+
return result;
|
|
496
560
|
}
|
|
497
561
|
//# sourceMappingURL=build-bdc.js.map
|