@mailwoman/bdc 9.0.0 → 9.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,8 +8,7 @@
8
8
  *
9
9
  * Mirrors `mailwoman/gazetteer-pipeline/poi/build-poi.ts`'s shape closely: the same build-tuning
10
10
  * pragmas, the same single-pass `Map<number, number>` coverage aggregation taken during the load
11
- * (no second scan), the same {@link asContractDB} Kysely-invariance cast for the shared
12
- * `@mailwoman/core/layers` calls, and the same `writeLayerManifest` → `sealDatabase` tail.
11
+ * (no second scan), and the same `writeLayerManifest` `sealDatabase` tail.
13
12
  *
14
13
  * Two differences from that precedent, both deliberate:
15
14
  *
@@ -181,11 +180,18 @@ export interface BuildBDCResult {
181
180
  */
182
181
  export declare function peekProviderID(csvBuffer: Buffer, csvPath?: string): ProviderID;
183
182
  /**
184
- * Naive (vertex-average, NOT area-weighted) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s) only
185
- * (interior rings/holes are ignored). A known simplification, not an oversight: census blocks are small relative to a
186
- * res-9 H3 cell (~174m edge), so the vertex-average and a proper area-weighted centroid land in the same cell for all
187
- * but pathologically elongated or holed block shapes. A precise area-weighted centroid is a reasonable future upgrade
188
- * if that ever proves wrong in practice no polygon-centroid library is pulled in for this first cut.
183
+ * Area-weighted (shoelace) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s), area-weighted across
184
+ * rings for a MultiPolygon. Interior rings/holes are still ignored a hole moves a block's centroid far less than the
185
+ * vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
186
+ *
187
+ * This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
188
+ * falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
189
+ * vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
190
+ * edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
191
+ * squiggly natural edge and drag a vertex-average toward it.
192
+ *
193
+ * A degenerate geometry with zero total ring area (a sliver the shoelace annihilates) falls back to the vertex average
194
+ * — a weaker answer beats none, and the fallback is exactly the old behavior.
189
195
  *
190
196
  * Returns `undefined` for anything that doesn't parse as one of the two geometry types (including `null` geometry).
191
197
  */
@@ -1 +1 @@
1
- {"version":3,"file":"build-bdc.d.ts","sourceRoot":"","sources":["../../sdk/build-bdc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAOH,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAY9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAKrD,OAAO,KAAK,EAAO,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAchE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAwB,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAQ5E;;;GAGG;AACH,eAAO,MAAM,eAAe,QAEqF,CAAA;AAEjH,MAAM,WAAW,eAAe;IAC/B;;;OAGG;IACH,IAAI,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACvE;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IACX;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAA;IAC3E,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IACtC;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,CAAA;IACtE;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,aAAa,CAAC,CAAA;IACvC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;OAGG;IACH,kBAAkB,EAAE,MAAM,CAAA;CAC1B;AAsDD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,UAAU,CA+B9E;AA8BD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAkCtG;AAED;;;;;;;GAOG;AACH,wBAAgB,8BAA8B,CAC7C,WAAW,EAAE,MAAM,GACjB,CAAC,KAAK,EAAE,MAAM,KAAK;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAS7D;AAmHD;;;;GAIG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CA6RxF"}
1
+ {"version":3,"file":"build-bdc.d.ts","sourceRoot":"","sources":["../../sdk/build-bdc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAOH,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAY9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAKrD,OAAO,KAAK,EAAO,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAchE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAwB,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAQ5E;;;GAGG;AACH,eAAO,MAAM,eAAe,QAEqF,CAAA;AAEjH,MAAM,WAAW,eAAe;IAC/B;;;OAGG;IACH,IAAI,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACvE;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IACX;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAA;IAC3E,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IACtC;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,CAAA;IACtE;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,aAAa,CAAC,CAAA;IACvC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;OAGG;IACH,kBAAkB,EAAE,MAAM,CAAA;CAC1B;AA4CD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,UAAU,CA+B9E;AAwDD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CA8DtG;AAED;;;;;;;GAOG;AACH,wBAAgB,8BAA8B,CAC7C,WAAW,EAAE,MAAM,GACjB,CAAC,KAAK,EAAE,MAAM,KAAK;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAS7D;AAmHD;;;;GAIG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAuTxF"}
@@ -8,8 +8,7 @@
8
8
  *
9
9
  * Mirrors `mailwoman/gazetteer-pipeline/poi/build-poi.ts`'s shape closely: the same build-tuning
10
10
  * pragmas, the same single-pass `Map<number, number>` coverage aggregation taken during the load
11
- * (no second scan), the same {@link asContractDB} Kysely-invariance cast for the shared
12
- * `@mailwoman/core/layers` calls, and the same `writeLayerManifest` → `sealDatabase` tail.
11
+ * (no second scan), and the same `writeLayerManifest` `sealDatabase` tail.
13
12
  *
14
13
  * Two differences from that precedent, both deliberate:
15
14
  *
@@ -50,18 +49,18 @@
50
49
  * mid-build crash mustn't cost the previously-good artifact, which is exactly the failure mode
51
50
  * the house rule exists for.
52
51
  */
53
- import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
54
- import { readFile } from "node:fs/promises";
55
- import { dirname } from "node:path";
52
+ import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs";
53
+ import { open } from "node:fs/promises";
54
+ import { basename, dirname, join } from "node:path";
56
55
  import { DatabaseSync } from "node:sqlite";
57
56
  import { DatabaseClient } from "@mailwoman/core/kysley/client";
58
- import { createLayerCoverageTable, createLayerManifestTable, LayerFreshnessPolicy, LayerTier, writeLayerCoverage, writeLayerManifest, } from "@mailwoman/core/layers";
57
+ import { CoverageBasis, createLayerCoverageTable, createLayerManifestTable, LayerFreshnessPolicy, LayerTier, writeLayerCoverage, writeLayerManifest, } from "@mailwoman/core/layers";
59
58
  import { tryParsingJSON } from "@mailwoman/core/objects";
60
- import { openBuiltDatabase, sealDatabase } from "@mailwoman/core/utils";
59
+ import { openBuiltDatabase, sealDatabase, swapDatabaseIntoPlace } from "@mailwoman/core/utils";
61
60
  import { shortCellToInt } from "@mailwoman/spatial";
62
61
  import { cellToParent, latLngToCell } from "h3-js";
63
62
  import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, createBDCAvailabilityTable, createBDCGeoidIndex, createBDCProviderTable, } from "../schema.js";
64
- import { takeAvailabilityLine } from "./parsing.js";
63
+ import { readAvailabilityRows } from "./parsing.js";
65
64
  /**
66
65
  * Rows committed per `BEGIN`/`COMMIT` batch during both the staging load and the materialize pass — matches
67
66
  * `build-poi.ts`'s `STAGE_BATCH_SIZE` discipline.
@@ -73,15 +72,6 @@ const STAGE_BATCH_SIZE = 10_000;
73
72
  */
74
73
  export const BDC_ATTRIBUTION = "FCC Broadband Data Collection. This workspace never ingests, ships, or derives data from the Fabric: " +
75
74
  "location_id is carried only as an opaque join key that a licensed user may join against their own Fabric copy.";
76
- /**
77
- * `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
78
- * in `DB` — narrows a `DatabaseClient<BDCDatabase>` handle back down for the `@mailwoman/core/layers` calls. Exact
79
- * precedent: `build-poi.ts`'s own `asContractDB`; see that file for the full rationale (tried widening the shared
80
- * package's signatures first — breaks THEIR internal `insertInto`/`selectFrom` calls instead).
81
- */
82
- function asContractDB(kdb) {
83
- return kdb;
84
- }
85
75
  /**
86
76
  * Create the build-only `bdc_stage` table — deliberately NOT part of the public {@link BDCDatabase} interface (it's
87
77
  * dropped before the artifact seals, so it never appears in the shipped schema). Built via Kysely's schema builder per
@@ -145,24 +135,54 @@ export function peekProviderID(csvBuffer, csvPath) {
145
135
  return providerID;
146
136
  }
147
137
  /**
148
- * Reads each of `csvPaths` fully into memory, peeks its `provider_id` ({@linkcode peekProviderID}, passing the path
149
- * through so a malformed file's error names it), then yields every row via `takeAvailabilityLine`. This is the
138
+ * Bytes read to peek the `provider_id`. Only the header row plus the first data row are needed and an FCC availability
139
+ * row is ~110 bytes, so this is three orders of magnitude of slack. A file shorter than this simply reads short —
140
+ * {@linkcode peekProviderID} already reports a header-only or empty file by message.
141
+ */
142
+ const PROVIDER_ID_PEEK_BYTES = 64 * 1024;
143
+ /**
144
+ * Read the head of a CSV, for {@linkcode peekProviderID}.
145
+ *
146
+ * The point is what it does NOT do. `provider_id` is a constant per file, so establishing it needs the first data row
147
+ * and nothing else; `readFile(csvPath)` was resident-loading the entire file to read one column of one row. The
148
+ * measured file that motivated this is 920 MB for a single state × technology.
149
+ */
150
+ async function readCSVHead(csvPath) {
151
+ const handle = await open(csvPath);
152
+ try {
153
+ const buffer = Buffer.allocUnsafe(PROVIDER_ID_PEEK_BYTES);
154
+ const { bytesRead } = await handle.read(buffer, 0, PROVIDER_ID_PEEK_BYTES, 0);
155
+ return buffer.subarray(0, bytesRead);
156
+ }
157
+ finally {
158
+ await handle.close();
159
+ }
160
+ }
161
+ /**
162
+ * Peeks each file's `provider_id` off its head ({@linkcode peekProviderID}, passing the path through so a malformed
163
+ * file's error names it), then STREAMS every row via `readAvailabilityRows` — the file is never resident. This is the
150
164
  * production counterpart to the test seam's injected `rows` — exercised by `build-bdc.test.ts` only for the
151
165
  * malformed-provider-id rejection path, same as `build-poi.ts`'s `readParquetRows`.
152
166
  */
153
167
  async function* readAvailabilityRowsFromCSVPaths(csvPaths) {
154
168
  for (const csvPath of csvPaths) {
155
- const buffer = await readFile(csvPath);
156
- const providerID = peekProviderID(buffer, csvPath);
157
- yield* takeAvailabilityLine(buffer, providerID);
169
+ const providerID = peekProviderID(await readCSVHead(csvPath), csvPath);
170
+ yield* readAvailabilityRows(csvPath, providerID);
158
171
  }
159
172
  }
160
173
  /**
161
- * Naive (vertex-average, NOT area-weighted) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s) only
162
- * (interior rings/holes are ignored). A known simplification, not an oversight: census blocks are small relative to a
163
- * res-9 H3 cell (~174m edge), so the vertex-average and a proper area-weighted centroid land in the same cell for all
164
- * but pathologically elongated or holed block shapes. A precise area-weighted centroid is a reasonable future upgrade
165
- * if that ever proves wrong in practice no polygon-centroid library is pulled in for this first cut.
174
+ * Area-weighted (shoelace) centroid of a GeoJSON `Polygon`/`MultiPolygon`'s EXTERIOR ring(s), area-weighted across
175
+ * rings for a MultiPolygon. Interior rings/holes are still ignored a hole moves a block's centroid far less than the
176
+ * vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
177
+ *
178
+ * This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
179
+ * falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
180
+ * vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
181
+ * edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
182
+ * squiggly natural edge and drag a vertex-average toward it.
183
+ *
184
+ * A degenerate geometry with zero total ring area (a sliver the shoelace annihilates) falls back to the vertex average
185
+ * — a weaker answer beats none, and the fallback is exactly the old behavior.
166
186
  *
167
187
  * Returns `undefined` for anything that doesn't parse as one of the two geometry types (including `null` geometry).
168
188
  */
@@ -177,18 +197,41 @@ export function geometryCentroid(geometryJSON) {
177
197
  : geometry.type === "MultiPolygon"
178
198
  ? geometry.coordinates.map((polygon) => polygon[0] ?? [])
179
199
  : [];
200
+ let totalArea = 0;
201
+ let weightedLon = 0;
202
+ let weightedLat = 0;
180
203
  let sumLon = 0;
181
204
  let sumLat = 0;
182
205
  let count = 0;
183
206
  for (const ring of exteriorRings) {
184
- for (const point of ring) {
185
- const [lon, lat] = point;
186
- if (typeof lon !== "number" || typeof lat !== "number")
207
+ let ringArea = 0;
208
+ let ringLon = 0;
209
+ let ringLat = 0;
210
+ for (let i = 0; i < ring.length - 1; i++) {
211
+ const [x1, y1] = ring[i];
212
+ const [x2, y2] = ring[i + 1];
213
+ if (typeof x1 !== "number" || typeof y1 !== "number" || typeof x2 !== "number" || typeof y2 !== "number") {
187
214
  continue;
188
- sumLon += lon;
189
- sumLat += lat;
215
+ }
216
+ const cross = x1 * y2 - x2 * y1;
217
+ ringArea += cross;
218
+ ringLon += (x1 + x2) * cross;
219
+ ringLat += (y1 + y2) * cross;
220
+ // The vertex-average fallback accumulates alongside — one pass, both answers.
221
+ sumLon += x1;
222
+ sumLat += y1;
190
223
  count++;
191
224
  }
225
+ ringArea /= 2;
226
+ if (ringArea === 0)
227
+ continue;
228
+ const weight = Math.abs(ringArea);
229
+ totalArea += weight;
230
+ weightedLon += (ringLon / (6 * ringArea)) * weight;
231
+ weightedLat += (ringLat / (6 * ringArea)) * weight;
232
+ }
233
+ if (totalArea > 0) {
234
+ return { lat: weightedLat / totalArea, lon: weightedLon / totalArea };
192
235
  }
193
236
  if (count === 0)
194
237
  return undefined;
@@ -313,185 +356,206 @@ export async function buildBDCDatabase(options) {
313
356
  rmSync(buildingPath);
314
357
  }
315
358
  mkdirSync(dirname(options.out), { recursive: true });
359
+ // A crash inside a PRIOR run's swap can leave the slot empty while the previous version sits
360
+ // parked aside — restore it before building, so a failure in THIS run still leaves an artifact
361
+ // serving. Both aside spellings: this builder's old `.prev` and swapDatabaseIntoPlace's `.old-<pid>`.
362
+ if (!existsSync(options.out)) {
363
+ const base = basename(options.out);
364
+ const parked = readdirSync(dirname(options.out)).find((name) => name === `${base}.prev` || name.startsWith(`${base}.old-`));
365
+ if (parked) {
366
+ renameSync(join(dirname(options.out), parked), options.out);
367
+ progress(`restored ${parked} into place (a prior run crashed mid-swap)`);
368
+ }
369
+ }
316
370
  const rowSource = options.rows ?? readAvailabilityRowsFromCSVPaths(options.csvPaths);
317
371
  const db = new DatabaseSync(buildingPath);
318
372
  // Build-tuning pragmas — identical to build-poi.ts's discipline.
319
373
  db.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;");
320
374
  const kdb = new DatabaseClient({ database: db });
321
- progress("creating manifest/coverage/availability/provider/stage tables");
322
- await createLayerManifestTable(asContractDB(kdb));
323
- await createLayerCoverageTable(asContractDB(kdb));
324
- await createBDCAvailabilityTable(kdb);
325
- await createBDCProviderTable(kdb);
326
- await createBDCStageTable(kdb);
327
- const insStage = db.prepare(`INSERT OR IGNORE INTO bdc_stage (
375
+ // Assigned at the end of the try — the tallies live inside its scope; the seal + swap do not.
376
+ let result;
377
+ try {
378
+ progress("creating manifest/coverage/availability/provider/stage tables");
379
+ await createLayerManifestTable(kdb);
380
+ await createLayerCoverageTable(kdb);
381
+ await createBDCAvailabilityTable(kdb);
382
+ await createBDCProviderTable(kdb);
383
+ await createBDCStageTable(kdb);
384
+ const insStage = db.prepare(`INSERT OR IGNORE INTO bdc_stage (
328
385
  geoid, provider_id, technology_code, location_id,
329
386
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
330
387
  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
331
- let staged = 0;
332
- let batch = 0;
333
- progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)");
334
- db.exec("BEGIN");
335
- for await (const row of rowSource) {
336
- insStage.run(row.geoid, row.provider_id, row.technology_code, row.location_id, row.max_advertised_download_speed, row.max_advertised_upload_speed, row.low_latency, row.business_residential_code);
337
- staged++;
338
- batch++;
339
- if (batch >= STAGE_BATCH_SIZE) {
340
- db.exec("COMMIT");
341
- db.exec("BEGIN");
342
- batch = 0;
388
+ let staged = 0;
389
+ let batch = 0;
390
+ progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)");
391
+ db.exec("BEGIN");
392
+ for await (const row of rowSource) {
393
+ insStage.run(row.geoid, row.provider_id, row.technology_code, row.location_id, row.max_advertised_download_speed, row.max_advertised_upload_speed, row.low_latency, row.business_residential_code);
394
+ staged++;
395
+ batch++;
396
+ if (batch >= STAGE_BATCH_SIZE) {
397
+ db.exec("COMMIT");
398
+ db.exec("BEGIN");
399
+ batch = 0;
400
+ }
343
401
  }
344
- }
345
- db.exec("COMMIT");
346
- const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get();
347
- const deduped = staged - stagedCountRow.staged_count;
348
- progress(`staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`);
349
- const centroidCache = new Map();
350
- /**
351
- * Res-6 short-cell int → observed row count, aggregated during materialize (one pass, no second scan) — matches
352
- * `build-poi.ts`'s `coverage` Map.
353
- */
354
- const coverage = new Map();
355
- const providers = new Set();
356
- let unknownGeoids = 0;
357
- let inserted = 0;
358
- const insAvailability = db.prepare(`INSERT INTO bdc_availability (
402
+ db.exec("COMMIT");
403
+ const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get();
404
+ const deduped = staged - stagedCountRow.staged_count;
405
+ progress(`staged ${stagedCountRow.staged_count.toLocaleString()} distinct row(s), ${deduped.toLocaleString()} deduped`);
406
+ const centroidCache = new Map();
407
+ /**
408
+ * Res-6 short-cell int → observed row count, aggregated during materialize (one pass, no second scan) — matches
409
+ * `build-poi.ts`'s `coverage` Map.
410
+ */
411
+ const coverage = new Map();
412
+ const providers = new Set();
413
+ let unknownGeoids = 0;
414
+ let inserted = 0;
415
+ const insAvailability = db.prepare(`INSERT INTO bdc_availability (
359
416
  h3_cell, geoid, wof_id, provider_id, technology_code,
360
417
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code, location_id
361
418
  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
362
- // The FCC's per-provider CSVs are per-BSL: the SAME (geoid, provider_id, technology_code, speeds, low_latency,
363
- // business_residential_code) tuple can repeat once per Broadband Serviceable Location within that block (a
364
- // dense urban block can carry ~100 BSLs) — `bdc_stage`'s natural key includes `location_id`, so those BSL rows
365
- // all survive the staging dedup as distinct staged rows. In `includeLocationIDs` mode that's correct: every BSL
366
- // is a real, distinct row the caller asked to keep. In the default (NULL `location_id`) mode, when those BSLs
367
- // ALSO share identical speeds/flags, they'd otherwise materialize as byte-identical rows, inflating `result.rows`
368
- // and `layer_coverage.observed_rows` by the BSL count (~100x at real scale) — `SELECT DISTINCT`
369
- // over every column EXCEPT `location_id` collapses those byte-identical BSL duplicates down to one row.
370
- // IMPORTANT — this is NOT a guarantee of one row per (geoid, provider_id, technology_code) triple: BSLs at the
371
- // same triple with DIFFERING speeds/flags are NOT the same tuple, so `SELECT DISTINCT` does not merge them —
372
- // they survive as multiple NULL-`location_id` rows at that one triple. Accepted, not a bug; see the module
373
- // docstring and `filing-landscape.ts`'s docstring for the read-side consequence.
374
- const stageStmt = options.includeLocationIDs
375
- ? db.prepare(`SELECT geoid, provider_id, technology_code, location_id,
419
+ // The FCC's per-provider CSVs are per-BSL: the SAME (geoid, provider_id, technology_code, speeds, low_latency,
420
+ // business_residential_code) tuple can repeat once per Broadband Serviceable Location within that block (a
421
+ // dense urban block can carry ~100 BSLs) — `bdc_stage`'s natural key includes `location_id`, so those BSL rows
422
+ // all survive the staging dedup as distinct staged rows. In `includeLocationIDs` mode that's correct: every BSL
423
+ // is a real, distinct row the caller asked to keep. In the default (NULL `location_id`) mode, when those BSLs
424
+ // ALSO share identical speeds/flags, they'd otherwise materialize as byte-identical rows, inflating `result.rows`
425
+ // and `layer_coverage.observed_rows` by the BSL count (~100x at real scale) — `SELECT DISTINCT`
426
+ // over every column EXCEPT `location_id` collapses those byte-identical BSL duplicates down to one row.
427
+ // IMPORTANT — this is NOT a guarantee of one row per (geoid, provider_id, technology_code) triple: BSLs at the
428
+ // same triple with DIFFERING speeds/flags are NOT the same tuple, so `SELECT DISTINCT` does not merge them —
429
+ // they survive as multiple NULL-`location_id` rows at that one triple. Accepted, not a bug; see the module
430
+ // docstring and `filing-landscape.ts`'s docstring for the read-side consequence.
431
+ const stageStmt = options.includeLocationIDs
432
+ ? db.prepare(`SELECT geoid, provider_id, technology_code, location_id,
376
433
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
377
434
  FROM bdc_stage`)
378
- : db.prepare(`SELECT DISTINCT geoid, provider_id, technology_code,
435
+ : db.prepare(`SELECT DISTINCT geoid, provider_id, technology_code,
379
436
  max_advertised_download_speed, max_advertised_upload_speed, low_latency, business_residential_code
380
437
  FROM bdc_stage`);
381
- progress("materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)");
382
- db.exec("BEGIN");
383
- batch = 0;
384
- for (const row of stageStmt.iterate()) {
385
- let resolved = centroidCache.get(row.geoid);
386
- if (resolved === undefined) {
387
- const centroid = options.blockCentroids(row.geoid);
388
- resolved = centroid
389
- ? (() => {
390
- // Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
391
- // independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
392
- // exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
393
- // disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
394
- // boundary artifacts). Deriving both `h3_cell` and the coverage cell from
395
- // the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
396
- // exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
397
- // applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
398
- // the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
399
- const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION);
400
- return {
401
- h3Cell: shortCellToInt(fullRes9Cell),
402
- coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION)),
403
- };
404
- })()
405
- : null;
406
- centroidCache.set(row.geoid, resolved);
407
- }
408
- if (!resolved) {
409
- unknownGeoids++;
410
- continue;
438
+ progress("materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)");
439
+ db.exec("BEGIN");
440
+ batch = 0;
441
+ for (const row of stageStmt.iterate()) {
442
+ let resolved = centroidCache.get(row.geoid);
443
+ if (resolved === undefined) {
444
+ const centroid = options.blockCentroids(row.geoid);
445
+ resolved = centroid
446
+ ? (() => {
447
+ // Coverage cell MUST be derived as the res-9 cell's H3 hierarchy parent — NOT a second,
448
+ // independent `latLngToCell(centroid, 6)` call. H3's cell hierarchy is not geometrically
449
+ // exact: a point's directly-indexed res-6 cell and its res-9 cell's `cellToParent(…, 6)`
450
+ // disagree for a real fraction of points (~6% empirically over CONUS — hexagon/pentagon
451
+ // boundary artifacts). Deriving both `h3_cell` and the coverage cell from
452
+ // the SAME full res-9 index is what lets `filing-landscape.ts`'s reader reconstruct this
453
+ // exact coverage cell from nothing but the stored `h3_cell` (its `res9ShortCellToRes6Parent`
454
+ // applies `cellToParent` to the reconstructed res-9 cell) — builder and reader must derive
455
+ // the res-6 parent identically, or a genuinely-surveyed block can read back as unknown.
456
+ const fullRes9Cell = latLngToCell(centroid.lat, centroid.lon, BDC_H3_RESOLUTION);
457
+ return {
458
+ h3Cell: shortCellToInt(fullRes9Cell),
459
+ coverageCell: shortCellToInt(cellToParent(fullRes9Cell, BDC_COVERAGE_H3_RESOLUTION)),
460
+ };
461
+ })()
462
+ : null;
463
+ centroidCache.set(row.geoid, resolved);
464
+ }
465
+ if (!resolved) {
466
+ unknownGeoids++;
467
+ continue;
468
+ }
469
+ insAvailability.run(resolved.h3Cell, row.geoid,
470
+ // wof_id stays NULL here — WOF point-in-polygon resolution against the block centroid is a later
471
+ // registry-join task, the same decision-8 scoping schema.ts documents for `bdc_provider`.
472
+ null, row.provider_id, row.technology_code, row.max_advertised_download_speed, row.max_advertised_upload_speed, row.low_latency, row.business_residential_code, options.includeLocationIDs ? (row.location_id ?? null) : null);
473
+ inserted++;
474
+ providers.add(row.provider_id);
475
+ coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1);
476
+ batch++;
477
+ if (batch >= STAGE_BATCH_SIZE) {
478
+ db.exec("COMMIT");
479
+ db.exec("BEGIN");
480
+ batch = 0;
481
+ }
411
482
  }
412
- insAvailability.run(resolved.h3Cell, row.geoid,
413
- // wof_id stays NULL here WOF point-in-polygon resolution against the block centroid is a later
414
- // registry-join task, the same decision-8 scoping schema.ts documents for `bdc_provider`.
415
- null, row.provider_id, row.technology_code, row.max_advertised_download_speed, row.max_advertised_upload_speed, row.low_latency, row.business_residential_code, options.includeLocationIDs ? (row.location_id ?? null) : null);
416
- inserted++;
417
- providers.add(row.provider_id);
418
- coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1);
419
- batch++;
420
- if (batch >= STAGE_BATCH_SIZE) {
421
- db.exec("COMMIT");
422
- db.exec("BEGIN");
423
- batch = 0;
483
+ db.exec("COMMIT");
484
+ progress(`materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
485
+ `(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`);
486
+ await kdb.schema.dropTable("bdc_stage").execute();
487
+ progress("geoid index (index-after-load — see schema.ts)");
488
+ await createBDCGeoidIndex(kdb);
489
+ // Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
490
+ // have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
491
+ // were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
492
+ const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
493
+ h3Cell,
494
+ completeness: 1,
495
+ basis: CoverageBasis.SourcePresent,
496
+ observedRows,
497
+ }));
498
+ await writeLayerCoverage(kdb, coverageCells);
499
+ progress("writing layer manifest");
500
+ await writeLayerManifest(kdb, {
501
+ name: "bdc",
502
+ version: options.asOfDate,
503
+ schemaVersion: 1,
504
+ tier: LayerTier.Shipped,
505
+ license: "public-domain",
506
+ attribution: BDC_ATTRIBUTION,
507
+ source: "fcc-bdc",
508
+ sourceVintage: options.asOfDate,
509
+ buildCmd: "mailwoman gazetteer build bdc",
510
+ buildSHA: options.buildSHA,
511
+ freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
512
+ spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
513
+ createdAt: new Date().toISOString(),
514
+ });
515
+ // bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
516
+ // `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
517
+ // `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
518
+ let providersPopulated = 0;
519
+ if (options.providers) {
520
+ progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)");
521
+ providersPopulated = await populateBDCProviderTable(kdb, options.providers, options.filerDB, options.primaryFRNAsOf ?? options.asOfDate);
522
+ progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`);
424
523
  }
524
+ progress("finalize: ANALYZE + VACUUM");
525
+ db.exec("ANALYZE");
526
+ // page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
527
+ // `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
528
+ // same discipline).
529
+ db.exec("PRAGMA page_size=8192");
530
+ db.exec("VACUUM");
531
+ await kdb.destroy();
532
+ result = {
533
+ out: options.out,
534
+ rows: inserted,
535
+ deduped,
536
+ providers: providers.size,
537
+ coverageCells: coverageCells.length,
538
+ unknownGeoids,
539
+ providersPopulated,
540
+ };
425
541
  }
426
- db.exec("COMMIT");
427
- progress(`materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
428
- `(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`);
429
- await kdb.schema.dropTable("bdc_stage").execute();
430
- progress("geoid index (index-after-load — see schema.ts)");
431
- await createBDCGeoidIndex(kdb);
432
- // Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
433
- // have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
434
- // were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
435
- const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
436
- h3Cell,
437
- completeness: 1,
438
- observedRows,
439
- }));
440
- await writeLayerCoverage(asContractDB(kdb), coverageCells);
441
- progress("writing layer manifest");
442
- await writeLayerManifest(asContractDB(kdb), {
443
- name: "bdc",
444
- version: options.asOfDate,
445
- schemaVersion: 1,
446
- tier: LayerTier.Shipped,
447
- license: "public-domain",
448
- attribution: BDC_ATTRIBUTION,
449
- source: "fcc-bdc",
450
- sourceVintage: options.asOfDate,
451
- buildCmd: "mailwoman gazetteer build bdc",
452
- buildSHA: options.buildSHA,
453
- freshnessPolicy: LayerFreshnessPolicy.VersionedRefresh,
454
- spineKeys: { h3: { column: "h3_cell", resolution: BDC_H3_RESOLUTION }, wofID: "wof_id" },
455
- createdAt: new Date().toISOString(),
456
- });
457
- // bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
458
- // `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
459
- // `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
460
- let providersPopulated = 0;
461
- if (options.providers) {
462
- progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)");
463
- providersPopulated = await populateBDCProviderTable(kdb, options.providers, options.filerDB, options.primaryFRNAsOf ?? options.asOfDate);
464
- progress(`bdc_provider: ${providersPopulated.toLocaleString()} provider(s) populated`);
542
+ catch (error) {
543
+ // A mid-build throw must not leak the handle or orphan the staging file. The original error
544
+ // always wins over anything the cleanup itself throws.
545
+ try {
546
+ await kdb.destroy();
547
+ }
548
+ catch {
549
+ // The handle may already be closed or mid-statement nothing more to release.
550
+ }
551
+ rmSync(buildingPath, { force: true });
552
+ throw error;
465
553
  }
466
- progress("finalize: ANALYZE + VACUUM");
467
- db.exec("ANALYZE");
468
- // page_size MUST be set right before VACUUM — node:sqlite initializes the file at the 4096 default on
469
- // `new DatabaseSync`, so the earlier pragma is a no-op until a VACUUM rebuilds at the new size (build-poi.ts's
470
- // same discipline).
471
- db.exec("PRAGMA page_size=8192");
472
- db.exec("VACUUM");
473
- await kdb.destroy();
474
554
  progress("seal");
475
555
  sealDatabase(buildingPath);
476
- // Atomic move-into-place the previous version is moved ASIDE FIRST, per the AGENTS.md database house rule
477
- // ("build successfully, then move the previous version to a temp directory, and then move the new version into
478
- // place"). Mirrors `mailwoman/eval-harness/gauntlet/build-regression-db.ts`'s `${output}.prev` swap. Deliberate
479
- // deviation from `build-poi.ts`'s direct-write — see the module docstring.
480
- if (existsSync(options.out)) {
481
- renameSync(options.out, `${options.out}.prev`);
482
- }
483
- renameSync(buildingPath, options.out);
484
- if (existsSync(`${options.out}.prev`)) {
485
- rmSync(`${options.out}.prev`);
486
- }
487
- return {
488
- out: options.out,
489
- rows: inserted,
490
- deduped,
491
- providers: providers.size,
492
- coverageCells: coverageCells.length,
493
- unknownGeoids,
494
- providersPopulated,
495
- };
556
+ // Atomic move-into-place via the shared helper (the AGENTS.md database house rule): prior
557
+ // version aside first, forward rename restored on failure so the slot is never left empty.
558
+ swapDatabaseIntoPlace(buildingPath, options.out);
559
+ return result;
496
560
  }
497
561
  //# sourceMappingURL=build-bdc.js.map