@mailwoman/bdc 9.2.0 → 9.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/lib/env.ts +30 -0
  2. package/lib/index.ts +2 -0
  3. package/{sdk → lib/sdk}/build-bdc.ts +86 -117
  4. package/{sdk → lib/sdk}/client.ts +24 -38
  5. package/{sdk → lib/sdk}/common.ts +1 -1
  6. package/{sdk → lib/sdk}/data-collection.ts +2 -2
  7. package/{sdk → lib/sdk}/download.ts +19 -15
  8. package/{sdk → lib/sdk}/filing-dates.ts +3 -3
  9. package/{sdk → lib/sdk}/filing-landscape.ts +8 -12
  10. package/lib/sdk/index.ts +14 -0
  11. package/{sdk → lib/sdk}/list-files.ts +2 -2
  12. package/{sdk → lib/sdk}/nearest-infrastructure.ts +5 -5
  13. package/{sdk → lib/sdk}/parsing.ts +3 -3
  14. package/{sdk → lib/sdk}/plausibility.ts +42 -23
  15. package/out/env.d.ts +24 -0
  16. package/out/env.d.ts.map +1 -0
  17. package/out/env.js +28 -0
  18. package/out/env.js.map +1 -0
  19. package/out/index.d.ts +2 -2
  20. package/out/index.d.ts.map +1 -1
  21. package/out/index.js +2 -2
  22. package/out/index.js.map +1 -1
  23. package/out/schema.d.ts.map +1 -1
  24. package/out/schema.js.map +1 -1
  25. package/out/sdk/build-bdc.d.ts +20 -19
  26. package/out/sdk/build-bdc.d.ts.map +1 -1
  27. package/out/sdk/build-bdc.js +70 -98
  28. package/out/sdk/build-bdc.js.map +1 -1
  29. package/out/sdk/buildings.d.ts.map +1 -1
  30. package/out/sdk/buildings.js.map +1 -1
  31. package/out/sdk/client.d.ts +7 -7
  32. package/out/sdk/client.d.ts.map +1 -1
  33. package/out/sdk/client.js +17 -29
  34. package/out/sdk/client.js.map +1 -1
  35. package/out/sdk/common.d.ts +1 -1
  36. package/out/sdk/common.d.ts.map +1 -1
  37. package/out/sdk/common.js.map +1 -1
  38. package/out/sdk/data-collection.d.ts +2 -2
  39. package/out/sdk/data-collection.d.ts.map +1 -1
  40. package/out/sdk/data-collection.js.map +1 -1
  41. package/out/sdk/download.d.ts +4 -3
  42. package/out/sdk/download.d.ts.map +1 -1
  43. package/out/sdk/download.js +12 -13
  44. package/out/sdk/download.js.map +1 -1
  45. package/out/sdk/filing-dates.d.ts +3 -3
  46. package/out/sdk/filing-dates.d.ts.map +1 -1
  47. package/out/sdk/filing-dates.js +1 -1
  48. package/out/sdk/filing-dates.js.map +1 -1
  49. package/out/sdk/filing-landscape.d.ts +4 -4
  50. package/out/sdk/filing-landscape.d.ts.map +1 -1
  51. package/out/sdk/filing-landscape.js +7 -10
  52. package/out/sdk/filing-landscape.js.map +1 -1
  53. package/out/sdk/index.d.ts +14 -14
  54. package/out/sdk/index.d.ts.map +1 -1
  55. package/out/sdk/index.js +14 -14
  56. package/out/sdk/index.js.map +1 -1
  57. package/out/sdk/list-files.d.ts +2 -2
  58. package/out/sdk/list-files.d.ts.map +1 -1
  59. package/out/sdk/list-files.js +1 -1
  60. package/out/sdk/list-files.js.map +1 -1
  61. package/out/sdk/location.d.ts.map +1 -1
  62. package/out/sdk/location.js.map +1 -1
  63. package/out/sdk/nearest-infrastructure.d.ts +4 -4
  64. package/out/sdk/nearest-infrastructure.d.ts.map +1 -1
  65. package/out/sdk/nearest-infrastructure.js +2 -2
  66. package/out/sdk/nearest-infrastructure.js.map +1 -1
  67. package/out/sdk/parsing.d.ts +3 -3
  68. package/out/sdk/parsing.d.ts.map +1 -1
  69. package/out/sdk/parsing.js +2 -2
  70. package/out/sdk/parsing.js.map +1 -1
  71. package/out/sdk/plausibility.d.ts +32 -14
  72. package/out/sdk/plausibility.d.ts.map +1 -1
  73. package/out/sdk/plausibility.js +15 -12
  74. package/out/sdk/plausibility.js.map +1 -1
  75. package/out/sdk/technologies.d.ts.map +1 -1
  76. package/out/sdk/technologies.js.map +1 -1
  77. package/package.json +65 -16
  78. package/index.ts +0 -2
  79. package/sdk/index.ts +0 -14
  80. /package/{schema.ts → lib/schema.ts} +0 -0
  81. /package/{sdk → lib/sdk}/buildings.ts +0 -0
  82. /package/{sdk → lib/sdk}/location.ts +0 -0
  83. /package/{sdk → lib/sdk}/technologies.ts +0 -0
package/lib/env.ts ADDED
@@ -0,0 +1,30 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * The FCC Broadband Map credentials the BDC client sends.
7
+ */
8
+
9
+ import { $private as corePrivate, liveEnv } from "@mailwoman/core/env"
10
+ import { z } from "zod"
11
+
12
+ /**
13
+ * FCC Broadband Map (BDC) public-API credentials (`bdc/lib/sdk/client.ts`) — username + hash_value header auth. Never
14
+ * log their values.
15
+ */
16
+ export const PrivateBDCEnvSchema = z.object({
17
+ FCC_MAP_USERNAME: z.string().optional().meta({
18
+ title: "FCC Broadband Map username",
19
+ description: "Username used to authenticate to the FCC Broadband Data Collection API.",
20
+ }),
21
+ FCC_MAP_API_KEY: z.string().optional().meta({
22
+ title: "FCC Broadband Map API key",
23
+ description: "API key hash used to authenticate to the FCC Broadband Data Collection API.",
24
+ }),
25
+ })
26
+
27
+ /**
28
+ * Live BDC credentials over core's. Never log their values.
29
+ */
30
+ export const $private = liveEnv(PrivateBDCEnvSchema, corePrivate)
package/lib/index.ts ADDED
@@ -0,0 +1,2 @@
1
+ export * from "#schema"
2
+ export * from "#sdk/index"
@@ -50,32 +50,33 @@
50
50
  * the house rule exists for.
51
51
  */
52
52
 
53
- import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs"
54
- import { open } from "node:fs/promises"
55
- import { basename, dirname, join } from "node:path"
56
- import { DatabaseSync } from "node:sqlite"
57
-
58
- import { DatabaseClient } from "@mailwoman/core/kysley/client"
53
+ import { pathExists, readDirectory, readFileRange } from "@mailwoman/core/fs/readers"
54
+ import { removePathIfPresent, movePath, makeDirectories, removePath } from "@mailwoman/core/fs/writers"
55
+ import { tryParsingJSON } from "@mailwoman/core/json"
59
56
  import {
60
- CoverageBasis,
61
57
  createLayerCoverageTable,
62
58
  createLayerManifestTable,
63
59
  LayerFreshnessPolicy,
64
60
  LayerTier,
61
+ sourcePresentCoverageCells,
65
62
  writeLayerCoverage,
66
63
  writeLayerManifest,
67
64
  } from "@mailwoman/core/layers"
68
- import { tryParsingJSON } from "@mailwoman/core/objects"
69
- import { openBuiltDatabase, sealDatabase, swapDatabaseIntoPlace } from "@mailwoman/core/utils"
70
65
  import type { FilerDatabase } from "@mailwoman/filer"
71
- // `pickPrimaryFRN`/`readFRNFilingCandidates` are loaded via a LAZY `await import("@mailwoman/filer/sdk")`
66
+ // `pickPrimaryFRN`/`readFRNFilingCandidates` are loaded via a LAZY `await import("@mailwoman/filer/filer-lookup")`
72
67
  // inside `populateBDCProviderTable`, not a top-level runtime import — see that function's docstring
73
- //. Only the TYPES are imported here; `import type` is fully erased, so
74
- // this line has zero runtime cost for every `@mailwoman/bdc` consumer that never populates providers.
75
- import type { FRN, ProviderListRow } from "@mailwoman/filer/sdk"
68
+ //
69
+ // Only the TYPES are imported here; `import type` is fully erased.
70
+ import type { FRN } from "@mailwoman/filer/frn"
71
+ import type { ProviderListRow } from "@mailwoman/filer/sdk"
76
72
  import { shortCellToInt, type H3Cell } from "@mailwoman/spatial"
73
+ import { beginBatched } from "@mailwoman/sqlite/batched"
74
+ import { DatabaseClient } from "@mailwoman/sqlite/client"
75
+ import { openBuiltClient } from "@mailwoman/sqlite/sealed"
76
+ import { sealDatabase, swapDatabaseIntoPlace } from "@mailwoman/sqlite/sealed-db"
77
77
  import { cellToParent, latLngToCell } from "h3-js"
78
78
  import type { Insertable, Kysely } from "kysely"
79
+ import { basename, dirname, join } from "path-ts"
79
80
 
80
81
  import {
81
82
  BDC_COVERAGE_H3_RESOLUTION,
@@ -85,9 +86,9 @@ import {
85
86
  createBDCProviderTable,
86
87
  type BDCDatabase,
87
88
  type BDCProviderTable,
88
- } from "../schema.ts"
89
- import type { ProviderID } from "./common.ts"
90
- import { readAvailabilityRows, type BDCAvailabilityRow } from "./parsing.ts"
89
+ } from "#schema"
90
+ import type { ProviderID } from "#sdk/common"
91
+ import { readAvailabilityRows, type BDCAvailabilityRow } from "#sdk/parsing"
91
92
 
92
93
  /**
93
94
  * Rows committed per `BEGIN`/`COMMIT` batch during both the staging load and the materialize pass — matches
@@ -105,8 +106,8 @@ export const BDC_ATTRIBUTION =
105
106
 
106
107
  export interface BuildBDCOptions {
107
108
  /**
108
- * Injected row source — the test seam (mirrors `BuildPOIOptions.rows`). When given, `csvPaths` is ignored and no
109
- * filesystem read happens.
109
+ * Injected row source — the TEST INJECTION POINT (mirrors `BuildPOIOptions.rows`). When given, `csvPaths` is ignored
110
+ * and no filesystem read happens.
110
111
  */
111
112
  rows?: Iterable<BDCAvailabilityRow> | AsyncIterable<BDCAvailabilityRow>
112
113
  /**
@@ -138,15 +139,15 @@ export interface BuildBDCOptions {
138
139
  * Resolve a 15-char census block GEOID to its centroid. Injected so tests supply a small fixture `Map` lookup instead
139
140
  * of touching a real TIGER database; the real (CLI-wired) implementation is
140
141
  * {@linkcode createTIGERBlockCentroidLookup}, which reads `tabblock20.GEOID` (uppercase — `TIGERBlockTable`) block
141
- * geometry. Returning `undefined` for an unknown geoid is load-bearing: the materialize pass counts it in
142
- * `unknownGeoids` and skips the row — it must NEVER guess a cell.
142
+ * geometry. Returning `undefined` for an unknown geoid is required: the materialize pass counts it in `unknownGeoids`
143
+ * and skips the row — it must NEVER guess a cell.
143
144
  */
144
145
  blockCentroids: (geoid: string) => { lat: number; lon: number } | undefined
145
146
  onProgress?: (message: string) => void
146
147
  /**
147
- * Provider-list rows ({@link ProviderListRow}, `@mailwoman/filer/sdk`'s `parseProviderList`) — the test/CLI seam for
148
- * populating `bdc_provider` (2a decision 8 / 3a decision 6). When ABSENT (the default), `bdc_provider` stays empty
149
- * and the rest of the build is untouched: every code path this option touches is gated behind `if
148
+ * Provider-list rows ({@link ProviderListRow}, `@mailwoman/filer/sdk`'s `parseProviderList`) — the test/CLI injection
149
+ * point for populating `bdc_provider` (2a decision 8 / 3a decision 6). When ABSENT (the default), `bdc_provider`
150
+ * stays empty and the rest of the build is untouched: every code path this option touches is conditioned behind `if
150
151
  * (options.providers)`, so omitting it changes nothing. When present, `buildBDCDatabase` groups rows by `providerID`
151
152
  * and inserts one `bdc_provider` row per distinct provider — see {@link BuildBDCOptions.filerDB} for how the primary
152
153
  * FRN is picked when a provider carries more than one, and `schema.ts`'s `BDCProviderTable` docstring for the full
@@ -257,7 +258,7 @@ interface BDCStageRow {
257
258
  *
258
259
  * `csvPath` is optional and used ONLY to name the offending file in a thrown error (the direct-buffer unit tests call
259
260
  * this without one; {@linkcode readAvailabilityRowsFromCSVPaths} always supplies it). The `Number.isSafeInteger` guard
260
- * below is load-bearing, not defensive dressing: `bdc_stage.provider_id` is `INTEGER NOT NULL`, and a bare
261
+ * below is required, not defensive dressing: `bdc_stage.provider_id` is `INTEGER NOT NULL`, and a bare
261
262
  * `Number.parseInt` on a non-numeric field (a malformed/re-headered/truncated CSV) silently produces `NaN`. `NaN` binds
262
263
  * to that NOT NULL column as SQLite `NULL`, `INSERT OR IGNORE` then drops the row without a constraint error, and every
263
264
  * dropped row gets counted as `deduped` — the ENTIRE file's rows vanish silently, misreported as ordinary dedup. A
@@ -300,38 +301,21 @@ export function peekProviderID(csvBuffer: Buffer, csvPath?: string): ProviderID
300
301
  * Bytes read to peek the `provider_id`. Only the header row plus the first data row are needed and an FCC availability
301
302
  * row is ~110 bytes, so this is three orders of magnitude of slack. A file shorter than this simply reads short —
302
303
  * {@linkcode peekProviderID} already reports a header-only or empty file by message.
303
- */
304
- const PROVIDER_ID_PEEK_BYTES = 64 * 1024
305
-
306
- /**
307
- * Read the head of a CSV, for {@linkcode peekProviderID}.
308
304
  *
309
- * The point is what it does NOT do. `provider_id` is a constant per file, so establishing it needs the first data row
310
- * and nothing else; `readFile(csvPath)` was resident-loading the entire file to read one column of one row. The
311
- * measured file that motivated this is 920 MB for a single state × technology.
305
+ * `provider_id` is a constant per file, so establishing it needs the first data row and nothing else; a whole-file read
306
+ * was resident-loading 920 MB (one state × technology) to read one column of one row.
312
307
  */
313
- async function readCSVHead(csvPath: string): Promise<Buffer> {
314
- const handle = await open(csvPath)
315
-
316
- try {
317
- const buffer = Buffer.allocUnsafe(PROVIDER_ID_PEEK_BYTES)
318
- const { bytesRead } = await handle.read(buffer, 0, PROVIDER_ID_PEEK_BYTES, 0)
319
-
320
- return buffer.subarray(0, bytesRead)
321
- } finally {
322
- await handle.close()
323
- }
324
- }
308
+ const PROVIDER_ID_PEEK_BYTES = 64 * 1024
325
309
 
326
310
  /**
327
311
  * Peeks each file's `provider_id` off its head ({@linkcode peekProviderID}, passing the path through so a malformed
328
312
  * file's error names it), then STREAMS every row via `readAvailabilityRows` — the file is never resident. This is the
329
- * production counterpart to the test seam's injected `rows` — exercised by `build-bdc.test.ts` only for the
330
- * malformed-provider-id rejection path, same as `build-poi.ts`'s `readParquetRows`.
313
+ * production counterpart to the TEST INJECTION POINT's injected `rows` — exercised by `build-bdc.test.ts` only for the
314
+ * malformed-provider-id rejection path, same as `build-poi.ts`'s Parquet reader.
331
315
  */
332
316
  async function* readAvailabilityRowsFromCSVPaths(csvPaths: readonly string[]): AsyncIterable<BDCAvailabilityRow> {
333
317
  for (const csvPath of csvPaths) {
334
- const providerID = peekProviderID(await readCSVHead(csvPath), csvPath)
318
+ const providerID = peekProviderID(await readFileRange(csvPath, 0, PROVIDER_ID_PEEK_BYTES), csvPath)
335
319
 
336
320
  yield* readAvailabilityRows(csvPath, providerID)
337
321
  }
@@ -355,7 +339,7 @@ interface GeoJSONMultiPolygon {
355
339
  * rings for a MultiPolygon. Interior rings/holes are still ignored — a hole moves a block's centroid far less than the
356
340
  * vertex-density skew this replaces, and only 1.0% of measured blocks carry one.
357
341
  *
358
- * This REPLACED the first cut's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
342
+ * This REPLACED the first version's vertex-average, whose "same res-9 cell for all but pathological shapes" claim was
359
343
  * falsified by measurement over every real TIGER 2020 block in LA + Orange county (118,360 blocks, 2026-08-11): the
360
344
  * vertex-average landed in a different res-9 cell for 11.6% of blocks, p99 displacement 286 m (past the ~174 m cell
361
345
  * edge), max 3.7 km — the tail is TIGER's elongated rural/mountain blocks, whose boundary vertices cluster on the
@@ -432,16 +416,17 @@ export function geometryCentroid(geometryJSON: string | null): { lat: number; lo
432
416
 
433
417
  /**
434
418
  * The production `blockCentroids` supplier: opens the TIGER blocks database READ-ONLY and probes `tabblock20.GEOID`
435
- * (uppercase) per lookup, decoding its GeoJSON `geometry` column via {@linkcode geometryCentroid}. Kept synchronous
436
- * `BuildBDCOptions.blockCentroids` is a plain sync function (the same sync-by-interface discipline AGENTS.md documents
437
- * for the resolver ladder), so this uses `node:sqlite`'s raw `.prepare()`/`.get()` directly rather than Kysely. The
438
- * connection is left open for the caller's process lifetime (a read-path lookup, not a build) same lifecycle as the
439
- * resolver-wof-sqlite lookups.
419
+ * (uppercase) per lookup, decoding its GeoJSON `geometry` column via {@linkcode geometryCentroid}. The factory awaits
420
+ * its read-only open; the per-lookup probe and the `BuildBDCOptions.blockCentroids` interface stay synchronous — a
421
+ * plain sync function (the same sync-by-interface discipline AGENTS.md documents for the resolver ladder), so the
422
+ * returned closure uses `node:sqlite`'s raw `.prepare()`/`.get()` directly rather than Kysely. The connection is left
423
+ * open for the caller's process lifetime (a read-path lookup, not a build) — same lifecycle as the resolver-wof-sqlite
424
+ * lookups.
440
425
  */
441
- export function createTIGERBlockCentroidLookup(
426
+ export async function createTIGERBlockCentroidLookup(
442
427
  tigerDBPath: string
443
- ): (geoid: string) => { lat: number; lon: number } | undefined {
444
- const db = openBuiltDatabase(tigerDBPath)
428
+ ): Promise<(geoid: string) => { lat: number; lon: number } | undefined> {
429
+ const db = await openBuiltClient(tigerDBPath)
445
430
  const stmt = db.prepare("SELECT geometry FROM tabblock20 WHERE GEOID = ?")
446
431
 
447
432
  return (geoid: string) => {
@@ -504,15 +489,18 @@ async function groupProviderListRows(
504
489
  * `brand_name` is always inserted NULL — the provider list carries no brand-name column at all, so there is nothing to
505
490
  * populate it from, primary or otherwise (see the schema docstring).
506
491
  *
507
- * **Lazy `@mailwoman/filer/sdk` import.** `readFRNFilingCandidates`/`pickPrimaryFRN` are loaded via `await
508
- * import("@mailwoman/filer/sdk")`, memoized in `filerSDK` below, rather than a top-level static import — that barrel
509
- * re-exports `cluster-filers.ts`, which pulls in `@mailwoman/match`/`record`/`registry`. A top-level import regressed
510
- * `@mailwoman/bdc`'s import time ~32% for EVERY consumer, including ones that never populate providers at all; the
511
- * dynamic import here only ever runs when a multi-FRN `provider_id` is actually encountered, so a `providers`-less
512
- * build (or one whose providers are all single-FRN) pays nothing.
492
+ * **Lazy `@mailwoman/filer/filer-lookup` import.** `readFRNFilingCandidates`/`pickPrimaryFRN` are loaded via `await
493
+ * import("@mailwoman/filer/filer-lookup")`, memoized in `filerSDK` below, rather than a top-level static import. The
494
+ * cost this avoids is smaller than it was: the specifier used to be the `@mailwoman/filer/sdk` BARREL, which `export
495
+ * *`s `cluster-filers.ts` and so pulls `@mailwoman/match`/`record`/`registry` in behind it a top-level import of that
496
+ * barrel regressed `@mailwoman/bdc`'s import time ~32% for EVERY consumer, including ones that never populate
497
+ * providers. `filer-lookup.ts` alone imports only `@mailwoman/sqlite/client`, `#schema` and `#frn` (measured
498
+ * 2026-09-01), so the heavy graph is no longer on this path at all. The laziness is kept because it also defers opening
499
+ * the filer database, and a static import here is now a viable simplification if someone wants to measure it — but it
500
+ * is no longer load-bearing for import time.
513
501
  */
514
502
  async function populateBDCProviderTable(
515
- kdb: DatabaseClient<BDCDatabase>,
503
+ db: DatabaseClient<BDCDatabase>,
516
504
  providers: Iterable<ProviderListRow> | AsyncIterable<ProviderListRow>,
517
505
  filerDB: DatabaseClient<FilerDatabase> | undefined,
518
506
  asOf: string
@@ -520,7 +508,7 @@ async function populateBDCProviderTable(
520
508
  const byProviderID = await groupProviderListRows(providers)
521
509
  const insertRows: Insertable<BDCProviderTable>[] = []
522
510
 
523
- let filerSDK: typeof import("@mailwoman/filer/sdk") | undefined
511
+ let filerSDK: typeof import("@mailwoman/filer/filer-lookup") | undefined
524
512
 
525
513
  for (const [providerID, rows] of byProviderID) {
526
514
  const distinctFRNs = [...new Set(rows.map((row) => row.frn))]
@@ -538,7 +526,7 @@ async function populateBDCProviderTable(
538
526
  )
539
527
  }
540
528
 
541
- filerSDK ??= await import("@mailwoman/filer/sdk")
529
+ filerSDK ??= await import("@mailwoman/filer/filer-lookup")
542
530
 
543
531
  const candidates = await filerSDK.readFRNFilingCandidates(filerDB, distinctFRNs, asOf)
544
532
 
@@ -555,7 +543,7 @@ async function populateBDCProviderTable(
555
543
  }
556
544
 
557
545
  for (let index = 0; index < insertRows.length; index += PROVIDER_INSERT_BATCH_SIZE) {
558
- await kdb
546
+ await db
559
547
  .insertInto("bdc_provider")
560
548
  .values(insertRows.slice(index, index + PROVIDER_INSERT_BATCH_SIZE))
561
549
  .execute()
@@ -580,24 +568,24 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
580
568
 
581
569
  const buildingPath = `${options.out}.building`
582
570
 
583
- if (existsSync(buildingPath)) {
584
- rmSync(buildingPath)
571
+ if (await pathExists(buildingPath)) {
572
+ await removePath(buildingPath)
585
573
  }
586
574
 
587
- mkdirSync(dirname(options.out), { recursive: true })
575
+ await makeDirectories(dirname(options.out))
588
576
 
589
577
  // A crash inside a PRIOR run's swap can leave the slot empty while the previous version sits
590
578
  // parked aside — restore it before building, so a failure in THIS run still leaves an artifact
591
579
  // serving. Both aside spellings: this builder's old `.prev` and swapDatabaseIntoPlace's `.old-<pid>`.
592
- if (!existsSync(options.out)) {
580
+ if (!(await pathExists(options.out))) {
593
581
  const base = basename(options.out)
594
582
 
595
- const parked = readdirSync(dirname(options.out)).find(
583
+ const parked = (await readDirectory(dirname(options.out))).find(
596
584
  (name) => name === `${base}.prev` || name.startsWith(`${base}.old-`)
597
585
  )
598
586
 
599
587
  if (parked) {
600
- renameSync(join(dirname(options.out), parked), options.out)
588
+ await movePath(join(dirname(options.out), parked), options.out)
601
589
  progress(`restored ${parked} into place (a prior run crashed mid-swap)`)
602
590
  }
603
591
  }
@@ -605,21 +593,20 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
605
593
  const rowSource: AsyncIterable<BDCAvailabilityRow> | Iterable<BDCAvailabilityRow> =
606
594
  options.rows ?? readAvailabilityRowsFromCSVPaths(options.csvPaths!)
607
595
 
608
- const db = new DatabaseSync(buildingPath)
596
+ const db = new DatabaseClient<BDCDatabase>(buildingPath)
609
597
  // Build-tuning pragmas — identical to build-poi.ts's discipline.
610
598
  db.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;")
611
- const kdb = new DatabaseClient<BDCDatabase>({ database: db })
612
599
 
613
600
  // Assigned at the end of the try — the tallies live inside its scope; the seal + swap do not.
614
601
  let result: BuildBDCResult
615
602
 
616
603
  try {
617
604
  progress("creating manifest/coverage/availability/provider/stage tables")
618
- await createLayerManifestTable(kdb)
619
- await createLayerCoverageTable(kdb)
620
- await createBDCAvailabilityTable(kdb)
621
- await createBDCProviderTable(kdb)
622
- await createBDCStageTable(kdb)
605
+ await createLayerManifestTable(db)
606
+ await createLayerCoverageTable(db)
607
+ await createBDCAvailabilityTable(db)
608
+ await createBDCProviderTable(db)
609
+ await createBDCStageTable(db)
623
610
 
624
611
  const insStage = db.prepare(
625
612
  `INSERT OR IGNORE INTO bdc_stage (
@@ -629,10 +616,10 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
629
616
  )
630
617
 
631
618
  let staged = 0
632
- let batch = 0
633
619
 
634
620
  progress("staging rows — raw prepared INSERT OR IGNORE on the natural key (the Redis-dedup replacement)")
635
- db.exec("BEGIN")
621
+
622
+ const stageBatch = beginBatched(db, { rowsPerCommit: STAGE_BATCH_SIZE })
636
623
 
637
624
  for await (const row of rowSource) {
638
625
  insStage.run(
@@ -648,16 +635,10 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
648
635
 
649
636
  staged++
650
637
 
651
- batch++
652
-
653
- if (batch >= STAGE_BATCH_SIZE) {
654
- db.exec("COMMIT")
655
- db.exec("BEGIN")
656
- batch = 0
657
- }
638
+ stageBatch.rowWritten()
658
639
  }
659
640
 
660
- db.exec("COMMIT")
641
+ stageBatch.commit()
661
642
 
662
643
  const stagedCountRow = db.prepare("SELECT COUNT(*) AS staged_count FROM bdc_stage").get() as {
663
644
  staged_count: number
@@ -714,8 +695,7 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
714
695
  "materializing bdc_availability — resolving block centroids to h3_cell (unknown geoids skipped, never guessed)"
715
696
  )
716
697
 
717
- db.exec("BEGIN")
718
- batch = 0
698
+ const materializeBatch = beginBatched(db, { rowsPerCommit: STAGE_BATCH_SIZE })
719
699
 
720
700
  for (const row of stageStmt.iterate() as IterableIterator<BDCStageRow>) {
721
701
  let resolved = centroidCache.get(row.geoid)
@@ -771,47 +751,36 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
771
751
  providers.add(row.provider_id)
772
752
  coverage.set(resolved.coverageCell, (coverage.get(resolved.coverageCell) ?? 0) + 1)
773
753
 
774
- batch++
775
-
776
- if (batch >= STAGE_BATCH_SIZE) {
777
- db.exec("COMMIT")
778
- db.exec("BEGIN")
779
- batch = 0
780
- }
754
+ materializeBatch.rowWritten()
781
755
  }
782
756
 
783
- db.exec("COMMIT")
757
+ materializeBatch.commit()
784
758
 
785
759
  progress(
786
760
  `materialized ${inserted.toLocaleString()} row(s) across ${providers.size} provider(s) ` +
787
761
  `(${unknownGeoids.toLocaleString()} unknown geoid(s) skipped)`
788
762
  )
789
763
 
790
- await kdb.schema.dropTable("bdc_stage").execute()
764
+ await db.schema.dropTable("bdc_stage").execute()
791
765
 
792
766
  progress("geoid index (index-after-load — see schema.ts)")
793
- await createBDCGeoidIndex(kdb)
767
+ await createBDCGeoidIndex(db)
794
768
 
795
769
  // Coverage is SOURCE-LEVEL, not survey completeness — same convention build-poi.ts documents: a res-6 cell we
796
770
  // have availability rows in is recorded at completeness 1.0. A cell absent from `layer_coverage` means no rows
797
771
  // were observed there at all (the meaning-of-zero rule — missing = unknown, never `{completeness: 0}`).
798
- const coverageCells = [...coverage.entries()].map(([h3Cell, observedRows]) => ({
799
- h3Cell,
800
- completeness: 1,
801
- basis: CoverageBasis.SourcePresent,
802
- observedRows,
803
- }))
772
+ const coverageCells = sourcePresentCoverageCells(coverage)
804
773
 
805
- await writeLayerCoverage(kdb, coverageCells)
774
+ await writeLayerCoverage(db, coverageCells)
806
775
 
807
776
  progress("writing layer manifest")
808
777
 
809
- await writeLayerManifest(kdb, {
778
+ await writeLayerManifest(db, {
810
779
  name: "bdc",
811
780
  version: options.asOfDate,
812
781
  schemaVersion: 1,
813
782
  tier: LayerTier.Shipped,
814
- license: "public-domain",
783
+ license: "LicenseRef-USGov-Public-Domain",
815
784
  attribution: BDC_ATTRIBUTION,
816
785
  source: "fcc-bdc",
817
786
  sourceVintage: options.asOfDate,
@@ -822,7 +791,7 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
822
791
  createdAt: new Date().toISOString(),
823
792
  })
824
793
 
825
- // bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and gated behind
794
+ // bdc_provider population (2a decision 8 / 3a decision 6) — entirely additive and conditioned on
826
795
  // `options.providers`: when absent, this block never runs and `bdc_provider` stays empty (see
827
796
  // `BuildBDCOptions.providers`'s docstring for the default-path guarantee).
828
797
  let providersPopulated = 0
@@ -831,7 +800,7 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
831
800
  progress("populating bdc_provider from the provider list (decision 6 — lossy denormalization, see schema.ts)")
832
801
 
833
802
  providersPopulated = await populateBDCProviderTable(
834
- kdb,
803
+ db,
835
804
  options.providers,
836
805
  options.filerDB,
837
806
  options.primaryFRNAsOf ?? options.asOfDate
@@ -847,7 +816,7 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
847
816
  // same discipline).
848
817
  db.exec("PRAGMA page_size=8192")
849
818
  db.exec("VACUUM")
850
- await kdb.destroy()
819
+ await db.destroy()
851
820
 
852
821
  result = {
853
822
  out: options.out,
@@ -862,22 +831,22 @@ export async function buildBDCDatabase(options: BuildBDCOptions): Promise<BuildB
862
831
  // A mid-build throw must not leak the handle or orphan the staging file. The original error
863
832
  // always wins over anything the cleanup itself throws.
864
833
  try {
865
- await kdb.destroy()
834
+ await db.destroy()
866
835
  } catch {
867
836
  // The handle may already be closed or mid-statement — nothing more to release.
868
837
  }
869
838
 
870
- rmSync(buildingPath, { force: true })
839
+ await removePathIfPresent(buildingPath)
871
840
 
872
841
  throw error
873
842
  }
874
843
 
875
844
  progress("seal")
876
- sealDatabase(buildingPath)
845
+ await sealDatabase(buildingPath)
877
846
 
878
847
  // Atomic move-into-place via the shared helper (the AGENTS.md database house rule): prior
879
848
  // version aside first, forward rename restored on failure so the slot is never left empty.
880
- swapDatabaseIntoPlace(buildingPath, options.out)
849
+ await swapDatabaseIntoPlace(buildingPath, options.out)
881
850
 
882
851
  return result
883
852
  }
@@ -16,7 +16,7 @@
16
16
  * 2. The `username` + `hash_value` PLAIN HEADER PAIR — read carefully off the Nexus original's
17
17
  * `axios.headers` config, this is NOT bearer or basic auth.
18
18
  * 3. The request budget: {@linkcode BDC_DEFAULT_REQUESTS_PER_MINUTE} requests per MINUTE, six
19
- * seconds apart. See that constant for the sourcing and for why the interval gate is set too.
19
+ * seconds apart. See that constant for the sourcing and for why the interval limit is set too.
20
20
  * 4. UN-UNWRAPPED response bodies. Every BDC endpoint nests its payload under a `data` key
21
21
  * (`{ data: [...] }`), and callers pluck `.data` themselves at the call site — `filing-dates.ts`
22
22
  * and `list-files.ts` both do. {@linkcode BDCClient.get} deliberately does NOT unwrap, so the
@@ -36,11 +36,12 @@
36
36
  * | undecodable body | programmer bug | `isTransientResourceError(error)` is false |
37
37
  */
38
38
 
39
- import { APIClient, type APIClientConfig, type ClockLike, systemClock } from "@mailwoman/core/api"
39
+ import { API_CLIENT_DEFAULTS, APIClient, type APIClientConfig, type ClockLike, systemClock } from "@mailwoman/core/api"
40
40
  import { buildDiskStorage } from "@mailwoman/core/api/disk-storage"
41
- import { $private } from "@mailwoman/core/env"
41
+ import { dataRootPath } from "@mailwoman/core/data-root"
42
42
  import { ResourceError } from "@mailwoman/core/errors"
43
- import { dataRootPath } from "@mailwoman/core/utils"
43
+
44
+ import { $private } from "#env"
44
45
 
45
46
  // Re-exported so a caller branching on this client's failures needs exactly one import.
46
47
  export { isTransientResourceError } from "@mailwoman/core/api"
@@ -103,21 +104,6 @@ const PERCENT = 100
103
104
  */
104
105
  const DEFAULT_CACHE_TTL_MS = 24 * 60 * 60 * 1000
105
106
 
106
- /**
107
- * Total attempts (including the first) before giving up on a 429/5xx or a network-class failure.
108
- */
109
- const DEFAULT_MAX_ATTEMPTS = 3
110
-
111
- /**
112
- * Base delay for the exponential backoff between retry attempts, in milliseconds.
113
- */
114
- const DEFAULT_BASE_RETRY_DELAY_MS = 500
115
-
116
- /**
117
- * Per-attempt socket-inactivity timeout for a JSON request, in milliseconds.
118
- */
119
- const DEFAULT_REQUEST_TIMEOUT_MS = 30_000
120
-
121
107
  /**
122
108
  * Per-attempt socket-inactivity timeout for a zip download, in milliseconds — deliberately far longer than the JSON
123
109
  * one. A BDC availability archive is routinely hundreds of megabytes, and Axios applies `timeout` through
@@ -191,9 +177,9 @@ export interface CreateBDCClientOptions {
191
177
  */
192
178
  downloadTimeoutMs?: number
193
179
  /**
194
- * Axios overrides, merged over this client's own defaults. THE TEST SEAM, replacing the old `fetchImpl` option: every
195
- * test passes an `adapter` here, so no test in this workspace ever performs a live network call. Overriding `headers`
196
- * wholesale would drop the credential pair, so don't.
180
+ * Axios overrides, merged over this client's own defaults. THE TEST INJECTION POINT, replacing the old `fetchImpl`
181
+ * option: every test passes an `adapter` here, so no test in this workspace ever performs a live network call.
182
+ * Overriding `headers` wholesale would drop the credential pair, so don't.
197
183
  */
198
184
  axios?: APIClientConfig["axios"]
199
185
  }
@@ -234,10 +220,10 @@ export interface BDCThrottleStats {
234
220
  */
235
221
  waits: number
236
222
  /**
237
- * How many times the per-minute BUDGET gate opened a cooldown, counted off `APIClient`'s `cooldown_start` event. With
238
- * the interval gate also configured this is one per budget's worth of requests, and each is a REAL wait, not a
223
+ * How many times the per-minute BUDGET limit opened a cooldown, counted off `APIClient`'s `cooldown_start` event.
224
+ * With the interval limit also configured this is one per budget's worth of requests, and each is a REAL wait, not a
239
225
  * zero-length window-rollover marker: the budget's cooldown runs to the end of the minute the window opened in
240
- * (`APIClient.#reserveCooldownSlot`), and the interval gate has by then spent only `(N-1) * 60000/N` ms of it. At
226
+ * (`APIClient.#reserveCooldownSlot`), and the interval limit has by then spent only `(N-1) * 60000/N` ms of it. At
241
227
  * 10/minute that is a 6 s cooldown per 10 requests. Some of {@linkcode BDCThrottleStats.waitingMs} is therefore
242
228
  * cooldown, not pacing. See {@linkcode createBDCClient} for the full arrival trace.
243
229
  */
@@ -289,7 +275,7 @@ type UncachedBDCRequestConfig = BDCRequestConfig & {
289
275
  /**
290
276
  * Build the absolute request URL for `path`, with `params` as its query string.
291
277
  *
292
- * `path` is APPENDED to {@linkcode BDC_API_BASE_URL}, never RESOLVED against it. That distinction is load-bearing: `new
278
+ * `path` is APPENDED to {@linkcode BDC_API_BASE_URL}, never RESOLVED against it. That distinction is required: `new
293
279
  * URL("https://elsewhere.example/x", BDC_API_BASE_URL)` would resolve to `elsewhere.example` and carry the credential
294
280
  * header pair there, while string concatenation can only ever produce a path under the BDC origin.
295
281
  */
@@ -466,9 +452,9 @@ function formatDuration(ms: number): string {
466
452
  /**
467
453
  * A {@linkcode ClockLike} that records how long the client spends asleep, plus the reader that snapshots it.
468
454
  *
469
- * The clock is the only seam `APIClient` exposes that every wait passes through — the pacer sleeps on it, the cooldown
470
- * timer sleeps on it, and the retry backoff sleeps on it — so wrapping it is how the waiting becomes visible without
471
- * touching `core/api`.
455
+ * The clock is the only injection point `APIClient` exposes that every wait passes through — the pacer sleeps on it,
456
+ * the cooldown timer sleeps on it, and the retry backoff sleeps on it — so wrapping it is how the waiting becomes
457
+ * visible without touching `core/api`.
472
458
  *
473
459
  * WAITS ARE UNIONED, NOT SUMMED, and that is the whole subtlety here. Under a concurrent fan-out every caller sleeps at
474
460
  * once, and each one's wait is longer than the last: 40 concurrent requests at a 6 s interval sleep 6 s, 12 s, … 234 s,
@@ -551,20 +537,20 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
551
537
  username,
552
538
  downloadTimeoutMs: options.downloadTimeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS,
553
539
  readThrottleStats: () => meter.read(cooldowns),
554
- // BOTH GATES, on purpose, and the interval is the one that holds the rate.
540
+ // BOTH LIMITS, on purpose, and the interval is the one that holds the rate.
555
541
  //
556
542
  // `requestsPerMinute` alone does NOT deliver N requests per minute. It is a budget model whose
557
543
  // cooldown is `MS_PER_MINUTE / N` minus the gap since the previous dispatch — so N dispatches go out
558
544
  // back to back and the client then waits 60000/N ms, i.e. N requests every 60/N SECONDS. Measured
559
545
  // against a bare `APIClient` at `requestsPerMinute: 10` with a 20-call fan-out on a virtual clock:
560
546
  // arrivals at `[0 x10, 6000 x10]`, i.e. 20 inside one sliding minute against a budget of 10, and a
561
- // sustained 100 requests/minute — ten times the published limit. `minRequestIntervalMs` is the gate
547
+ // sustained 100 requests/minute — ten times the published limit. `minRequestIntervalMs` is the limit
562
548
  // that actually spaces dispatches, and it is what makes this client honor 10/minute.
563
549
  //
564
- // The budget is still declared rather than dropped — and it is NOT free. The two gates compose (both
550
+ // The budget is still declared rather than dropped — and it is NOT free. The two limits compose (both
565
551
  // must clear), so the budget's cooldown still fires, and it is a REAL wait: `APIClient` measures that
566
- // cooldown to the end of the MINUTE the window opened in, while the interval gate has by then spent
567
- // only `(N-1) * 60000/N` ms of it. With both gates on 10/minute, arrivals run `0, 6, …, 54 s`; the
552
+ // cooldown to the end of the MINUTE the window opened in, while the interval limit has by then spent
553
+ // only `(N-1) * 60000/N` ms of it. With both limits on 10/minute, arrivals run `0, 6, …, 54 s`; the
568
554
  // 10th dispatch opens a `60000 - 54000 = 6000 ms` cooldown; the pacer's grant for #11 is discarded
569
555
  // across that wait (`acquireDispatchSlot` re-acquires rather than holding a stale grant, under-issuing
570
556
  // by one — the safe direction), so #11 lands at 66 s and the pattern repeats. Steady state is 10
@@ -574,12 +560,12 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
574
560
  // Declaring the budget states the intent in the option whose name matches the published limit. If
575
561
  // `requestsPerMinute` is ever corrected in `core/api` to mean what it says, this client already
576
562
  // declares the right budget and the interval becomes a redundant second ceiling rather than the
577
- // load-bearing one.
563
+ // required one.
578
564
  requestsPerMinute,
579
565
  minRequestIntervalMs: Math.ceil(MS_PER_MINUTE / requestsPerMinute),
580
566
  retry: {
581
- maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
582
- baseDelayMs: options.baseRetryDelayMs ?? DEFAULT_BASE_RETRY_DELAY_MS,
567
+ maxAttempts: options.maxAttempts ?? API_CLIENT_DEFAULTS.maxAttempts,
568
+ baseDelayMs: options.baseRetryDelayMs ?? API_CLIENT_DEFAULTS.baseRetryDelayMs,
583
569
  },
584
570
  clock: meter.clock,
585
571
  caching: {
@@ -611,7 +597,7 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
611
597
  username,
612
598
  hash_value: apiKey,
613
599
  },
614
- timeout: options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
600
+ timeout: options.requestTimeoutMs ?? API_CLIENT_DEFAULTS.requestTimeoutMs,
615
601
  responseType: "json",
616
602
  // `silentJSONParsing` defaults to TRUE, which makes Axios hand back the RAW STRING when a body
617
603
  // fails to parse instead of raising. An upstream serving an HTML error page under a 200 would
@@ -8,7 +8,7 @@
8
8
 
9
9
  import type { Tagged } from "type-fest"
10
10
 
11
- import type { BroadbandTechnologyCode } from "./technologies.ts"
11
+ import type { BroadbandTechnologyCode } from "#sdk/technologies"
12
12
 
13
13
  /**
14
14
  * Unique identifier for an FCC BDC broadband provider.
@@ -5,8 +5,8 @@
5
5
  * @file FCC Broadband Data Collection System
6
6
  */
7
7
 
8
- import type { BuildingTypeCode } from "./buildings.ts"
9
- import type { BroadbandServicableLocationID } from "./location.ts"
8
+ import type { BuildingTypeCode } from "#sdk/buildings"
9
+ import type { BroadbandServicableLocationID } from "#sdk/location"
10
10
 
11
11
  /**
12
12
  * Snake_case identifier for a US State or Territory.