@mailwoman/bdc 9.1.0 → 9.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/lib/env.ts +30 -0
  2. package/lib/index.ts +2 -0
  3. package/{sdk → lib/sdk}/build-bdc.ts +343 -298
  4. package/{sdk → lib/sdk}/client.ts +24 -38
  5. package/{sdk → lib/sdk}/common.ts +1 -1
  6. package/{sdk → lib/sdk}/data-collection.ts +2 -2
  7. package/{sdk → lib/sdk}/download.ts +19 -15
  8. package/{sdk → lib/sdk}/filing-dates.ts +3 -3
  9. package/{sdk → lib/sdk}/filing-landscape.ts +12 -24
  10. package/lib/sdk/index.ts +14 -0
  11. package/{sdk → lib/sdk}/list-files.ts +2 -2
  12. package/{sdk → lib/sdk}/nearest-infrastructure.ts +5 -5
  13. package/lib/sdk/parsing.ts +157 -0
  14. package/{sdk → lib/sdk}/plausibility.ts +43 -29
  15. package/out/env.d.ts +24 -0
  16. package/out/env.d.ts.map +1 -0
  17. package/out/env.js +28 -0
  18. package/out/env.js.map +1 -0
  19. package/out/index.d.ts +2 -2
  20. package/out/index.d.ts.map +1 -1
  21. package/out/index.js +2 -2
  22. package/out/index.js.map +1 -1
  23. package/out/schema.d.ts.map +1 -1
  24. package/out/schema.js.map +1 -1
  25. package/out/sdk/build-bdc.d.ts +32 -25
  26. package/out/sdk/build-bdc.d.ts.map +1 -1
  27. package/out/sdk/build-bdc.js +255 -219
  28. package/out/sdk/build-bdc.js.map +1 -1
  29. package/out/sdk/buildings.d.ts.map +1 -1
  30. package/out/sdk/buildings.js.map +1 -1
  31. package/out/sdk/client.d.ts +7 -7
  32. package/out/sdk/client.d.ts.map +1 -1
  33. package/out/sdk/client.js +17 -29
  34. package/out/sdk/client.js.map +1 -1
  35. package/out/sdk/common.d.ts +1 -1
  36. package/out/sdk/common.d.ts.map +1 -1
  37. package/out/sdk/common.js.map +1 -1
  38. package/out/sdk/data-collection.d.ts +2 -2
  39. package/out/sdk/data-collection.d.ts.map +1 -1
  40. package/out/sdk/data-collection.js.map +1 -1
  41. package/out/sdk/download.d.ts +4 -3
  42. package/out/sdk/download.d.ts.map +1 -1
  43. package/out/sdk/download.js +12 -13
  44. package/out/sdk/download.js.map +1 -1
  45. package/out/sdk/filing-dates.d.ts +3 -3
  46. package/out/sdk/filing-dates.d.ts.map +1 -1
  47. package/out/sdk/filing-dates.js +1 -1
  48. package/out/sdk/filing-dates.js.map +1 -1
  49. package/out/sdk/filing-landscape.d.ts +4 -4
  50. package/out/sdk/filing-landscape.d.ts.map +1 -1
  51. package/out/sdk/filing-landscape.js +9 -19
  52. package/out/sdk/filing-landscape.js.map +1 -1
  53. package/out/sdk/index.d.ts +14 -14
  54. package/out/sdk/index.d.ts.map +1 -1
  55. package/out/sdk/index.js +14 -14
  56. package/out/sdk/index.js.map +1 -1
  57. package/out/sdk/list-files.d.ts +2 -2
  58. package/out/sdk/list-files.d.ts.map +1 -1
  59. package/out/sdk/list-files.js +1 -1
  60. package/out/sdk/list-files.js.map +1 -1
  61. package/out/sdk/location.d.ts.map +1 -1
  62. package/out/sdk/location.js.map +1 -1
  63. package/out/sdk/nearest-infrastructure.d.ts +4 -4
  64. package/out/sdk/nearest-infrastructure.d.ts.map +1 -1
  65. package/out/sdk/nearest-infrastructure.js +2 -2
  66. package/out/sdk/nearest-infrastructure.js.map +1 -1
  67. package/out/sdk/parsing.d.ts +66 -28
  68. package/out/sdk/parsing.d.ts.map +1 -1
  69. package/out/sdk/parsing.js +103 -70
  70. package/out/sdk/parsing.js.map +1 -1
  71. package/out/sdk/plausibility.d.ts +32 -14
  72. package/out/sdk/plausibility.d.ts.map +1 -1
  73. package/out/sdk/plausibility.js +15 -20
  74. package/out/sdk/plausibility.js.map +1 -1
  75. package/out/sdk/technologies.d.ts.map +1 -1
  76. package/out/sdk/technologies.js.map +1 -1
  77. package/package.json +164 -22
  78. package/index.ts +0 -2
  79. package/sdk/index.ts +0 -14
  80. package/sdk/parsing.ts +0 -124
  81. /package/{schema.ts → lib/schema.ts} +0 -0
  82. /package/{sdk → lib/sdk}/buildings.ts +0 -0
  83. /package/{sdk → lib/sdk}/location.ts +0 -0
  84. /package/{sdk → lib/sdk}/technologies.ts +0 -0
@@ -16,7 +16,7 @@
16
16
  * 2. The `username` + `hash_value` PLAIN HEADER PAIR — read carefully off the Nexus original's
17
17
  * `axios.headers` config, this is NOT bearer or basic auth.
18
18
  * 3. The request budget: {@linkcode BDC_DEFAULT_REQUESTS_PER_MINUTE} requests per MINUTE, six
19
- * seconds apart. See that constant for the sourcing and for why the interval gate is set too.
19
+ * seconds apart. See that constant for the sourcing and for why the interval limit is set too.
20
20
  * 4. UN-UNWRAPPED response bodies. Every BDC endpoint nests its payload under a `data` key
21
21
  * (`{ data: [...] }`), and callers pluck `.data` themselves at the call site — `filing-dates.ts`
22
22
  * and `list-files.ts` both do. {@linkcode BDCClient.get} deliberately does NOT unwrap, so the
@@ -36,11 +36,12 @@
36
36
  * | undecodable body | programmer bug | `isTransientResourceError(error)` is false |
37
37
  */
38
38
 
39
- import { APIClient, type APIClientConfig, type ClockLike, systemClock } from "@mailwoman/core/api"
39
+ import { API_CLIENT_DEFAULTS, APIClient, type APIClientConfig, type ClockLike, systemClock } from "@mailwoman/core/api"
40
40
  import { buildDiskStorage } from "@mailwoman/core/api/disk-storage"
41
- import { $private } from "@mailwoman/core/env"
41
+ import { dataRootPath } from "@mailwoman/core/data-root"
42
42
  import { ResourceError } from "@mailwoman/core/errors"
43
- import { dataRootPath } from "@mailwoman/core/utils"
43
+
44
+ import { $private } from "#env"
44
45
 
45
46
  // Re-exported so a caller branching on this client's failures needs exactly one import.
46
47
  export { isTransientResourceError } from "@mailwoman/core/api"
@@ -103,21 +104,6 @@ const PERCENT = 100
103
104
  */
104
105
  const DEFAULT_CACHE_TTL_MS = 24 * 60 * 60 * 1000
105
106
 
106
- /**
107
- * Total attempts (including the first) before giving up on a 429/5xx or a network-class failure.
108
- */
109
- const DEFAULT_MAX_ATTEMPTS = 3
110
-
111
- /**
112
- * Base delay for the exponential backoff between retry attempts, in milliseconds.
113
- */
114
- const DEFAULT_BASE_RETRY_DELAY_MS = 500
115
-
116
- /**
117
- * Per-attempt socket-inactivity timeout for a JSON request, in milliseconds.
118
- */
119
- const DEFAULT_REQUEST_TIMEOUT_MS = 30_000
120
-
121
107
  /**
122
108
  * Per-attempt socket-inactivity timeout for a zip download, in milliseconds — deliberately far longer than the JSON
123
109
  * one. A BDC availability archive is routinely hundreds of megabytes, and Axios applies `timeout` through
@@ -191,9 +177,9 @@ export interface CreateBDCClientOptions {
191
177
  */
192
178
  downloadTimeoutMs?: number
193
179
  /**
194
- * Axios overrides, merged over this client's own defaults. THE TEST SEAM, replacing the old `fetchImpl` option: every
195
- * test passes an `adapter` here, so no test in this workspace ever performs a live network call. Overriding `headers`
196
- * wholesale would drop the credential pair, so don't.
180
+ * Axios overrides, merged over this client's own defaults. THE TEST INJECTION POINT, replacing the old `fetchImpl`
181
+ * option: every test passes an `adapter` here, so no test in this workspace ever performs a live network call.
182
+ * Overriding `headers` wholesale would drop the credential pair, so don't.
197
183
  */
198
184
  axios?: APIClientConfig["axios"]
199
185
  }
@@ -234,10 +220,10 @@ export interface BDCThrottleStats {
234
220
  */
235
221
  waits: number
236
222
  /**
237
- * How many times the per-minute BUDGET gate opened a cooldown, counted off `APIClient`'s `cooldown_start` event. With
238
- * the interval gate also configured this is one per budget's worth of requests, and each is a REAL wait, not a
223
+ * How many times the per-minute BUDGET limit opened a cooldown, counted off `APIClient`'s `cooldown_start` event.
224
+ * With the interval limit also configured this is one per budget's worth of requests, and each is a REAL wait, not a
239
225
  * zero-length window-rollover marker: the budget's cooldown runs to the end of the minute the window opened in
240
- * (`APIClient.#reserveCooldownSlot`), and the interval gate has by then spent only `(N-1) * 60000/N` ms of it. At
226
+ * (`APIClient.#reserveCooldownSlot`), and the interval limit has by then spent only `(N-1) * 60000/N` ms of it. At
241
227
  * 10/minute that is a 6 s cooldown per 10 requests. Some of {@linkcode BDCThrottleStats.waitingMs} is therefore
242
228
  * cooldown, not pacing. See {@linkcode createBDCClient} for the full arrival trace.
243
229
  */
@@ -289,7 +275,7 @@ type UncachedBDCRequestConfig = BDCRequestConfig & {
289
275
  /**
290
276
  * Build the absolute request URL for `path`, with `params` as its query string.
291
277
  *
292
- * `path` is APPENDED to {@linkcode BDC_API_BASE_URL}, never RESOLVED against it. That distinction is load-bearing: `new
278
+ * `path` is APPENDED to {@linkcode BDC_API_BASE_URL}, never RESOLVED against it. That distinction is required: `new
293
279
  * URL("https://elsewhere.example/x", BDC_API_BASE_URL)` would resolve to `elsewhere.example` and carry the credential
294
280
  * header pair there, while string concatenation can only ever produce a path under the BDC origin.
295
281
  */
@@ -466,9 +452,9 @@ function formatDuration(ms: number): string {
466
452
  /**
467
453
  * A {@linkcode ClockLike} that records how long the client spends asleep, plus the reader that snapshots it.
468
454
  *
469
- * The clock is the only seam `APIClient` exposes that every wait passes through — the pacer sleeps on it, the cooldown
470
- * timer sleeps on it, and the retry backoff sleeps on it — so wrapping it is how the waiting becomes visible without
471
- * touching `core/api`.
455
+ * The clock is the only injection point `APIClient` exposes that every wait passes through — the pacer sleeps on it,
456
+ * the cooldown timer sleeps on it, and the retry backoff sleeps on it — so wrapping it is how the waiting becomes
457
+ * visible without touching `core/api`.
472
458
  *
473
459
  * WAITS ARE UNIONED, NOT SUMMED, and that is the whole subtlety here. Under a concurrent fan-out every caller sleeps at
474
460
  * once, and each one's wait is longer than the last: 40 concurrent requests at a 6 s interval sleep 6 s, 12 s, … 234 s,
@@ -551,20 +537,20 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
551
537
  username,
552
538
  downloadTimeoutMs: options.downloadTimeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS,
553
539
  readThrottleStats: () => meter.read(cooldowns),
554
- // BOTH GATES, on purpose, and the interval is the one that holds the rate.
540
+ // BOTH LIMITS, on purpose, and the interval is the one that holds the rate.
555
541
  //
556
542
  // `requestsPerMinute` alone does NOT deliver N requests per minute. It is a budget model whose
557
543
  // cooldown is `MS_PER_MINUTE / N` minus the gap since the previous dispatch — so N dispatches go out
558
544
  // back to back and the client then waits 60000/N ms, i.e. N requests every 60/N SECONDS. Measured
559
545
  // against a bare `APIClient` at `requestsPerMinute: 10` with a 20-call fan-out on a virtual clock:
560
546
  // arrivals at `[0 x10, 6000 x10]`, i.e. 20 inside one sliding minute against a budget of 10, and a
561
- // sustained 100 requests/minute — ten times the published limit. `minRequestIntervalMs` is the gate
547
+ // sustained 100 requests/minute — ten times the published limit. `minRequestIntervalMs` is the limit
562
548
  // that actually spaces dispatches, and it is what makes this client honor 10/minute.
563
549
  //
564
- // The budget is still declared rather than dropped — and it is NOT free. The two gates compose (both
550
+ // The budget is still declared rather than dropped — and it is NOT free. The two limits compose (both
565
551
  // must clear), so the budget's cooldown still fires, and it is a REAL wait: `APIClient` measures that
566
- // cooldown to the end of the MINUTE the window opened in, while the interval gate has by then spent
567
- // only `(N-1) * 60000/N` ms of it. With both gates on 10/minute, arrivals run `0, 6, …, 54 s`; the
552
+ // cooldown to the end of the MINUTE the window opened in, while the interval limit has by then spent
553
+ // only `(N-1) * 60000/N` ms of it. With both limits on 10/minute, arrivals run `0, 6, …, 54 s`; the
568
554
  // 10th dispatch opens a `60000 - 54000 = 6000 ms` cooldown; the pacer's grant for #11 is discarded
569
555
  // across that wait (`acquireDispatchSlot` re-acquires rather than holding a stale grant, under-issuing
570
556
  // by one — the safe direction), so #11 lands at 66 s and the pattern repeats. Steady state is 10
@@ -574,12 +560,12 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
574
560
  // Declaring the budget states the intent in the option whose name matches the published limit. If
575
561
  // `requestsPerMinute` is ever corrected in `core/api` to mean what it says, this client already
576
562
  // declares the right budget and the interval becomes a redundant second ceiling rather than the
577
- // load-bearing one.
563
+ // required one.
578
564
  requestsPerMinute,
579
565
  minRequestIntervalMs: Math.ceil(MS_PER_MINUTE / requestsPerMinute),
580
566
  retry: {
581
- maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
582
- baseDelayMs: options.baseRetryDelayMs ?? DEFAULT_BASE_RETRY_DELAY_MS,
567
+ maxAttempts: options.maxAttempts ?? API_CLIENT_DEFAULTS.maxAttempts,
568
+ baseDelayMs: options.baseRetryDelayMs ?? API_CLIENT_DEFAULTS.baseRetryDelayMs,
583
569
  },
584
570
  clock: meter.clock,
585
571
  caching: {
@@ -611,7 +597,7 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
611
597
  username,
612
598
  hash_value: apiKey,
613
599
  },
614
- timeout: options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
600
+ timeout: options.requestTimeoutMs ?? API_CLIENT_DEFAULTS.requestTimeoutMs,
615
601
  responseType: "json",
616
602
  // `silentJSONParsing` defaults to TRUE, which makes Axios hand back the RAW STRING when a body
617
603
  // fails to parse instead of raising. An upstream serving an HTML error page under a 200 would
@@ -8,7 +8,7 @@
8
8
 
9
9
  import type { Tagged } from "type-fest"
10
10
 
11
- import type { BroadbandTechnologyCode } from "./technologies.ts"
11
+ import type { BroadbandTechnologyCode } from "#sdk/technologies"
12
12
 
13
13
  /**
14
14
  * Unique identifier for an FCC BDC broadband provider.
@@ -5,8 +5,8 @@
5
5
  * @file FCC Broadband Data Collection System
6
6
  */
7
7
 
8
- import type { BuildingTypeCode } from "./buildings.ts"
9
- import type { BroadbandServicableLocationID } from "./location.ts"
8
+ import type { BuildingTypeCode } from "#sdk/buildings"
9
+ import type { BroadbandServicableLocationID } from "#sdk/location"
10
10
 
11
11
  /**
12
12
  * Snake_case identifier for a US State or Territory.
@@ -5,13 +5,14 @@
5
5
  * @file FCC BDC availability-file download + zip extraction.
6
6
  */
7
7
 
8
- import * as fs from "node:fs/promises"
9
- import * as path from "node:path"
10
-
8
+ import { tryStat } from "@mailwoman/core/fs/readers"
9
+ import { makeDirectories, writeLocalFile } from "@mailwoman/core/fs/writers"
11
10
  import { extractSingleFileZip } from "@mailwoman/core/fs/zip"
11
+ import type { PathBuilderLike } from "path-ts"
12
+ import { resolvePath } from "path-ts/resolve"
12
13
 
13
- import type { BDCClient } from "./client.ts"
14
- import { BDCFilingDataType, type BDCFile } from "./common.ts"
14
+ import type { BDCClient } from "#sdk/client"
15
+ import { BDCFilingDataType, type BDCFile } from "#sdk/common"
15
16
 
16
17
  /**
17
18
  * Download and cache an FCC BDC availability file, extracting its zip-wrapped CSV to `destinationDir`.
@@ -28,15 +29,18 @@ import { BDCFilingDataType, type BDCFile } from "./common.ts"
28
29
  *
29
30
  * @returns The path of the extracted (and now cached) CSV file.
30
31
  */
31
- export async function downloadBDCFile(client: BDCClient, file: BDCFile, destinationDir: string): Promise<string> {
32
- const csvPath = path.join(destinationDir, `${file.fileName}.csv`)
32
+ export async function downloadBDCFile(
33
+ client: BDCClient,
34
+ file: BDCFile,
35
+ destinationDir: PathBuilderLike
36
+ ): Promise<string> {
37
+ const csvPath = resolvePath(destinationDir, `${file.fileName}.csv`)
33
38
 
34
- const alreadyCached = await fs
35
- .access(csvPath)
36
- .then(() => true)
37
- .catch(() => false)
39
+ const alreadyCached = await tryStat(csvPath)
38
40
 
39
- if (alreadyCached) return csvPath
41
+ if (alreadyCached) {
42
+ return csvPath.toString()
43
+ }
40
44
 
41
45
  const zippedArrayBuffer = await client.getArrayBuffer(
42
46
  `/map/downloads/downloadFile/${BDCFilingDataType.Availability}/${file.fileID}`
@@ -44,8 +48,8 @@ export async function downloadBDCFile(client: BDCClient, file: BDCFile, destinat
44
48
 
45
49
  const csvBuffer = await extractSingleFileZip(Buffer.from(zippedArrayBuffer))
46
50
 
47
- await fs.mkdir(destinationDir, { recursive: true })
48
- await fs.writeFile(csvPath, csvBuffer)
51
+ await makeDirectories(destinationDir.toString())
52
+ await writeLocalFile(csvBuffer, csvPath.toString())
49
53
 
50
- return csvPath
54
+ return csvPath.toString()
51
55
  }
@@ -9,7 +9,7 @@
9
9
  * {@linkcode BDCClient}.
10
10
  *
11
11
  * CACHING MOVED TO THE CLIENT. Both the Nexus original (one `<filingType>-dates.json` per filing
12
- * type) and this port's first cut (`dataRootPath("bdc", "cache", "filing-dates.json")`, unfiltered)
12
+ * type) and this port's first version (`dataRootPath("bdc", "cache", "filing-dates.json")`, unfiltered)
13
13
  * hand-rolled a JSON file cache here. `BDCClient` is built on `APIClient` and now carries an on-disk
14
14
  * response cache of its own, so the hand-rolled one was the exact duplication that migration exists to
15
15
  * remove — and it was worse than what replaced it: it had NO expiry, so a machine that resolved a
@@ -18,8 +18,8 @@
18
18
  * per-request cache bypass with the same meaning it always had.
19
19
  */
20
20
 
21
- import type { BDCClient } from "./client.ts"
22
- import type { BDCFilingDataType } from "./common.ts"
21
+ import type { BDCClient } from "#sdk/client"
22
+ import type { BDCFilingDataType } from "#sdk/common"
23
23
 
24
24
  /**
25
25
  * One entry from the FCC BDC's `/map/listAsOfDates` endpoint.
@@ -3,8 +3,8 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * `filing_landscape` reader — the FOUR PRE-REGISTERED ACCEPTANCE GATES this whole phase
7
- * is judged by. See `filing-landscape.test.ts` for the gate tests; this module is only the reader.
6
+ * `filing_landscape` reader — the FOUR PRE-REGISTERED ACCEPTANCE CRITERIA this whole phase
7
+ * is judged by. See `filing-landscape.test.ts` for the criterion tests; this module is only the reader.
8
8
  *
9
9
  * Coverage check (the meaning-of-zero rule): a queried block counts as SURVEYED only when its res-6
10
10
  * coverage cell is present in `layer_coverage` (via `readLayerCoverage`) — `undefined` means the area
@@ -33,13 +33,12 @@
33
33
  * block excluded from `surveyed_block_count` never contributes to `filings` either.
34
34
  */
35
35
 
36
- import type { DatabaseClient } from "@mailwoman/core/kysley/client"
37
- import { readLayerCoverage, readLayerManifest, type LayerContractDatabase } from "@mailwoman/core/layers"
38
- import { expandH3Cell, shortCellToInt, type H3Cell, type H3CellShort } from "@mailwoman/spatial"
39
- import { cellToParent } from "h3-js"
40
- import { sql, type Kysely } from "kysely"
36
+ import { readLayerCoverage, readLayerManifest } from "@mailwoman/core/layers"
37
+ import { shortCellToParentInt } from "@mailwoman/spatial"
38
+ import type { DatabaseClient } from "@mailwoman/sqlite/client"
39
+ import { sql } from "kysely"
41
40
 
42
- import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, type BDCDatabase } from "../schema.ts"
41
+ import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, type BDCDatabase } from "#schema"
43
42
 
44
43
  /**
45
44
  * Exactly one of `geoids` or `h3Cells` is required — `filingLandscape` throws otherwise.
@@ -139,24 +138,13 @@ const speedBucketCaseSQL = sql<string>`CASE
139
138
  ELSE ${BDC_SPEED_BUCKET_GIGABIT}
140
139
  END`
141
140
 
142
- /**
143
- * `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
144
- * in `DB` — same narrowing cast as `build-bdc.ts`'s `asContractDB`.
145
- */
146
- function asContractDB(kdb: DatabaseClient<BDCDatabase>): Kysely<LayerContractDatabase> {
147
- return kdb as unknown as Kysely<LayerContractDatabase>
148
- }
149
-
150
141
  /**
151
142
  * Reconstruct the res-6 ancestor of a res-9 short-cell int WITHOUT a centroid — see the module docstring for why the
152
143
  * centroid is the wrong input. Exported so tests can assert this agrees, cell-for-cell, with `build-bdc.ts`'s own
153
144
  * coverage-cell derivation (the two MUST share this derivation — see that file's docstring).
154
145
  */
155
146
  export function res9ShortCellToRes6Parent(h3CellShortInt: number): number {
156
- const fullCell = expandH3Cell(h3CellShortInt.toString(16) as H3CellShort, BDC_H3_RESOLUTION)
157
- const parentCell = cellToParent(fullCell, BDC_COVERAGE_H3_RESOLUTION) as H3Cell
158
-
159
- return shortCellToInt(parentCell)
147
+ return shortCellToParentInt(h3CellShortInt, BDC_H3_RESOLUTION, BDC_COVERAGE_H3_RESOLUTION)
160
148
  }
161
149
 
162
150
  /**
@@ -183,8 +171,8 @@ export async function filingLandscape(
183
171
  }
184
172
 
185
173
  // Read (and validate) the manifest FIRST — a broken/missing manifest must throw before any block is
186
- // classified, never fall through to an "unstamped" answer (gate 4).
187
- const manifest = await readLayerManifest(asContractDB(db))
174
+ // classified, never fall through to an "unstamped" answer (criterion 4).
175
+ const manifest = await readLayerManifest(db)
188
176
 
189
177
  const requestedUnits: ReadonlyArray<string | number> = query.geoids ?? query.h3Cells!
190
178
  const unitColumn = query.geoids ? ("geoid" as const) : ("h3_cell" as const)
@@ -214,7 +202,7 @@ export async function filingLandscape(
214
202
  let surveyedBlockCount = 0
215
203
  let unknownBlockCount = 0
216
204
  // Only units that PASS the coverage check feed the census below — a unit with rows but no coverage evidence
217
- // (a corrupted/inconsistent db — see filing-landscape.test.ts's "coverage row deleted" gate) is `unknown`, and
205
+ // (a corrupted/inconsistent db — see filing-landscape.test.ts's "coverage row deleted" case) is `unknown`, and
218
206
  // its rows must not leak into `filings` either: `surveyed_block_count` and the blocks backing `filings` must
219
207
  // always agree, or a caller cross-referencing the two gets a contradiction (an "unknown" block whose filings
220
208
  // still show up looks exactly like the false-negative bug this reader exists to prevent).
@@ -230,7 +218,7 @@ export async function filingLandscape(
230
218
  }
231
219
 
232
220
  const res6Parent = res9ShortCellToRes6Parent(candidateCell)
233
- const coverage = await readLayerCoverage(asContractDB(db), res6Parent)
221
+ const coverage = await readLayerCoverage(db, res6Parent)
234
222
 
235
223
  if (coverage === undefined) {
236
224
  unknownBlockCount++
@@ -0,0 +1,14 @@
1
+ export * from "#sdk/build-bdc"
2
+ export * from "#sdk/buildings"
3
+ export * from "#sdk/client"
4
+ export * from "#sdk/common"
5
+ export * from "#sdk/data-collection"
6
+ export * from "#sdk/download"
7
+ export * from "#sdk/filing-dates"
8
+ export * from "#sdk/filing-landscape"
9
+ export * from "#sdk/list-files"
10
+ export * from "#sdk/location"
11
+ export * from "#sdk/nearest-infrastructure"
12
+ export * from "#sdk/parsing"
13
+ export * from "#sdk/plausibility"
14
+ export * from "#sdk/technologies"
@@ -11,7 +11,7 @@
11
11
  * ascending by revision (`compareRevisionAsc`) rather than in API order.
12
12
  */
13
13
 
14
- import type { BDCClient } from "./client.ts"
14
+ import type { BDCClient } from "#sdk/client"
15
15
  import {
16
16
  compareRevisionAsc,
17
17
  parseRawBDCFile,
@@ -21,7 +21,7 @@ import {
21
21
  type BDCStateSubCategory,
22
22
  type BDCSummarySubCategory,
23
23
  type RawBDCFile,
24
- } from "./common.ts"
24
+ } from "#sdk/common"
25
25
 
26
26
  export interface RetrieveProviderAvailabilityFilesParams {
27
27
  /**
@@ -34,13 +34,13 @@
34
34
  * spines agree without this module hardcoding a resolution of its own.
35
35
  */
36
36
 
37
- import type { DatabaseClient } from "@mailwoman/core/kysley/client"
38
- import { readLayerCoverage, type CoverageCell, type LayerContractDatabase } from "@mailwoman/core/layers"
39
- import { POI_H3_RESOLUTION, type POILookup } from "@mailwoman/resolver-wof-sqlite/poi-lookup"
37
+ import { readLayerCoverage, type CoverageCell, type LayerContractHandle } from "@mailwoman/core/layers"
38
+ import { POI_H3_RESOLUTION, type POILookup } from "@mailwoman/resolver-wof-sqlite/poi"
40
39
  import { shortCellToInt, type H3Cell, type PointLiteral } from "@mailwoman/spatial"
40
+ import type { DatabaseClient } from "@mailwoman/sqlite/client"
41
41
  import { latLngToCell } from "h3-js"
42
42
 
43
- import { res9ShortCellToRes6Parent } from "./filing-landscape.ts"
43
+ import { res9ShortCellToRes6Parent } from "#sdk/filing-landscape"
44
44
 
45
45
  /**
46
46
  * Ring budget default for {@link nearestInfrastructure} — wider than `POILookup`'s own internal `DEFAULT_MAX_RINGS`
@@ -89,7 +89,7 @@ export interface NearestInfrastructureOptions {
89
89
  */
90
90
  export async function nearestInfrastructure(
91
91
  poiLookup: POILookup,
92
- contractDB: DatabaseClient<LayerContractDatabase>,
92
+ contractDB: LayerContractHandle & Pick<DatabaseClient, "destroy">,
93
93
  options: NearestInfrastructureOptions
94
94
  ): Promise<InfrastructureHit[]> {
95
95
  const [longitude, latitude] = options.center.coordinates
@@ -0,0 +1,157 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file FCC BDC availability CSV row reader.
6
+ *
7
+ * Streams the FCC's 12-column availability CSV through `CSVSpliterator` in `array` mode and projects the
8
+ * seven columns 2a keeps. Columns 0-2 (`frn`, `provider_id`, `brand_name`) and 9, 11 (`state_usps`,
9
+ * `h3_res8_id`) are read past and never emitted: `provider_id` comes from the {@linkcode ProviderID}
10
+ * parameter instead (the FCC partitions availability files per provider, so the caller already knows it),
11
+ * and FRN/brand/state/H3 join concerns are a 2c registry-join boundary.
12
+ *
13
+ * Two projection decisions are required and pre-registered. `location_id` (column 3) stays a STRING —
14
+ * the FCC's values are zero-padded 10-digit strings and `parseInt` would lose the leading zeros (decision
15
+ * 1). `geoid` (column 10) is a string joining `TIGERBlockTable.GEOID` (decision 3).
16
+ *
17
+ * ## Why the source is a resource, not a Buffer
18
+ *
19
+ * This read is STREAMING because the files do not fit the alternative. One state × one technology —
20
+ * `bdc_48_FibertothePremises_fixed_broadband_D25` — is 920 MB and 10,369,043 rows, and a national run
21
+ * spans every state × every technology. The previous byte scanner took a `Buffer`, so its caller opened
22
+ * with `readFile(csvPath)` and held the whole file resident per file.
23
+ *
24
+ * ## Why quoting is not optional here
25
+ *
26
+ * Measured on that same file: 421,555 rows carry one embedded comma inside a quoted `brand_name` and 327
27
+ * carry two ("FiberFirst, LLC", "Valor Telecommunications of Texas, LP"). A delimiter scan blind to quotes
28
+ * shifts every column right of `brand_name` on 4% of rows — measured exactly, a quote-blind
29
+ * `String.split(",")` mismatches 81,095 of 2,000,000 real rows. `enableQuoteHandling` also keeps an
30
+ * embedded NEWLINE inside its row — no row in that file needs it (the 12/13/14-field line counts sum
31
+ * exactly to `wc -l`, so no record is split across lines), but the guarantee is what makes the reader safe
32
+ * on a file nobody has measured yet.
33
+ *
34
+ * ## What this costs
35
+ *
36
+ * Measured on the full 920 MB / 10,369,042-row file, projecting these 7 of 12 columns, identical checksums
37
+ * across every arm. One arm per process — arms sharing a process mis-rank, because the JIT warms across
38
+ * them:
39
+ *
40
+ * ```
41
+ * hand-rolled byte scan, whole buffer 778 ns/row peak RSS 978 MB
42
+ * spliterator 6.2.0, streaming 2,435 ns/row peak RSS 112 MB
43
+ * spliterator patched, streaming 860 ns/row peak RSS 100 MB
44
+ * ```
45
+ *
46
+ * The 6.2.0 row is why this looked like a regression when it landed: CSVSpliterator decoded once per
47
+ * COLUMN, and `TextDecoder`'s per-call overhead dominates at column sizes. Fixed upstream in
48
+ * sister-software/spliterator#6 by decoding the row once — so the streaming path now runs within 10% of a
49
+ * whole-buffer byte scan while holding 9.8x less memory. **Requires a spliterator release carrying that
50
+ * fix**; on 6.2.0 this reader is correct and roughly 3x slower.
51
+ *
52
+ * Do not route around it by hand-rolling a splitter here again. The staging insert this feeds measures 615
53
+ * ns/row, so the parse does sit on the critical path — but a local fast-and-quote-blind parser is exactly
54
+ * the trade that produced the scanner this replaced, and the quote-blind version is 4% wrong.
55
+ */
56
+
57
+ import type { AsyncDataResource } from "spliterator"
58
+ import { CSVSpliterator } from "spliterator"
59
+
60
+ import type { ProviderID } from "#sdk/common"
61
+
62
+ /**
63
+ * Column positions in the FCC's 12-column availability CSV. Named rather than sliced by offset so a reader can check
64
+ * them against the header row without counting commas:
65
+ *
66
+ * `frn,provider_id,brand_name,location_id,technology,max_advertised_download_speed,`
67
+ * `max_advertised_upload_speed,low_latency,business_residential_code,state_usps,block_geoid,h3_res8_id`
68
+ */
69
+ const Column = {
70
+ LocationID: 3,
71
+ Technology: 4,
72
+ MaxAdvertisedDownloadSpeed: 5,
73
+ MaxAdvertisedUploadSpeed: 6,
74
+ LowLatency: 7,
75
+ BusinessResidentialCode: 8,
76
+ BlockGeoID: 10,
77
+ } as const
78
+
79
+ /**
80
+ * A single parsed row of FCC BDC availability data.
81
+ *
82
+ * @see {@linkcode readAvailabilityRows}
83
+ */
84
+ export interface BDCAvailabilityRow {
85
+ provider_id: number
86
+ /**
87
+ * Kept as a string — the FCC's `location_id` values are zero-padded 10-digit strings; `parseInt`ing would lose
88
+ * leading zeros (2a decision 1).
89
+ */
90
+ location_id: string
91
+ technology_code: number
92
+ max_advertised_download_speed: number
93
+ max_advertised_upload_speed: number
94
+ low_latency: 0 | 1
95
+ business_residential_code: string
96
+ /**
97
+ * Joins `TIGERBlockTable.GEOID` (note: uppercase column on that side) — 2a decision 3.
98
+ */
99
+ geoid: string
100
+ }
101
+
102
+ /**
103
+ * Project one already-split CSV row onto {@linkcode BDCAvailabilityRow}.
104
+ *
105
+ * Separate from the iteration so the sync and async readers cannot drift in what they emit — the one-function
106
+ * discipline. Not exported: a caller with a split row wants {@linkcode readAvailabilityRowsSync}.
107
+ */
108
+ function projectRow(columns: readonly string[], providerID: ProviderID): BDCAvailabilityRow {
109
+ return {
110
+ provider_id: providerID,
111
+ location_id: columns[Column.LocationID] ?? "",
112
+ technology_code: Number.parseInt(columns[Column.Technology] ?? "", 10),
113
+ max_advertised_download_speed: Number.parseInt(columns[Column.MaxAdvertisedDownloadSpeed] ?? "", 10),
114
+ max_advertised_upload_speed: Number.parseInt(columns[Column.MaxAdvertisedUploadSpeed] ?? "", 10),
115
+ low_latency: columns[Column.LowLatency] === "1" ? 1 : 0,
116
+ business_residential_code: columns[Column.BusinessResidentialCode] ?? "",
117
+ geoid: columns[Column.BlockGeoID] ?? "",
118
+ }
119
+ }
120
+
121
+ /**
122
+ * Shared reader options. `header: true` consumes the first row as the header even in `array` mode;
123
+ * `enableQuoteHandling` is what makes the 421,882 quoted-brand rows keep their column alignment. `crlf` already
124
+ * defaults to `true` for CSV (RFC 4180), so a CRLF file does not leak `\r` into the last column.
125
+ */
126
+ const READER_OPTIONS = { mode: "array", enableQuoteHandling: true } as const
127
+
128
+ /**
129
+ * Stream an FCC BDC availability CSV, yielding every data row. The header row is consumed, never emitted.
130
+ *
131
+ * `source` is anything `spliterator` can open asynchronously — a path string, a `path-ts` builder, a URL, a file
132
+ * handle, or an async chunk iterator. Prefer handing it the path and letting it own the read: that is what keeps a 920
133
+ * MB file off the heap.
134
+ */
135
+ export async function* readAvailabilityRows(
136
+ source: AsyncDataResource,
137
+ providerID: ProviderID
138
+ ): AsyncIterable<BDCAvailabilityRow> {
139
+ for await (const columns of CSVSpliterator.fromAsync<string[]>(source, READER_OPTIONS)) {
140
+ yield projectRow(columns, providerID)
141
+ }
142
+ }
143
+
144
+ /**
145
+ * Synchronous sibling for an in-memory buffer — fixtures and tests, never the build path.
146
+ *
147
+ * Kept because a test that must assert on a literal CSV should not have to stand up a stream to do it. If you are
148
+ * reaching for this against a file on disk, reach for {@linkcode readAvailabilityRows} instead.
149
+ */
150
+ export function* readAvailabilityRowsSync(
151
+ csvBuffer: Buffer | string,
152
+ providerID: ProviderID
153
+ ): Iterable<BDCAvailabilityRow> {
154
+ for (const columns of CSVSpliterator.from<string[]>(csvBuffer, READER_OPTIONS)) {
155
+ yield projectRow(columns, providerID)
156
+ }
157
+ }