@mailwoman/bdc 9.1.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/env.ts +30 -0
- package/lib/index.ts +2 -0
- package/{sdk → lib/sdk}/build-bdc.ts +343 -298
- package/{sdk → lib/sdk}/client.ts +24 -38
- package/{sdk → lib/sdk}/common.ts +1 -1
- package/{sdk → lib/sdk}/data-collection.ts +2 -2
- package/{sdk → lib/sdk}/download.ts +19 -15
- package/{sdk → lib/sdk}/filing-dates.ts +3 -3
- package/{sdk → lib/sdk}/filing-landscape.ts +12 -24
- package/lib/sdk/index.ts +14 -0
- package/{sdk → lib/sdk}/list-files.ts +2 -2
- package/{sdk → lib/sdk}/nearest-infrastructure.ts +5 -5
- package/lib/sdk/parsing.ts +157 -0
- package/{sdk → lib/sdk}/plausibility.ts +43 -29
- package/out/env.d.ts +24 -0
- package/out/env.d.ts.map +1 -0
- package/out/env.js +28 -0
- package/out/env.js.map +1 -0
- package/out/index.d.ts +2 -2
- package/out/index.d.ts.map +1 -1
- package/out/index.js +2 -2
- package/out/index.js.map +1 -1
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js.map +1 -1
- package/out/sdk/build-bdc.d.ts +32 -25
- package/out/sdk/build-bdc.d.ts.map +1 -1
- package/out/sdk/build-bdc.js +255 -219
- package/out/sdk/build-bdc.js.map +1 -1
- package/out/sdk/buildings.d.ts.map +1 -1
- package/out/sdk/buildings.js.map +1 -1
- package/out/sdk/client.d.ts +7 -7
- package/out/sdk/client.d.ts.map +1 -1
- package/out/sdk/client.js +17 -29
- package/out/sdk/client.js.map +1 -1
- package/out/sdk/common.d.ts +1 -1
- package/out/sdk/common.d.ts.map +1 -1
- package/out/sdk/common.js.map +1 -1
- package/out/sdk/data-collection.d.ts +2 -2
- package/out/sdk/data-collection.d.ts.map +1 -1
- package/out/sdk/data-collection.js.map +1 -1
- package/out/sdk/download.d.ts +4 -3
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +12 -13
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/filing-dates.d.ts +3 -3
- package/out/sdk/filing-dates.d.ts.map +1 -1
- package/out/sdk/filing-dates.js +1 -1
- package/out/sdk/filing-dates.js.map +1 -1
- package/out/sdk/filing-landscape.d.ts +4 -4
- package/out/sdk/filing-landscape.d.ts.map +1 -1
- package/out/sdk/filing-landscape.js +9 -19
- package/out/sdk/filing-landscape.js.map +1 -1
- package/out/sdk/index.d.ts +14 -14
- package/out/sdk/index.d.ts.map +1 -1
- package/out/sdk/index.js +14 -14
- package/out/sdk/index.js.map +1 -1
- package/out/sdk/list-files.d.ts +2 -2
- package/out/sdk/list-files.d.ts.map +1 -1
- package/out/sdk/list-files.js +1 -1
- package/out/sdk/list-files.js.map +1 -1
- package/out/sdk/location.d.ts.map +1 -1
- package/out/sdk/location.js.map +1 -1
- package/out/sdk/nearest-infrastructure.d.ts +4 -4
- package/out/sdk/nearest-infrastructure.d.ts.map +1 -1
- package/out/sdk/nearest-infrastructure.js +2 -2
- package/out/sdk/nearest-infrastructure.js.map +1 -1
- package/out/sdk/parsing.d.ts +66 -28
- package/out/sdk/parsing.d.ts.map +1 -1
- package/out/sdk/parsing.js +103 -70
- package/out/sdk/parsing.js.map +1 -1
- package/out/sdk/plausibility.d.ts +32 -14
- package/out/sdk/plausibility.d.ts.map +1 -1
- package/out/sdk/plausibility.js +15 -20
- package/out/sdk/plausibility.js.map +1 -1
- package/out/sdk/technologies.d.ts.map +1 -1
- package/out/sdk/technologies.js.map +1 -1
- package/package.json +164 -22
- package/index.ts +0 -2
- package/sdk/index.ts +0 -14
- package/sdk/parsing.ts +0 -124
- /package/{schema.ts → lib/schema.ts} +0 -0
- /package/{sdk → lib/sdk}/buildings.ts +0 -0
- /package/{sdk → lib/sdk}/location.ts +0 -0
- /package/{sdk → lib/sdk}/technologies.ts +0 -0
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* 2. The `username` + `hash_value` PLAIN HEADER PAIR — read carefully off the Nexus original's
|
|
17
17
|
* `axios.headers` config, this is NOT bearer or basic auth.
|
|
18
18
|
* 3. The request budget: {@linkcode BDC_DEFAULT_REQUESTS_PER_MINUTE} requests per MINUTE, six
|
|
19
|
-
* seconds apart. See that constant for the sourcing and for why the interval
|
|
19
|
+
* seconds apart. See that constant for the sourcing and for why the interval limit is set too.
|
|
20
20
|
* 4. UN-UNWRAPPED response bodies. Every BDC endpoint nests its payload under a `data` key
|
|
21
21
|
* (`{ data: [...] }`), and callers pluck `.data` themselves at the call site — `filing-dates.ts`
|
|
22
22
|
* and `list-files.ts` both do. {@linkcode BDCClient.get} deliberately does NOT unwrap, so the
|
|
@@ -36,11 +36,12 @@
|
|
|
36
36
|
* | undecodable body | programmer bug | `isTransientResourceError(error)` is false |
|
|
37
37
|
*/
|
|
38
38
|
|
|
39
|
-
import { APIClient, type APIClientConfig, type ClockLike, systemClock } from "@mailwoman/core/api"
|
|
39
|
+
import { API_CLIENT_DEFAULTS, APIClient, type APIClientConfig, type ClockLike, systemClock } from "@mailwoman/core/api"
|
|
40
40
|
import { buildDiskStorage } from "@mailwoman/core/api/disk-storage"
|
|
41
|
-
import {
|
|
41
|
+
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
42
42
|
import { ResourceError } from "@mailwoman/core/errors"
|
|
43
|
-
|
|
43
|
+
|
|
44
|
+
import { $private } from "#env"
|
|
44
45
|
|
|
45
46
|
// Re-exported so a caller branching on this client's failures needs exactly one import.
|
|
46
47
|
export { isTransientResourceError } from "@mailwoman/core/api"
|
|
@@ -103,21 +104,6 @@ const PERCENT = 100
|
|
|
103
104
|
*/
|
|
104
105
|
const DEFAULT_CACHE_TTL_MS = 24 * 60 * 60 * 1000
|
|
105
106
|
|
|
106
|
-
/**
|
|
107
|
-
* Total attempts (including the first) before giving up on a 429/5xx or a network-class failure.
|
|
108
|
-
*/
|
|
109
|
-
const DEFAULT_MAX_ATTEMPTS = 3
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* Base delay for the exponential backoff between retry attempts, in milliseconds.
|
|
113
|
-
*/
|
|
114
|
-
const DEFAULT_BASE_RETRY_DELAY_MS = 500
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* Per-attempt socket-inactivity timeout for a JSON request, in milliseconds.
|
|
118
|
-
*/
|
|
119
|
-
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000
|
|
120
|
-
|
|
121
107
|
/**
|
|
122
108
|
* Per-attempt socket-inactivity timeout for a zip download, in milliseconds — deliberately far longer than the JSON
|
|
123
109
|
* one. A BDC availability archive is routinely hundreds of megabytes, and Axios applies `timeout` through
|
|
@@ -191,9 +177,9 @@ export interface CreateBDCClientOptions {
|
|
|
191
177
|
*/
|
|
192
178
|
downloadTimeoutMs?: number
|
|
193
179
|
/**
|
|
194
|
-
* Axios overrides, merged over this client's own defaults. THE TEST
|
|
195
|
-
* test passes an `adapter` here, so no test in this workspace ever performs a live network call.
|
|
196
|
-
* wholesale would drop the credential pair, so don't.
|
|
180
|
+
* Axios overrides, merged over this client's own defaults. THE TEST INJECTION POINT, replacing the old `fetchImpl`
|
|
181
|
+
* option: every test passes an `adapter` here, so no test in this workspace ever performs a live network call.
|
|
182
|
+
* Overriding `headers` wholesale would drop the credential pair, so don't.
|
|
197
183
|
*/
|
|
198
184
|
axios?: APIClientConfig["axios"]
|
|
199
185
|
}
|
|
@@ -234,10 +220,10 @@ export interface BDCThrottleStats {
|
|
|
234
220
|
*/
|
|
235
221
|
waits: number
|
|
236
222
|
/**
|
|
237
|
-
* How many times the per-minute BUDGET
|
|
238
|
-
* the interval
|
|
223
|
+
* How many times the per-minute BUDGET limit opened a cooldown, counted off `APIClient`'s `cooldown_start` event.
|
|
224
|
+
* With the interval limit also configured this is one per budget's worth of requests, and each is a REAL wait, not a
|
|
239
225
|
* zero-length window-rollover marker: the budget's cooldown runs to the end of the minute the window opened in
|
|
240
|
-
* (`APIClient.#reserveCooldownSlot`), and the interval
|
|
226
|
+
* (`APIClient.#reserveCooldownSlot`), and the interval limit has by then spent only `(N-1) * 60000/N` ms of it. At
|
|
241
227
|
* 10/minute that is a 6 s cooldown per 10 requests. Some of {@linkcode BDCThrottleStats.waitingMs} is therefore
|
|
242
228
|
* cooldown, not pacing. See {@linkcode createBDCClient} for the full arrival trace.
|
|
243
229
|
*/
|
|
@@ -289,7 +275,7 @@ type UncachedBDCRequestConfig = BDCRequestConfig & {
|
|
|
289
275
|
/**
|
|
290
276
|
* Build the absolute request URL for `path`, with `params` as its query string.
|
|
291
277
|
*
|
|
292
|
-
* `path` is APPENDED to {@linkcode BDC_API_BASE_URL}, never RESOLVED against it. That distinction is
|
|
278
|
+
* `path` is APPENDED to {@linkcode BDC_API_BASE_URL}, never RESOLVED against it. That distinction is required: `new
|
|
293
279
|
* URL("https://elsewhere.example/x", BDC_API_BASE_URL)` would resolve to `elsewhere.example` and carry the credential
|
|
294
280
|
* header pair there, while string concatenation can only ever produce a path under the BDC origin.
|
|
295
281
|
*/
|
|
@@ -466,9 +452,9 @@ function formatDuration(ms: number): string {
|
|
|
466
452
|
/**
|
|
467
453
|
* A {@linkcode ClockLike} that records how long the client spends asleep, plus the reader that snapshots it.
|
|
468
454
|
*
|
|
469
|
-
* The clock is the only
|
|
470
|
-
* timer sleeps on it, and the retry backoff sleeps on it — so wrapping it is how the waiting becomes
|
|
471
|
-
* touching `core/api`.
|
|
455
|
+
* The clock is the only injection point `APIClient` exposes that every wait passes through — the pacer sleeps on it,
|
|
456
|
+
* the cooldown timer sleeps on it, and the retry backoff sleeps on it — so wrapping it is how the waiting becomes
|
|
457
|
+
* visible without touching `core/api`.
|
|
472
458
|
*
|
|
473
459
|
* WAITS ARE UNIONED, NOT SUMMED, and that is the whole subtlety here. Under a concurrent fan-out every caller sleeps at
|
|
474
460
|
* once, and each one's wait is longer than the last: 40 concurrent requests at a 6 s interval sleep 6 s, 12 s, … 234 s,
|
|
@@ -551,20 +537,20 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
|
|
|
551
537
|
username,
|
|
552
538
|
downloadTimeoutMs: options.downloadTimeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS,
|
|
553
539
|
readThrottleStats: () => meter.read(cooldowns),
|
|
554
|
-
// BOTH
|
|
540
|
+
// BOTH LIMITS, on purpose, and the interval is the one that holds the rate.
|
|
555
541
|
//
|
|
556
542
|
// `requestsPerMinute` alone does NOT deliver N requests per minute. It is a budget model whose
|
|
557
543
|
// cooldown is `MS_PER_MINUTE / N` minus the gap since the previous dispatch — so N dispatches go out
|
|
558
544
|
// back to back and the client then waits 60000/N ms, i.e. N requests every 60/N SECONDS. Measured
|
|
559
545
|
// against a bare `APIClient` at `requestsPerMinute: 10` with a 20-call fan-out on a virtual clock:
|
|
560
546
|
// arrivals at `[0 x10, 6000 x10]`, i.e. 20 inside one sliding minute against a budget of 10, and a
|
|
561
|
-
// sustained 100 requests/minute — ten times the published limit. `minRequestIntervalMs` is the
|
|
547
|
+
// sustained 100 requests/minute — ten times the published limit. `minRequestIntervalMs` is the limit
|
|
562
548
|
// that actually spaces dispatches, and it is what makes this client honor 10/minute.
|
|
563
549
|
//
|
|
564
|
-
// The budget is still declared rather than dropped — and it is NOT free. The two
|
|
550
|
+
// The budget is still declared rather than dropped — and it is NOT free. The two limits compose (both
|
|
565
551
|
// must clear), so the budget's cooldown still fires, and it is a REAL wait: `APIClient` measures that
|
|
566
|
-
// cooldown to the end of the MINUTE the window opened in, while the interval
|
|
567
|
-
// only `(N-1) * 60000/N` ms of it. With both
|
|
552
|
+
// cooldown to the end of the MINUTE the window opened in, while the interval limit has by then spent
|
|
553
|
+
// only `(N-1) * 60000/N` ms of it. With both limits on 10/minute, arrivals run `0, 6, …, 54 s`; the
|
|
568
554
|
// 10th dispatch opens a `60000 - 54000 = 6000 ms` cooldown; the pacer's grant for #11 is discarded
|
|
569
555
|
// across that wait (`acquireDispatchSlot` re-acquires rather than holding a stale grant, under-issuing
|
|
570
556
|
// by one — the safe direction), so #11 lands at 66 s and the pattern repeats. Steady state is 10
|
|
@@ -574,12 +560,12 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
|
|
|
574
560
|
// Declaring the budget states the intent in the option whose name matches the published limit. If
|
|
575
561
|
// `requestsPerMinute` is ever corrected in `core/api` to mean what it says, this client already
|
|
576
562
|
// declares the right budget and the interval becomes a redundant second ceiling rather than the
|
|
577
|
-
//
|
|
563
|
+
// required one.
|
|
578
564
|
requestsPerMinute,
|
|
579
565
|
minRequestIntervalMs: Math.ceil(MS_PER_MINUTE / requestsPerMinute),
|
|
580
566
|
retry: {
|
|
581
|
-
maxAttempts: options.maxAttempts ??
|
|
582
|
-
baseDelayMs: options.baseRetryDelayMs ??
|
|
567
|
+
maxAttempts: options.maxAttempts ?? API_CLIENT_DEFAULTS.maxAttempts,
|
|
568
|
+
baseDelayMs: options.baseRetryDelayMs ?? API_CLIENT_DEFAULTS.baseRetryDelayMs,
|
|
583
569
|
},
|
|
584
570
|
clock: meter.clock,
|
|
585
571
|
caching: {
|
|
@@ -611,7 +597,7 @@ export function createBDCClient(options: CreateBDCClientOptions = {}): BDCClient
|
|
|
611
597
|
username,
|
|
612
598
|
hash_value: apiKey,
|
|
613
599
|
},
|
|
614
|
-
timeout: options.requestTimeoutMs ??
|
|
600
|
+
timeout: options.requestTimeoutMs ?? API_CLIENT_DEFAULTS.requestTimeoutMs,
|
|
615
601
|
responseType: "json",
|
|
616
602
|
// `silentJSONParsing` defaults to TRUE, which makes Axios hand back the RAW STRING when a body
|
|
617
603
|
// fails to parse instead of raising. An upstream serving an HTML error page under a 200 would
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* @file FCC Broadband Data Collection System
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import type { BuildingTypeCode } from "
|
|
9
|
-
import type { BroadbandServicableLocationID } from "
|
|
8
|
+
import type { BuildingTypeCode } from "#sdk/buildings"
|
|
9
|
+
import type { BroadbandServicableLocationID } from "#sdk/location"
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* Snake_case identifier for a US State or Territory.
|
|
@@ -5,13 +5,14 @@
|
|
|
5
5
|
* @file FCC BDC availability-file download + zip extraction.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
|
|
8
|
+
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
9
|
+
import { makeDirectories, writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
11
10
|
import { extractSingleFileZip } from "@mailwoman/core/fs/zip"
|
|
11
|
+
import type { PathBuilderLike } from "path-ts"
|
|
12
|
+
import { resolvePath } from "path-ts/resolve"
|
|
12
13
|
|
|
13
|
-
import type { BDCClient } from "
|
|
14
|
-
import { BDCFilingDataType, type BDCFile } from "
|
|
14
|
+
import type { BDCClient } from "#sdk/client"
|
|
15
|
+
import { BDCFilingDataType, type BDCFile } from "#sdk/common"
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
18
|
* Download and cache an FCC BDC availability file, extracting its zip-wrapped CSV to `destinationDir`.
|
|
@@ -28,15 +29,18 @@ import { BDCFilingDataType, type BDCFile } from "./common.ts"
|
|
|
28
29
|
*
|
|
29
30
|
* @returns The path of the extracted (and now cached) CSV file.
|
|
30
31
|
*/
|
|
31
|
-
export async function downloadBDCFile(
|
|
32
|
-
|
|
32
|
+
export async function downloadBDCFile(
|
|
33
|
+
client: BDCClient,
|
|
34
|
+
file: BDCFile,
|
|
35
|
+
destinationDir: PathBuilderLike
|
|
36
|
+
): Promise<string> {
|
|
37
|
+
const csvPath = resolvePath(destinationDir, `${file.fileName}.csv`)
|
|
33
38
|
|
|
34
|
-
const alreadyCached = await
|
|
35
|
-
.access(csvPath)
|
|
36
|
-
.then(() => true)
|
|
37
|
-
.catch(() => false)
|
|
39
|
+
const alreadyCached = await tryStat(csvPath)
|
|
38
40
|
|
|
39
|
-
if (alreadyCached)
|
|
41
|
+
if (alreadyCached) {
|
|
42
|
+
return csvPath.toString()
|
|
43
|
+
}
|
|
40
44
|
|
|
41
45
|
const zippedArrayBuffer = await client.getArrayBuffer(
|
|
42
46
|
`/map/downloads/downloadFile/${BDCFilingDataType.Availability}/${file.fileID}`
|
|
@@ -44,8 +48,8 @@ export async function downloadBDCFile(client: BDCClient, file: BDCFile, destinat
|
|
|
44
48
|
|
|
45
49
|
const csvBuffer = await extractSingleFileZip(Buffer.from(zippedArrayBuffer))
|
|
46
50
|
|
|
47
|
-
await
|
|
48
|
-
await
|
|
51
|
+
await makeDirectories(destinationDir.toString())
|
|
52
|
+
await writeLocalFile(csvBuffer, csvPath.toString())
|
|
49
53
|
|
|
50
|
-
return csvPath
|
|
54
|
+
return csvPath.toString()
|
|
51
55
|
}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* {@linkcode BDCClient}.
|
|
10
10
|
*
|
|
11
11
|
* CACHING MOVED TO THE CLIENT. Both the Nexus original (one `<filingType>-dates.json` per filing
|
|
12
|
-
* type) and this port's first
|
|
12
|
+
* type) and this port's first version (`dataRootPath("bdc", "cache", "filing-dates.json")`, unfiltered)
|
|
13
13
|
* hand-rolled a JSON file cache here. `BDCClient` is built on `APIClient` and now carries an on-disk
|
|
14
14
|
* response cache of its own, so the hand-rolled one was the exact duplication that migration exists to
|
|
15
15
|
* remove — and it was worse than what replaced it: it had NO expiry, so a machine that resolved a
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
* per-request cache bypass with the same meaning it always had.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import type { BDCClient } from "
|
|
22
|
-
import type { BDCFilingDataType } from "
|
|
21
|
+
import type { BDCClient } from "#sdk/client"
|
|
22
|
+
import type { BDCFilingDataType } from "#sdk/common"
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
25
|
* One entry from the FCC BDC's `/map/listAsOfDates` endpoint.
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* `filing_landscape` reader — the FOUR PRE-REGISTERED ACCEPTANCE
|
|
7
|
-
* is judged by. See `filing-landscape.test.ts` for the
|
|
6
|
+
* `filing_landscape` reader — the FOUR PRE-REGISTERED ACCEPTANCE CRITERIA this whole phase
|
|
7
|
+
* is judged by. See `filing-landscape.test.ts` for the criterion tests; this module is only the reader.
|
|
8
8
|
*
|
|
9
9
|
* Coverage check (the meaning-of-zero rule): a queried block counts as SURVEYED only when its res-6
|
|
10
10
|
* coverage cell is present in `layer_coverage` (via `readLayerCoverage`) — `undefined` means the area
|
|
@@ -33,13 +33,12 @@
|
|
|
33
33
|
* block excluded from `surveyed_block_count` never contributes to `filings` either.
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
|
-
import
|
|
37
|
-
import {
|
|
38
|
-
import
|
|
39
|
-
import {
|
|
40
|
-
import { sql, type Kysely } from "kysely"
|
|
36
|
+
import { readLayerCoverage, readLayerManifest } from "@mailwoman/core/layers"
|
|
37
|
+
import { shortCellToParentInt } from "@mailwoman/spatial"
|
|
38
|
+
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
39
|
+
import { sql } from "kysely"
|
|
41
40
|
|
|
42
|
-
import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, type BDCDatabase } from "
|
|
41
|
+
import { BDC_COVERAGE_H3_RESOLUTION, BDC_H3_RESOLUTION, type BDCDatabase } from "#schema"
|
|
43
42
|
|
|
44
43
|
/**
|
|
45
44
|
* Exactly one of `geoids` or `h3Cells` is required — `filingLandscape` throws otherwise.
|
|
@@ -139,24 +138,13 @@ const speedBucketCaseSQL = sql<string>`CASE
|
|
|
139
138
|
ELSE ${BDC_SPEED_BUCKET_GIGABIT}
|
|
140
139
|
END`
|
|
141
140
|
|
|
142
|
-
/**
|
|
143
|
-
* `BDCDatabase extends LayerContractDatabase` structurally, but Kysely's `transaction()` makes `Kysely<DB>` INVARIANT
|
|
144
|
-
* in `DB` — same narrowing cast as `build-bdc.ts`'s `asContractDB`.
|
|
145
|
-
*/
|
|
146
|
-
function asContractDB(kdb: DatabaseClient<BDCDatabase>): Kysely<LayerContractDatabase> {
|
|
147
|
-
return kdb as unknown as Kysely<LayerContractDatabase>
|
|
148
|
-
}
|
|
149
|
-
|
|
150
141
|
/**
|
|
151
142
|
* Reconstruct the res-6 ancestor of a res-9 short-cell int WITHOUT a centroid — see the module docstring for why the
|
|
152
143
|
* centroid is the wrong input. Exported so tests can assert this agrees, cell-for-cell, with `build-bdc.ts`'s own
|
|
153
144
|
* coverage-cell derivation (the two MUST share this derivation — see that file's docstring).
|
|
154
145
|
*/
|
|
155
146
|
export function res9ShortCellToRes6Parent(h3CellShortInt: number): number {
|
|
156
|
-
|
|
157
|
-
const parentCell = cellToParent(fullCell, BDC_COVERAGE_H3_RESOLUTION) as H3Cell
|
|
158
|
-
|
|
159
|
-
return shortCellToInt(parentCell)
|
|
147
|
+
return shortCellToParentInt(h3CellShortInt, BDC_H3_RESOLUTION, BDC_COVERAGE_H3_RESOLUTION)
|
|
160
148
|
}
|
|
161
149
|
|
|
162
150
|
/**
|
|
@@ -183,8 +171,8 @@ export async function filingLandscape(
|
|
|
183
171
|
}
|
|
184
172
|
|
|
185
173
|
// Read (and validate) the manifest FIRST — a broken/missing manifest must throw before any block is
|
|
186
|
-
// classified, never fall through to an "unstamped" answer (
|
|
187
|
-
const manifest = await readLayerManifest(
|
|
174
|
+
// classified, never fall through to an "unstamped" answer (criterion 4).
|
|
175
|
+
const manifest = await readLayerManifest(db)
|
|
188
176
|
|
|
189
177
|
const requestedUnits: ReadonlyArray<string | number> = query.geoids ?? query.h3Cells!
|
|
190
178
|
const unitColumn = query.geoids ? ("geoid" as const) : ("h3_cell" as const)
|
|
@@ -214,7 +202,7 @@ export async function filingLandscape(
|
|
|
214
202
|
let surveyedBlockCount = 0
|
|
215
203
|
let unknownBlockCount = 0
|
|
216
204
|
// Only units that PASS the coverage check feed the census below — a unit with rows but no coverage evidence
|
|
217
|
-
// (a corrupted/inconsistent db — see filing-landscape.test.ts's "coverage row deleted"
|
|
205
|
+
// (a corrupted/inconsistent db — see filing-landscape.test.ts's "coverage row deleted" case) is `unknown`, and
|
|
218
206
|
// its rows must not leak into `filings` either: `surveyed_block_count` and the blocks backing `filings` must
|
|
219
207
|
// always agree, or a caller cross-referencing the two gets a contradiction (an "unknown" block whose filings
|
|
220
208
|
// still show up looks exactly like the false-negative bug this reader exists to prevent).
|
|
@@ -230,7 +218,7 @@ export async function filingLandscape(
|
|
|
230
218
|
}
|
|
231
219
|
|
|
232
220
|
const res6Parent = res9ShortCellToRes6Parent(candidateCell)
|
|
233
|
-
const coverage = await readLayerCoverage(
|
|
221
|
+
const coverage = await readLayerCoverage(db, res6Parent)
|
|
234
222
|
|
|
235
223
|
if (coverage === undefined) {
|
|
236
224
|
unknownBlockCount++
|
package/lib/sdk/index.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export * from "#sdk/build-bdc"
|
|
2
|
+
export * from "#sdk/buildings"
|
|
3
|
+
export * from "#sdk/client"
|
|
4
|
+
export * from "#sdk/common"
|
|
5
|
+
export * from "#sdk/data-collection"
|
|
6
|
+
export * from "#sdk/download"
|
|
7
|
+
export * from "#sdk/filing-dates"
|
|
8
|
+
export * from "#sdk/filing-landscape"
|
|
9
|
+
export * from "#sdk/list-files"
|
|
10
|
+
export * from "#sdk/location"
|
|
11
|
+
export * from "#sdk/nearest-infrastructure"
|
|
12
|
+
export * from "#sdk/parsing"
|
|
13
|
+
export * from "#sdk/plausibility"
|
|
14
|
+
export * from "#sdk/technologies"
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* ascending by revision (`compareRevisionAsc`) rather than in API order.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import type { BDCClient } from "
|
|
14
|
+
import type { BDCClient } from "#sdk/client"
|
|
15
15
|
import {
|
|
16
16
|
compareRevisionAsc,
|
|
17
17
|
parseRawBDCFile,
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
type BDCStateSubCategory,
|
|
22
22
|
type BDCSummarySubCategory,
|
|
23
23
|
type RawBDCFile,
|
|
24
|
-
} from "
|
|
24
|
+
} from "#sdk/common"
|
|
25
25
|
|
|
26
26
|
export interface RetrieveProviderAvailabilityFilesParams {
|
|
27
27
|
/**
|
|
@@ -34,13 +34,13 @@
|
|
|
34
34
|
* spines agree without this module hardcoding a resolution of its own.
|
|
35
35
|
*/
|
|
36
36
|
|
|
37
|
-
import type
|
|
38
|
-
import {
|
|
39
|
-
import { POI_H3_RESOLUTION, type POILookup } from "@mailwoman/resolver-wof-sqlite/poi-lookup"
|
|
37
|
+
import { readLayerCoverage, type CoverageCell, type LayerContractHandle } from "@mailwoman/core/layers"
|
|
38
|
+
import { POI_H3_RESOLUTION, type POILookup } from "@mailwoman/resolver-wof-sqlite/poi"
|
|
40
39
|
import { shortCellToInt, type H3Cell, type PointLiteral } from "@mailwoman/spatial"
|
|
40
|
+
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
41
41
|
import { latLngToCell } from "h3-js"
|
|
42
42
|
|
|
43
|
-
import { res9ShortCellToRes6Parent } from "
|
|
43
|
+
import { res9ShortCellToRes6Parent } from "#sdk/filing-landscape"
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
46
|
* Ring budget default for {@link nearestInfrastructure} — wider than `POILookup`'s own internal `DEFAULT_MAX_RINGS`
|
|
@@ -89,7 +89,7 @@ export interface NearestInfrastructureOptions {
|
|
|
89
89
|
*/
|
|
90
90
|
export async function nearestInfrastructure(
|
|
91
91
|
poiLookup: POILookup,
|
|
92
|
-
contractDB: DatabaseClient
|
|
92
|
+
contractDB: LayerContractHandle & Pick<DatabaseClient, "destroy">,
|
|
93
93
|
options: NearestInfrastructureOptions
|
|
94
94
|
): Promise<InfrastructureHit[]> {
|
|
95
95
|
const [longitude, latitude] = options.center.coordinates
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software.
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file FCC BDC availability CSV row reader.
|
|
6
|
+
*
|
|
7
|
+
* Streams the FCC's 12-column availability CSV through `CSVSpliterator` in `array` mode and projects the
|
|
8
|
+
* seven columns 2a keeps. Columns 0-2 (`frn`, `provider_id`, `brand_name`) and 9, 11 (`state_usps`,
|
|
9
|
+
* `h3_res8_id`) are read past and never emitted: `provider_id` comes from the {@linkcode ProviderID}
|
|
10
|
+
* parameter instead (the FCC partitions availability files per provider, so the caller already knows it),
|
|
11
|
+
* and FRN/brand/state/H3 join concerns are a 2c registry-join boundary.
|
|
12
|
+
*
|
|
13
|
+
* Two projection decisions are required and pre-registered. `location_id` (column 3) stays a STRING —
|
|
14
|
+
* the FCC's values are zero-padded 10-digit strings and `parseInt` would lose the leading zeros (decision
|
|
15
|
+
* 1). `geoid` (column 10) is a string joining `TIGERBlockTable.GEOID` (decision 3).
|
|
16
|
+
*
|
|
17
|
+
* ## Why the source is a resource, not a Buffer
|
|
18
|
+
*
|
|
19
|
+
* This read is STREAMING because the files do not fit the alternative. One state × one technology —
|
|
20
|
+
* `bdc_48_FibertothePremises_fixed_broadband_D25` — is 920 MB and 10,369,043 rows, and a national run
|
|
21
|
+
* spans every state × every technology. The previous byte scanner took a `Buffer`, so its caller opened
|
|
22
|
+
* with `readFile(csvPath)` and held the whole file resident per file.
|
|
23
|
+
*
|
|
24
|
+
* ## Why quoting is not optional here
|
|
25
|
+
*
|
|
26
|
+
* Measured on that same file: 421,555 rows carry one embedded comma inside a quoted `brand_name` and 327
|
|
27
|
+
* carry two ("FiberFirst, LLC", "Valor Telecommunications of Texas, LP"). A delimiter scan blind to quotes
|
|
28
|
+
* shifts every column right of `brand_name` on 4% of rows — measured exactly, a quote-blind
|
|
29
|
+
* `String.split(",")` mismatches 81,095 of 2,000,000 real rows. `enableQuoteHandling` also keeps an
|
|
30
|
+
* embedded NEWLINE inside its row — no row in that file needs it (the 12/13/14-field line counts sum
|
|
31
|
+
* exactly to `wc -l`, so no record is split across lines), but the guarantee is what makes the reader safe
|
|
32
|
+
* on a file nobody has measured yet.
|
|
33
|
+
*
|
|
34
|
+
* ## What this costs
|
|
35
|
+
*
|
|
36
|
+
* Measured on the full 920 MB / 10,369,042-row file, projecting these 7 of 12 columns, identical checksums
|
|
37
|
+
* across every arm. One arm per process — arms sharing a process mis-rank, because the JIT warms across
|
|
38
|
+
* them:
|
|
39
|
+
*
|
|
40
|
+
* ```
|
|
41
|
+
* hand-rolled byte scan, whole buffer 778 ns/row peak RSS 978 MB
|
|
42
|
+
* spliterator 6.2.0, streaming 2,435 ns/row peak RSS 112 MB
|
|
43
|
+
* spliterator patched, streaming 860 ns/row peak RSS 100 MB
|
|
44
|
+
* ```
|
|
45
|
+
*
|
|
46
|
+
* The 6.2.0 row is why this looked like a regression when it landed: CSVSpliterator decoded once per
|
|
47
|
+
* COLUMN, and `TextDecoder`'s per-call overhead dominates at column sizes. Fixed upstream in
|
|
48
|
+
* sister-software/spliterator#6 by decoding the row once — so the streaming path now runs within 10% of a
|
|
49
|
+
* whole-buffer byte scan while holding 9.8x less memory. **Requires a spliterator release carrying that
|
|
50
|
+
* fix**; on 6.2.0 this reader is correct and roughly 3x slower.
|
|
51
|
+
*
|
|
52
|
+
* Do not route around it by hand-rolling a splitter here again. The staging insert this feeds measures 615
|
|
53
|
+
* ns/row, so the parse does sit on the critical path — but a local fast-and-quote-blind parser is exactly
|
|
54
|
+
* the trade that produced the scanner this replaced, and the quote-blind version is 4% wrong.
|
|
55
|
+
*/
|
|
56
|
+
|
|
57
|
+
import type { AsyncDataResource } from "spliterator"
|
|
58
|
+
import { CSVSpliterator } from "spliterator"
|
|
59
|
+
|
|
60
|
+
import type { ProviderID } from "#sdk/common"
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Column positions in the FCC's 12-column availability CSV. Named rather than sliced by offset so a reader can check
|
|
64
|
+
* them against the header row without counting commas:
|
|
65
|
+
*
|
|
66
|
+
* `frn,provider_id,brand_name,location_id,technology,max_advertised_download_speed,`
|
|
67
|
+
* `max_advertised_upload_speed,low_latency,business_residential_code,state_usps,block_geoid,h3_res8_id`
|
|
68
|
+
*/
|
|
69
|
+
const Column = {
|
|
70
|
+
LocationID: 3,
|
|
71
|
+
Technology: 4,
|
|
72
|
+
MaxAdvertisedDownloadSpeed: 5,
|
|
73
|
+
MaxAdvertisedUploadSpeed: 6,
|
|
74
|
+
LowLatency: 7,
|
|
75
|
+
BusinessResidentialCode: 8,
|
|
76
|
+
BlockGeoID: 10,
|
|
77
|
+
} as const
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* A single parsed row of FCC BDC availability data.
|
|
81
|
+
*
|
|
82
|
+
* @see {@linkcode readAvailabilityRows}
|
|
83
|
+
*/
|
|
84
|
+
export interface BDCAvailabilityRow {
|
|
85
|
+
provider_id: number
|
|
86
|
+
/**
|
|
87
|
+
* Kept as a string — the FCC's `location_id` values are zero-padded 10-digit strings; `parseInt`ing would lose
|
|
88
|
+
* leading zeros (2a decision 1).
|
|
89
|
+
*/
|
|
90
|
+
location_id: string
|
|
91
|
+
technology_code: number
|
|
92
|
+
max_advertised_download_speed: number
|
|
93
|
+
max_advertised_upload_speed: number
|
|
94
|
+
low_latency: 0 | 1
|
|
95
|
+
business_residential_code: string
|
|
96
|
+
/**
|
|
97
|
+
* Joins `TIGERBlockTable.GEOID` (note: uppercase column on that side) — 2a decision 3.
|
|
98
|
+
*/
|
|
99
|
+
geoid: string
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Project one already-split CSV row onto {@linkcode BDCAvailabilityRow}.
|
|
104
|
+
*
|
|
105
|
+
* Separate from the iteration so the sync and async readers cannot drift in what they emit — the one-function
|
|
106
|
+
* discipline. Not exported: a caller with a split row wants {@linkcode readAvailabilityRowsSync}.
|
|
107
|
+
*/
|
|
108
|
+
function projectRow(columns: readonly string[], providerID: ProviderID): BDCAvailabilityRow {
|
|
109
|
+
return {
|
|
110
|
+
provider_id: providerID,
|
|
111
|
+
location_id: columns[Column.LocationID] ?? "",
|
|
112
|
+
technology_code: Number.parseInt(columns[Column.Technology] ?? "", 10),
|
|
113
|
+
max_advertised_download_speed: Number.parseInt(columns[Column.MaxAdvertisedDownloadSpeed] ?? "", 10),
|
|
114
|
+
max_advertised_upload_speed: Number.parseInt(columns[Column.MaxAdvertisedUploadSpeed] ?? "", 10),
|
|
115
|
+
low_latency: columns[Column.LowLatency] === "1" ? 1 : 0,
|
|
116
|
+
business_residential_code: columns[Column.BusinessResidentialCode] ?? "",
|
|
117
|
+
geoid: columns[Column.BlockGeoID] ?? "",
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Shared reader options. `header: true` consumes the first row as the header even in `array` mode;
|
|
123
|
+
* `enableQuoteHandling` is what makes the 421,882 quoted-brand rows keep their column alignment. `crlf` already
|
|
124
|
+
* defaults to `true` for CSV (RFC 4180), so a CRLF file does not leak `\r` into the last column.
|
|
125
|
+
*/
|
|
126
|
+
const READER_OPTIONS = { mode: "array", enableQuoteHandling: true } as const
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Stream an FCC BDC availability CSV, yielding every data row. The header row is consumed, never emitted.
|
|
130
|
+
*
|
|
131
|
+
* `source` is anything `spliterator` can open asynchronously — a path string, a `path-ts` builder, a URL, a file
|
|
132
|
+
* handle, or an async chunk iterator. Prefer handing it the path and letting it own the read: that is what keeps a 920
|
|
133
|
+
* MB file off the heap.
|
|
134
|
+
*/
|
|
135
|
+
export async function* readAvailabilityRows(
|
|
136
|
+
source: AsyncDataResource,
|
|
137
|
+
providerID: ProviderID
|
|
138
|
+
): AsyncIterable<BDCAvailabilityRow> {
|
|
139
|
+
for await (const columns of CSVSpliterator.fromAsync<string[]>(source, READER_OPTIONS)) {
|
|
140
|
+
yield projectRow(columns, providerID)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Synchronous sibling for an in-memory buffer — fixtures and tests, never the build path.
|
|
146
|
+
*
|
|
147
|
+
* Kept because a test that must assert on a literal CSV should not have to stand up a stream to do it. If you are
|
|
148
|
+
* reaching for this against a file on disk, reach for {@linkcode readAvailabilityRows} instead.
|
|
149
|
+
*/
|
|
150
|
+
export function* readAvailabilityRowsSync(
|
|
151
|
+
csvBuffer: Buffer | string,
|
|
152
|
+
providerID: ProviderID
|
|
153
|
+
): Iterable<BDCAvailabilityRow> {
|
|
154
|
+
for (const columns of CSVSpliterator.from<string[]>(csvBuffer, READER_OPTIONS)) {
|
|
155
|
+
yield projectRow(columns, providerID)
|
|
156
|
+
}
|
|
157
|
+
}
|