@mailwoman/bdc 9.0.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/sdk/download.d.ts +0 -11
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +1 -37
- package/out/sdk/download.js.map +1 -1
- package/package.json +6 -6
- package/sdk/download.ts +1 -41
package/out/sdk/download.d.ts
CHANGED
|
@@ -3,17 +3,6 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
* @file FCC BDC availability-file download + zip extraction.
|
|
6
|
-
*
|
|
7
|
-
* Re-homed from Nexus's `sync/fcc/bdc/download-file.ts` (relicense-by-copy, no provenance headers),
|
|
8
|
-
* trimmed hard: the Nexus original downloaded AND cached the `.zip`, extracted it, THEN wrote a Parquet
|
|
9
|
-
* file with a row-count integrity check. All Parquet machinery is dropped here — `bdc.db` is
|
|
10
|
-
* SQLite, not Parquet-backed (`bdc/schema.ts`) — and the `.zip` itself isn't cached either; only the
|
|
11
|
-
* extracted CSV is written to `destinationDir`, and its presence alone is the cache check.
|
|
12
|
-
*
|
|
13
|
-
* The zip-extraction library also changes: the Nexus original's `extractSingleFileZip` used `adm-zip`
|
|
14
|
-
* (a repo-wide Nexus dependency). No unzip dependency exists anywhere in this repo — every workspace
|
|
15
|
-
* `package.json` was checked, `tiger/` and `osm/` included — so `yauzl-promise` is a `bdc`-only
|
|
16
|
-
* dependency.
|
|
17
6
|
*/
|
|
18
7
|
import type { BDCClient } from "./client.ts";
|
|
19
8
|
import { type BDCFile } from "./common.ts";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAqB,KAAK,OAAO,EAAE,MAAM,aAAa,CAAA;AAE7D;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAoB/G"}
|
package/out/sdk/download.js
CHANGED
|
@@ -3,47 +3,11 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
* @file FCC BDC availability-file download + zip extraction.
|
|
6
|
-
*
|
|
7
|
-
* Re-homed from Nexus's `sync/fcc/bdc/download-file.ts` (relicense-by-copy, no provenance headers),
|
|
8
|
-
* trimmed hard: the Nexus original downloaded AND cached the `.zip`, extracted it, THEN wrote a Parquet
|
|
9
|
-
* file with a row-count integrity check. All Parquet machinery is dropped here — `bdc.db` is
|
|
10
|
-
* SQLite, not Parquet-backed (`bdc/schema.ts`) — and the `.zip` itself isn't cached either; only the
|
|
11
|
-
* extracted CSV is written to `destinationDir`, and its presence alone is the cache check.
|
|
12
|
-
*
|
|
13
|
-
* The zip-extraction library also changes: the Nexus original's `extractSingleFileZip` used `adm-zip`
|
|
14
|
-
* (a repo-wide Nexus dependency). No unzip dependency exists anywhere in this repo — every workspace
|
|
15
|
-
* `package.json` was checked, `tiger/` and `osm/` included — so `yauzl-promise` is a `bdc`-only
|
|
16
|
-
* dependency.
|
|
17
6
|
*/
|
|
18
7
|
import * as fs from "node:fs/promises";
|
|
19
8
|
import * as path from "node:path";
|
|
20
|
-
import {
|
|
9
|
+
import { extractSingleFileZip } from "@mailwoman/core/fs/zip";
|
|
21
10
|
import { BDCFilingDataType } from "./common.js";
|
|
22
|
-
/**
|
|
23
|
-
* Extract the first file entry of a zip archive buffer into a single in-memory `Buffer`.
|
|
24
|
-
*
|
|
25
|
-
* BDC availability downloads are always a single zip-wrapped CSV, so — like the Nexus original — this doesn't walk
|
|
26
|
-
* every entry, just the first non-directory one.
|
|
27
|
-
*/
|
|
28
|
-
async function extractSingleFileZip(zippedBuffer) {
|
|
29
|
-
const zip = await fromBuffer(zippedBuffer);
|
|
30
|
-
try {
|
|
31
|
-
for await (const entry of zip) {
|
|
32
|
-
if (entry.filename.endsWith("/"))
|
|
33
|
-
continue;
|
|
34
|
-
const readStream = await entry.openReadStream();
|
|
35
|
-
const chunks = [];
|
|
36
|
-
for await (const chunk of readStream) {
|
|
37
|
-
chunks.push(chunk);
|
|
38
|
-
}
|
|
39
|
-
return Buffer.concat(chunks);
|
|
40
|
-
}
|
|
41
|
-
throw new Error("extractSingleFileZip: no file entries found in zip archive.");
|
|
42
|
-
}
|
|
43
|
-
finally {
|
|
44
|
-
await zip.close();
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
11
|
/**
|
|
48
12
|
* Download and cache an FCC BDC availability file, extracting its zip-wrapped CSV to `destinationDir`.
|
|
49
13
|
*
|
package/out/sdk/download.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AAEjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAG7D,OAAO,EAAE,iBAAiB,EAAgB,MAAM,aAAa,CAAA;AAE7D;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAiB,EAAE,IAAa,EAAE,cAAsB;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,QAAQ,MAAM,CAAC,CAAA;IAEjE,MAAM,aAAa,GAAG,MAAM,EAAE;SAC5B,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;IAEpB,IAAI,aAAa;QAAE,OAAO,OAAO,CAAA;IAEjC,MAAM,iBAAiB,GAAG,MAAM,MAAM,CAAC,cAAc,CACpD,+BAA+B,iBAAiB,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,EAAE,CAC9E,CAAA;IAED,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAA;IAE5E,MAAM,EAAE,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;IAEtC,OAAO,OAAO,CAAA;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/bdc",
|
|
3
|
-
"version": "9.
|
|
3
|
+
"version": "9.1.0",
|
|
4
4
|
"description": "FCC Broadband Data Collection (BDC) availability data — fetch/parse/ingest provider and bdc.db layer reader",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"bdc",
|
|
@@ -78,11 +78,11 @@
|
|
|
78
78
|
"test": "vitest run"
|
|
79
79
|
},
|
|
80
80
|
"dependencies": {
|
|
81
|
-
"@mailwoman/core": "9.
|
|
82
|
-
"@mailwoman/filer": "9.
|
|
83
|
-
"@mailwoman/resolver-wof-sqlite": "9.
|
|
84
|
-
"@mailwoman/spatial": "9.
|
|
85
|
-
"@mailwoman/tiger": "9.
|
|
81
|
+
"@mailwoman/core": "9.1.0",
|
|
82
|
+
"@mailwoman/filer": "9.1.0",
|
|
83
|
+
"@mailwoman/resolver-wof-sqlite": "9.1.0",
|
|
84
|
+
"@mailwoman/spatial": "9.1.0",
|
|
85
|
+
"@mailwoman/tiger": "9.1.0",
|
|
86
86
|
"h3-js": "^4.5.0",
|
|
87
87
|
"kysely": "^0.29.4",
|
|
88
88
|
"type-fest": "^5.8.0",
|
package/sdk/download.ts
CHANGED
|
@@ -3,56 +3,16 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
* @file FCC BDC availability-file download + zip extraction.
|
|
6
|
-
*
|
|
7
|
-
* Re-homed from Nexus's `sync/fcc/bdc/download-file.ts` (relicense-by-copy, no provenance headers),
|
|
8
|
-
* trimmed hard: the Nexus original downloaded AND cached the `.zip`, extracted it, THEN wrote a Parquet
|
|
9
|
-
* file with a row-count integrity check. All Parquet machinery is dropped here — `bdc.db` is
|
|
10
|
-
* SQLite, not Parquet-backed (`bdc/schema.ts`) — and the `.zip` itself isn't cached either; only the
|
|
11
|
-
* extracted CSV is written to `destinationDir`, and its presence alone is the cache check.
|
|
12
|
-
*
|
|
13
|
-
* The zip-extraction library also changes: the Nexus original's `extractSingleFileZip` used `adm-zip`
|
|
14
|
-
* (a repo-wide Nexus dependency). No unzip dependency exists anywhere in this repo — every workspace
|
|
15
|
-
* `package.json` was checked, `tiger/` and `osm/` included — so `yauzl-promise` is a `bdc`-only
|
|
16
|
-
* dependency.
|
|
17
6
|
*/
|
|
18
7
|
|
|
19
8
|
import * as fs from "node:fs/promises"
|
|
20
9
|
import * as path from "node:path"
|
|
21
10
|
|
|
22
|
-
import {
|
|
11
|
+
import { extractSingleFileZip } from "@mailwoman/core/fs/zip"
|
|
23
12
|
|
|
24
13
|
import type { BDCClient } from "./client.ts"
|
|
25
14
|
import { BDCFilingDataType, type BDCFile } from "./common.ts"
|
|
26
15
|
|
|
27
|
-
/**
|
|
28
|
-
* Extract the first file entry of a zip archive buffer into a single in-memory `Buffer`.
|
|
29
|
-
*
|
|
30
|
-
* BDC availability downloads are always a single zip-wrapped CSV, so — like the Nexus original — this doesn't walk
|
|
31
|
-
* every entry, just the first non-directory one.
|
|
32
|
-
*/
|
|
33
|
-
async function extractSingleFileZip(zippedBuffer: Buffer): Promise<Buffer> {
|
|
34
|
-
const zip = await fromBuffer(zippedBuffer)
|
|
35
|
-
|
|
36
|
-
try {
|
|
37
|
-
for await (const entry of zip) {
|
|
38
|
-
if (entry.filename.endsWith("/")) continue
|
|
39
|
-
|
|
40
|
-
const readStream = await entry.openReadStream()
|
|
41
|
-
const chunks: Buffer[] = []
|
|
42
|
-
|
|
43
|
-
for await (const chunk of readStream) {
|
|
44
|
-
chunks.push(chunk)
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
return Buffer.concat(chunks)
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
throw new Error("extractSingleFileZip: no file entries found in zip archive.")
|
|
51
|
-
} finally {
|
|
52
|
-
await zip.close()
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
16
|
/**
|
|
57
17
|
* Download and cache an FCC BDC availability file, extracting its zip-wrapped CSV to `destinationDir`.
|
|
58
18
|
*
|